1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::MeerkatId;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images, send_message_on_mob_with_mode,
24};
25use crate::runtime::ConsoleMember;
26use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
27
28pub use state::{
29 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
30 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
31 SourceIngestionTransition,
32};
33pub use store::{
34 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
35 SqliteConsoleLogStore,
36};
37pub use types::{
38 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
39 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
40 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
41 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
42 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
43};
44
45const TIMELINE_CHANNEL_CAP: usize = 1024;
46const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
47const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
48const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
49const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
50const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
51const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
52const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
53const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
54const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57enum IdentityCollectionMode {
58 CachedOnly,
59 IncludeLiveMembers,
60}
61
62#[derive(Clone)]
63pub struct MobKitConsoleAggregator {
64 inner: Arc<AggregatorInner>,
65}
66
67#[derive(Debug, Clone, Copy)]
68pub struct ConsoleAggregatorOptions {
69 pub session_history_backfill_enabled: bool,
70 pub max_concurrent_session_backfills: usize,
71}
72
73impl Default for ConsoleAggregatorOptions {
74 fn default() -> Self {
75 Self {
76 session_history_backfill_enabled: true,
77 max_concurrent_session_backfills: 16,
78 }
79 }
80}
81
82struct AggregatorInner {
83 store: Arc<dyn ConsoleLogStore>,
84 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
85 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
86 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
87 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 session_backfill_permits: Arc<Semaphore>,
89 identity_read_model: ConsoleIdentityReadModel,
90 options: ConsoleAggregatorOptions,
91}
92
93#[derive(Clone)]
94struct ConsoleIdentityReadModel {
95 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
96 refresh_lock: Arc<tokio::sync::Mutex<()>>,
97 primed: Arc<AtomicBool>,
98}
99
100impl Default for ConsoleIdentityReadModel {
101 fn default() -> Self {
102 Self {
103 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
104 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
105 primed: Arc::new(AtomicBool::new(false)),
106 }
107 }
108}
109
110impl ConsoleIdentityReadModel {
111 async fn snapshot(
112 &self,
113 inner: Arc<AggregatorInner>,
114 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
115 if !self.primed.load(Ordering::Acquire) {
116 self.prime_now(inner).await?;
117 }
118 Ok(self.inner.read().await.clone())
119 }
120
121 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
122 self.inner.read().await.clone()
123 }
124
125 async fn refresh_now_if_idle(
126 &self,
127 inner: Arc<AggregatorInner>,
128 mode: IdentityCollectionMode,
129 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
130 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
131 return None;
132 };
133 let result = collect_identity_records(&inner, mode).await;
134 drop(guard);
135 match result {
136 Ok(identities) => {
137 self.replace(identities.clone()).await;
138 Some(Ok(identities))
139 }
140 Err(err) => Some(Err(err)),
141 }
142 }
143
144 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
145 if self.primed.load(Ordering::Acquire) {
146 return Ok(());
147 }
148 let _guard = self.refresh_lock.clone().lock_owned().await;
149 if self.primed.load(Ordering::Acquire) {
150 return Ok(());
151 }
152 let identities =
153 collect_identity_records(&inner, IdentityCollectionMode::CachedOnly).await?;
154 *self.inner.write().await = identities;
155 self.primed.store(true, Ordering::Release);
156 Ok(())
157 }
158
159 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
160 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
161 return;
162 };
163 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
164 return;
165 };
166 let read_model = self.clone();
167 runtime_handle.spawn(async move {
168 let _guard = guard;
169 match collect_identity_records(&inner, IdentityCollectionMode::IncludeLiveMembers).await
170 {
171 Ok(identities) => {
172 *read_model.inner.write().await = identities;
173 read_model.primed.store(true, Ordering::Release);
174 }
175 Err(err) => {
176 tracing::warn!(error = %err, "console identity read-model refresh failed");
177 }
178 }
179 });
180 }
181
182 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
183 *self.inner.write().await = identities;
184 self.primed.store(true, Ordering::Release);
185 }
186}
187
188#[derive(Clone)]
189struct RuntimeEntry {
190 runtime_key: String,
191 identity_namespace: String,
192 runtime: MobRuntime,
193 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
194 console_events: ConsoleEventStore,
195 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
196}
197
198#[derive(Clone)]
199struct ResolvedConsoleMember {
200 entry: RuntimeEntry,
201 handle: MobHandle,
202 member: MobMemberListEntry,
203 source_mob_id: String,
204 runtime_identity: String,
205}
206
207fn console_member_for_resolved_member(
208 resolved: &ResolvedConsoleMember,
209 record: &ConsoleIdentityRecord,
210) -> ConsoleMember {
211 ConsoleMember {
212 agent_identity: resolved.member.agent_identity.to_string(),
213 role: resolved.member.role.to_string(),
214 state: format!("{:?}", resolved.member.state),
215 model_capabilities: Default::default(),
216 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
217 session_id: record.session_id.clone(),
218 wired_to: resolved
219 .member
220 .wired_to
221 .iter()
222 .map(ToString::to_string)
223 .collect(),
224 labels: record.labels.clone(),
225 }
226}
227
228async fn resolved_member_visible(
229 resolved: &ResolvedConsoleMember,
230 record: &ConsoleIdentityRecord,
231) -> bool {
232 if !raw_resolved_member_visible(resolved, record) {
233 return false;
234 }
235 !live_record_shadowed_by_hidden_durable_binding(resolved, record).await
236}
237
238fn raw_resolved_member_visible(
239 resolved: &ResolvedConsoleMember,
240 record: &ConsoleIdentityRecord,
241) -> bool {
242 resolved.entry.visibility_policy.identity_visible(record)
243 && resolved
244 .entry
245 .visibility_policy
246 .member_visible(&console_member_for_resolved_member(resolved, record))
247}
248
249pub trait ConsoleVisibilityPolicy: Send + Sync {
250 fn include_implicit_delegate_members(&self) -> bool {
251 true
252 }
253
254 fn member_visible(&self, _member: &ConsoleMember) -> bool {
255 true
256 }
257
258 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
259 true
260 }
261
262 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
263 true
264 }
265
266 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
267 None
268 }
269}
270
271#[derive(Debug, Default)]
272pub struct AllowAllConsoleVisibilityPolicy;
273
274impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
275
276#[derive(Debug, Default)]
277pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
278
279impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
280 fn include_implicit_delegate_members(&self) -> bool {
281 false
282 }
283
284 fn member_visible(&self, member: &ConsoleMember) -> bool {
285 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
286 }
287
288 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
289 !is_implicit_delegate_member(
290 record
291 .labels
292 .get("role")
293 .map(String::as_str)
294 .unwrap_or_default(),
295 &record.labels,
296 )
297 }
298}
299
300#[derive(Clone)]
301pub struct ConsoleRuntimeRegistration {
302 pub runtime_key: String,
303 pub runtime: Arc<UnifiedRuntime>,
304 pub identity_namespace: String,
305 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
306}
307
308impl MobKitConsoleAggregator {
309 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
310 Self::new_with_options(store, ConsoleAggregatorOptions::default())
311 }
312
313 pub fn new_with_options(
314 store: Arc<dyn ConsoleLogStore>,
315 mut options: ConsoleAggregatorOptions,
316 ) -> Self {
317 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
318 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
319 Self {
320 inner: Arc::new(AggregatorInner {
321 store,
322 runtimes: RwLock::new(BTreeMap::new()),
323 event_tx,
324 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
325 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
326 session_backfill_permits: Arc::new(Semaphore::new(
327 options.max_concurrent_session_backfills,
328 )),
329 identity_read_model: ConsoleIdentityReadModel::default(),
330 options,
331 }),
332 }
333 }
334
335 pub fn in_memory() -> Self {
336 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
337 }
338
339 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
340 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
341 }
342
343 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
344 self.inner.event_tx.subscribe()
345 }
346
347 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
348 self.inner.store.clone()
349 }
350
351 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
352 let identity_runtime = registration.runtime.identity_runtime().cloned();
353 self.register_runtime_handles_with_policy(
354 registration.runtime_key,
355 registration.identity_namespace,
356 registration.runtime.mob_runtime().clone(),
357 identity_runtime,
358 registration.runtime.console_events(),
359 registration.visibility_policy,
360 );
361 }
362
363 pub(crate) fn register_runtime_handles_with_policy(
364 &self,
365 runtime_key: impl Into<String>,
366 identity_namespace: impl Into<String>,
367 runtime: MobRuntime,
368 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
369 console_events: ConsoleEventStore,
370 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
371 ) {
372 let runtime_key = runtime_key.into();
373 let identity_namespace = identity_namespace.into();
374 let entry = RuntimeEntry {
375 runtime_key: runtime_key.clone(),
376 identity_namespace,
377 runtime,
378 identity_runtime,
379 console_events: console_events.clone(),
380 visibility_policy,
381 };
382 if let Ok(mut runtimes) = self.inner.runtimes.write() {
383 runtimes.insert(runtime_key.clone(), entry);
384 }
385 self.inner
386 .identity_read_model
387 .refresh_soon(self.inner.clone());
388 let inner = self.inner.clone();
389 let events_for_live = console_events.clone();
390 let events_for_live_recovery = console_events.clone();
391 let runtime_key_for_live = runtime_key.clone();
392 tokio::spawn(async move {
393 let mut rx = events_for_live.subscribe();
394 loop {
395 match rx.recv().await {
396 Ok(envelope) => {
397 let _ =
398 project_console_event(inner.clone(), &runtime_key_for_live, envelope)
399 .await;
400 }
401 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
402 let _ = recover_lagged_source_events(
403 inner.clone(),
404 &runtime_key_for_live,
405 &events_for_live_recovery,
406 )
407 .await;
408 }
409 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
410 }
411 }
412 });
413
414 let inner = self.inner.clone();
415 let events_for_replay = console_events;
416 let runtime_key_for_replay = runtime_key;
417 tokio::spawn(async move {
418 let mut ingestion_state = SourceIngestionState::Registered;
419 if let Ok((next, _effects)) =
420 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
421 {
422 ingestion_state = next;
423 }
424 if let Ok(events) = events_for_replay.replay_all(None).await {
425 for envelope in events {
426 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
427 .await;
428 }
429 }
430 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
431 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
432 if let Ok((next, _effects)) =
433 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
434 {
435 ingestion_state = next;
436 }
437 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
438 });
439 }
440
441 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
442 if let Some(identities) = self
443 .inner
444 .identity_read_model
445 .refresh_now_if_idle(
446 self.inner.clone(),
447 IdentityCollectionMode::IncludeLiveMembers,
448 )
449 .await
450 {
451 let identities = identities?;
452 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
453 return Ok(identities);
454 }
455 self.inner
456 .identity_read_model
457 .refresh_soon(self.inner.clone());
458 let identities = self
459 .inner
460 .identity_read_model
461 .snapshot(self.inner.clone())
462 .await?;
463 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464 Ok(identities)
465 }
466
467 #[cfg(test)]
468 pub(crate) async fn list_identities_fresh(
469 &self,
470 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
471 let _guard = self
472 .inner
473 .identity_read_model
474 .refresh_lock
475 .clone()
476 .lock_owned()
477 .await;
478 let identities =
479 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
480 .await?;
481 self.inner
482 .identity_read_model
483 .replace(identities.clone())
484 .await;
485 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
486 Ok(identities)
487 }
488
489 pub async fn inspect_identity(
490 &self,
491 identity: &str,
492 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
493 let entries = self
494 .inner
495 .runtimes
496 .read()
497 .map_err(|_| runtime_registry_lock_error())?
498 .values()
499 .cloned()
500 .collect::<Vec<_>>();
501 let live_records = self.live_records_for_identity(identity).await;
502 let mut durable_matches = Vec::new();
503 if !requested_identity_has_runtime_member_alias(identity, &entries) {
504 for entry in &entries {
505 let Some(identity_runtime) = entry.identity_runtime.clone() else {
506 continue;
507 };
508 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
509 {
510 let Ok(parsed_identity) =
511 crate::identity_first::AgentIdentity::parse(&raw_identity)
512 else {
513 continue;
514 };
515 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
516 continue;
517 };
518 let mut record = identity_record_for_status(entry, &status);
519 let topology_peers =
520 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
521 record.topology_peers = topology_peers
522 .get(&record.identity)
523 .cloned()
524 .unwrap_or_default();
525 if console_identity_record_visible(entry, &record).await {
526 durable_matches.push((entry.clone(), record));
527 }
528 }
529 }
530 }
531 if durable_matches.len() > 1 {
532 let candidates = durable_matches
533 .iter()
534 .map(|(_, record)| record.runtime_member_id.clone())
535 .collect::<Vec<_>>()
536 .join(", ");
537 return Err(format!(
538 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
539 )
540 .into());
541 }
542 if let Some((entry, record)) = durable_matches.into_iter().next() {
543 let durable_live_records = self
544 .live_records_for_durable_record(&entry, identity, &record, &live_records)
545 .await;
546 let visible_durable_live_records =
547 visible_live_records_for_entry(&entry, &durable_live_records).await;
548 if let Some(ambiguous_error) =
549 ambiguous_live_alias_error(identity, &visible_durable_live_records)
550 {
551 return Err(ambiguous_error.into());
552 }
553 if let Some(stale_error) =
554 stale_durable_record_error(identity, &record, &durable_live_records)
555 {
556 return Err(stale_error.into());
557 }
558 if let Some(session_id) = record.session_id.clone() {
559 spawn_session_history_backfill_target(
560 self.inner.clone(),
561 SessionBackfillTarget {
562 entry,
563 record: record.clone(),
564 session_id,
565 },
566 false,
567 );
568 }
569 return Ok(Some(ConsoleIdentityInspection {
570 peers: record.topology_peers.clone(),
571 identity: record,
572 }));
573 }
574 let mut runtime_id_matches = Vec::new();
575 for entry in &entries {
576 let Some(identity_runtime) = entry.identity_runtime.clone() else {
577 continue;
578 };
579 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
580 continue;
581 };
582 let Some(status) = identity_runtime
583 .statuses()
584 .await
585 .into_iter()
586 .find(|status| {
587 status
588 .agent_runtime_id
589 .as_ref()
590 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
591 })
592 else {
593 continue;
594 };
595 let mut record = identity_record_for_status(entry, &status);
596 let topology_peers =
597 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
598 record.topology_peers = topology_peers
599 .get(&record.identity)
600 .cloned()
601 .unwrap_or_default();
602 if console_identity_record_visible(entry, &record).await {
603 runtime_id_matches.push((entry.clone(), record));
604 }
605 }
606 if runtime_id_matches.len() > 1 {
607 let candidates = runtime_id_matches
608 .iter()
609 .map(|(_, record)| record.identity.clone())
610 .collect::<Vec<_>>()
611 .join(", ");
612 return Err(format!(
613 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
614 )
615 .into());
616 }
617 if let Some((entry, record)) = runtime_id_matches.into_iter().next() {
618 let durable_live_records = self
619 .live_records_for_durable_record(&entry, identity, &record, &live_records)
620 .await;
621 let visible_durable_live_records =
622 visible_live_records_for_entry(&entry, &durable_live_records).await;
623 if let Some(ambiguous_error) =
624 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
625 {
626 return Err(ambiguous_error.into());
627 }
628 if let Some(stale_error) =
629 stale_durable_record_error(identity, &record, &durable_live_records)
630 {
631 return Err(stale_error.into());
632 }
633 if let Some(session_id) = record.session_id.clone() {
634 spawn_session_history_backfill_target(
635 self.inner.clone(),
636 SessionBackfillTarget {
637 entry: entry.clone(),
638 record: record.clone(),
639 session_id,
640 },
641 false,
642 );
643 }
644 return Ok(Some(ConsoleIdentityInspection {
645 peers: record.topology_peers.clone(),
646 identity: record,
647 }));
648 }
649
650 let live_matches = self.resolve_visible_members(identity).await;
651 if live_matches.len() > 1 {
652 let candidates = live_matches
653 .iter()
654 .map(|resolved| resolved.runtime_identity.clone())
655 .collect::<Vec<_>>()
656 .join(", ");
657 return Err(format!(
658 "ambiguous live identity alias {identity}: candidates [{candidates}]"
659 )
660 .into());
661 }
662 if let Some(resolved) = live_matches.into_iter().next() {
663 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
664 return Ok(None);
665 };
666 if let Some(stale_error) =
667 stale_live_record_binding_error(&resolved.entry, identity, &record).await
668 {
669 return Err(stale_error.into());
670 }
671 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
672 && let Ok(parsed_identity) =
673 crate::identity_first::AgentIdentity::parse(&record.identity)
674 && let Ok(status) = identity_runtime.status(&parsed_identity).await
675 {
676 let durable_record = identity_record_for_status(&resolved.entry, &status);
677 let durable_live_records = self
678 .live_records_for_durable_record(
679 &resolved.entry,
680 identity,
681 &durable_record,
682 &live_records,
683 )
684 .await;
685 let visible_durable_live_records =
686 visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
687 if let Some(ambiguous_error) =
688 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
689 {
690 return Err(ambiguous_error.into());
691 }
692 if let Some(stale_error) =
693 stale_durable_record_error(identity, &durable_record, &durable_live_records)
694 {
695 return Err(stale_error.into());
696 }
697 if status
698 .agent_runtime_id
699 .as_ref()
700 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
701 {
702 return Ok(None);
703 }
704 }
705 if !resolved_member_visible(&resolved, &record).await {
706 return Ok(None);
707 }
708 if let Some(session_id) = record.session_id.clone() {
709 spawn_session_history_backfill_target(
710 self.inner.clone(),
711 SessionBackfillTarget {
712 entry: resolved.entry.clone(),
713 record: record.clone(),
714 session_id,
715 },
716 false,
717 );
718 }
719 let peers = resolved
720 .member
721 .wired_to
722 .iter()
723 .map(ToString::to_string)
724 .collect();
725 return Ok(Some(ConsoleIdentityInspection {
726 identity: record,
727 peers,
728 }));
729 }
730
731 Ok(None)
732 }
733
734 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
735 let entries = self
736 .inner
737 .runtimes
738 .read()
739 .map_err(|_| runtime_registry_lock_error())?
740 .clone();
741 let live_records = self.live_records_for_identity(identity).await;
742 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
743 let mut durable_matches = Vec::new();
744 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
745 for entry in entries.values() {
746 let Some(identity_runtime) = entry.identity_runtime.clone() else {
747 continue;
748 };
749 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
750 {
751 let Ok(parsed_identity) =
752 crate::identity_first::AgentIdentity::parse(&raw_identity)
753 else {
754 continue;
755 };
756 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
757 continue;
758 };
759 let record = identity_record_for_status(entry, &status);
760 if !console_identity_record_visible(entry, &record).await {
761 continue;
762 }
763 durable_matches.push((
764 entry.clone(),
765 identity_runtime.clone(),
766 parsed_identity,
767 record,
768 ));
769 }
770 }
771 }
772 if durable_matches.len() > 1 {
773 let candidates = durable_matches
774 .iter()
775 .map(|(_, _, identity, _)| identity.as_str().to_string())
776 .collect::<Vec<_>>()
777 .join(", ");
778 return Err(format!(
779 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
780 )
781 .into());
782 }
783 if let Some((entry, identity_runtime, parsed_identity, record)) =
784 durable_matches.into_iter().next()
785 {
786 let durable_live_records = self
787 .live_records_for_durable_record(&entry, identity, &record, &live_records)
788 .await;
789 let visible_durable_live_records =
790 visible_live_records_for_entry(&entry, &durable_live_records).await;
791 if let Some(ambiguous_error) =
792 ambiguous_live_alias_error(identity, &visible_durable_live_records)
793 {
794 return Err(ambiguous_error.into());
795 }
796 if let Some(stale_error) =
797 stale_durable_record_error(identity, &record, &durable_live_records)
798 {
799 return Err(stale_error.into());
800 }
801 identity_runtime
802 .retire(&parsed_identity)
803 .await
804 .map_err(|err| -> ConsoleLogError {
805 format!("retire failed for {identity}: {err}").into()
806 })?;
807 {
808 let mut durable_identities =
809 namespace_match_candidates(identity, &entry.identity_namespace);
810 durable_identities.push(parsed_identity.as_str().to_string());
811 durable_identities.sort();
812 durable_identities.dedup();
813 if let Err(err) =
814 retire_stale_console_members_for_runtime_entry(&entry, &durable_identities)
815 .await
816 {
817 tracing::warn!(
818 identity,
819 error = %err,
820 "stale console member cleanup failed after durable identity retire"
821 );
822 }
823 }
824 return Ok(true);
825 }
826 let matches = self.resolve_visible_members(identity).await;
827 if matches.len() > 1 {
828 let candidates = matches
829 .iter()
830 .map(|resolved| resolved.runtime_identity.clone())
831 .collect::<Vec<_>>()
832 .join(", ");
833 return Err(format!(
834 "ambiguous live identity alias {identity}: candidates [{candidates}]"
835 )
836 .into());
837 }
838 let mut live_retired_any = false;
839 for resolved in matches {
840 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
841 continue;
842 };
843 if !resolved_member_visible(&resolved, &record).await {
844 continue;
845 }
846 if let Some(stale_error) =
847 stale_live_record_binding_error(&resolved.entry, identity, &record).await
848 {
849 return Err(stale_error.into());
850 }
851 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
852 && let Ok(parsed_identity) =
853 crate::identity_first::AgentIdentity::parse(&record.identity)
854 && let Ok(status) = identity_runtime.status(&parsed_identity).await
855 {
856 let durable_record = identity_record_for_status(&resolved.entry, &status);
857 let durable_live_records = self
858 .live_records_for_durable_record(
859 &resolved.entry,
860 identity,
861 &durable_record,
862 &live_records,
863 )
864 .await;
865 let visible_durable_live_records =
866 visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
867 if let Some(ambiguous_error) =
868 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
869 {
870 return Err(ambiguous_error.into());
871 }
872 if let Some(stale_error) =
873 stale_durable_record_error(identity, &durable_record, &durable_live_records)
874 {
875 return Err(stale_error.into());
876 }
877 if status
878 .agent_runtime_id
879 .as_ref()
880 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
881 {
882 continue;
883 }
884 }
885 resolved
886 .handle
887 .retire(MeerkatId::from(resolved.runtime_identity.as_str()))
888 .await
889 .map_err(|err| -> ConsoleLogError {
890 format!("retire failed for {identity}: {err}").into()
891 })
892 .or_else(|err| {
893 if lifecycle_archive_cleanup_completed(&err.to_string()) {
894 Ok(())
895 } else {
896 Err(err)
897 }
898 })?;
899 live_retired_any = true;
900 }
901 Ok(live_retired_any)
902 }
903
904 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
905 self.inner.store.clear_frames().await
906 }
907
908 pub async fn query_timeline(
909 &self,
910 query: ConsoleTimelineQuery,
911 ) -> ConsoleLogResult<ConsoleTimelinePage> {
912 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
913 Ok(ConsoleTimelinePage {
914 frames: page.frames,
915 next_cursor: page.next_cursor,
916 })
917 }
918
919 pub async fn query_timeline_windowed(
920 &self,
921 query: ConsoleTimelineWindowQuery,
922 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
923 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
924 .await?;
925 if query.after.is_none()
926 && query.before.is_none()
927 && let Some(identity) = query.identity.clone()
928 {
929 let mut probe_query = query.clone();
930 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
931 let page = self.inner.store.query_windowed_frames(probe_query).await?;
932 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
933 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
934 }
935 }
936 Box::pin(self.query_timeline_visible(query)).await
937 }
938
939 async fn reject_after_cursor_beyond_store_frontier(
940 &self,
941 after: Option<&ConsoleCursor>,
942 ) -> ConsoleLogResult<()> {
943 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
944 return Ok(());
945 };
946 let latest_seq = self
947 .inner
948 .store
949 .latest_cursor()
950 .await?
951 .and_then(|cursor| cursor.seq())
952 .unwrap_or(0);
953 if after_seq > latest_seq {
954 return Err(std::io::Error::other(
955 "timeline replay cursor is beyond the current store frontier",
956 )
957 .into());
958 }
959 Ok(())
960 }
961
962 async fn query_timeline_visible(
963 &self,
964 query: ConsoleTimelineWindowQuery,
965 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
966 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
967 let mut scan_query = query.clone();
968 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
969 let mut visible_frames = Vec::with_capacity(requested_limit);
970 let mut anchor_frames = Vec::new();
971 let mut identity_visibility_cache = HashMap::new();
972 let identity_records = self.inner.identity_read_model.current().await;
973 let mut next_cursor = query.after.clone();
974 let mut since_last_scanned_cursor = query.after.clone();
975 let mut since_last_delivered_cursor = None;
976 let mut since_stopped_at_visible_limit = false;
977 let mut latest_cursor = None;
978 let mut exhausted = false;
979 let mut scanned = 0usize;
980
981 loop {
982 let page = self
983 .inner
984 .store
985 .query_windowed_frames(scan_query.clone())
986 .await?;
987 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
988 if page.frames.is_empty() {
989 exhausted = true;
990 break;
991 }
992 let raw_len = page.frames.len();
993 scanned = scanned.saturating_add(raw_len);
994 match query.mode {
995 ConsoleTimelineMode::Since => {
996 if let Some(cursor) = page.next_cursor.clone() {
997 since_last_scanned_cursor = Some(cursor.clone());
998 scan_query.after = Some(cursor);
999 }
1000 }
1001 ConsoleTimelineMode::Recent => {
1002 if let Some(first) = page.frames.first() {
1003 scan_query.before = Some(first.cursor.clone());
1004 }
1005 next_cursor = page.next_cursor.clone().or(next_cursor);
1006 }
1007 }
1008
1009 for frame in page.frames {
1010 let allow_historical_identity =
1011 query.identity.as_deref() == Some(frame.identity.as_str());
1012 if frame_is_visible_cached(
1013 &self.inner,
1014 &frame,
1015 allow_historical_identity,
1016 &mut identity_visibility_cache,
1017 &identity_records,
1018 )
1019 .await
1020 .unwrap_or(false)
1021 {
1022 if query.mode == ConsoleTimelineMode::Recent
1023 && query.identity.is_some()
1024 && is_identity_timeline_anchor_frame(&frame)
1025 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1026 {
1027 anchor_frames.push(frame.clone());
1028 }
1029 match query.mode {
1030 ConsoleTimelineMode::Since => {
1031 if visible_frames.len() >= requested_limit {
1032 since_stopped_at_visible_limit = true;
1033 break;
1034 }
1035 since_last_delivered_cursor = Some(frame.cursor.clone());
1036 visible_frames.push(frame);
1037 if visible_frames.len() >= requested_limit {
1038 since_stopped_at_visible_limit = true;
1039 exhausted = false;
1040 break;
1041 }
1042 }
1043 ConsoleTimelineMode::Recent => {
1044 visible_frames.push(frame);
1045 }
1046 }
1047 }
1048 }
1049 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1050 && query.identity.is_some()
1051 && anchor_frames.is_empty()
1052 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1053 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1054 break;
1055 }
1056 if since_stopped_at_visible_limit {
1057 break;
1058 }
1059 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1060 exhausted = true;
1061 break;
1062 }
1063 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1064 break;
1065 }
1066 }
1067
1068 if query.mode == ConsoleTimelineMode::Recent {
1069 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1070 if visible_frames.len() > requested_limit {
1071 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1072 }
1073 if !anchor_frames.is_empty() {
1074 let anchor_cursors = anchor_frames
1075 .iter()
1076 .map(|frame| frame.cursor.clone())
1077 .collect::<Vec<_>>();
1078 let mut merged = anchor_frames;
1079 for frame in visible_frames {
1080 if !merged
1081 .iter()
1082 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1083 {
1084 merged.push(frame);
1085 }
1086 }
1087 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1088 visible_frames = merged;
1089 if visible_frames.len() > requested_limit {
1090 let mut anchors = Vec::new();
1091 let mut tail = Vec::new();
1092 for frame in visible_frames {
1093 if anchor_cursors.contains(&frame.cursor) {
1094 anchors.push(frame);
1095 } else {
1096 tail.push(frame);
1097 }
1098 }
1099 visible_frames = if anchors.len() >= requested_limit {
1100 anchors.split_off(anchors.len() - requested_limit)
1101 } else {
1102 let keep_tail = requested_limit.saturating_sub(anchors.len());
1103 if tail.len() > keep_tail {
1104 tail = tail.split_off(tail.len() - keep_tail);
1105 }
1106 anchors.extend(tail);
1107 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1108 anchors
1109 };
1110 }
1111 }
1112 }
1113
1114 Ok(ConsoleTimelineWindowPage {
1115 frames: visible_frames,
1116 next_cursor: match query.mode {
1117 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1118 since_last_delivered_cursor.or(since_last_scanned_cursor)
1119 }
1120 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1121 ConsoleTimelineMode::Recent => next_cursor,
1122 },
1123 latest_cursor,
1124 exhausted,
1125 })
1126 }
1127
1128 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1129 let runtime_keys = self
1130 .inner
1131 .runtimes
1132 .read()
1133 .map_err(|_| runtime_registry_lock_error())?
1134 .keys()
1135 .cloned()
1136 .collect::<Vec<_>>();
1137 let results =
1138 join_all(runtime_keys.into_iter().map(|runtime_key| {
1139 backfill_session_history(self.inner.clone(), runtime_key, true)
1140 }))
1141 .await;
1142 for result in results {
1143 result?;
1144 }
1145 Ok(())
1146 }
1147
1148 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1149 self.inner.store.latest_cursor().await
1150 }
1151
1152 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1153 match event {
1154 ConsoleTimelineEvent::ConsoleFrame { frame }
1155 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1156 let identity_records = self.inner.identity_read_model.current().await;
1157 frame_is_visible(&self.inner, frame, false, &identity_records)
1158 .await
1159 .unwrap_or(false)
1160 }
1161 ConsoleTimelineEvent::SnapshotStarted { .. }
1162 | ConsoleTimelineEvent::SnapshotComplete { .. }
1163 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1164 }
1165 }
1166
1167 pub async fn timeline_frame_visible_for_query(
1168 &self,
1169 frame: &ConsoleFrame,
1170 identity: Option<&str>,
1171 ) -> bool {
1172 let allow_historical_identity = identity == Some(frame.identity.as_str());
1173 let identity_records = self.inner.identity_read_model.current().await;
1174 frame_is_visible(
1175 &self.inner,
1176 frame,
1177 allow_historical_identity,
1178 &identity_records,
1179 )
1180 .await
1181 .unwrap_or(false)
1182 }
1183
1184 pub async fn send(
1185 &self,
1186 request: ConsoleSendRequest,
1187 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1188 validate_send_request(&request)?;
1189 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1190 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1191 };
1192 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1193 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1194 };
1195 if !resolved_member_visible(&resolved, &record).await {
1196 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1197 }
1198 if !member_is_addressable(&resolved.member) {
1199 return Err(ConsoleSendError::NotAddressable(request.identity));
1200 }
1201 if resolved.member.state == meerkat_mob::MemberState::Retiring {
1202 return Err(ConsoleSendError::Retired(request.identity));
1203 }
1204
1205 let content = content_input_from_value(&request.content)?;
1206 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1207 assert_member_accepts_images(
1208 &resolved.handle,
1209 resolved.entry.runtime.session_service(),
1210 &resolved.runtime_identity,
1211 &content,
1212 )
1213 .await
1214 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1215
1216 let dedupe_key = send_dedupe_key(
1217 &resolved.entry.runtime_key,
1218 &request.identity,
1219 &request.origin,
1220 &request.idempotency_key,
1221 );
1222 let handling_mode_value = request
1223 .handling_mode
1224 .as_deref()
1225 .unwrap_or("queue")
1226 .to_string();
1227 let request_fingerprint =
1228 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1229 if let Some(existing) = self
1230 .inner
1231 .store
1232 .frame_by_dedupe_key(&dedupe_key)
1233 .await
1234 .map_err(ConsoleSendError::Log)?
1235 {
1236 let same_request = existing.source.source_cursor.as_deref()
1237 == Some(request_fingerprint.as_str())
1238 || existing.source.source_cursor.is_none()
1239 && existing.payload.get("origin").and_then(Value::as_str)
1240 == Some(request.origin.as_str())
1241 && existing.payload.get("content") == Some(&request.content)
1242 && existing
1243 .payload
1244 .get("handling_mode")
1245 .and_then(Value::as_str)
1246 == Some(handling_mode_value.as_str());
1247 if !same_request {
1248 return Err(ConsoleSendError::IdempotencyConflict(
1249 request.idempotency_key,
1250 ));
1251 }
1252 return Ok(accepted_from_frame(&existing));
1253 }
1254
1255 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1256 resolved
1257 .entry
1258 .console_events
1259 .reserve_interaction_value(
1260 &resolved.runtime_identity,
1261 Some(resolved.runtime_identity.as_str()),
1262 &interaction_id,
1263 &request.origin,
1264 request.content.clone(),
1265 )
1266 .await
1267 .map_err(ConsoleSendError::State)?;
1268 let session_id = resolved
1269 .handle
1270 .resolve_bridge_session_id_observation(&MeerkatId::from(
1271 resolved.runtime_identity.as_str(),
1272 ))
1273 .await
1274 .map(|sid| sid.to_string());
1275 let mut new_frame = NewConsoleFrame {
1276 id: None,
1277 dedupe_key,
1278 timestamp_ms: current_time_ms(),
1279 runtime_key: resolved.entry.runtime_key.clone(),
1280 identity: request.identity.clone(),
1281 conversation_id: Some(request.identity.clone()),
1282 session_id: session_id.clone(),
1283 kind: "user_input".to_string(),
1284 status: ConsoleFrameStatus::Accepted,
1285 payload: json!({
1286 "content": request.content,
1287 "origin": request.origin,
1288 "idempotency_key": request.idempotency_key,
1289 "handling_mode": handling_mode_value,
1290 }),
1291 source: ConsoleFrameSource {
1292 kind: ConsoleFrameSourceKind::Send,
1293 source_cursor: Some(request_fingerprint),
1294 },
1295 source_event_id: None,
1296 interaction_id: Some(interaction_id.clone()),
1297 turn_id: None,
1298 run_id: None,
1299 parent_frame_id: None,
1300 caused_by_frame_id: None,
1301 };
1302 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1303 new_frame.payload = redacted;
1304 new_frame.status = ConsoleFrameStatus::Redacted;
1305 }
1306
1307 let _ = SendState::Requested
1308 .apply(SendTransition::PersistAccepted)
1309 .map_err(ConsoleSendError::State)?;
1310 let outcome = self
1311 .inner
1312 .store
1313 .append_if_absent(new_frame)
1314 .await
1315 .map_err(ConsoleSendError::Log)?;
1316 if outcome.disposition == AppendDisposition::Existing {
1317 return Ok(accepted_from_frame(&outcome.frame));
1318 }
1319 let _ = self
1320 .inner
1321 .event_tx
1322 .send(ConsoleTimelineEvent::ConsoleFrame {
1323 frame: outcome.frame.clone(),
1324 });
1325 let accepted = accepted_from_frame(&outcome.frame);
1326
1327 let (dispatching, _effects) = SendState::AcceptedPersisted
1328 .apply(SendTransition::StartDispatch)
1329 .map_err(ConsoleSendError::State)?;
1330 update_frame_status_and_emit(
1331 &self.inner,
1332 &outcome.frame.id,
1333 ConsoleFrameStatus::Dispatching,
1334 )
1335 .await
1336 .map_err(ConsoleSendError::Log)?;
1337
1338 spawn_console_send_dispatch(
1339 self.inner.clone(),
1340 resolved,
1341 content,
1342 handling_mode,
1343 dispatching,
1344 outcome.frame,
1345 interaction_id,
1346 );
1347 Ok(accepted)
1348 }
1349
1350 pub async fn reserve_identity_first_interaction(
1351 &self,
1352 request: ConsoleSendRequest,
1353 session_id: Option<&str>,
1354 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1355 validate_send_request(&request)?;
1356 let _content = content_input_from_value(&request.content)?;
1357 let runtime_key =
1358 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1359 let handling_mode_value = request
1360 .handling_mode
1361 .as_deref()
1362 .unwrap_or("queue")
1363 .to_string();
1364 let dedupe_key = send_dedupe_key(
1365 &runtime_key,
1366 &request.identity,
1367 &request.origin,
1368 &request.idempotency_key,
1369 );
1370 let request_fingerprint =
1371 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1372 if let Some(existing) = self
1373 .inner
1374 .store
1375 .frame_by_dedupe_key(&dedupe_key)
1376 .await
1377 .map_err(ConsoleSendError::Log)?
1378 {
1379 let same_request = existing.source.source_cursor.as_deref()
1380 == Some(request_fingerprint.as_str())
1381 || existing.source.source_cursor.is_none()
1382 && existing.payload.get("origin").and_then(Value::as_str)
1383 == Some(request.origin.as_str())
1384 && existing.payload.get("content") == Some(&request.content)
1385 && existing
1386 .payload
1387 .get("handling_mode")
1388 .and_then(Value::as_str)
1389 == Some(handling_mode_value.as_str());
1390 if !same_request {
1391 return Err(ConsoleSendError::IdempotencyConflict(
1392 request.idempotency_key,
1393 ));
1394 }
1395 return Ok(accepted_from_frame(&existing));
1396 }
1397
1398 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1399 let new_frame = NewConsoleFrame {
1400 id: None,
1401 dedupe_key,
1402 timestamp_ms: current_time_ms(),
1403 runtime_key,
1404 identity: request.identity.clone(),
1405 conversation_id: Some(request.identity.clone()),
1406 session_id: session_id.map(ToString::to_string),
1407 kind: "user_input".to_string(),
1408 status: ConsoleFrameStatus::Accepted,
1409 payload: json!({
1410 "content": request.content,
1411 "origin": request.origin,
1412 "idempotency_key": request.idempotency_key,
1413 "handling_mode": handling_mode_value,
1414 }),
1415 source: ConsoleFrameSource {
1416 kind: ConsoleFrameSourceKind::Send,
1417 source_cursor: Some(request_fingerprint),
1418 },
1419 source_event_id: None,
1420 interaction_id: Some(interaction_id),
1421 turn_id: None,
1422 run_id: None,
1423 parent_frame_id: None,
1424 caused_by_frame_id: None,
1425 };
1426 let outcome = self
1427 .inner
1428 .store
1429 .append_if_absent(new_frame)
1430 .await
1431 .map_err(ConsoleSendError::Log)?;
1432 let _ = self
1433 .inner
1434 .event_tx
1435 .send(ConsoleTimelineEvent::ConsoleFrame {
1436 frame: outcome.frame.clone(),
1437 });
1438 Ok(accepted_from_frame(&outcome.frame))
1439 }
1440
1441 async fn runtime_key_for_identity_first_send(
1442 &self,
1443 identity: &str,
1444 ) -> Result<String, ConsoleSendError> {
1445 let entries = self
1446 .inner
1447 .runtimes
1448 .read()
1449 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1450 .clone();
1451 if entries.is_empty() {
1452 return Ok("identity-first".to_string());
1453 }
1454
1455 let mut identity_runtime_keys = Vec::new();
1456 let mut hidden_durable_match = false;
1457 let mut durable_matches = Vec::new();
1458 for entry in entries.values() {
1459 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1460 continue;
1461 };
1462 identity_runtime_keys.push(entry.runtime_key.clone());
1463 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1464 continue;
1465 }
1466 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1467 {
1468 let Ok(parsed_identity) =
1469 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1470 else {
1471 continue;
1472 };
1473 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1474 let record = identity_record_for_status(entry, &status);
1475 if !console_identity_record_visible(entry, &record).await {
1476 hidden_durable_match = true;
1477 continue;
1478 }
1479 durable_matches.push((entry.clone(), record));
1480 }
1481 }
1482 }
1483 if durable_matches.len() > 1 {
1484 let candidates = durable_matches
1485 .iter()
1486 .map(|(entry, record)| format!("{}@{}", record.identity, entry.runtime_key))
1487 .collect::<Vec<_>>()
1488 .join(", ");
1489 return Err(ConsoleSendError::InvalidRequest(format!(
1490 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1491 )));
1492 }
1493 if let Some((entry, record)) = durable_matches.into_iter().next() {
1494 let live_records = self.live_records_for_identity(identity).await;
1495 let durable_live_records = self
1496 .live_records_for_durable_record(&entry, identity, &record, &live_records)
1497 .await;
1498 let visible_durable_live_records =
1499 visible_live_records_for_entry(&entry, &durable_live_records).await;
1500 if let Some(ambiguous_error) =
1501 ambiguous_live_alias_error(identity, &visible_durable_live_records)
1502 {
1503 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1504 }
1505 if let Some(stale_error) =
1506 stale_durable_record_error(identity, &record, &durable_live_records)
1507 {
1508 return Err(ConsoleSendError::InvalidRequest(stale_error));
1509 }
1510 return Ok(entry.runtime_key);
1511 }
1512 if entries.values().any(|entry| {
1513 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1514 }) {
1515 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1516 }
1517 if hidden_durable_match {
1518 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1519 }
1520
1521 match identity_runtime_keys.as_slice() {
1522 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1523 [runtime_key] => Ok(runtime_key.clone()),
1524 _ => Err(ConsoleSendError::InvalidRequest(format!(
1525 "identity-first send for '{identity}' did not match exactly one registered runtime"
1526 ))),
1527 }
1528 }
1529
1530 pub async fn mark_interaction_delivery_failed(
1531 &self,
1532 input_frame_id: &str,
1533 ) -> Result<(), ConsoleSendError> {
1534 update_frame_status_and_emit(
1535 &self.inner,
1536 input_frame_id,
1537 ConsoleFrameStatus::DeliveryFailed,
1538 )
1539 .await
1540 .map_err(ConsoleSendError::Log)?;
1541 Ok(())
1542 }
1543
1544 pub async fn mark_interaction_delivered(
1545 &self,
1546 input_frame_id: &str,
1547 ) -> Result<(), ConsoleSendError> {
1548 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1549 .await
1550 .map_err(ConsoleSendError::Log)?;
1551 Ok(())
1552 }
1553
1554 pub async fn mark_steer_interaction_delivered(
1555 &self,
1556 input_frame_id: &str,
1557 interaction_id: &str,
1558 ) -> Result<(), ConsoleSendError> {
1559 let Some(updated) = update_frame_status_and_emit(
1560 &self.inner,
1561 input_frame_id,
1562 ConsoleFrameStatus::Delivered,
1563 )
1564 .await
1565 .map_err(ConsoleSendError::Log)?
1566 else {
1567 return Ok(());
1568 };
1569 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1570 .await
1571 .map_err(ConsoleSendError::Log)?;
1572 Ok(())
1573 }
1574
1575 pub async fn binary_blob_store_for_identity(
1576 &self,
1577 identity: &str,
1578 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1579 if identity.trim().is_empty() {
1580 return Err(ConsoleSendError::InvalidRequest(
1581 "identity must be non-empty".to_string(),
1582 ));
1583 }
1584 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1585 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1586 };
1587 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1588 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1589 };
1590 if !resolved_member_visible(&resolved, &record).await {
1591 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1592 }
1593 if !member_is_addressable(&resolved.member) {
1594 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1595 }
1596 if resolved.member.state == meerkat_mob::MemberState::Retiring {
1597 return Err(ConsoleSendError::Retired(identity.to_string()));
1598 }
1599 Ok(resolved.entry.runtime.binary_blob_store())
1600 }
1601
1602 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1603 self.inner
1604 .runtimes
1605 .read()
1606 .map(|entries| {
1607 entries
1608 .values()
1609 .filter_map(|entry| entry.runtime.binary_blob_store())
1610 .collect()
1611 })
1612 .unwrap_or_default()
1613 }
1614
1615 async fn resolve_send_member(
1616 &self,
1617 identity: &str,
1618 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1619 let matches = self.resolve_visible_members(identity).await;
1620 if matches.len() > 1 {
1621 let candidates = matches
1622 .iter()
1623 .map(|resolved| resolved.runtime_identity.clone())
1624 .collect::<Vec<_>>()
1625 .join(", ");
1626 return Err(ConsoleSendError::AmbiguousIdentity {
1627 identity: identity.to_string(),
1628 candidates,
1629 });
1630 }
1631 let Some(resolved) = matches.into_iter().next() else {
1632 if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1633 return Err(ConsoleSendError::InvalidRequest(stale_error));
1634 }
1635 return Ok(None);
1636 };
1637 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1638 && let Some(stale_error) =
1639 stale_live_record_binding_error(&resolved.entry, identity, &record).await
1640 {
1641 return Err(ConsoleSendError::InvalidRequest(stale_error));
1642 }
1643 if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1644 return Err(ConsoleSendError::InvalidRequest(stale_error));
1645 }
1646 Ok(Some(resolved))
1647 }
1648
1649 async fn durable_send_binding_error(
1650 &self,
1651 identity: &str,
1652 ) -> Result<Option<String>, ConsoleSendError> {
1653 let entries = self
1654 .inner
1655 .runtimes
1656 .read()
1657 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1658 .clone();
1659 if entries.values().any(|entry| {
1660 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1661 }) {
1662 return Ok(None);
1663 }
1664 let live_records = self.live_records_for_identity(identity).await;
1665 let mut durable_matches = Vec::new();
1666 for entry in entries.values() {
1667 let Some(identity_runtime) = entry.identity_runtime.clone() else {
1668 continue;
1669 };
1670 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1671 let Ok(parsed_identity) =
1672 crate::identity_first::AgentIdentity::parse(&raw_identity)
1673 else {
1674 continue;
1675 };
1676 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1677 continue;
1678 };
1679 let record = identity_record_for_status(entry, &status);
1680 if !console_identity_record_visible(entry, &record).await {
1681 continue;
1682 }
1683 durable_matches.push((entry.clone(), record));
1684 }
1685 }
1686 if durable_matches.len() > 1 {
1687 let candidates = durable_matches
1688 .iter()
1689 .map(|(_, record)| record.runtime_member_id.clone())
1690 .collect::<Vec<_>>()
1691 .join(", ");
1692 return Err(ConsoleSendError::AmbiguousIdentity {
1693 identity: identity.to_string(),
1694 candidates,
1695 });
1696 }
1697 let Some((entry, durable_record)) = durable_matches.into_iter().next() else {
1698 return Ok(None);
1699 };
1700 let durable_live_records = self
1701 .live_records_for_durable_record(&entry, identity, &durable_record, &live_records)
1702 .await;
1703 let visible_durable_live_records =
1704 visible_live_records_for_entry(&entry, &durable_live_records).await;
1705 if let Some(ambiguous_error) =
1706 ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1707 {
1708 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1709 }
1710 Ok(stale_durable_record_error(
1711 identity,
1712 &durable_record,
1713 &durable_live_records,
1714 ))
1715 }
1716
1717 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1718 let entries = self
1719 .inner
1720 .runtimes
1721 .read()
1722 .ok()
1723 .map(|entries| entries.clone())
1724 .unwrap_or_default();
1725 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1726 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1727 for entry in entries.values() {
1728 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1729 if raw_identities.is_empty() {
1730 continue;
1731 }
1732 let mids = raw_identities
1733 .iter()
1734 .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
1735 .collect::<Vec<_>>();
1736 for resolved in member_sources_for_entry(entry).await {
1737 let session_id = resolved
1738 .handle
1739 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1740 .await
1741 .map(|sid| sid.to_string())
1742 .unwrap_or_default();
1743 if mids.contains(&resolved.member.agent_identity) {
1744 exact_matches.push((session_id, resolved));
1745 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1746 {
1747 label_matches.push((session_id, resolved));
1748 }
1749 }
1750 }
1751 let mut matches = exact_matches;
1752 matches.extend(label_matches);
1753 let mut seen_members = BTreeSet::new();
1754 matches.retain(|(session_id, resolved)| {
1755 seen_members.insert((
1756 resolved.entry.runtime_key.clone(),
1757 resolved.source_mob_id.clone(),
1758 session_id.clone(),
1759 resolved.runtime_identity.clone(),
1760 ))
1761 });
1762 matches.sort_by(|left, right| right.0.cmp(&left.0));
1763 matches.into_iter().map(|(_, resolved)| resolved).collect()
1764 }
1765
1766 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1767 let mut visible = Vec::new();
1768 for resolved in self.resolve_members(identity).await {
1769 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1770 continue;
1771 };
1772 if resolved_member_visible(&resolved, &record).await {
1773 visible.push(resolved);
1774 }
1775 }
1776 visible
1777 }
1778
1779 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1780 let mut records = Vec::new();
1781 for resolved in self.resolve_members(identity).await {
1782 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1783 continue;
1784 };
1785 if resolved_member_visible(&resolved, &record).await {
1786 records.push(record);
1787 }
1788 }
1789 records
1790 }
1791
1792 async fn live_records_for_durable_record(
1793 &self,
1794 entry: &RuntimeEntry,
1795 requested_identity: &str,
1796 durable: &ConsoleIdentityRecord,
1797 requested_live_records: &[ConsoleIdentityRecord],
1798 ) -> Vec<ConsoleIdentityRecord> {
1799 let mut records = if durable.identity == requested_identity {
1800 requested_live_records.to_vec()
1801 } else {
1802 self.live_records_for_identity(&durable.identity).await
1803 };
1804 for runtime_record in
1805 live_records_for_runtime_member(entry, &durable.runtime_member_id).await
1806 {
1807 if records.iter().any(|record| {
1808 record.runtime_key == runtime_record.runtime_key
1809 && record.runtime_member_id == runtime_record.runtime_member_id
1810 && record.session_id == runtime_record.session_id
1811 && record.identity == runtime_record.identity
1812 }) {
1813 continue;
1814 }
1815 records.push(runtime_record);
1816 }
1817 records
1818 }
1819}
1820
1821async fn console_identity_record_visible(
1822 entry: &RuntimeEntry,
1823 record: &ConsoleIdentityRecord,
1824) -> bool {
1825 if !entry.visibility_policy.identity_visible(record) {
1826 return false;
1827 }
1828 let mut bound_live_member_seen = false;
1829 let mut wrong_live_projection_seen = false;
1830 for resolved in member_sources_for_entry(entry).await {
1831 if resolved.runtime_identity != record.runtime_member_id {
1832 continue;
1833 }
1834 bound_live_member_seen = true;
1835 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
1836 continue;
1837 };
1838 if live_record.identity != record.identity {
1839 wrong_live_projection_seen = true;
1840 continue;
1841 }
1842 if raw_resolved_member_visible(&resolved, &live_record) {
1843 return true;
1844 }
1845 }
1846 if wrong_live_projection_seen {
1847 return true;
1848 }
1849 !bound_live_member_seen
1850}
1851
1852async fn live_record_shadowed_by_hidden_durable_binding(
1853 resolved: &ResolvedConsoleMember,
1854 record: &ConsoleIdentityRecord,
1855) -> bool {
1856 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
1857 return false;
1858 };
1859 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
1860 return false;
1861 };
1862 let Ok(status) = identity_runtime.status(&identity).await else {
1863 return false;
1864 };
1865 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
1866 return false;
1867 };
1868 if bound_runtime_id.as_str() == resolved.runtime_identity {
1869 return false;
1870 }
1871 for candidate in member_sources_for_entry(&resolved.entry).await {
1872 if candidate.runtime_identity != bound_runtime_id.as_str() {
1873 continue;
1874 }
1875 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
1876 continue;
1877 };
1878 if bound_record.identity == record.identity
1879 && !raw_resolved_member_visible(&candidate, &bound_record)
1880 {
1881 return true;
1882 }
1883 }
1884 false
1885}
1886
1887async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
1888 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
1889 .unwrap_or_else(|| frame.identity.clone());
1890 let frame_session_id = frame.session_id.as_deref();
1891 let mut saw_hidden = false;
1892 let mut saw_visible = false;
1893 for resolved in member_sources_for_entry(entry).await {
1894 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1895 continue;
1896 };
1897 let session_matches =
1898 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
1899 let runtime_matches = record.runtime_member_id == runtime_member_id;
1900 let identity_matches = record.identity == frame.identity;
1901 if !session_matches && !runtime_matches && !identity_matches {
1902 continue;
1903 }
1904 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
1905 return true;
1906 }
1907 if resolved_member_visible(&resolved, &record).await {
1908 saw_visible = true;
1909 } else {
1910 saw_hidden = true;
1911 }
1912 }
1913 saw_hidden && !saw_visible
1914}
1915
1916async fn live_records_for_runtime_member(
1917 entry: &RuntimeEntry,
1918 runtime_member_id: &str,
1919) -> Vec<ConsoleIdentityRecord> {
1920 let mut records = Vec::new();
1921 for resolved in member_sources_for_entry(entry).await {
1922 if resolved.runtime_identity != runtime_member_id {
1923 continue;
1924 }
1925 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1926 continue;
1927 };
1928 records.push(record);
1929 }
1930 records
1931}
1932
1933async fn visible_live_records_for_entry(
1934 entry: &RuntimeEntry,
1935 records: &[ConsoleIdentityRecord],
1936) -> Vec<ConsoleIdentityRecord> {
1937 let mut visible = Vec::new();
1938 for record in records {
1939 if record.runtime_key != entry.runtime_key
1940 || console_identity_record_visible(entry, record).await
1941 {
1942 visible.push(record.clone());
1943 }
1944 }
1945 visible
1946}
1947
1948async fn stale_live_record_binding_error(
1949 entry: &RuntimeEntry,
1950 requested_identity: &str,
1951 live_record: &ConsoleIdentityRecord,
1952) -> Option<String> {
1953 let identity_runtime = entry.identity_runtime.as_ref()?;
1954 for status in identity_runtime.statuses().await {
1955 let matches_runtime = status
1956 .agent_runtime_id
1957 .as_ref()
1958 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
1959 if !matches_runtime {
1960 continue;
1961 }
1962 let durable_record = identity_record_for_status(entry, &status);
1963 if let Some(stale_error) = stale_durable_record_error(
1964 requested_identity,
1965 &durable_record,
1966 std::slice::from_ref(live_record),
1967 ) {
1968 return Some(stale_error);
1969 }
1970 }
1971 None
1972}
1973
1974fn stale_durable_record_error(
1975 requested_identity: &str,
1976 durable: &ConsoleIdentityRecord,
1977 live_records: &[ConsoleIdentityRecord],
1978) -> Option<String> {
1979 let matching_live = live_records
1980 .iter()
1981 .filter(|record| record.identity == durable.identity)
1982 .collect::<Vec<_>>();
1983 if let Some(wrong_projection) = live_records.iter().find(|record| {
1984 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
1985 }) {
1986 return Some(format!(
1987 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
1988 durable.identity, durable.runtime_member_id, wrong_projection.identity
1989 ));
1990 }
1991 if matching_live.is_empty() {
1992 return None;
1993 }
1994 if let Some(session_mismatch) = matching_live.iter().find(|record| {
1995 record.runtime_member_id == durable.runtime_member_id
1996 && durable.session_id.is_some()
1997 && record.session_id.is_some()
1998 && durable.session_id != record.session_id
1999 }) {
2000 return Some(format!(
2001 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2002 durable.identity,
2003 durable.runtime_member_id,
2004 durable.session_id.as_deref().unwrap_or("<none>"),
2005 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2006 ));
2007 }
2008 if matching_live
2009 .iter()
2010 .any(|record| record.runtime_member_id == durable.runtime_member_id)
2011 {
2012 return None;
2013 }
2014 let live_candidates = matching_live
2015 .iter()
2016 .map(|record| record.runtime_member_id.as_str())
2017 .collect::<Vec<_>>()
2018 .join(", ");
2019 Some(format!(
2020 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2021 durable.identity, durable.runtime_member_id, live_candidates
2022 ))
2023}
2024
2025fn ambiguous_live_alias_error(
2026 requested_identity: &str,
2027 live_records: &[ConsoleIdentityRecord],
2028) -> Option<String> {
2029 if live_records.len() <= 1 {
2030 return None;
2031 }
2032 let candidates = live_records
2033 .iter()
2034 .map(|record| {
2035 format!(
2036 "{}@{}",
2037 record.runtime_member_id,
2038 record
2039 .labels
2040 .get("source_mob_id")
2041 .map(String::as_str)
2042 .unwrap_or(record.runtime_key.as_str())
2043 )
2044 })
2045 .collect::<Vec<_>>()
2046 .join(", ");
2047 Some(format!(
2048 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2049 ))
2050}
2051
2052fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2053 member_id == durable_identity
2054}
2055
2056async fn retire_stale_console_members_for_identity(
2057 handle: &MobHandle,
2058 durable_identities: &[String],
2059 keep_runtime_member_id: Option<&str>,
2060) -> Result<(), String> {
2061 let stale_members = handle
2062 .list_members_including_retiring()
2063 .await
2064 .into_iter()
2065 .filter(|member| {
2066 (durable_identities.iter().any(|durable_identity| {
2067 member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2068 }) || member.labels.get("agent_identity").is_some_and(|identity| {
2069 durable_identities
2070 .iter()
2071 .any(|durable_identity| identity == durable_identity)
2072 })) && keep_runtime_member_id
2073 .map(|keep| member.agent_identity.as_str() != keep)
2074 .unwrap_or(true)
2075 })
2076 .map(|member| member.agent_identity)
2077 .collect::<Vec<_>>();
2078 for member_id in stale_members {
2079 match handle.retire(member_id).await {
2080 Ok(()) => {}
2081 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2082 Err(err) => return Err(err.to_string()),
2083 }
2084 }
2085 Ok(())
2086}
2087
2088async fn retire_stale_console_members_for_runtime_entry(
2089 entry: &RuntimeEntry,
2090 durable_identities: &[String],
2091) -> Result<(), String> {
2092 let primary_handle = entry.runtime.handle();
2093 let primary_mob_id = primary_handle.mob_id().to_string();
2094 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2095 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2096 for (mob_id, handle) in state.mob_handles_snapshot().await {
2097 if mob_id.as_str() != primary_mob_id {
2098 handles.push((mob_id.to_string(), handle));
2099 }
2100 }
2101 }
2102 let mut seen_handles = BTreeSet::new();
2103 for (mob_id, handle) in handles {
2104 if !seen_handles.insert(mob_id) {
2105 continue;
2106 }
2107 retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2108 }
2109 Ok(())
2110}
2111
2112fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2113 error.contains("disposal completed but ArchiveSession failed")
2114 && error.contains("cancel-before-retire failed")
2115 && error.contains("Runtime not ready: running")
2116}
2117
2118fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2119 if frames.is_empty() {
2120 return true;
2121 }
2122 let latest_session_history_timestamp_ms = frames
2123 .iter()
2124 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2125 .map(|frame| frame.timestamp_ms)
2126 .max();
2127 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2128 return current_time_ms().saturating_sub(latest_timestamp_ms)
2129 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2130 }
2131 frames.iter().any(|frame| {
2132 matches!(
2133 frame.kind.as_str(),
2134 "turn_started"
2135 | "run_started"
2136 | "reasoning_delta"
2137 | "reasoning_complete"
2138 | "tool_call_requested"
2139 | "tool_call"
2140 | "tool_execution_started"
2141 | "tool_execution_completed"
2142 | "tool_result_received"
2143 )
2144 })
2145}
2146
2147fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2148 match frame.kind.as_str() {
2149 "user_input" | "run_started" => true,
2150 "interaction_started" => frame
2151 .payload
2152 .get("content")
2153 .or_else(|| frame.payload.get("prompt"))
2154 .is_some(),
2155 _ => false,
2156 }
2157}
2158
2159fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2160 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2161 for record in records {
2162 by_identity
2163 .entry(identity_record_dedupe_key(&record))
2164 .and_modify(|current| {
2165 if identity_record_prefer(&record, current) {
2166 *current = record.clone();
2167 }
2168 })
2169 .or_insert(record);
2170 }
2171 by_identity.into_values().collect()
2172}
2173
2174fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2175 format!(
2176 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2177 record.identity,
2178 record.runtime_key,
2179 record
2180 .labels
2181 .get("source_mob_id")
2182 .map(String::as_str)
2183 .unwrap_or(""),
2184 record.runtime_member_id,
2185 record.session_id.as_deref().unwrap_or("")
2186 )
2187}
2188
2189fn identity_record_prefer(
2190 candidate: &ConsoleIdentityRecord,
2191 current: &ConsoleIdentityRecord,
2192) -> bool {
2193 let candidate_is_live_label_projection =
2194 candidate
2195 .labels
2196 .get("agent_identity")
2197 .is_some_and(|identity| {
2198 identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2199 || candidate.runtime_member_id != candidate.identity
2200 });
2201 let current_is_live_label_projection =
2202 current
2203 .labels
2204 .get("agent_identity")
2205 .is_some_and(|identity| {
2206 identity == &strip_namespace(¤t.identity, "").unwrap_or_default()
2207 || current.runtime_member_id != current.identity
2208 });
2209 if candidate_is_live_label_projection != current_is_live_label_projection {
2210 return candidate_is_live_label_projection;
2211 }
2212 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2213 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2214 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2215 return candidate_has_distinct_runtime_binding;
2216 }
2217 let candidate_live = candidate.addressable && candidate.health != "retired";
2218 let current_live = current.addressable && current.health != "retired";
2219 if candidate_live != current_live {
2220 return candidate_live;
2221 }
2222 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2223}
2224
2225async fn collect_identity_records(
2226 inner: &Arc<AggregatorInner>,
2227 mode: IdentityCollectionMode,
2228) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2229 let entries = inner
2230 .runtimes
2231 .read()
2232 .map_err(|_| runtime_registry_lock_error())?
2233 .clone();
2234 let mut identities = Vec::new();
2235 for entry in entries.values() {
2236 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2237 let topology_peers =
2238 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2239 for status in identity_runtime.statuses().await {
2240 let mut record = identity_record_for_status(entry, &status);
2241 record.topology_peers = topology_peers
2242 .get(&record.identity)
2243 .cloned()
2244 .unwrap_or_default();
2245 if console_identity_record_visible(entry, &record).await {
2246 identities.push(record);
2247 }
2248 }
2249 if mode == IdentityCollectionMode::CachedOnly {
2250 continue;
2251 }
2252 }
2253 let members = if entry.identity_runtime.is_some() {
2254 match tokio::time::timeout(
2255 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2256 member_sources_for_entry(entry),
2257 )
2258 .await
2259 {
2260 Ok(members) => members,
2261 Err(_) => {
2262 tracing::warn!(
2263 runtime_key = %entry.runtime_key,
2264 "identity-first live member refresh timed out; keeping cached identity read model"
2265 );
2266 Vec::new()
2267 }
2268 }
2269 } else {
2270 member_sources_for_entry(entry).await
2271 };
2272 for resolved in members {
2273 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2274 && resolved_member_visible(&resolved, &record).await
2275 {
2276 identities.push(record);
2277 }
2278 }
2279 }
2280 Ok(dedupe_identity_records(identities))
2281}
2282
2283async fn identity_runtime_topology_peers(
2284 entry: &RuntimeEntry,
2285 identity_runtime: &crate::identity_first::IdentityRuntime,
2286) -> BTreeMap<String, Vec<String>> {
2287 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2288 for edge in identity_runtime.desired_peer_edges().await {
2289 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2290 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2291 peers.entry(a.clone()).or_default().insert(b.clone());
2292 peers.entry(b).or_default().insert(a);
2293 }
2294 peers
2295 .into_iter()
2296 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2297 .collect()
2298}
2299
2300fn spawn_identity_backfills_for_records(
2301 inner: Arc<AggregatorInner>,
2302 records: &[ConsoleIdentityRecord],
2303) {
2304 if !inner.options.session_history_backfill_enabled {
2305 return;
2306 }
2307 let entries = match inner.runtimes.read() {
2308 Ok(entries) => entries.clone(),
2309 Err(_) => return,
2310 };
2311 for record in records {
2312 if record.health == "dormant" || record.health == "uninitialized" {
2313 continue;
2314 }
2315 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2316 continue;
2317 };
2318 let Some(session_id) = record.session_id.clone() else {
2319 continue;
2320 };
2321 spawn_session_history_backfill_target(
2322 inner.clone(),
2323 SessionBackfillTarget {
2324 entry,
2325 record: record.clone(),
2326 session_id,
2327 },
2328 false,
2329 );
2330 }
2331}
2332
2333async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2334 let mut resolved = Vec::new();
2335 let primary_handle = entry.runtime.handle();
2336 let primary_mob_id = primary_handle.mob_id().to_string();
2337 for member in primary_handle.list_members_observation_snapshot().await {
2338 resolved.push(ResolvedConsoleMember {
2339 entry: entry.clone(),
2340 handle: primary_handle.clone(),
2341 runtime_identity: member.agent_identity.to_string(),
2342 source_mob_id: primary_mob_id.clone(),
2343 member,
2344 });
2345 }
2346
2347 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2348 return resolved;
2349 };
2350 if !entry.visibility_policy.include_implicit_delegate_members() {
2351 return resolved;
2352 }
2353 for (mob_id, handle) in state.mob_handles_snapshot().await {
2354 if mob_id.as_str() == primary_mob_id {
2355 continue;
2356 }
2357 for member in handle.list_members_observation_snapshot().await {
2358 resolved.push(ResolvedConsoleMember {
2359 entry: entry.clone(),
2360 handle: handle.clone(),
2361 runtime_identity: member.agent_identity.to_string(),
2362 source_mob_id: mob_id.to_string(),
2363 member,
2364 });
2365 }
2366 }
2367 resolved
2368}
2369
2370async fn dispatch_message_to_resolved_member(
2371 resolved: &ResolvedConsoleMember,
2372 content: ContentInput,
2373 handling_mode: meerkat_core::types::HandlingMode,
2374) -> Result<String, String> {
2375 let mid = MeerkatId::from(resolved.runtime_identity.as_str());
2376 match send_message_on_mob_with_mode(
2377 &resolved.handle,
2378 &resolved.runtime_identity,
2379 content.clone(),
2380 handling_mode,
2381 )
2382 .await
2383 {
2384 Ok(session_id) => Ok(session_id),
2385 Err(err) if is_not_externally_addressable(&err) => {
2386 let member = resolved
2387 .handle
2388 .member(&mid)
2389 .await
2390 .map_err(|err| err.to_string())?;
2391 let _receipt = member
2392 .internal_turn(content)
2393 .await
2394 .map_err(|err| err.to_string())?;
2395 resolved
2396 .handle
2397 .resolve_bridge_session_id_observation(&mid)
2398 .await
2399 .map(|sid| sid.to_string())
2400 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2401 }
2402 Err(err) => Err(err.to_string()),
2403 }
2404}
2405
2406fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2407 matches!(
2408 err,
2409 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2410 )
2411}
2412
2413fn spawn_console_send_dispatch(
2414 inner: Arc<AggregatorInner>,
2415 resolved: ResolvedConsoleMember,
2416 content: ContentInput,
2417 handling_mode: meerkat_core::types::HandlingMode,
2418 dispatching: SendState,
2419 user_frame: ConsoleFrame,
2420 interaction_id: String,
2421) {
2422 tokio::spawn(async move {
2423 match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2424 Ok(_) => {
2425 let _ = dispatching.apply(SendTransition::MarkDelivered);
2426 if let Err(err) = update_frame_status_and_emit(
2427 &inner,
2428 &user_frame.id,
2429 ConsoleFrameStatus::Delivered,
2430 )
2431 .await
2432 {
2433 tracing::warn!(
2434 frame_id = %user_frame.id,
2435 error = %err,
2436 "failed to update console send delivery status"
2437 );
2438 }
2439 if handling_mode == HandlingMode::Steer
2440 && let Err(err) =
2441 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2442 {
2443 tracing::warn!(
2444 frame_id = %user_frame.id,
2445 interaction_id = %interaction_id,
2446 error = %err,
2447 "failed to append console steer terminal frame"
2448 );
2449 }
2450 }
2451 Err(err) => {
2452 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2453 if let Err(update_err) = update_frame_status_and_emit(
2454 &inner,
2455 &user_frame.id,
2456 ConsoleFrameStatus::DeliveryFailed,
2457 )
2458 .await
2459 {
2460 tracing::warn!(
2461 frame_id = %user_frame.id,
2462 error = %update_err,
2463 "failed to update console send failure status"
2464 );
2465 }
2466 let failure_frame = NewConsoleFrame {
2467 id: None,
2468 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2469 timestamp_ms: current_time_ms(),
2470 runtime_key: user_frame.runtime_key,
2471 identity: user_frame.identity,
2472 conversation_id: user_frame.conversation_id,
2473 session_id: user_frame.session_id,
2474 kind: "message_delivery_failed".to_string(),
2475 status: ConsoleFrameStatus::DeliveryFailed,
2476 payload: json!({ "reason": err }),
2477 source: ConsoleFrameSource {
2478 kind: ConsoleFrameSourceKind::Synthetic,
2479 source_cursor: None,
2480 },
2481 source_event_id: None,
2482 interaction_id: Some(interaction_id),
2483 turn_id: None,
2484 run_id: None,
2485 parent_frame_id: Some(user_frame.id.clone()),
2486 caused_by_frame_id: Some(user_frame.id),
2487 };
2488 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2489 tracing::warn!(
2490 error = %append_err,
2491 "failed to append console send failure frame"
2492 );
2493 }
2494 }
2495 }
2496 });
2497}
2498
2499async fn append_steer_delivery_terminal(
2500 inner: &AggregatorInner,
2501 user_frame: &ConsoleFrame,
2502 interaction_id: &str,
2503) -> ConsoleLogResult<AppendOutcome> {
2504 append_and_emit(
2505 inner,
2506 NewConsoleFrame {
2507 id: None,
2508 dedupe_key: format!("steer-delivered:{}", user_frame.id),
2509 timestamp_ms: current_time_ms(),
2510 runtime_key: user_frame.runtime_key.clone(),
2511 identity: user_frame.identity.clone(),
2512 conversation_id: user_frame.conversation_id.clone(),
2513 session_id: user_frame.session_id.clone(),
2514 kind: "interaction_complete".to_string(),
2515 status: ConsoleFrameStatus::Completed,
2516 payload: json!({
2517 "reason": "steer_delivered",
2518 "handling_mode": "steer",
2519 }),
2520 source: ConsoleFrameSource {
2521 kind: ConsoleFrameSourceKind::Synthetic,
2522 source_cursor: None,
2523 },
2524 source_event_id: None,
2525 interaction_id: Some(interaction_id.to_string()),
2526 turn_id: None,
2527 run_id: None,
2528 parent_frame_id: Some(user_frame.id.clone()),
2529 caused_by_frame_id: Some(user_frame.id.clone()),
2530 },
2531 )
2532 .await
2533}
2534
2535#[derive(Debug)]
2536pub enum ConsoleSendError {
2537 UnknownIdentity(String),
2538 AmbiguousIdentity {
2539 identity: String,
2540 candidates: String,
2541 },
2542 NotAddressable(String),
2543 Retired(String),
2544 InvalidContent(String),
2545 InvalidHandlingMode(String),
2546 InvalidRequest(String),
2547 IdempotencyConflict(String),
2548 State(&'static str),
2549 Dispatch(String),
2550 Log(ConsoleLogError),
2551}
2552
2553impl std::fmt::Display for ConsoleSendError {
2554 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2555 match self {
2556 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2557 Self::AmbiguousIdentity {
2558 identity,
2559 candidates,
2560 } => write!(
2561 f,
2562 "ambiguous live identity alias {identity}: candidates [{candidates}]"
2563 ),
2564 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2565 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2566 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2567 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2568 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2569 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2570 Self::State(message) => write!(f, "console send state error: {message}"),
2571 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2572 Self::Log(err) => write!(f, "console log error: {err}"),
2573 }
2574 }
2575}
2576
2577impl std::error::Error for ConsoleSendError {}
2578
2579async fn backfill_session_history(
2580 inner: Arc<AggregatorInner>,
2581 runtime_key: String,
2582 force_refresh: bool,
2583) -> ConsoleLogResult<()> {
2584 if !inner.options.session_history_backfill_enabled {
2585 return Ok(());
2586 }
2587 let Some(entry) = inner
2588 .runtimes
2589 .read()
2590 .ok()
2591 .and_then(|entries| entries.get(&runtime_key).cloned())
2592 else {
2593 return Ok(());
2594 };
2595 let members = member_sources_for_entry(&entry).await;
2596 let mut targets = Vec::new();
2597 for resolved in members {
2598 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2599 continue;
2600 };
2601 if !resolved_member_visible(&resolved, &record).await {
2602 continue;
2603 }
2604 let Some(session_id) = record.session_id.clone() else {
2605 continue;
2606 };
2607 targets.push(SessionBackfillTarget {
2608 entry: entry.clone(),
2609 record,
2610 session_id,
2611 });
2612 }
2613 backfill_session_history_targets(inner, targets, force_refresh).await
2614}
2615
2616#[derive(Clone)]
2617struct SessionBackfillTarget {
2618 entry: RuntimeEntry,
2619 record: ConsoleIdentityRecord,
2620 session_id: String,
2621}
2622
2623async fn backfill_session_history_targets(
2624 inner: Arc<AggregatorInner>,
2625 targets: Vec<SessionBackfillTarget>,
2626 force_refresh: bool,
2627) -> ConsoleLogResult<()> {
2628 let mut tasks = tokio::task::JoinSet::new();
2629 for target in targets {
2630 tasks.spawn(backfill_one_session_history(
2631 inner.clone(),
2632 target,
2633 force_refresh,
2634 ));
2635 }
2636 let mut first_error = None;
2637 while let Some(result) = tasks.join_next().await {
2638 match result {
2639 Ok(Ok(())) => {}
2640 Ok(Err(err)) => {
2641 if first_error.is_none() {
2642 first_error = Some(err);
2643 }
2644 }
2645 Err(err) => {
2646 if first_error.is_none() {
2647 first_error = Some(Box::new(std::io::Error::other(format!(
2648 "session backfill task failed: {err}"
2649 ))) as ConsoleLogError);
2650 }
2651 }
2652 }
2653 }
2654 if let Some(err) = first_error {
2655 Err(err)
2656 } else {
2657 Ok(())
2658 }
2659}
2660
2661async fn backfill_one_session_history(
2662 inner: Arc<AggregatorInner>,
2663 target: SessionBackfillTarget,
2664 force_refresh: bool,
2665) -> ConsoleLogResult<()> {
2666 let _permit = inner
2667 .session_backfill_permits
2668 .clone()
2669 .acquire_owned()
2670 .await
2671 .map_err(|err| -> ConsoleLogError {
2672 Box::new(std::io::Error::other(format!(
2673 "session backfill limiter closed: {err}"
2674 )))
2675 })?;
2676 let SessionBackfillTarget {
2677 entry,
2678 record,
2679 session_id,
2680 } = target;
2681 let watermark_runtime_key =
2682 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2683 let watermark = inner
2684 .store
2685 .source_watermark(
2686 &watermark_runtime_key,
2687 ConsoleFrameSourceKind::SessionHistory,
2688 )
2689 .await?;
2690 let now_ms = current_time_ms();
2691 let mut offset = watermark
2692 .as_deref()
2693 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2694 .unwrap_or(0);
2695 if !force_refresh
2696 && watermark.as_deref().is_some_and(|watermark| {
2697 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2698 })
2699 {
2700 return Ok(());
2701 }
2702 loop {
2703 let page = match entry
2704 .runtime
2705 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2706 .await
2707 {
2708 Ok(page) => page,
2709 Err(err) => {
2710 append_backfill_gap(
2711 &inner,
2712 &entry.runtime_key,
2713 &record.identity,
2714 err.to_string(),
2715 )
2716 .await?;
2717 break;
2718 }
2719 };
2720 let page_value = match serde_json::to_value(page) {
2721 Ok(value) => value,
2722 Err(err) => {
2723 append_backfill_gap(
2724 &inner,
2725 &entry.runtime_key,
2726 &record.identity,
2727 err.to_string(),
2728 )
2729 .await?;
2730 break;
2731 }
2732 };
2733 let base_offset = page_value
2734 .get("offset")
2735 .and_then(Value::as_u64)
2736 .unwrap_or(offset as u64) as usize;
2737 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
2738 append_backfill_gap(
2739 &inner,
2740 &entry.runtime_key,
2741 &record.identity,
2742 "session history page missing messages".to_string(),
2743 )
2744 .await?;
2745 break;
2746 };
2747 if messages.is_empty() {
2748 if offset > 0 {
2749 record_session_history_watermark(
2750 &inner,
2751 &watermark_runtime_key,
2752 &session_id,
2753 offset,
2754 )
2755 .await?;
2756 }
2757 break;
2758 }
2759 for (idx, message) in messages.iter().enumerate() {
2760 let absolute_offset = base_offset + idx;
2761 let frames = frames_from_session_history_message_with_namespace(
2762 &entry.runtime_key,
2763 &record.identity,
2764 &entry.identity_namespace,
2765 &session_id,
2766 absolute_offset,
2767 message.clone(),
2768 );
2769 for mut frame in frames {
2770 if history_frame_has_existing_counterpart(&inner, &frame).await? {
2771 continue;
2772 }
2773 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
2774 frame.payload = redacted;
2775 frame.status = ConsoleFrameStatus::Redacted;
2776 }
2777 append_and_emit(&inner, frame).await?;
2778 }
2779 }
2780 offset = base_offset + messages.len();
2781 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
2782 .await?;
2783 let has_more = page_value
2784 .get("has_more")
2785 .and_then(Value::as_bool)
2786 .unwrap_or(false);
2787 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
2788 break;
2789 }
2790 }
2791 Ok(())
2792}
2793
2794async fn record_session_history_watermark(
2795 inner: &AggregatorInner,
2796 watermark_runtime_key: &str,
2797 session_id: &str,
2798 offset: usize,
2799) -> ConsoleLogResult<()> {
2800 inner
2801 .store
2802 .record_source_watermark(
2803 watermark_runtime_key,
2804 ConsoleFrameSourceKind::SessionHistory,
2805 &format_session_history_watermark(session_id, offset, current_time_ms()),
2806 )
2807 .await
2808}
2809
2810fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
2811 if !inner.options.session_history_backfill_enabled {
2812 return;
2813 }
2814 tokio::spawn(async move {
2815 {
2816 let mut active = inner.active_session_backfills.lock().await;
2817 if !active.insert(runtime_key.clone()) {
2818 return;
2819 }
2820 }
2821 let result = backfill_session_history(inner.clone(), runtime_key.clone(), false).await;
2822 let mut active = inner.active_session_backfills.lock().await;
2823 active.remove(&runtime_key);
2824 drop(active);
2825 if let Err(err) = result {
2826 tracing::warn!(
2827 runtime_key = %runtime_key,
2828 error = %err,
2829 "console session-history backfill failed"
2830 );
2831 }
2832 });
2833}
2834
2835fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
2836 if !inner.options.session_history_backfill_enabled {
2837 return;
2838 }
2839 tokio::spawn(async move {
2840 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
2841 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2842 loop {
2843 interval.tick().await;
2844 let runtime_still_registered = inner
2845 .runtimes
2846 .read()
2847 .ok()
2848 .is_some_and(|entries| entries.contains_key(&runtime_key));
2849 if !runtime_still_registered {
2850 break;
2851 }
2852 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
2853 }
2854 });
2855}
2856
2857fn spawn_session_history_backfill_target(
2858 inner: Arc<AggregatorInner>,
2859 target: SessionBackfillTarget,
2860 force_refresh: bool,
2861) {
2862 if !inner.options.session_history_backfill_enabled {
2863 return;
2864 }
2865 tokio::spawn(async move {
2866 let active_key = targeted_session_history_active_key(&target, force_refresh);
2867 let result =
2868 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
2869 if let Err(err) = result {
2870 tracing::warn!(
2871 active_key = %active_key,
2872 error = %err,
2873 "console targeted session-history backfill failed"
2874 );
2875 }
2876 });
2877}
2878
2879async fn run_targeted_session_history_backfill(
2880 inner: Arc<AggregatorInner>,
2881 target: SessionBackfillTarget,
2882 force_refresh: bool,
2883) -> ConsoleLogResult<()> {
2884 let active_key = targeted_session_history_active_key(&target, force_refresh);
2885 {
2886 let mut active = inner.active_session_backfills.lock().await;
2887 if !active.insert(active_key.clone()) {
2888 return Ok(());
2889 }
2890 }
2891 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
2892 let mut active = inner.active_session_backfills.lock().await;
2893 active.remove(&active_key);
2894 result
2895}
2896
2897fn targeted_session_history_active_key(
2898 target: &SessionBackfillTarget,
2899 force_refresh: bool,
2900) -> String {
2901 let mode = if force_refresh { "force" } else { "refresh" };
2902 format!(
2903 "{}:session-history:{}:{mode}",
2904 target.entry.runtime_key, target.session_id
2905 )
2906}
2907
2908fn spawn_session_history_backfill_for_identity(
2909 inner: Arc<AggregatorInner>,
2910 identity: String,
2911 force_refresh: bool,
2912) {
2913 if !inner.options.session_history_backfill_enabled {
2914 return;
2915 }
2916 tokio::spawn(async move {
2917 for target in session_backfill_targets_for_identity(&inner, &identity).await {
2918 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
2919 }
2920 });
2921}
2922
2923fn spawn_opportunistic_session_history_backfill_for_identity(
2924 inner: Arc<AggregatorInner>,
2925 identity: String,
2926) {
2927 if !inner.options.session_history_backfill_enabled {
2928 return;
2929 }
2930 tokio::spawn(async move {
2931 for target in session_backfill_targets_for_identity(&inner, &identity).await {
2932 let active_key = format!(
2933 "{}:session-history:{}",
2934 target.entry.runtime_key, target.session_id
2935 );
2936 {
2937 let mut seen = inner.opportunistic_session_backfills.lock().await;
2938 if !seen.insert(active_key) {
2939 continue;
2940 }
2941 }
2942 spawn_session_history_backfill_target(inner.clone(), target, false);
2943 }
2944 });
2945}
2946
2947#[cfg(test)]
2948async fn session_backfill_target_for_identity(
2949 inner: &AggregatorInner,
2950 identity: &str,
2951) -> Option<SessionBackfillTarget> {
2952 session_backfill_targets_for_identity(inner, identity)
2953 .await
2954 .into_iter()
2955 .next()
2956}
2957
2958async fn session_backfill_targets_for_identity(
2959 inner: &AggregatorInner,
2960 identity: &str,
2961) -> Vec<SessionBackfillTarget> {
2962 let entries = inner
2963 .runtimes
2964 .read()
2965 .ok()
2966 .map(|entries| entries.clone())
2967 .unwrap_or_default();
2968 let mut targets = Vec::new();
2969 for entry in entries.values() {
2970 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2971 if raw_identities.is_empty() {
2972 continue;
2973 }
2974 let mids = raw_identities
2975 .iter()
2976 .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
2977 .collect::<Vec<_>>();
2978 for resolved in member_sources_for_entry(entry)
2979 .await
2980 .into_iter()
2981 .filter(|candidate| {
2982 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
2983 })
2984 {
2985 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2986 continue;
2987 };
2988 if !resolved_member_visible(&resolved, &record).await {
2989 continue;
2990 }
2991 let Some(session_id) = record.session_id.clone() else {
2992 continue;
2993 };
2994 targets.push(SessionBackfillTarget {
2995 entry: entry.clone(),
2996 record,
2997 session_id,
2998 });
2999 }
3000 }
3001 targets
3002}
3003
3004fn resolved_member_matches_raw_identities(
3005 resolved: &ResolvedConsoleMember,
3006 raw_identities: &[String],
3007 mids: &[MeerkatId],
3008) -> bool {
3009 mids.contains(&resolved.member.agent_identity)
3010 || resolved
3011 .member
3012 .labels
3013 .get("agent_identity")
3014 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3015}
3016
3017async fn recover_lagged_source_events(
3018 inner: Arc<AggregatorInner>,
3019 runtime_key: &str,
3020 console_events: &ConsoleEventStore,
3021) -> ConsoleLogResult<()> {
3022 let watermark = inner
3023 .store
3024 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3025 .await?;
3026 match console_events.replay_all(watermark.as_deref()).await {
3027 Ok(events) => {
3028 for envelope in events {
3029 project_console_event(inner.clone(), runtime_key, envelope).await?;
3030 }
3031 }
3032 Err(err) => {
3033 append_source_gap(
3034 &inner,
3035 runtime_key,
3036 format!(
3037 "{}:{}:{}",
3038 err.error, err.stream, err.requested_last_event_id
3039 ),
3040 )
3041 .await?;
3042 }
3043 }
3044 Ok(())
3045}
3046
3047async fn append_source_gap(
3048 inner: &AggregatorInner,
3049 runtime_key: &str,
3050 reason: String,
3051) -> ConsoleLogResult<()> {
3052 append_and_emit(
3053 inner,
3054 NewConsoleFrame {
3055 id: None,
3056 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3057 timestamp_ms: current_time_ms(),
3058 runtime_key: runtime_key.to_string(),
3059 identity: "__console__".to_string(),
3060 conversation_id: None,
3061 session_id: None,
3062 kind: "replay_unavailable".to_string(),
3063 status: ConsoleFrameStatus::DeliveryFailed,
3064 payload: json!({
3065 "reason": reason,
3066 "source_kind": "console_event",
3067 }),
3068 source: ConsoleFrameSource {
3069 kind: ConsoleFrameSourceKind::Synthetic,
3070 source_cursor: None,
3071 },
3072 source_event_id: None,
3073 interaction_id: None,
3074 turn_id: None,
3075 run_id: None,
3076 parent_frame_id: None,
3077 caused_by_frame_id: None,
3078 },
3079 )
3080 .await?;
3081 let _ = inner
3082 .event_tx
3083 .send(ConsoleTimelineEvent::ReplayUnavailable {
3084 requested_cursor: format!("source-gap:{runtime_key}"),
3085 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3086 });
3087 Ok(())
3088}
3089
3090async fn append_backfill_gap(
3091 inner: &AggregatorInner,
3092 runtime_key: &str,
3093 identity: &str,
3094 reason: String,
3095) -> ConsoleLogResult<()> {
3096 append_and_emit(
3097 inner,
3098 NewConsoleFrame {
3099 id: None,
3100 dedupe_key: format!(
3101 "session-backfill-gap:{runtime_key}:{identity}:{}",
3102 current_time_ms()
3103 ),
3104 timestamp_ms: current_time_ms(),
3105 runtime_key: runtime_key.to_string(),
3106 identity: identity.to_string(),
3107 conversation_id: Some(identity.to_string()),
3108 session_id: None,
3109 kind: "replay_unavailable".to_string(),
3110 status: ConsoleFrameStatus::DeliveryFailed,
3111 payload: json!({
3112 "reason": reason,
3113 "source_kind": "session_history",
3114 }),
3115 source: ConsoleFrameSource {
3116 kind: ConsoleFrameSourceKind::Synthetic,
3117 source_cursor: None,
3118 },
3119 source_event_id: None,
3120 interaction_id: None,
3121 turn_id: None,
3122 run_id: None,
3123 parent_frame_id: None,
3124 caused_by_frame_id: None,
3125 },
3126 )
3127 .await?;
3128 Ok(())
3129}
3130
3131async fn project_console_event(
3132 inner: Arc<AggregatorInner>,
3133 runtime_key: &str,
3134 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3135) -> ConsoleLogResult<()> {
3136 let Some(entry) = inner
3137 .runtimes
3138 .read()
3139 .ok()
3140 .and_then(|entries| entries.get(runtime_key).cloned())
3141 else {
3142 return Ok(());
3143 };
3144 let mut frame = frame_from_console_event(&entry, envelope);
3145 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3146 frame.payload = redacted;
3147 frame.status = ConsoleFrameStatus::Redacted;
3148 }
3149 let source_cursor = frame
3150 .source_event_id
3151 .clone()
3152 .unwrap_or_else(|| frame.dedupe_key.clone());
3153 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3154 Some(frame.identity.clone())
3155 } else {
3156 None
3157 };
3158 let opportunistic_refresh_identity = if refresh_identity.is_none()
3159 && console_event_should_start_session_history_backfill(&frame)
3160 {
3161 Some(frame.identity.clone())
3162 } else {
3163 None
3164 };
3165 append_and_emit(&inner, frame).await?;
3166 inner
3167 .store
3168 .record_source_watermark(
3169 &entry.runtime_key,
3170 ConsoleFrameSourceKind::ConsoleEvent,
3171 &source_cursor,
3172 )
3173 .await?;
3174 if let Some(identity) = refresh_identity {
3175 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3176 } else if let Some(identity) = opportunistic_refresh_identity {
3177 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3178 }
3179 Ok(())
3180}
3181
3182fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3183 matches!(
3184 frame.kind.as_str(),
3185 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3186 ) || frame.session_id.is_some()
3187}
3188
3189fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3190 if frame.identity == SYSTEM_EVENT_IDENTITY {
3191 return false;
3192 }
3193 matches!(
3194 frame.kind.as_str(),
3195 "turn_started"
3196 | "run_started"
3197 | "reasoning_complete"
3198 | "tool_call_requested"
3199 | "tool_call"
3200 | "tool_execution_started"
3201 | "text_delta"
3202 | "system_notice"
3203 )
3204}
3205
3206async fn append_and_emit(
3207 inner: &AggregatorInner,
3208 frame: NewConsoleFrame,
3209) -> ConsoleLogResult<AppendOutcome> {
3210 let outcome = inner.store.append_if_absent(frame).await?;
3211 if outcome.disposition == AppendDisposition::Inserted {
3212 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3213 frame: outcome.frame.clone(),
3214 });
3215 }
3216 Ok(outcome)
3217}
3218
3219async fn update_frame_status_and_emit(
3220 inner: &AggregatorInner,
3221 frame_id: &str,
3222 status: ConsoleFrameStatus,
3223) -> ConsoleLogResult<Option<ConsoleFrame>> {
3224 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3225 return Ok(None);
3226 };
3227 let update_marker = NewConsoleFrame {
3228 id: None,
3229 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3230 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3231 runtime_key: updated.runtime_key.clone(),
3232 identity: updated.identity.clone(),
3233 conversation_id: updated.conversation_id.clone(),
3234 session_id: updated.session_id.clone(),
3235 kind: "frame_updated".to_string(),
3236 status: updated.status,
3237 payload: json!({ "frame": updated.clone() }),
3238 source: ConsoleFrameSource {
3239 kind: ConsoleFrameSourceKind::Synthetic,
3240 source_cursor: None,
3241 },
3242 source_event_id: None,
3243 interaction_id: updated.interaction_id.clone(),
3244 turn_id: updated.turn_id.clone(),
3245 run_id: updated.run_id.clone(),
3246 parent_frame_id: Some(updated.id.clone()),
3247 caused_by_frame_id: Some(updated.id.clone()),
3248 };
3249 let outcome = inner.store.append_if_absent(update_marker).await?;
3250 if outcome.disposition == AppendDisposition::Inserted {
3251 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3252 frame: outcome.frame,
3253 });
3254 }
3255 Ok(Some(updated))
3256}
3257
3258fn frame_from_console_event(
3259 entry: &RuntimeEntry,
3260 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3261) -> NewConsoleFrame {
3262 let turn_id = envelope
3263 .data
3264 .get("turn_id")
3265 .and_then(Value::as_str)
3266 .map(ToString::to_string);
3267 let run_id = envelope
3268 .data
3269 .get("run_id")
3270 .and_then(Value::as_str)
3271 .map(ToString::to_string);
3272 let status = match envelope.event_type.as_str() {
3273 "interaction_started" => ConsoleFrameStatus::Accepted,
3274 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3275 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3276 _ => ConsoleFrameStatus::Delivered,
3277 };
3278 let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3279 NewConsoleFrame {
3280 id: Some(envelope.event_id.clone()),
3281 dedupe_key: format!("console-event:{}:{}", entry.runtime_key, envelope.event_id),
3282 timestamp_ms: envelope.timestamp_ms,
3283 runtime_key: entry.runtime_key.clone(),
3284 identity: identity.clone(),
3285 conversation_id: Some(identity),
3286 session_id: envelope
3287 .data
3288 .get("session_id")
3289 .and_then(Value::as_str)
3290 .map(ToString::to_string),
3291 kind: envelope.event_type,
3292 status,
3293 payload: envelope.data,
3294 source: ConsoleFrameSource {
3295 kind: ConsoleFrameSourceKind::ConsoleEvent,
3296 source_cursor: None,
3297 },
3298 source_event_id: Some(envelope.event_id),
3299 interaction_id: envelope.interaction_id,
3300 turn_id,
3301 run_id,
3302 parent_frame_id: None,
3303 caused_by_frame_id: None,
3304 }
3305}
3306
3307#[cfg(test)]
3308fn frame_from_session_history_message(
3309 runtime_key: &str,
3310 identity: &str,
3311 session_id: &str,
3312 offset: usize,
3313 message: Value,
3314) -> Option<NewConsoleFrame> {
3315 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3316 .into_iter()
3317 .next()
3318}
3319
3320#[cfg(test)]
3321fn frames_from_session_history_message(
3322 runtime_key: &str,
3323 identity: &str,
3324 session_id: &str,
3325 offset: usize,
3326 message: Value,
3327) -> Vec<NewConsoleFrame> {
3328 frames_from_session_history_message_with_namespace(
3329 runtime_key,
3330 identity,
3331 "",
3332 session_id,
3333 offset,
3334 message,
3335 )
3336}
3337
3338fn frames_from_session_history_message_with_namespace(
3339 runtime_key: &str,
3340 identity: &str,
3341 identity_namespace: &str,
3342 session_id: &str,
3343 offset: usize,
3344 message: Value,
3345) -> Vec<NewConsoleFrame> {
3346 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3347 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3348 return Vec::new();
3349 };
3350 if let Message::ToolResults {
3351 results,
3352 created_at,
3353 } = parsed
3354 {
3355 return results
3356 .into_iter()
3357 .enumerate()
3358 .map(|(idx, result)| {
3359 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3360 let result_text = result.text_content();
3361 let tool_use_id = result.tool_use_id.clone();
3362 NewConsoleFrame {
3363 id: None,
3364 dedupe_key: format!(
3365 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3366 ),
3367 timestamp_ms: created_at.timestamp_millis().max(0) as u64,
3368 runtime_key: runtime_key.to_string(),
3369 identity: identity.to_string(),
3370 conversation_id: Some(identity.to_string()),
3371 session_id: Some(session_id.to_string()),
3372 kind: "tool_execution_completed".to_string(),
3373 status: ConsoleFrameStatus::Completed,
3374 payload: json!({
3375 "id": tool_use_id,
3376 "tool_call_id": tool_use_id,
3377 "result": result_text,
3378 "content": content,
3379 "is_error": result.is_error,
3380 "source_event_type": "session_history",
3381 "type": "session_history",
3382 }),
3383 source: ConsoleFrameSource {
3384 kind: ConsoleFrameSourceKind::SessionHistory,
3385 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3386 },
3387 source_event_id: None,
3388 interaction_id: None,
3389 turn_id: None,
3390 run_id: None,
3391 parent_frame_id: None,
3392 caused_by_frame_id: None,
3393 }
3394 })
3395 .collect();
3396 }
3397 let (kind, timestamp_ms, payload) = match &parsed {
3398 Message::User(user) => {
3399 if session_history_user_message_is_scaffold(&message) {
3400 return Vec::new();
3401 }
3402 (
3403 "user_input",
3404 user.created_at.timestamp_millis().max(0) as u64,
3405 json!({
3406 "content": user.content,
3407 "message": message,
3408 }),
3409 )
3410 }
3411 Message::Assistant(assistant) => (
3412 "interaction_complete",
3413 assistant.created_at.timestamp_millis().max(0) as u64,
3414 json!({
3415 "result": assistant.content,
3416 "text": assistant.content,
3417 "message": message,
3418 "source_event_type": "session_history",
3419 "type": "session_history",
3420 }),
3421 ),
3422 Message::BlockAssistant(assistant) => {
3423 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3424 (
3425 "interaction_complete",
3426 assistant.created_at.timestamp_millis().max(0) as u64,
3427 json!({
3428 "result": text,
3429 "text": text,
3430 "message": message,
3431 "source_event_type": "session_history",
3432 "type": "session_history",
3433 }),
3434 )
3435 }
3436 Message::SystemNotice(notice) => (
3437 "system_notice",
3438 notice.created_at.timestamp_millis().max(0) as u64,
3439 json!({
3440 "message": message,
3441 "kind": notice.kind,
3442 "render_class": notice.kind.render_class(),
3443 "body": notice.body,
3444 "blocks": notice.blocks,
3445 "source_event_type": "session_history",
3446 "type": "session_history",
3447 }),
3448 ),
3449 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3450 };
3451 let mut frames = vec![NewConsoleFrame {
3452 id: None,
3453 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3454 timestamp_ms,
3455 runtime_key: runtime_key.to_string(),
3456 identity: identity.to_string(),
3457 conversation_id: Some(identity.to_string()),
3458 session_id: Some(session_id.to_string()),
3459 kind: kind.to_string(),
3460 status: ConsoleFrameStatus::Completed,
3461 payload,
3462 source: ConsoleFrameSource {
3463 kind: ConsoleFrameSourceKind::SessionHistory,
3464 source_cursor: Some(format!("{session_id}:{offset}")),
3465 },
3466 source_event_id: None,
3467 interaction_id: None,
3468 turn_id: None,
3469 run_id: None,
3470 parent_frame_id: None,
3471 caused_by_frame_id: None,
3472 }];
3473 if let Message::BlockAssistant(assistant) = &parsed {
3474 frames.extend(spawn_initial_message_frames_from_assistant(
3475 runtime_key,
3476 identity,
3477 identity_namespace,
3478 session_id,
3479 offset,
3480 assistant,
3481 &payload_hash,
3482 ));
3483 }
3484 frames
3485}
3486
3487fn spawn_initial_message_frames_from_assistant(
3488 runtime_key: &str,
3489 parent_identity: &str,
3490 identity_namespace: &str,
3491 session_id: &str,
3492 offset: usize,
3493 assistant: &meerkat_core::types::BlockAssistantMessage,
3494 payload_hash: &str,
3495) -> Vec<NewConsoleFrame> {
3496 let mut frames = Vec::new();
3497 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3498 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3499 continue;
3500 }
3501 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3502 continue;
3503 };
3504 for (spawn_idx, (target_identity, initial_message)) in
3505 spawn_initial_messages_from_tool_args(&args)
3506 .into_iter()
3507 .enumerate()
3508 {
3509 let target_identity = apply_namespace(&target_identity, identity_namespace);
3510 let message_content = match &initial_message {
3511 Value::String(text) => json!([
3512 {
3513 "type": "text",
3514 "text": text,
3515 }
3516 ]),
3517 other => other.clone(),
3518 };
3519 let message = match &initial_message {
3520 Value::String(text) => json!({
3521 "role": "user",
3522 "content": text,
3523 "created_at": assistant.created_at,
3524 }),
3525 other => json!({
3526 "role": "user",
3527 "content": other,
3528 "created_at": assistant.created_at,
3529 }),
3530 };
3531 frames.push(NewConsoleFrame {
3532 id: None,
3533 dedupe_key: format!(
3534 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3535 ),
3536 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3537 runtime_key: runtime_key.to_string(),
3538 identity: target_identity.clone(),
3539 conversation_id: Some(target_identity),
3540 session_id: None,
3541 kind: "user_input".to_string(),
3542 status: ConsoleFrameStatus::Completed,
3543 payload: json!({
3544 "content": message_content,
3545 "message": message,
3546 "source_event_type": "session_history_spawn_initial_message",
3547 "type": "session_history",
3548 "tool_call_id": tool.id,
3549 "parent_identity": parent_identity,
3550 "via_tool": tool.name,
3551 }),
3552 source: ConsoleFrameSource {
3553 kind: ConsoleFrameSourceKind::SessionHistory,
3554 source_cursor: Some(format!(
3555 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
3556 )),
3557 },
3558 source_event_id: None,
3559 interaction_id: None,
3560 turn_id: None,
3561 run_id: None,
3562 parent_frame_id: None,
3563 caused_by_frame_id: None,
3564 });
3565 }
3566 }
3567 frames
3568}
3569
3570fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
3571 let mut messages = Vec::new();
3572 if let Some(message) = spawn_initial_message_from_record(args) {
3573 messages.push(message);
3574 }
3575 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
3576 for spec in specs {
3577 if let Some(message) = spawn_initial_message_from_record(spec) {
3578 messages.push(message);
3579 }
3580 }
3581 }
3582 messages
3583}
3584
3585fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
3586 let target_identity = record
3587 .get("member_id")
3588 .or_else(|| record.get("agent_identity"))
3589 .and_then(Value::as_str)?
3590 .trim();
3591 if target_identity.is_empty() {
3592 return None;
3593 }
3594 let initial_message = record
3595 .get("initial_message")
3596 .or_else(|| record.get("task"))?
3597 .clone();
3598 Some((target_identity.to_string(), initial_message))
3599}
3600
3601fn session_history_user_message_is_scaffold(message: &Value) -> bool {
3602 message
3603 .get("content")
3604 .is_some_and(scaffold_message_content_is_noise)
3605}
3606
3607fn scaffold_message_content_is_noise(value: &Value) -> bool {
3608 match value {
3609 Value::String(text) => scaffold_message_text_is_noise(text),
3610 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
3611 Value::Object(map) => ["text", "content", "message"]
3612 .iter()
3613 .filter_map(|key| map.get(*key))
3614 .any(scaffold_message_content_is_noise),
3615 _ => false,
3616 }
3617}
3618
3619fn scaffold_message_text_is_noise(text: &str) -> bool {
3620 let trimmed = text.trim_start();
3621 trimmed.starts_with("[PEER UPDATE]")
3622 || trimmed
3623 .to_ascii_lowercase()
3624 .starts_with("you have been spawned")
3625}
3626
3627fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
3628 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
3629 rest.split(':').next()?.parse().ok()
3630}
3631
3632fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
3633 format!("{session_id}:{offset}:{checked_at_ms}")
3634}
3635
3636fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
3637 let Some(checked_at_ms) = watermark
3638 .rsplit_once(':')
3639 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
3640 else {
3641 return false;
3642 };
3643 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
3644 let ttl_ms = if offset > 0 {
3645 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
3646 } else {
3647 SESSION_HISTORY_REFRESH_TTL_MS
3648 };
3649 now_ms.saturating_sub(checked_at_ms) < ttl_ms
3650}
3651
3652async fn history_frame_has_existing_counterpart(
3653 inner: &AggregatorInner,
3654 frame: &NewConsoleFrame,
3655) -> ConsoleLogResult<bool> {
3656 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
3657 let Some(fingerprint) = fingerprint else {
3658 return Ok(false);
3659 };
3660 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
3661 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
3662 let mut after = None;
3663 loop {
3664 let page = inner
3665 .store
3666 .query_frames(ConsoleTimelineQuery {
3667 identity: Some(frame.identity.clone()),
3668 conversation_id: frame.conversation_id.clone(),
3669 after,
3670 limit: 1_000,
3671 })
3672 .await?;
3673 for existing in &page.frames {
3674 let same_session = existing.session_id == frame.session_id
3675 || existing.session_id.is_none()
3676 || frame.session_id.is_none();
3677 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
3678 continue;
3679 }
3680 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
3681 == Some(&fingerprint)
3682 {
3683 return Ok(true);
3684 }
3685 if assistant_terminal
3686 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
3687 {
3688 let turn_key = existing
3689 .interaction_id
3690 .as_deref()
3691 .or(existing.turn_id.as_deref())
3692 .or(existing.run_id.as_deref())
3693 .unwrap_or("session");
3694 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
3695 aggregated.push_str(delta);
3696 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
3697 return Ok(true);
3698 }
3699 }
3700 }
3701 if page.frames.is_empty() || page.next_cursor.is_none() {
3702 return Ok(false);
3703 }
3704 after = page.next_cursor;
3705 }
3706}
3707
3708fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
3709 format!("{runtime_key}:session-history:{session_id}")
3710}
3711
3712fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3713 match kind {
3714 "user_input" | "interaction_started" => payload
3715 .get("content")
3716 .map(stable_value_fingerprint)
3717 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
3718 "tool_execution_completed" => tool_result_fingerprint(payload),
3719 "text_delta" => {
3720 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
3721 }
3722 "text_complete" | "interaction_complete" | "run_completed" => {
3723 assistant_terminal_fingerprint(kind, payload)
3724 }
3725 _ => None,
3726 }
3727}
3728
3729fn tool_result_fingerprint(payload: &Value) -> Option<String> {
3730 let id = payload
3731 .get("tool_call_id")
3732 .or_else(|| payload.get("id"))
3733 .and_then(Value::as_str)
3734 .unwrap_or("");
3735 let result = payload
3736 .get("result")
3737 .or_else(|| payload.get("content"))
3738 .map(stable_value_fingerprint)?;
3739 Some(format!("{id}:{result}"))
3740}
3741
3742fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3743 match kind {
3744 "text_complete" | "interaction_complete" | "run_completed" => payload
3745 .get("text")
3746 .or_else(|| payload.get("result"))
3747 .or_else(|| payload.get("content"))
3748 .map(stable_value_fingerprint),
3749 _ => None,
3750 }
3751}
3752
3753fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
3754 if kind != "text_delta" {
3755 return None;
3756 }
3757 payload
3758 .get("delta")
3759 .or_else(|| payload.get("text"))
3760 .or_else(|| payload.get("content"))
3761 .and_then(Value::as_str)
3762 .or_else(|| payload.as_str())
3763}
3764
3765fn stable_value_fingerprint(value: &Value) -> String {
3766 if let Some(text) = content_value_text(value) {
3767 return normalize_transcript_fingerprint_text(&text);
3768 }
3769 match value {
3770 Value::String(text) => normalize_transcript_fingerprint_text(text),
3771 other => serde_json::to_string(other).unwrap_or_default(),
3772 }
3773}
3774
3775fn content_value_text(value: &Value) -> Option<String> {
3776 match value {
3777 Value::String(text) => Some(text.clone()),
3778 Value::Array(items) => {
3779 let text = items
3780 .iter()
3781 .filter_map(content_value_text)
3782 .collect::<String>();
3783 (!text.is_empty()).then_some(text)
3784 }
3785 Value::Object(map) => ["text", "content", "message", "blocks"]
3786 .iter()
3787 .filter_map(|key| map.get(*key))
3788 .find_map(content_value_text),
3789 _ => None,
3790 }
3791}
3792
3793fn normalize_transcript_fingerprint_text(text: &str) -> String {
3794 let trimmed = text.trim();
3795 trimmed
3796 .strip_prefix("[EVENT via rpc] ")
3797 .unwrap_or(trimmed)
3798 .trim()
3799 .to_string()
3800}
3801
3802#[derive(Debug, Clone, Copy)]
3803enum CachedIdentityVisibility {
3804 Visible,
3805 Hidden,
3806 Missing,
3807}
3808
3809async fn frame_is_visible(
3810 inner: &AggregatorInner,
3811 frame: &ConsoleFrame,
3812 allow_historical_identity: bool,
3813 identity_records: &[ConsoleIdentityRecord],
3814) -> ConsoleLogResult<bool> {
3815 let mut identity_visibility_cache = HashMap::new();
3816 frame_is_visible_cached(
3817 inner,
3818 frame,
3819 allow_historical_identity,
3820 &mut identity_visibility_cache,
3821 identity_records,
3822 )
3823 .await
3824}
3825
3826async fn frame_is_visible_cached(
3827 inner: &AggregatorInner,
3828 frame: &ConsoleFrame,
3829 allow_historical_identity: bool,
3830 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
3831 identity_records: &[ConsoleIdentityRecord],
3832) -> ConsoleLogResult<bool> {
3833 let entry = {
3834 let entries = inner
3835 .runtimes
3836 .read()
3837 .map_err(|_| runtime_registry_lock_error())?;
3838 if entries.is_empty() {
3839 return Ok(true);
3840 }
3841 let Some(entry) = entries.get(&frame.runtime_key) else {
3842 return Ok(false);
3843 };
3844 entry.clone()
3845 };
3846 if frame.identity != "__console__" {
3847 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
3848 let identity_visibility =
3849 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
3850 cached
3851 } else {
3852 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
3853 .unwrap_or_else(|| frame.identity.clone());
3854 let visibility = match identity_records.iter().find(|record| {
3855 record.runtime_key == frame.runtime_key
3856 && (record.identity == frame.identity
3857 || record.runtime_member_id == runtime_member_id)
3858 }) {
3859 Some(record) => {
3860 if console_identity_record_visible(&entry, record).await {
3861 CachedIdentityVisibility::Visible
3862 } else {
3863 CachedIdentityVisibility::Hidden
3864 }
3865 }
3866 None => CachedIdentityVisibility::Missing,
3867 };
3868 identity_visibility_cache.insert(cache_key, visibility);
3869 visibility
3870 };
3871 match identity_visibility {
3872 CachedIdentityVisibility::Visible => {
3873 if frame_matches_hidden_member(&entry, frame).await {
3874 return Ok(false);
3875 }
3876 }
3877 CachedIdentityVisibility::Hidden => return Ok(false),
3878 CachedIdentityVisibility::Missing => {
3879 if frame_matches_hidden_member(&entry, frame).await {
3880 return Ok(false);
3881 }
3882 return Ok(
3883 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
3884 );
3885 }
3886 }
3887 }
3888 Ok(entry.visibility_policy.frame_visible(frame))
3889}
3890
3891async fn identity_record_for_member(
3892 entry: &RuntimeEntry,
3893 handle: &MobHandle,
3894 member: &MobMemberListEntry,
3895) -> Option<ConsoleIdentityRecord> {
3896 let runtime_member_id = member.agent_identity.to_string();
3897 let durable_identity = member
3898 .labels
3899 .get("agent_identity")
3900 .filter(|value| !value.trim().is_empty())
3901 .map_or(runtime_member_id.as_str(), String::as_str);
3902 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
3903 let addressable = member_is_addressable(member);
3904 let visibility = if member.state == meerkat_mob::MemberState::Retiring {
3905 ConsoleVisibility::RetiredReadable
3906 } else if addressable {
3907 ConsoleVisibility::Addressable
3908 } else {
3909 ConsoleVisibility::Hidden
3910 };
3911 let session_id = handle
3912 .resolve_bridge_session_id_observation(&member.agent_identity)
3913 .await
3914 .map(|sid| sid.to_string());
3915 let display_name = member
3916 .labels
3917 .get("display_name")
3918 .cloned()
3919 .unwrap_or_else(|| runtime_member_id.clone());
3920 let mut labels = member.labels.clone();
3921 labels
3922 .entry("role".to_string())
3923 .or_insert_with(|| member.role.to_string());
3924 Some(ConsoleIdentityRecord {
3925 identity,
3926 display_name,
3927 runtime_key: entry.runtime_key.clone(),
3928 runtime_member_id,
3929 session_id,
3930 visibility,
3931 addressable,
3932 health: if addressable {
3933 "ready"
3934 } else {
3935 "hidden_by_policy"
3936 }
3937 .to_string(),
3938 topology_peers: member.wired_to.iter().map(ToString::to_string).collect(),
3939 labels,
3940 })
3941}
3942
3943async fn identity_record_for_resolved_member(
3944 resolved: &ResolvedConsoleMember,
3945) -> Option<ConsoleIdentityRecord> {
3946 let mut record =
3947 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
3948 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
3949 record
3950 .labels
3951 .entry("source_mob_id".to_string())
3952 .or_insert_with(|| resolved.source_mob_id.clone());
3953 }
3954 Some(record)
3955}
3956
3957fn identity_record_for_status(
3958 entry: &RuntimeEntry,
3959 status: &crate::identity_first::IdentityStatus,
3960) -> ConsoleIdentityRecord {
3961 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
3962 let runtime_member_id = status
3963 .agent_runtime_id
3964 .as_ref()
3965 .map(crate::identity_first::AgentRuntimeId::as_str)
3966 .unwrap_or_else(|| status.identity.as_str())
3967 .to_string();
3968 let addressable = status.addressability
3969 == crate::identity_first::AgentAddressability::Addressable
3970 && matches!(
3971 status.state,
3972 crate::identity_first::IdentityLifecycleState::Active
3973 | crate::identity_first::IdentityLifecycleState::Dormant
3974 | crate::identity_first::IdentityLifecycleState::Uninitialized
3975 );
3976 let visibility = match status.state {
3977 crate::identity_first::IdentityLifecycleState::Retiring => {
3978 ConsoleVisibility::RetiredReadable
3979 }
3980 crate::identity_first::IdentityLifecycleState::Broken
3981 | crate::identity_first::IdentityLifecycleState::Suspended => {
3982 ConsoleVisibility::Unreachable
3983 }
3984 _ if addressable => ConsoleVisibility::Addressable,
3985 _ => ConsoleVisibility::Hidden,
3986 };
3987 let health = match status.state {
3988 crate::identity_first::IdentityLifecycleState::Active => "ready",
3989 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
3990 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
3991 crate::identity_first::IdentityLifecycleState::Broken => "broken",
3992 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
3993 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
3994 }
3995 .to_string();
3996 let display_name = status
3997 .display_name
3998 .as_ref()
3999 .map(crate::identity_first::DisplayName::as_str)
4000 .unwrap_or_else(|| status.identity.as_str())
4001 .to_string();
4002 ConsoleIdentityRecord {
4003 identity,
4004 display_name,
4005 runtime_key: entry.runtime_key.clone(),
4006 runtime_member_id,
4007 session_id: status.session_id.as_ref().map(ToString::to_string),
4008 visibility,
4009 addressable,
4010 health,
4011 topology_peers: Vec::new(),
4012 labels: status.labels.clone(),
4013 }
4014}
4015
4016pub(crate) fn is_implicit_delegate_member(
4017 role: &str,
4018 labels: &std::collections::BTreeMap<String, String>,
4019) -> bool {
4020 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4021}
4022
4023fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4024 member
4025 .labels
4026 .get("addressable")
4027 .map(|value| !value.eq_ignore_ascii_case("false"))
4028 .unwrap_or(true)
4029}
4030
4031fn apply_namespace(identity: &str, namespace: &str) -> String {
4032 let namespace = namespace.trim().trim_matches('/');
4033 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4034 identity.to_string()
4035 } else {
4036 format!("{namespace}/{identity}")
4037 }
4038}
4039
4040fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4041 let namespace = namespace.trim().trim_matches('/');
4042 if namespace.is_empty() {
4043 return Some(identity.to_string());
4044 }
4045 identity
4046 .strip_prefix(namespace)
4047 .and_then(|rest| rest.strip_prefix('/'))
4048 .map(ToString::to_string)
4049}
4050
4051fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4052 let namespace = namespace.trim().trim_matches('/');
4053 if namespace.is_empty() {
4054 return vec![identity.to_string()];
4055 }
4056 let mut candidates = Vec::new();
4057 if let Some(stripped) = strip_namespace(identity, namespace) {
4058 candidates.push(stripped);
4059 candidates.push(identity.to_string());
4060 }
4061 candidates.sort();
4062 candidates.dedup();
4063 candidates
4064}
4065
4066fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4067 namespace_match_candidates(identity, namespace)
4068 .iter()
4069 .any(|candidate| candidate.starts_with("rt:"))
4070}
4071
4072fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4073 entries.iter().any(|entry| {
4074 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4075 })
4076}
4077
4078fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4079 if request.identity.trim().is_empty() {
4080 return Err(ConsoleSendError::InvalidRequest(
4081 "identity must be non-empty".to_string(),
4082 ));
4083 }
4084 if request.origin.trim().is_empty() {
4085 return Err(ConsoleSendError::InvalidRequest(
4086 "origin must be non-empty".to_string(),
4087 ));
4088 }
4089 if request.idempotency_key.trim().is_empty() {
4090 return Err(ConsoleSendError::InvalidRequest(
4091 "idempotency_key must be non-empty".to_string(),
4092 ));
4093 }
4094 Ok(())
4095}
4096
4097fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4098 let content: ContentInput = serde_json::from_value(value.clone())
4099 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4100 match &content {
4101 ContentInput::Text(text) if text.trim().is_empty() => Err(
4102 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4103 ),
4104 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4105 "content blocks must be non-empty".to_string(),
4106 )),
4107 _ => Ok(content),
4108 }
4109}
4110
4111fn parse_handling_mode(
4112 value: Option<&str>,
4113) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4114 match value.unwrap_or("queue") {
4115 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4116 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4117 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4118 }
4119}
4120
4121fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4122 ConsoleInteractionAccepted {
4123 interaction_id: frame
4124 .interaction_id
4125 .clone()
4126 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4127 identity: frame.identity.clone(),
4128 conversation_id: frame.conversation_id.clone(),
4129 session_id: frame.session_id.clone(),
4130 input_frame_id: frame.id.clone(),
4131 cursor: frame.cursor.clone(),
4132 status: frame.status,
4133 }
4134}
4135
4136fn send_dedupe_key(
4137 runtime_key: &str,
4138 identity: &str,
4139 origin: &str,
4140 idempotency_key: &str,
4141) -> String {
4142 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4143}
4144
4145fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4146 let content_json = serde_json::to_string(content).unwrap_or_default();
4147 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4148}
4149
4150fn hash_short(value: &str) -> String {
4151 let mut hasher = Sha256::new();
4152 hasher.update(value.as_bytes());
4153 let digest = hasher.finalize();
4154 to_hex(&digest[..8])
4155}
4156
4157fn to_hex(bytes: &[u8]) -> String {
4158 const HEX: &[u8; 16] = b"0123456789abcdef";
4159 let mut out = String::with_capacity(bytes.len() * 2);
4160 for byte in bytes {
4161 out.push(HEX[(byte >> 4) as usize] as char);
4162 out.push(HEX[(byte & 0x0f) as usize] as char);
4163 }
4164 out
4165}
4166
4167fn current_time_ms() -> u64 {
4168 match SystemTime::now().duration_since(UNIX_EPOCH) {
4169 Ok(duration) => duration.as_millis() as u64,
4170 Err(_) => 0,
4171 }
4172}
4173
4174fn runtime_registry_lock_error() -> ConsoleLogError {
4175 Box::new(std::io::Error::other(
4176 "console runtime registry lock poisoned",
4177 ))
4178}
4179
4180#[cfg(test)]
4181#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4182mod tests {
4183 use std::sync::Arc;
4184 use std::sync::atomic::{AtomicUsize, Ordering};
4185 use std::time::Duration;
4186 use std::time::Instant;
4187
4188 use futures::StreamExt;
4189 use meerkat::{AgentFactory, Config, build_ephemeral_service};
4190 use meerkat_client::types::LlmStream;
4191 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4192 use meerkat_core::{
4193 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4194 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4195 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4196 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4197 StreamError,
4198 };
4199 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4200 use serde_json::json;
4201
4202 use super::*;
4203 use crate::mob_handle_runtime::MobBootstrapSpec;
4204
4205 #[derive(Debug)]
4206 struct HideHiddenNoiseFrames;
4207
4208 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4209 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4210 frame.kind != "hidden_noise"
4211 }
4212 }
4213
4214 #[derive(Debug)]
4215 struct HideRuntimeMemberIdentity(&'static str);
4216
4217 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4218 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4219 record.runtime_member_id != self.0
4220 }
4221 }
4222
4223 #[derive(Debug)]
4224 struct HideRuntimeMemberOnly(&'static str);
4225
4226 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4227 fn member_visible(&self, member: &ConsoleMember) -> bool {
4228 member.agent_identity != self.0
4229 }
4230 }
4231
4232 #[derive(Debug)]
4233 struct HideConsoleIdentity(&'static str);
4234
4235 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4236 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4237 record.identity != self.0
4238 }
4239 }
4240
4241 struct CountingConsoleLogStore {
4242 inner: InMemoryConsoleLogStore,
4243 source_watermark_calls: AtomicUsize,
4244 record_watermark_calls: AtomicUsize,
4245 }
4246
4247 impl CountingConsoleLogStore {
4248 fn new() -> Self {
4249 Self {
4250 inner: InMemoryConsoleLogStore::new(),
4251 source_watermark_calls: AtomicUsize::new(0),
4252 record_watermark_calls: AtomicUsize::new(0),
4253 }
4254 }
4255
4256 fn source_watermark_calls(&self) -> usize {
4257 self.source_watermark_calls.load(Ordering::SeqCst)
4258 }
4259 }
4260
4261 struct SlowTestClient {
4262 delay: Duration,
4263 }
4264
4265 #[async_trait::async_trait]
4266 impl LlmClient for SlowTestClient {
4267 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4268 Ok(messages.to_vec())
4269 }
4270
4271 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4272 let delay = self.delay;
4273 let delayed_text = futures::stream::once(async move {
4274 tokio::time::sleep(delay).await;
4275 Ok(LlmEvent::TextDelta {
4276 delta: "slow ok".to_string(),
4277 meta: None,
4278 })
4279 });
4280 let done = futures::stream::once(async {
4281 Ok(LlmEvent::Done {
4282 outcome: LlmDoneOutcome::Success {
4283 stop_reason: StopReason::EndTurn,
4284 },
4285 })
4286 });
4287 Box::pin(delayed_text.chain(done))
4288 }
4289
4290 fn provider(&self) -> &'static str {
4291 "slow-test"
4292 }
4293
4294 async fn health_check(&self) -> Result<(), LlmError> {
4295 Ok(())
4296 }
4297 }
4298
4299 #[derive(Clone)]
4300 struct DelayedHistorySessionService {
4301 inner: Arc<dyn MobSessionService>,
4302 delay: Duration,
4303 read_calls: Arc<AtomicUsize>,
4304 active_reads: Arc<AtomicUsize>,
4305 max_active_reads: Arc<AtomicUsize>,
4306 }
4307
4308 impl DelayedHistorySessionService {
4309 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4310 Self {
4311 inner,
4312 delay,
4313 read_calls: Arc::new(AtomicUsize::new(0)),
4314 active_reads: Arc::new(AtomicUsize::new(0)),
4315 max_active_reads: Arc::new(AtomicUsize::new(0)),
4316 }
4317 }
4318
4319 fn read_calls(&self) -> usize {
4320 self.read_calls.load(Ordering::SeqCst)
4321 }
4322
4323 fn max_active_reads(&self) -> usize {
4324 self.max_active_reads.load(Ordering::SeqCst)
4325 }
4326 }
4327
4328 #[async_trait::async_trait]
4329 impl SessionService for DelayedHistorySessionService {
4330 async fn create_session(
4331 &self,
4332 req: meerkat_core::CreateSessionRequest,
4333 ) -> Result<RunResult, SessionError> {
4334 self.inner.create_session(req).await
4335 }
4336
4337 async fn start_turn(
4338 &self,
4339 id: &SessionId,
4340 req: StartTurnRequest,
4341 ) -> Result<RunResult, SessionError> {
4342 self.inner.start_turn(id, req).await
4343 }
4344
4345 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4346 self.inner.interrupt(id).await
4347 }
4348
4349 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4350 self.inner.read(id).await
4351 }
4352
4353 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4354 self.inner.list(query).await
4355 }
4356
4357 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4358 self.inner.archive(id).await
4359 }
4360
4361 async fn subscribe_session_events(
4362 &self,
4363 id: &SessionId,
4364 ) -> Result<EventStream, StreamError> {
4365 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4366 }
4367 }
4368
4369 #[async_trait::async_trait]
4370 impl SessionServiceCommsExt for DelayedHistorySessionService {
4371 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4372 self.inner.comms_runtime(session_id).await
4373 }
4374 }
4375
4376 #[async_trait::async_trait]
4377 impl SessionServiceControlExt for DelayedHistorySessionService {
4378 async fn append_system_context(
4379 &self,
4380 id: &SessionId,
4381 req: AppendSystemContextRequest,
4382 ) -> Result<AppendSystemContextResult, SessionControlError> {
4383 self.inner.append_system_context(id, req).await
4384 }
4385 }
4386
4387 #[async_trait::async_trait]
4388 impl SessionServiceHistoryExt for DelayedHistorySessionService {
4389 async fn read_history(
4390 &self,
4391 id: &SessionId,
4392 query: SessionHistoryQuery,
4393 ) -> Result<SessionHistoryPage, SessionError> {
4394 self.read_calls.fetch_add(1, Ordering::SeqCst);
4395 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4396 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4397 tokio::time::sleep(self.delay).await;
4398 let result = self.inner.read_history(id, query).await;
4399 self.active_reads.fetch_sub(1, Ordering::SeqCst);
4400 result
4401 }
4402 }
4403
4404 #[async_trait::async_trait]
4405 impl MobSessionService for DelayedHistorySessionService {
4406 fn supports_persistent_sessions(&self) -> bool {
4407 self.inner.supports_persistent_sessions()
4408 }
4409
4410 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4411 self.inner.runtime_adapter()
4412 }
4413
4414 async fn session_belongs_to_mob(
4415 &self,
4416 session_id: &SessionId,
4417 mob_id: &meerkat_mob::MobId,
4418 ) -> bool {
4419 self.inner.session_belongs_to_mob(session_id, mob_id).await
4420 }
4421
4422 async fn cancel_all_checkpointers(&self) {
4423 self.inner.cancel_all_checkpointers().await;
4424 }
4425
4426 async fn rearm_all_checkpointers(&self) {
4427 self.inner.rearm_all_checkpointers().await;
4428 }
4429 }
4430
4431 #[async_trait::async_trait]
4432 impl ConsoleLogStore for CountingConsoleLogStore {
4433 async fn append_if_absent(
4434 &self,
4435 frame: NewConsoleFrame,
4436 ) -> ConsoleLogResult<AppendOutcome> {
4437 self.inner.append_if_absent(frame).await
4438 }
4439
4440 async fn update_frame_status(
4441 &self,
4442 frame_id: &str,
4443 status: ConsoleFrameStatus,
4444 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4445 self.inner.update_frame_status(frame_id, status).await
4446 }
4447
4448 async fn query_frames(
4449 &self,
4450 query: ConsoleTimelineQuery,
4451 ) -> ConsoleLogResult<ConsoleTimelinePage> {
4452 self.inner.query_frames(query).await
4453 }
4454
4455 async fn query_windowed_frames(
4456 &self,
4457 query: ConsoleTimelineWindowQuery,
4458 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4459 self.inner.query_windowed_frames(query).await
4460 }
4461
4462 async fn frame_by_dedupe_key(
4463 &self,
4464 dedupe_key: &str,
4465 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4466 self.inner.frame_by_dedupe_key(dedupe_key).await
4467 }
4468
4469 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4470 self.inner.latest_cursor().await
4471 }
4472
4473 async fn clear_frames(&self) -> ConsoleLogResult<()> {
4474 self.inner.clear_frames().await
4475 }
4476
4477 async fn record_source_watermark(
4478 &self,
4479 runtime_key: &str,
4480 source_kind: ConsoleFrameSourceKind,
4481 source_cursor: &str,
4482 ) -> ConsoleLogResult<()> {
4483 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
4484 self.inner
4485 .record_source_watermark(runtime_key, source_kind, source_cursor)
4486 .await
4487 }
4488
4489 async fn source_watermark(
4490 &self,
4491 runtime_key: &str,
4492 source_kind: ConsoleFrameSourceKind,
4493 ) -> ConsoleLogResult<Option<String>> {
4494 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
4495 self.inner.source_watermark(runtime_key, source_kind).await
4496 }
4497 }
4498
4499 async fn build_single_member_runtime() -> UnifiedRuntime {
4500 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
4501 }
4502
4503 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
4504 let definition = MobDefinition::from_toml(
4505 r#"
4506[mob]
4507id = "console-aggregator-perf-test"
4508
4509[profiles.worker]
4510model = "gpt-5.5"
4511external_addressable = true
4512
4513[profiles.worker.tools]
4514comms = true
4515"#,
4516 )
4517 .expect("definition parses");
4518 let runtime = UnifiedRuntime::builder()
4519 .definition(definition)
4520 .default_llm_client(client)
4521 .build()
4522 .await
4523 .expect("runtime builds");
4524 runtime
4525 .spawn(SpawnMemberSpec::from_wire(
4526 "worker".to_string(),
4527 "agent-a".to_string(),
4528 Some("You are agent-a.".into()),
4529 None,
4530 None,
4531 ))
4532 .await
4533 .expect("member spawns");
4534 runtime
4535 }
4536
4537 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
4538 let definition = MobDefinition::from_toml(&format!(
4539 r#"
4540[mob]
4541id = "{mob_id}"
4542
4543[profiles.worker]
4544model = "gpt-5.5"
4545external_addressable = true
4546
4547[profiles.worker.tools]
4548comms = true
4549"#
4550 ))
4551 .expect("definition parses");
4552 UnifiedRuntime::builder()
4553 .definition(definition)
4554 .default_llm_client(Arc::new(TestClient::default()))
4555 .build()
4556 .await
4557 .expect("runtime builds")
4558 }
4559
4560 async fn build_stress_runtime(
4561 member_count: usize,
4562 history_delay: Duration,
4563 ) -> (
4564 tempfile::TempDir,
4565 Arc<UnifiedRuntime>,
4566 DelayedHistorySessionService,
4567 ) {
4568 let temp_dir = tempfile::tempdir().expect("temp dir");
4569 let session_path = temp_dir.path().join("sessions");
4570 std::fs::create_dir_all(&session_path).expect("session path");
4571 let factory = AgentFactory::new(&session_path).comms(true);
4572 let base_service = Arc::new(build_ephemeral_service(
4573 factory,
4574 Config::default(),
4575 member_count + 8,
4576 ));
4577 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
4578 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
4579 let definition = MobDefinition::from_toml(
4580 r#"
4581[mob]
4582id = "console-aggregator-stress-test"
4583
4584[profiles.worker]
4585model = "gpt-5.5"
4586external_addressable = true
4587
4588[profiles.worker.tools]
4589comms = true
4590"#,
4591 )
4592 .expect("definition parses");
4593 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
4594 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
4595 allow_ephemeral_sessions: true,
4596 notify_orchestrator_on_resume: true,
4597 default_llm_client: Some(Arc::new(TestClient::default())),
4598 });
4599 let runtime = Arc::new(
4600 UnifiedRuntime::bootstrap(
4601 spec,
4602 crate::types::MobKitConfig {
4603 modules: Vec::new(),
4604 discovery: crate::types::DiscoverySpec {
4605 namespace: "stress".to_string(),
4606 modules: Vec::new(),
4607 },
4608 pre_spawn: Vec::new(),
4609 },
4610 Duration::from_secs(5),
4611 )
4612 .await
4613 .expect("runtime boots"),
4614 );
4615 for idx in 0..member_count {
4616 runtime
4617 .spawn(SpawnMemberSpec::from_wire(
4618 "worker".to_string(),
4619 format!("agent-{idx}"),
4620 Some(format!("You are agent-{idx}.").into()),
4621 None,
4622 None,
4623 ))
4624 .await
4625 .expect("member spawns");
4626 }
4627 (temp_dir, runtime, delayed_service)
4628 }
4629
4630 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
4631 RuntimeEntry {
4632 runtime_key: runtime_key.to_string(),
4633 identity_namespace: "test".to_string(),
4634 runtime: runtime.mob_runtime().clone(),
4635 identity_runtime: runtime.identity_runtime().cloned(),
4636 console_events: runtime.console_events(),
4637 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4638 }
4639 }
4640
4641 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
4642 ConsoleIdentityRecord {
4643 identity: identity.to_string(),
4644 display_name: identity.to_string(),
4645 runtime_key: "runtime-cache".to_string(),
4646 runtime_member_id: identity.to_string(),
4647 session_id: Some(format!("session-{identity}")),
4648 visibility: ConsoleVisibility::Addressable,
4649 addressable: true,
4650 health: "ready".to_string(),
4651 topology_peers: Vec::new(),
4652 labels: BTreeMap::new(),
4653 }
4654 }
4655
4656 #[test]
4657 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
4658 let mut durable = identity_record_for_test("test/review:singleton");
4659 durable.runtime_member_id = "rt:review:singleton:1".to_string();
4660 durable.session_id = Some("session-stale".to_string());
4661 durable
4662 .labels
4663 .insert("agent_identity".to_string(), "review:singleton".to_string());
4664
4665 let mut live = identity_record_for_test("test/review:singleton");
4666 live.runtime_member_id = "rt:review:singleton:0".to_string();
4667 live.session_id = Some("session-live".to_string());
4668 live.labels
4669 .insert("agent_identity".to_string(), "review:singleton".to_string());
4670
4671 let records = dedupe_identity_records(vec![durable, live]);
4672
4673 assert_eq!(
4674 records.len(),
4675 2,
4676 "list projection must expose stale durable/live alias split-brain instead of picking one record"
4677 );
4678 assert!(
4679 records
4680 .iter()
4681 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
4682 );
4683 assert!(
4684 records
4685 .iter()
4686 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
4687 );
4688 }
4689
4690 #[test]
4691 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
4692 let mut first = identity_record_for_test("test/review:singleton");
4693 first.runtime_member_id = "rt:review:singleton:0".to_string();
4694 first
4695 .labels
4696 .insert("agent_identity".to_string(), "review:singleton".to_string());
4697
4698 let mut second = identity_record_for_test("test/review:singleton");
4699 second.runtime_member_id = "rt:review:singleton:1".to_string();
4700 second
4701 .labels
4702 .insert("agent_identity".to_string(), "review:singleton".to_string());
4703
4704 let records = dedupe_identity_records(vec![first, second]);
4705
4706 assert_eq!(
4707 records.len(),
4708 2,
4709 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
4710 );
4711 }
4712
4713 #[test]
4714 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
4715 let mut first = identity_record_for_test("test/review:singleton");
4716 first.runtime_member_id = "rt:review:singleton:0".to_string();
4717 first.session_id = None;
4718 first
4719 .labels
4720 .insert("agent_identity".to_string(), "review:singleton".to_string());
4721 first
4722 .labels
4723 .insert("source_mob_id".to_string(), "mob-a".to_string());
4724
4725 let mut second = first.clone();
4726 second
4727 .labels
4728 .insert("source_mob_id".to_string(), "mob-b".to_string());
4729
4730 let records = dedupe_identity_records(vec![first, second]);
4731
4732 assert_eq!(
4733 records.len(),
4734 2,
4735 "same member/session aliases from different source mobs must remain visible"
4736 );
4737 }
4738
4739 #[test]
4740 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
4741 let durable = identity_record_for_test("test/review:singleton");
4742 let live = durable.clone();
4743
4744 let records = dedupe_identity_records(vec![durable, live]);
4745
4746 assert_eq!(
4747 records.len(),
4748 1,
4749 "healthy durable and primary live projections should not duplicate the roster"
4750 );
4751 }
4752
4753 #[test]
4754 fn stale_durable_record_error_detects_session_split_brain() {
4755 let mut durable = identity_record_for_test("test/review:singleton");
4756 durable.runtime_member_id = "rt:review:singleton:0".to_string();
4757 durable.session_id = Some("session-durable".to_string());
4758
4759 let mut live = durable.clone();
4760 live.session_id = Some("session-live".to_string());
4761
4762 let err = stale_durable_record_error("review:singleton", &durable, &[live])
4763 .expect("session mismatch must be stale split-brain");
4764
4765 assert!(
4766 err.contains("stale durable identity alias"),
4767 "unexpected error: {err}"
4768 );
4769 assert!(
4770 err.contains("session-durable") && err.contains("session-live"),
4771 "error should name both sessions: {err}"
4772 );
4773 }
4774
4775 async fn identity_runtime_for_test(
4776 identities: &[&str],
4777 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
4778 {
4779 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
4780 crate::identity_first::IdentityRuntimeConfig {
4781 continuity_store: Arc::new(
4782 crate::identity_first::LocalContinuityStore::in_memory()?
4783 ),
4784 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
4785 runtime_instance_id: "console-aggregator-identity-test".to_string(),
4786 has_runtime_store: true,
4787 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
4788 bridge: None,
4789 default_timeout: None,
4790 },
4791 ));
4792 for identity in identities {
4793 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
4794 let record = crate::identity_first::ContinuityRecord {
4795 identity: identity.clone(),
4796 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
4797 "rt:{}:0",
4798 identity.as_str()
4799 ))?,
4800 session_id: SessionId::new(),
4801 generation: crate::identity_first::ContinuityGeneration::new(0),
4802 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
4803 };
4804 runtime
4805 .register(
4806 crate::identity_first::DurableAgentSpec {
4807 identity: identity.clone(),
4808 profile: meerkat_mob::ProfileName::from("worker"),
4809 addressability: crate::identity_first::AgentAddressability::Addressable,
4810 display_name: None,
4811 labels: BTreeMap::new(),
4812 context: None,
4813 additional_instructions: Vec::new(),
4814 initial_message: None,
4815 runtime_mode_override: None,
4816 },
4817 crate::identity_first::IdentityLifecycleState::Active,
4818 Some(record),
4819 Some(crate::identity_first::LeaseGrant {
4820 identity,
4821 fencing_token: crate::identity_first::FencingToken::new(1),
4822 ttl: Duration::from_mins(5),
4823 }),
4824 )
4825 .await;
4826 }
4827 Ok(runtime)
4828 }
4829
4830 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4831 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
4832 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4833 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
4834 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
4835 let aggregator = MobKitConsoleAggregator::in_memory();
4836 aggregator.register_runtime_handles_with_policy(
4837 "runtime-a",
4838 "",
4839 runtime.mob_runtime().clone(),
4840 Some(identity_runtime),
4841 runtime.console_events(),
4842 Arc::new(AllowAllConsoleVisibilityPolicy),
4843 );
4844
4845 assert!(
4846 aggregator
4847 .inspect_identity("rt:review:singleton:0")
4848 .await?
4849 .is_none(),
4850 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
4851 );
4852 assert!(
4853 !aggregator.retire_identity("rt:review:singleton:0").await?,
4854 "runtime-member-id shaped requests must not retire a durable rt:* identity"
4855 );
4856 let reserve_err = aggregator
4857 .reserve_identity_first_interaction(
4858 ConsoleSendRequest {
4859 identity: "rt:review:singleton:0".to_string(),
4860 content: json!("hello"),
4861 origin: "console:test".to_string(),
4862 idempotency_key: "rt-shadow-reserve".to_string(),
4863 handling_mode: Some("queue".to_string()),
4864 },
4865 None,
4866 )
4867 .await
4868 .expect_err(
4869 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
4870 );
4871 assert!(
4872 reserve_err.to_string().contains("unknown identity"),
4873 "unexpected reserve error: {reserve_err}"
4874 );
4875 let send_err = aggregator
4876 .send(ConsoleSendRequest {
4877 identity: "rt:review:singleton:0".to_string(),
4878 content: json!("hello"),
4879 origin: "console:test".to_string(),
4880 idempotency_key: "rt-shadow-send".to_string(),
4881 handling_mode: Some("queue".to_string()),
4882 })
4883 .await
4884 .expect_err(
4885 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
4886 );
4887 assert!(
4888 send_err.to_string().contains("unknown identity"),
4889 "unexpected send error: {send_err}"
4890 );
4891 let blob_err = match aggregator
4892 .binary_blob_store_for_identity("rt:review:singleton:0")
4893 .await
4894 {
4895 Ok(_) => panic!(
4896 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
4897 ),
4898 Err(err) => err,
4899 };
4900 assert!(
4901 blob_err.to_string().contains("unknown identity"),
4902 "unexpected blob error: {blob_err}"
4903 );
4904
4905 let _ = runtime.mob_handle().stop().await;
4906 Ok(())
4907 }
4908
4909 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4910 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
4911 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4912 let runtime = build_empty_runtime("identity-label-test").await;
4913 let mut labels = BTreeMap::new();
4914 labels.insert(
4915 "agent_identity".to_string(),
4916 "channel:C0SMOKEOB3".to_string(),
4917 );
4918 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
4919 runtime
4920 .spawn(
4921 SpawnMemberSpec::from_wire(
4922 "worker".to_string(),
4923 "rt:channel:C0SMOKEOB3:0".to_string(),
4924 Some("You are C0SMOKEOB3.".into()),
4925 None,
4926 None,
4927 )
4928 .with_labels(labels),
4929 )
4930 .await
4931 .expect("member spawns");
4932
4933 let entry = RuntimeEntry {
4934 runtime_key: "runtime-a".to_string(),
4935 identity_namespace: String::new(),
4936 runtime: runtime.mob_runtime().clone(),
4937 identity_runtime: runtime.identity_runtime().cloned(),
4938 console_events: runtime.console_events(),
4939 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4940 };
4941 let aggregator = MobKitConsoleAggregator::in_memory();
4942 aggregator
4943 .inner
4944 .runtimes
4945 .write()
4946 .expect("runtime registry")
4947 .insert("runtime-a".to_string(), entry);
4948
4949 let records = aggregator.list_identities_fresh().await?;
4950 let record = records
4951 .iter()
4952 .find(|record| record.identity == "channel:C0SMOKEOB3")
4953 .expect("durable identity is exposed");
4954 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
4955
4956 let inspection = aggregator
4957 .inspect_identity("channel:C0SMOKEOB3")
4958 .await?
4959 .expect("durable identity resolves back to runtime member");
4960 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
4961 assert_eq!(
4962 inspection.identity.runtime_member_id,
4963 "rt:channel:C0SMOKEOB3:0"
4964 );
4965
4966 let _ = runtime.mob_handle().stop().await;
4967 Ok(())
4968 }
4969
4970 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4971 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
4972 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4973 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
4974 let mut labels = BTreeMap::new();
4975 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4976 runtime
4977 .spawn(
4978 SpawnMemberSpec::from_wire(
4979 "worker".to_string(),
4980 "rt:review:singleton:0".to_string(),
4981 Some("You are the Review Agent.".into()),
4982 None,
4983 None,
4984 )
4985 .with_labels(labels),
4986 )
4987 .await?;
4988 let hidden_session_id = runtime
4989 .mob_handle()
4990 .resolve_bridge_session_id_observation(&meerkat_mob::ids::MeerkatId::from(
4991 "rt:review:singleton:0",
4992 ))
4993 .await
4994 .map(|sid| sid.to_string());
4995 let mut duplicate_labels = BTreeMap::new();
4996 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4997 runtime
4998 .spawn(
4999 SpawnMemberSpec::from_wire(
5000 "worker".to_string(),
5001 "rt:review:singleton:1".to_string(),
5002 Some("You are a visible duplicate Review Agent.".into()),
5003 None,
5004 None,
5005 )
5006 .with_labels(duplicate_labels),
5007 )
5008 .await?;
5009
5010 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5011 let aggregator = MobKitConsoleAggregator::in_memory();
5012 aggregator.register_runtime_handles_with_policy(
5013 "runtime-a",
5014 "",
5015 runtime.mob_runtime().clone(),
5016 Some(identity_runtime),
5017 runtime.console_events(),
5018 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5019 );
5020
5021 assert!(
5022 aggregator
5023 .inspect_identity("review:singleton")
5024 .await?
5025 .is_none(),
5026 "member-hidden live alias must not inspect through aggregator"
5027 );
5028 let records = aggregator.list_identities_fresh().await?;
5029 assert!(
5030 records
5031 .iter()
5032 .all(|record| record.identity != "review:singleton"),
5033 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5034 );
5035 let appended = aggregator
5036 .store()
5037 .append_if_absent(NewConsoleFrame {
5038 id: None,
5039 dedupe_key: "member-hidden-frame".to_string(),
5040 timestamp_ms: 1,
5041 runtime_key: "runtime-a".to_string(),
5042 identity: "review:singleton".to_string(),
5043 conversation_id: Some("review:singleton".to_string()),
5044 session_id: hidden_session_id,
5045 kind: "text_delta".to_string(),
5046 status: ConsoleFrameStatus::Delivered,
5047 payload: json!({ "delta": "hidden" }),
5048 source: ConsoleFrameSource {
5049 kind: ConsoleFrameSourceKind::ConsoleEvent,
5050 source_cursor: None,
5051 },
5052 source_event_id: Some("member-hidden-frame".to_string()),
5053 interaction_id: None,
5054 turn_id: None,
5055 run_id: None,
5056 parent_frame_id: None,
5057 caused_by_frame_id: None,
5058 })
5059 .await?;
5060 assert!(
5061 !aggregator
5062 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5063 frame: appended.frame.clone()
5064 })
5065 .await,
5066 "member-hidden live alias must not pass timeline event visibility"
5067 );
5068 let identity_only = aggregator
5069 .store()
5070 .append_if_absent(NewConsoleFrame {
5071 id: None,
5072 dedupe_key: "member-hidden-identity-only-frame".to_string(),
5073 timestamp_ms: 2,
5074 runtime_key: "runtime-a".to_string(),
5075 identity: "review:singleton".to_string(),
5076 conversation_id: Some("review:singleton".to_string()),
5077 session_id: None,
5078 kind: "text_delta".to_string(),
5079 status: ConsoleFrameStatus::Delivered,
5080 payload: json!({ "delta": "identity-only-hidden" }),
5081 source: ConsoleFrameSource {
5082 kind: ConsoleFrameSourceKind::ConsoleEvent,
5083 source_cursor: None,
5084 },
5085 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5086 interaction_id: None,
5087 turn_id: None,
5088 run_id: None,
5089 parent_frame_id: None,
5090 caused_by_frame_id: None,
5091 })
5092 .await?;
5093 assert!(
5094 !aggregator
5095 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5096 frame: identity_only.frame.clone()
5097 })
5098 .await,
5099 "member-hidden durable alias must not leak identity-only timeline frames"
5100 );
5101 let page = aggregator
5102 .query_timeline(ConsoleTimelineQuery {
5103 identity: Some("review:singleton".to_string()),
5104 limit: 10,
5105 ..ConsoleTimelineQuery::default()
5106 })
5107 .await?;
5108 assert!(
5109 page.frames.is_empty(),
5110 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5111 );
5112 assert!(
5113 !aggregator.retire_identity("review:singleton").await?,
5114 "member-hidden live alias must not retire through aggregator"
5115 );
5116 let reserve_err = aggregator
5117 .reserve_identity_first_interaction(
5118 ConsoleSendRequest {
5119 identity: "review:singleton".to_string(),
5120 content: json!("hello"),
5121 origin: "console:test".to_string(),
5122 idempotency_key: "member-hidden-reserve".to_string(),
5123 handling_mode: Some("queue".to_string()),
5124 },
5125 None,
5126 )
5127 .await
5128 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5129 assert!(
5130 reserve_err.to_string().contains("unknown identity"),
5131 "unexpected reserve error: {reserve_err}"
5132 );
5133 let send_err = aggregator
5134 .send(ConsoleSendRequest {
5135 identity: "review:singleton".to_string(),
5136 content: json!("hello"),
5137 origin: "console:test".to_string(),
5138 idempotency_key: "member-hidden-send".to_string(),
5139 handling_mode: Some("queue".to_string()),
5140 })
5141 .await
5142 .expect_err("member-hidden live alias must not receive sends");
5143 assert!(
5144 send_err.to_string().contains("unknown identity"),
5145 "unexpected send error: {send_err}"
5146 );
5147 let blob_result = aggregator
5148 .binary_blob_store_for_identity("review:singleton")
5149 .await;
5150 let Err(blob_err) = blob_result else {
5151 panic!("member-hidden live alias must not expose blob store");
5152 };
5153 assert!(
5154 blob_err.to_string().contains("unknown identity"),
5155 "unexpected blob error: {blob_err}"
5156 );
5157
5158 let _ = runtime.mob_handle().stop().await;
5159 Ok(())
5160 }
5161
5162 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5163 async fn explicit_session_backfill_target_resolves_durable_identity_label()
5164 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5165 let runtime = build_empty_runtime("identity-label-backfill-test").await;
5166 let mut labels = BTreeMap::new();
5167 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5168 runtime
5169 .spawn(
5170 SpawnMemberSpec::from_wire(
5171 "worker".to_string(),
5172 "rt:review:singleton:0".to_string(),
5173 Some("You are the Review Agent.".into()),
5174 None,
5175 None,
5176 )
5177 .with_labels(labels),
5178 )
5179 .await
5180 .expect("member spawns");
5181
5182 let entry = RuntimeEntry {
5183 runtime_key: "runtime-a".to_string(),
5184 identity_namespace: String::new(),
5185 runtime: runtime.mob_runtime().clone(),
5186 identity_runtime: runtime.identity_runtime().cloned(),
5187 console_events: runtime.console_events(),
5188 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5189 };
5190 let aggregator = MobKitConsoleAggregator::in_memory();
5191 aggregator
5192 .inner
5193 .runtimes
5194 .write()
5195 .expect("runtime registry")
5196 .insert("runtime-a".to_string(), entry);
5197
5198 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5199 .await
5200 .expect("durable label should resolve targeted session backfill");
5201 assert_eq!(target.record.identity, "review:singleton");
5202 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5203
5204 let _ = runtime.mob_handle().stop().await;
5205 Ok(())
5206 }
5207
5208 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5209 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5210 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5211 let runtime = build_empty_runtime("identity-stale-durable-test").await;
5212 let mut labels = BTreeMap::new();
5213 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5214 runtime
5215 .spawn(
5216 SpawnMemberSpec::from_wire(
5217 "worker".to_string(),
5218 "rt:review:singleton:0".to_string(),
5219 Some("You are the live Review Agent.".into()),
5220 None,
5221 None,
5222 )
5223 .with_labels(labels),
5224 )
5225 .await
5226 .expect("member spawns");
5227
5228 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5229 crate::identity_first::IdentityRuntimeConfig {
5230 continuity_store: Arc::new(
5231 crate::identity_first::LocalContinuityStore::in_memory()?
5232 ),
5233 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5234 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5235 has_runtime_store: true,
5236 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5237 bridge: None,
5238 default_timeout: None,
5239 },
5240 ));
5241 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5242 let record = crate::identity_first::ContinuityRecord {
5243 identity: identity.clone(),
5244 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5245 "rt:review:singleton:1",
5246 )?,
5247 session_id: SessionId::new(),
5248 generation: crate::identity_first::ContinuityGeneration::new(1),
5249 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5250 };
5251 identity_runtime
5252 .register(
5253 crate::identity_first::DurableAgentSpec {
5254 identity: identity.clone(),
5255 profile: meerkat_mob::ProfileName::from("worker"),
5256 addressability: crate::identity_first::AgentAddressability::Addressable,
5257 display_name: None,
5258 labels: BTreeMap::new(),
5259 context: None,
5260 additional_instructions: Vec::new(),
5261 initial_message: None,
5262 runtime_mode_override: None,
5263 },
5264 crate::identity_first::IdentityLifecycleState::Active,
5265 Some(record),
5266 Some(crate::identity_first::LeaseGrant {
5267 identity,
5268 fencing_token: crate::identity_first::FencingToken::new(7),
5269 ttl: Duration::from_mins(5),
5270 }),
5271 )
5272 .await;
5273
5274 let aggregator = MobKitConsoleAggregator::in_memory();
5275 aggregator
5276 .inner
5277 .runtimes
5278 .write()
5279 .expect("runtime registry")
5280 .insert(
5281 "runtime-a".to_string(),
5282 RuntimeEntry {
5283 runtime_key: "runtime-a".to_string(),
5284 identity_namespace: String::new(),
5285 runtime: runtime.mob_runtime().clone(),
5286 identity_runtime: Some(identity_runtime),
5287 console_events: runtime.console_events(),
5288 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5289 },
5290 );
5291
5292 let err = aggregator
5293 .inspect_identity("review:singleton")
5294 .await
5295 .expect_err("stale durable binding must not inspect as healthy");
5296 assert!(
5297 err.to_string().contains("stale durable identity alias"),
5298 "unexpected error: {err}"
5299 );
5300 let retire_err = aggregator
5301 .retire_identity("review:singleton")
5302 .await
5303 .expect_err("stale durable binding must not retire as healthy");
5304 assert!(
5305 retire_err
5306 .to_string()
5307 .contains("stale durable identity alias"),
5308 "unexpected retire error: {retire_err}"
5309 );
5310 let send_err = aggregator
5311 .send(ConsoleSendRequest {
5312 identity: "review:singleton".to_string(),
5313 content: json!("hello"),
5314 origin: "console:test".to_string(),
5315 idempotency_key: "stale-durable-send".to_string(),
5316 handling_mode: Some("queue".to_string()),
5317 })
5318 .await
5319 .expect_err("stale durable binding must not send to the wrong live member");
5320 assert!(
5321 send_err
5322 .to_string()
5323 .contains("stale durable identity alias"),
5324 "unexpected send error: {send_err}"
5325 );
5326 let reserve_err = aggregator
5327 .reserve_identity_first_interaction(
5328 ConsoleSendRequest {
5329 identity: "review:singleton".to_string(),
5330 content: json!("hello"),
5331 origin: "console:test".to_string(),
5332 idempotency_key: "stale-durable-reserve".to_string(),
5333 handling_mode: Some("queue".to_string()),
5334 },
5335 None,
5336 )
5337 .await
5338 .expect_err("identity-first reserve must not bypass stale durable binding");
5339 assert!(
5340 reserve_err
5341 .to_string()
5342 .contains("stale durable identity alias"),
5343 "unexpected reserve error: {reserve_err}"
5344 );
5345 let blob_err = match aggregator
5346 .binary_blob_store_for_identity("review:singleton")
5347 .await
5348 {
5349 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5350 Err(err) => err,
5351 };
5352 assert!(
5353 blob_err
5354 .to_string()
5355 .contains("stale durable identity alias"),
5356 "unexpected blob error: {blob_err}"
5357 );
5358 let records = aggregator.list_identities_fresh().await?;
5359 assert!(
5360 records
5361 .iter()
5362 .any(|record| record.identity == "review:singleton"
5363 && record.runtime_member_id == "rt:review:singleton:0"),
5364 "live alias should still be listed; records: {records:#?}"
5365 );
5366 assert!(
5367 records
5368 .iter()
5369 .any(|record| record.identity == "review:singleton"
5370 && record.runtime_member_id == "rt:review:singleton:1"),
5371 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5372 );
5373
5374 let _ = runtime.mob_handle().stop().await;
5375 Ok(())
5376 }
5377
5378 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5379 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
5380 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5381 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
5382 let mut labels = BTreeMap::new();
5383 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5384 runtime
5385 .spawn(
5386 SpawnMemberSpec::from_wire(
5387 "worker".to_string(),
5388 "rt:review:singleton:0".to_string(),
5389 Some("You are the mislabeled Review Agent.".into()),
5390 None,
5391 None,
5392 )
5393 .with_labels(labels),
5394 )
5395 .await
5396 .expect("member spawns");
5397
5398 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5399 crate::identity_first::IdentityRuntimeConfig {
5400 continuity_store: Arc::new(
5401 crate::identity_first::LocalContinuityStore::in_memory()?
5402 ),
5403 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5404 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
5405 has_runtime_store: true,
5406 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5407 bridge: None,
5408 default_timeout: None,
5409 },
5410 ));
5411 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5412 let record = crate::identity_first::ContinuityRecord {
5413 identity: identity.clone(),
5414 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5415 "rt:review:singleton:0",
5416 )?,
5417 session_id: SessionId::new(),
5418 generation: crate::identity_first::ContinuityGeneration::new(1),
5419 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5420 };
5421 identity_runtime
5422 .register(
5423 crate::identity_first::DurableAgentSpec {
5424 identity: identity.clone(),
5425 profile: meerkat_mob::ProfileName::from("worker"),
5426 addressability: crate::identity_first::AgentAddressability::Addressable,
5427 display_name: None,
5428 labels: BTreeMap::new(),
5429 context: None,
5430 additional_instructions: Vec::new(),
5431 initial_message: None,
5432 runtime_mode_override: None,
5433 },
5434 crate::identity_first::IdentityLifecycleState::Active,
5435 Some(record),
5436 Some(crate::identity_first::LeaseGrant {
5437 identity,
5438 fencing_token: crate::identity_first::FencingToken::new(7),
5439 ttl: Duration::from_mins(5),
5440 }),
5441 )
5442 .await;
5443
5444 let aggregator = MobKitConsoleAggregator::in_memory();
5445 aggregator
5446 .inner
5447 .runtimes
5448 .write()
5449 .expect("runtime registry")
5450 .insert(
5451 "runtime-a".to_string(),
5452 RuntimeEntry {
5453 runtime_key: "runtime-a".to_string(),
5454 identity_namespace: String::new(),
5455 runtime: runtime.mob_runtime().clone(),
5456 identity_runtime: Some(identity_runtime),
5457 console_events: runtime.console_events(),
5458 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5459 },
5460 );
5461
5462 let err = aggregator
5463 .inspect_identity("review:singleton")
5464 .await
5465 .expect_err("wrong projected live identity must not inspect as healthy");
5466 assert!(
5467 err.to_string()
5468 .contains("projects identity other:singleton"),
5469 "unexpected error: {err}"
5470 );
5471 let retire_err = aggregator
5472 .retire_identity("review:singleton")
5473 .await
5474 .expect_err("wrong projected live identity must not retire as healthy");
5475 assert!(
5476 retire_err
5477 .to_string()
5478 .contains("projects identity other:singleton"),
5479 "unexpected retire error: {retire_err}"
5480 );
5481 let send_err = aggregator
5482 .send(ConsoleSendRequest {
5483 identity: "review:singleton".to_string(),
5484 content: json!("hello"),
5485 origin: "console:test".to_string(),
5486 idempotency_key: "wrong-projection-send".to_string(),
5487 handling_mode: Some("queue".to_string()),
5488 })
5489 .await
5490 .expect_err("wrong projected live identity must not send as unknown");
5491 assert!(
5492 send_err
5493 .to_string()
5494 .contains("projects identity other:singleton"),
5495 "unexpected send error: {send_err}"
5496 );
5497 let reserve_err = aggregator
5498 .reserve_identity_first_interaction(
5499 ConsoleSendRequest {
5500 identity: "review:singleton".to_string(),
5501 content: json!("hello"),
5502 origin: "console:test".to_string(),
5503 idempotency_key: "wrong-projection-reserve".to_string(),
5504 handling_mode: Some("queue".to_string()),
5505 },
5506 None,
5507 )
5508 .await
5509 .expect_err("identity-first reserve must not bypass wrong live projection");
5510 assert!(
5511 reserve_err
5512 .to_string()
5513 .contains("projects identity other:singleton"),
5514 "unexpected reserve error: {reserve_err}"
5515 );
5516 let blob_err = match aggregator
5517 .binary_blob_store_for_identity("review:singleton")
5518 .await
5519 {
5520 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
5521 Err(err) => err,
5522 };
5523 assert!(
5524 blob_err
5525 .to_string()
5526 .contains("projects identity other:singleton"),
5527 "unexpected blob error: {blob_err}"
5528 );
5529 let projected_err = aggregator
5530 .inspect_identity("other:singleton")
5531 .await
5532 .expect_err("wrong projected alias must not inspect through live-only fallback");
5533 assert!(
5534 projected_err
5535 .to_string()
5536 .contains("identity runtime binding for review:singleton"),
5537 "unexpected projected-alias error: {projected_err}"
5538 );
5539 let projected_retire_err = aggregator
5540 .retire_identity("other:singleton")
5541 .await
5542 .expect_err("wrong projected alias must not retire through live-only fallback");
5543 assert!(
5544 projected_retire_err
5545 .to_string()
5546 .contains("identity runtime binding for review:singleton"),
5547 "unexpected projected-alias retire error: {projected_retire_err}"
5548 );
5549
5550 let _ = runtime.mob_handle().stop().await;
5551 Ok(())
5552 }
5553
5554 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5555 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
5556 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5557 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
5558 let mut labels = BTreeMap::new();
5559 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5560 runtime
5561 .spawn(
5562 SpawnMemberSpec::from_wire(
5563 "worker".to_string(),
5564 "rt:review:singleton:0".to_string(),
5565 Some("You are a hidden wrong projection.".into()),
5566 None,
5567 None,
5568 )
5569 .with_labels(labels),
5570 )
5571 .await
5572 .expect("member spawns");
5573
5574 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5575 crate::identity_first::IdentityRuntimeConfig {
5576 continuity_store: Arc::new(
5577 crate::identity_first::LocalContinuityStore::in_memory()?
5578 ),
5579 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5580 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
5581 has_runtime_store: true,
5582 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5583 bridge: None,
5584 default_timeout: None,
5585 },
5586 ));
5587 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5588 identity_runtime
5589 .register(
5590 crate::identity_first::DurableAgentSpec {
5591 identity: identity.clone(),
5592 profile: meerkat_mob::ProfileName::from("worker"),
5593 addressability: crate::identity_first::AgentAddressability::Addressable,
5594 display_name: None,
5595 labels: BTreeMap::new(),
5596 context: None,
5597 additional_instructions: Vec::new(),
5598 initial_message: None,
5599 runtime_mode_override: None,
5600 },
5601 crate::identity_first::IdentityLifecycleState::Active,
5602 Some(crate::identity_first::ContinuityRecord {
5603 identity: identity.clone(),
5604 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5605 "rt:review:singleton:0",
5606 )?,
5607 session_id: SessionId::new(),
5608 generation: crate::identity_first::ContinuityGeneration::new(1),
5609 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5610 }),
5611 Some(crate::identity_first::LeaseGrant {
5612 identity,
5613 fencing_token: crate::identity_first::FencingToken::new(7),
5614 ttl: Duration::from_mins(5),
5615 }),
5616 )
5617 .await;
5618
5619 let aggregator = MobKitConsoleAggregator::in_memory();
5620 aggregator
5621 .inner
5622 .runtimes
5623 .write()
5624 .expect("runtime registry")
5625 .insert(
5626 "runtime-a".to_string(),
5627 RuntimeEntry {
5628 runtime_key: "runtime-a".to_string(),
5629 identity_namespace: String::new(),
5630 runtime: runtime.mob_runtime().clone(),
5631 identity_runtime: Some(identity_runtime),
5632 console_events: runtime.console_events(),
5633 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
5634 },
5635 );
5636
5637 let err = aggregator
5638 .inspect_identity("review:singleton")
5639 .await
5640 .expect_err("hidden wrong projection must still fail durable binding validation");
5641 assert!(
5642 err.to_string()
5643 .contains("projects identity other:singleton"),
5644 "unexpected hidden wrong-projection error: {err}"
5645 );
5646 let reserve_err = aggregator
5647 .reserve_identity_first_interaction(
5648 ConsoleSendRequest {
5649 identity: "review:singleton".to_string(),
5650 content: json!("hello"),
5651 origin: "console:test".to_string(),
5652 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
5653 handling_mode: Some("queue".to_string()),
5654 },
5655 None,
5656 )
5657 .await
5658 .expect_err("reserve must not hide wrong projection from durable validation");
5659 assert!(
5660 reserve_err
5661 .to_string()
5662 .contains("projects identity other:singleton"),
5663 "unexpected reserve error: {reserve_err}"
5664 );
5665
5666 let _ = runtime.mob_handle().stop().await;
5667 Ok(())
5668 }
5669
5670 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5671 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
5672 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5673 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
5674 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
5675 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5676 crate::identity_first::IdentityRuntimeConfig {
5677 continuity_store: Arc::new(
5678 crate::identity_first::LocalContinuityStore::in_memory()?
5679 ),
5680 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5681 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
5682 has_runtime_store: true,
5683 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5684 bridge: None,
5685 default_timeout: None,
5686 },
5687 ));
5688 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5689 crate::identity_first::IdentityRuntimeConfig {
5690 continuity_store: Arc::new(
5691 crate::identity_first::LocalContinuityStore::in_memory()?
5692 ),
5693 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5694 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
5695 has_runtime_store: true,
5696 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5697 bridge: None,
5698 default_timeout: None,
5699 },
5700 ));
5701 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
5702 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5703 identity_runtime
5704 .register(
5705 crate::identity_first::DurableAgentSpec {
5706 identity: identity.clone(),
5707 profile: meerkat_mob::ProfileName::from("worker"),
5708 addressability: crate::identity_first::AgentAddressability::Addressable,
5709 display_name: None,
5710 labels: BTreeMap::new(),
5711 context: None,
5712 additional_instructions: Vec::new(),
5713 initial_message: None,
5714 runtime_mode_override: None,
5715 },
5716 crate::identity_first::IdentityLifecycleState::Active,
5717 Some(crate::identity_first::ContinuityRecord {
5718 identity: identity.clone(),
5719 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5720 "rt:review:singleton:0",
5721 )?,
5722 session_id: SessionId::new(),
5723 generation: crate::identity_first::ContinuityGeneration::new(1),
5724 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5725 }),
5726 Some(crate::identity_first::LeaseGrant {
5727 identity,
5728 fencing_token: crate::identity_first::FencingToken::new(7),
5729 ttl: Duration::from_mins(5),
5730 }),
5731 )
5732 .await;
5733 }
5734
5735 let aggregator = MobKitConsoleAggregator::in_memory();
5736 {
5737 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
5738 runtimes.insert(
5739 "a-hidden".to_string(),
5740 RuntimeEntry {
5741 runtime_key: "a-hidden".to_string(),
5742 identity_namespace: String::new(),
5743 runtime: hidden_runtime.mob_runtime().clone(),
5744 identity_runtime: Some(hidden_identity_runtime),
5745 console_events: hidden_runtime.console_events(),
5746 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
5747 },
5748 );
5749 runtimes.insert(
5750 "b-visible".to_string(),
5751 RuntimeEntry {
5752 runtime_key: "b-visible".to_string(),
5753 identity_namespace: String::new(),
5754 runtime: visible_runtime.mob_runtime().clone(),
5755 identity_runtime: Some(visible_identity_runtime),
5756 console_events: visible_runtime.console_events(),
5757 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5758 },
5759 );
5760 }
5761
5762 let inspection = aggregator
5763 .inspect_identity("rt:review:singleton:0")
5764 .await?
5765 .expect("visible runtime-id match should be returned");
5766 assert_eq!(inspection.identity.runtime_key, "b-visible");
5767
5768 let _ = hidden_runtime.mob_handle().stop().await;
5769 let _ = visible_runtime.mob_handle().stop().await;
5770 Ok(())
5771 }
5772
5773 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5774 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
5775 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5776 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
5777 let mut primary_labels = BTreeMap::new();
5778 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5779 runtime
5780 .spawn(
5781 SpawnMemberSpec::from_wire(
5782 "worker".to_string(),
5783 "rt:review:singleton:0".to_string(),
5784 Some("You are the primary Review Agent.".into()),
5785 None,
5786 None,
5787 )
5788 .with_labels(primary_labels),
5789 )
5790 .await
5791 .expect("primary member spawns");
5792
5793 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
5794 let mut delegate_labels = BTreeMap::new();
5795 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5796 delegate_runtime
5797 .spawn(
5798 SpawnMemberSpec::from_wire(
5799 "worker".to_string(),
5800 "rt:review:singleton:0".to_string(),
5801 Some("You are the wrong projected Review Agent.".into()),
5802 None,
5803 None,
5804 )
5805 .with_labels(delegate_labels),
5806 )
5807 .await
5808 .expect("delegate member spawns");
5809 runtime
5810 .mob_runtime()
5811 .agent_mob_mcp_state()
5812 .expect("runtime should expose mob MCP state")
5813 .mob_insert_handle(
5814 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
5815 delegate_runtime.mob_handle().clone(),
5816 )
5817 .await;
5818
5819 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5820 crate::identity_first::IdentityRuntimeConfig {
5821 continuity_store: Arc::new(
5822 crate::identity_first::LocalContinuityStore::in_memory()?
5823 ),
5824 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5825 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
5826 has_runtime_store: true,
5827 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5828 bridge: None,
5829 default_timeout: None,
5830 },
5831 ));
5832 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5833 let record = crate::identity_first::ContinuityRecord {
5834 identity: identity.clone(),
5835 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5836 "rt:review:singleton:0",
5837 )?,
5838 session_id: SessionId::new(),
5839 generation: crate::identity_first::ContinuityGeneration::new(1),
5840 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5841 };
5842 identity_runtime
5843 .register(
5844 crate::identity_first::DurableAgentSpec {
5845 identity: identity.clone(),
5846 profile: meerkat_mob::ProfileName::from("worker"),
5847 addressability: crate::identity_first::AgentAddressability::Addressable,
5848 display_name: None,
5849 labels: BTreeMap::new(),
5850 context: None,
5851 additional_instructions: Vec::new(),
5852 initial_message: None,
5853 runtime_mode_override: None,
5854 },
5855 crate::identity_first::IdentityLifecycleState::Active,
5856 Some(record),
5857 Some(crate::identity_first::LeaseGrant {
5858 identity,
5859 fencing_token: crate::identity_first::FencingToken::new(7),
5860 ttl: Duration::from_mins(5),
5861 }),
5862 )
5863 .await;
5864
5865 let aggregator = MobKitConsoleAggregator::in_memory();
5866 aggregator
5867 .inner
5868 .runtimes
5869 .write()
5870 .expect("runtime registry")
5871 .insert(
5872 "runtime-a".to_string(),
5873 RuntimeEntry {
5874 runtime_key: "runtime-a".to_string(),
5875 identity_namespace: String::new(),
5876 runtime: runtime.mob_runtime().clone(),
5877 identity_runtime: Some(identity_runtime),
5878 console_events: runtime.console_events(),
5879 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5880 },
5881 );
5882
5883 let err = aggregator
5884 .inspect_identity("review:singleton")
5885 .await
5886 .expect_err("cross-source wrong projection must not be hidden by primary source");
5887 assert!(
5888 err.to_string()
5889 .contains("projects identity other:singleton")
5890 || err.to_string().contains("ambiguous live identity alias"),
5891 "unexpected error: {err}"
5892 );
5893
5894 let _ = delegate_runtime.mob_handle().stop().await;
5895 let _ = runtime.mob_handle().stop().await;
5896 Ok(())
5897 }
5898
5899 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5900 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
5901 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5902 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
5903 let mut labels = BTreeMap::new();
5904 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5905 runtime
5906 .spawn(
5907 SpawnMemberSpec::from_wire(
5908 "worker".to_string(),
5909 "rt:review:singleton:0".to_string(),
5910 Some("You are the first Review Agent.".into()),
5911 None,
5912 None,
5913 )
5914 .with_labels(labels.clone()),
5915 )
5916 .await
5917 .expect("first member spawns");
5918 runtime
5919 .spawn(
5920 SpawnMemberSpec::from_wire(
5921 "worker".to_string(),
5922 "rt:review:singleton:1".to_string(),
5923 Some("You are the second Review Agent.".into()),
5924 None,
5925 None,
5926 )
5927 .with_labels(labels),
5928 )
5929 .await
5930 .expect("second member spawns");
5931
5932 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5933 crate::identity_first::IdentityRuntimeConfig {
5934 continuity_store: Arc::new(
5935 crate::identity_first::LocalContinuityStore::in_memory()?
5936 ),
5937 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5938 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
5939 has_runtime_store: true,
5940 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5941 bridge: None,
5942 default_timeout: None,
5943 },
5944 ));
5945 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5946 identity_runtime
5947 .register(
5948 crate::identity_first::DurableAgentSpec {
5949 identity: identity.clone(),
5950 profile: meerkat_mob::ProfileName::from("worker"),
5951 addressability: crate::identity_first::AgentAddressability::Addressable,
5952 display_name: None,
5953 labels: BTreeMap::new(),
5954 context: None,
5955 additional_instructions: Vec::new(),
5956 initial_message: None,
5957 runtime_mode_override: None,
5958 },
5959 crate::identity_first::IdentityLifecycleState::Active,
5960 Some(crate::identity_first::ContinuityRecord {
5961 identity: identity.clone(),
5962 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5963 "rt:review:singleton:0",
5964 )?,
5965 session_id: SessionId::new(),
5966 generation: crate::identity_first::ContinuityGeneration::new(1),
5967 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5968 }),
5969 Some(crate::identity_first::LeaseGrant {
5970 identity,
5971 fencing_token: crate::identity_first::FencingToken::new(8),
5972 ttl: Duration::from_mins(5),
5973 }),
5974 )
5975 .await;
5976
5977 let aggregator = MobKitConsoleAggregator::in_memory();
5978 aggregator
5979 .inner
5980 .runtimes
5981 .write()
5982 .expect("runtime registry")
5983 .insert(
5984 "runtime-a".to_string(),
5985 RuntimeEntry {
5986 runtime_key: "runtime-a".to_string(),
5987 identity_namespace: String::new(),
5988 runtime: runtime.mob_runtime().clone(),
5989 identity_runtime: Some(identity_runtime),
5990 console_events: runtime.console_events(),
5991 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5992 },
5993 );
5994
5995 let err = aggregator
5996 .inspect_identity("review:singleton")
5997 .await
5998 .expect_err("duplicate live alias must not inspect via durable fast path");
5999 assert!(
6000 err.to_string().contains("ambiguous live identity alias"),
6001 "unexpected error: {err}"
6002 );
6003 let retire_err = aggregator
6004 .retire_identity("review:singleton")
6005 .await
6006 .expect_err("duplicate live alias must not retire via durable fast path");
6007 assert!(
6008 retire_err
6009 .to_string()
6010 .contains("ambiguous live identity alias"),
6011 "unexpected retire error: {retire_err}"
6012 );
6013 let runtime_id_err = aggregator
6014 .inspect_identity("rt:review:singleton:0")
6015 .await
6016 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6017 assert!(
6018 runtime_id_err
6019 .to_string()
6020 .contains("ambiguous live identity alias"),
6021 "unexpected runtime-id inspect error: {runtime_id_err}"
6022 );
6023 let runtime_id_retire_err = aggregator
6024 .retire_identity("rt:review:singleton:0")
6025 .await
6026 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6027 assert!(
6028 runtime_id_retire_err
6029 .to_string()
6030 .contains("ambiguous live identity alias")
6031 || runtime_id_retire_err
6032 .to_string()
6033 .contains("stale durable identity alias"),
6034 "unexpected runtime-id retire error: {runtime_id_retire_err}"
6035 );
6036
6037 let _ = runtime.mob_handle().stop().await;
6038 Ok(())
6039 }
6040
6041 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6042 async fn inspect_identity_ignores_hidden_live_alias_candidates()
6043 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6044 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6045 let mut labels = BTreeMap::new();
6046 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6047 runtime
6048 .spawn(
6049 SpawnMemberSpec::from_wire(
6050 "worker".to_string(),
6051 "rt:review:singleton:0".to_string(),
6052 Some("You are the visible Review Agent.".into()),
6053 None,
6054 None,
6055 )
6056 .with_labels(labels.clone()),
6057 )
6058 .await
6059 .expect("visible member spawns");
6060 runtime
6061 .spawn(
6062 SpawnMemberSpec::from_wire(
6063 "worker".to_string(),
6064 "rt:review:singleton:1".to_string(),
6065 Some("You are hidden maintenance noise.".into()),
6066 None,
6067 None,
6068 )
6069 .with_labels(labels),
6070 )
6071 .await
6072 .expect("hidden member spawns");
6073
6074 let aggregator = MobKitConsoleAggregator::in_memory();
6075 aggregator.register_runtime_handles_with_policy(
6076 "runtime-a",
6077 "",
6078 runtime.mob_runtime().clone(),
6079 None,
6080 runtime.console_events(),
6081 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6082 );
6083
6084 let inspection = aggregator
6085 .inspect_identity("review:singleton")
6086 .await?
6087 .expect("visible live alias should inspect without hidden ambiguity");
6088 assert_eq!(inspection.identity.identity, "review:singleton");
6089 assert_eq!(
6090 inspection.identity.runtime_member_id,
6091 "rt:review:singleton:0"
6092 );
6093
6094 let retired = aggregator.retire_identity("review:singleton").await?;
6095 assert!(
6096 retired,
6097 "visible live alias should retire without hidden ambiguity"
6098 );
6099
6100 let _ = runtime.mob_handle().stop().await;
6101 Ok(())
6102 }
6103
6104 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6105 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6106 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6107 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6108 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6109 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6110 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6111
6112 let aggregator = MobKitConsoleAggregator::in_memory();
6113 aggregator.register_runtime_handles_with_policy(
6114 "a-hidden",
6115 "",
6116 hidden_runtime.mob_runtime().clone(),
6117 Some(hidden_identity_runtime),
6118 hidden_runtime.console_events(),
6119 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6120 );
6121 aggregator.register_runtime_handles_with_policy(
6122 "b-visible",
6123 "",
6124 visible_runtime.mob_runtime().clone(),
6125 Some(visible_identity_runtime),
6126 visible_runtime.console_events(),
6127 Arc::new(AllowAllConsoleVisibilityPolicy),
6128 );
6129
6130 let accepted = aggregator
6131 .reserve_identity_first_interaction(
6132 ConsoleSendRequest {
6133 identity: "review:singleton".to_string(),
6134 content: json!("hello"),
6135 origin: "console:test".to_string(),
6136 idempotency_key: "visible-after-hidden-reserve".to_string(),
6137 handling_mode: Some("queue".to_string()),
6138 },
6139 None,
6140 )
6141 .await?;
6142 let page = aggregator
6143 .query_timeline(ConsoleTimelineQuery {
6144 identity: Some("review:singleton".to_string()),
6145 ..ConsoleTimelineQuery::default()
6146 })
6147 .await?;
6148 let frame = page
6149 .frames
6150 .iter()
6151 .find(|frame| frame.id == accepted.input_frame_id)
6152 .ok_or("accepted frame missing from timeline")?;
6153 assert_eq!(frame.runtime_key, "b-visible");
6154
6155 let _ = hidden_runtime.mob_handle().stop().await;
6156 let _ = visible_runtime.mob_handle().stop().await;
6157 Ok(())
6158 }
6159
6160 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6161 async fn identity_first_list_identities_projects_cached_topology_peers()
6162 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6163 let runtime = build_empty_runtime("identity-topology-cache-test").await;
6164 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6165 identity_runtime
6166 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6167 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6168 crate::identity_first::AgentIdentity::parse("agent:beta")?,
6169 )?])
6170 .await;
6171
6172 let aggregator = MobKitConsoleAggregator::in_memory();
6173 aggregator.register_runtime_handles_with_policy(
6174 "identity-first",
6175 "",
6176 runtime.mob_runtime().clone(),
6177 Some(identity_runtime),
6178 runtime.console_events(),
6179 Arc::new(AllowAllConsoleVisibilityPolicy),
6180 );
6181
6182 let identities = aggregator.list_identities().await?;
6183 let alpha = identities
6184 .iter()
6185 .find(|record| record.identity == "agent:alpha")
6186 .ok_or("agent:alpha identity missing")?;
6187 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6188 let inspection = aggregator
6189 .inspect_identity("agent:alpha")
6190 .await?
6191 .ok_or("agent:alpha inspection missing")?;
6192 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6193
6194 let _ = runtime.mob_handle().stop().await;
6195 Ok(())
6196 }
6197
6198 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6199 async fn identity_first_list_identities_includes_member_only_spawned_workers()
6200 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6201 let runtime = build_empty_runtime("identity-member-only-test").await;
6202 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6203 let aggregator = MobKitConsoleAggregator::in_memory();
6204 aggregator.register_runtime_handles_with_policy(
6205 "identity-first",
6206 "",
6207 runtime.mob_runtime().clone(),
6208 Some(identity_runtime),
6209 runtime.console_events(),
6210 Arc::new(AllowAllConsoleVisibilityPolicy),
6211 );
6212
6213 runtime
6214 .spawn(SpawnMemberSpec::from_wire(
6215 "worker".to_string(),
6216 "agent:beta".to_string(),
6217 Some("You are beta.".into()),
6218 None,
6219 None,
6220 ))
6221 .await?;
6222
6223 let identities = aggregator.list_identities().await?;
6224 assert!(
6225 identities
6226 .iter()
6227 .any(|record| record.identity == "agent:beta"),
6228 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6229 );
6230
6231 let _ = runtime.mob_handle().stop().await;
6232 Ok(())
6233 }
6234
6235 #[tokio::test]
6236 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6237 let aggregator = MobKitConsoleAggregator::in_memory();
6238 let record = identity_record_for_test("agent-cached");
6239 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6240 aggregator
6241 .inner
6242 .identity_read_model
6243 .primed
6244 .store(true, Ordering::Release);
6245 let _guard = aggregator
6246 .inner
6247 .identity_read_model
6248 .refresh_lock
6249 .clone()
6250 .lock_owned()
6251 .await;
6252
6253 let identities =
6254 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6255 .await
6256 .expect("hot identity list should not wait for refresh lock")
6257 .expect("identity list succeeds");
6258
6259 assert_eq!(identities, vec![record]);
6260 }
6261
6262 #[tokio::test]
6263 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6264 let aggregator = MobKitConsoleAggregator::in_memory();
6265 let guard = aggregator
6266 .inner
6267 .identity_read_model
6268 .refresh_lock
6269 .clone()
6270 .lock_owned()
6271 .await;
6272 let waiter = tokio::spawn({
6273 let aggregator = aggregator.clone();
6274 async move { aggregator.list_identities().await }
6275 });
6276
6277 tokio::time::sleep(Duration::from_millis(20)).await;
6278 assert!(
6279 !waiter.is_finished(),
6280 "cold identity list should wait for the in-flight refresh to finish"
6281 );
6282
6283 let record = identity_record_for_test("agent-primed");
6284 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6285 aggregator
6286 .inner
6287 .identity_read_model
6288 .primed
6289 .store(true, Ordering::Release);
6290 drop(guard);
6291
6292 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6293 .await
6294 .expect("cold identity list waiter should resume")
6295 .expect("waiter joins")
6296 .expect("identity list succeeds");
6297 assert_eq!(identities, vec![record]);
6298 }
6299
6300 #[tokio::test]
6301 async fn query_timeline_reads_from_aggregate_store() {
6302 let aggregator = MobKitConsoleAggregator::in_memory();
6303 let frame = NewConsoleFrame {
6304 id: None,
6305 dedupe_key: "event-1".to_string(),
6306 timestamp_ms: 1,
6307 runtime_key: "runtime-a".to_string(),
6308 identity: "agent-a".to_string(),
6309 conversation_id: Some("agent-a".to_string()),
6310 session_id: None,
6311 kind: "text_delta".to_string(),
6312 status: ConsoleFrameStatus::Delivered,
6313 payload: json!({ "delta": "hello" }),
6314 source: ConsoleFrameSource {
6315 kind: ConsoleFrameSourceKind::ConsoleEvent,
6316 source_cursor: None,
6317 },
6318 source_event_id: Some("event-1".to_string()),
6319 interaction_id: None,
6320 turn_id: None,
6321 run_id: None,
6322 parent_frame_id: None,
6323 caused_by_frame_id: None,
6324 };
6325 aggregator
6326 .store()
6327 .append_if_absent(frame)
6328 .await
6329 .expect("append frame");
6330
6331 let page = aggregator
6332 .query_timeline(ConsoleTimelineQuery {
6333 identity: Some("agent-a".to_string()),
6334 limit: 10,
6335 ..ConsoleTimelineQuery::default()
6336 })
6337 .await
6338 .expect("query timeline");
6339 assert_eq!(page.frames.len(), 1);
6340 assert_eq!(page.frames[0].kind, "text_delta");
6341 }
6342
6343 #[tokio::test]
6344 async fn identity_recent_anchor_respects_query_limit() {
6345 let aggregator = MobKitConsoleAggregator::in_memory();
6346 aggregator
6347 .store()
6348 .append_if_absent(NewConsoleFrame {
6349 id: None,
6350 dedupe_key: "anchored-user-input".to_string(),
6351 timestamp_ms: 1,
6352 runtime_key: "runtime-a".to_string(),
6353 identity: "agent-a".to_string(),
6354 conversation_id: Some("agent-a".to_string()),
6355 session_id: None,
6356 kind: "user_input".to_string(),
6357 status: ConsoleFrameStatus::Delivered,
6358 payload: json!({ "content": "anchor me" }),
6359 source: ConsoleFrameSource {
6360 kind: ConsoleFrameSourceKind::Synthetic,
6361 source_cursor: None,
6362 },
6363 source_event_id: Some("anchored-user-input".to_string()),
6364 interaction_id: Some("turn-a".to_string()),
6365 turn_id: None,
6366 run_id: None,
6367 parent_frame_id: None,
6368 caused_by_frame_id: None,
6369 })
6370 .await
6371 .expect("append anchor");
6372 for idx in 2..=40 {
6373 aggregator
6374 .store()
6375 .append_if_absent(NewConsoleFrame {
6376 id: None,
6377 dedupe_key: format!("noisy-tail-{idx}"),
6378 timestamp_ms: idx,
6379 runtime_key: "runtime-a".to_string(),
6380 identity: "agent-a".to_string(),
6381 conversation_id: Some("agent-a".to_string()),
6382 session_id: None,
6383 kind: "reasoning_delta".to_string(),
6384 status: ConsoleFrameStatus::Delivered,
6385 payload: json!({ "delta": idx }),
6386 source: ConsoleFrameSource {
6387 kind: ConsoleFrameSourceKind::ConsoleEvent,
6388 source_cursor: None,
6389 },
6390 source_event_id: Some(format!("noisy-tail-{idx}")),
6391 interaction_id: Some("turn-a".to_string()),
6392 turn_id: None,
6393 run_id: None,
6394 parent_frame_id: None,
6395 caused_by_frame_id: None,
6396 })
6397 .await
6398 .expect("append noisy tail");
6399 }
6400
6401 let page = aggregator
6402 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6403 identity: Some("agent-a".to_string()),
6404 mode: ConsoleTimelineMode::Recent,
6405 limit: 5,
6406 ..ConsoleTimelineWindowQuery::default()
6407 })
6408 .await
6409 .expect("query recent identity timeline");
6410
6411 assert_eq!(
6412 page.frames.len(),
6413 5,
6414 "identity anchor merge must not exceed the requested limit"
6415 );
6416 assert!(
6417 page.frames
6418 .iter()
6419 .any(|frame| frame.dedupe_key == "anchored-user-input"),
6420 "the bounded result should still retain the useful turn anchor: {:#?}",
6421 page.frames
6422 );
6423 assert_eq!(
6424 page.frames.last().and_then(|frame| frame.cursor.seq()),
6425 Some(40)
6426 );
6427 }
6428
6429 #[tokio::test]
6430 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
6431 let aggregator = MobKitConsoleAggregator::in_memory();
6432 let runtime = build_single_member_runtime().await;
6433 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
6434 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
6435 aggregator
6436 .inner
6437 .runtimes
6438 .write()
6439 .expect("runtime registry")
6440 .insert("runtime-a".to_string(), entry);
6441
6442 for idx in 0..1_500 {
6443 aggregator
6444 .store()
6445 .append_if_absent(NewConsoleFrame {
6446 id: None,
6447 dedupe_key: format!("hidden-gap-{idx}"),
6448 timestamp_ms: idx,
6449 runtime_key: "runtime-a".to_string(),
6450 identity: "agent-a".to_string(),
6451 conversation_id: Some("agent-a".to_string()),
6452 session_id: None,
6453 kind: "hidden_noise".to_string(),
6454 status: ConsoleFrameStatus::Delivered,
6455 payload: json!({ "delta": idx }),
6456 source: ConsoleFrameSource {
6457 kind: ConsoleFrameSourceKind::ConsoleEvent,
6458 source_cursor: None,
6459 },
6460 source_event_id: Some(format!("hidden-gap-{idx}")),
6461 interaction_id: None,
6462 turn_id: None,
6463 run_id: None,
6464 parent_frame_id: None,
6465 caused_by_frame_id: None,
6466 })
6467 .await
6468 .expect("append hidden frame");
6469 }
6470 aggregator
6471 .store()
6472 .append_if_absent(NewConsoleFrame {
6473 id: None,
6474 dedupe_key: "visible-after-gap".to_string(),
6475 timestamp_ms: 1_501,
6476 runtime_key: "runtime-a".to_string(),
6477 identity: "agent-a".to_string(),
6478 conversation_id: Some("agent-a".to_string()),
6479 session_id: None,
6480 kind: "interaction_complete".to_string(),
6481 status: ConsoleFrameStatus::Delivered,
6482 payload: json!({ "text": "visible after hidden gap" }),
6483 source: ConsoleFrameSource {
6484 kind: ConsoleFrameSourceKind::ConsoleEvent,
6485 source_cursor: None,
6486 },
6487 source_event_id: Some("visible-after-gap".to_string()),
6488 interaction_id: Some("turn-visible".to_string()),
6489 turn_id: None,
6490 run_id: None,
6491 parent_frame_id: None,
6492 caused_by_frame_id: None,
6493 })
6494 .await
6495 .expect("append visible frame");
6496
6497 let page = aggregator
6498 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6499 identity: Some("agent-a".to_string()),
6500 mode: ConsoleTimelineMode::Since,
6501 limit: 1,
6502 ..ConsoleTimelineWindowQuery::default()
6503 })
6504 .await
6505 .expect("query timeline");
6506 assert_eq!(page.frames.len(), 1);
6507 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
6508 assert_eq!(
6509 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6510 Some(1_501)
6511 );
6512 let _ = runtime.mob_handle().stop().await;
6513 }
6514
6515 #[tokio::test]
6516 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
6517 let aggregator = MobKitConsoleAggregator::in_memory();
6518 for idx in 1..=300 {
6519 aggregator
6520 .store()
6521 .append_if_absent(NewConsoleFrame {
6522 id: None,
6523 dedupe_key: format!("visible-{idx}"),
6524 timestamp_ms: idx,
6525 runtime_key: "runtime-a".to_string(),
6526 identity: "agent-a".to_string(),
6527 conversation_id: Some("agent-a".to_string()),
6528 session_id: None,
6529 kind: "interaction_complete".to_string(),
6530 status: ConsoleFrameStatus::Delivered,
6531 payload: json!({ "text": format!("visible {idx}") }),
6532 source: ConsoleFrameSource {
6533 kind: ConsoleFrameSourceKind::ConsoleEvent,
6534 source_cursor: None,
6535 },
6536 source_event_id: Some(format!("visible-{idx}")),
6537 interaction_id: Some(format!("turn-{idx}")),
6538 turn_id: None,
6539 run_id: None,
6540 parent_frame_id: None,
6541 caused_by_frame_id: None,
6542 })
6543 .await
6544 .expect("append visible frame");
6545 }
6546
6547 let first = aggregator
6548 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6549 identity: Some("agent-a".to_string()),
6550 mode: ConsoleTimelineMode::Since,
6551 limit: 10,
6552 ..ConsoleTimelineWindowQuery::default()
6553 })
6554 .await
6555 .expect("query first page");
6556 assert_eq!(first.frames.len(), 10);
6557 assert_eq!(
6558 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6559 Some(10)
6560 );
6561
6562 let second = aggregator
6563 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6564 identity: Some("agent-a".to_string()),
6565 mode: ConsoleTimelineMode::Since,
6566 after: first.next_cursor,
6567 limit: 10,
6568 ..ConsoleTimelineWindowQuery::default()
6569 })
6570 .await
6571 .expect("query second page");
6572 assert_eq!(second.frames.len(), 10);
6573 assert_eq!(second.frames[0].dedupe_key, "visible-11");
6574 assert_eq!(
6575 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6576 Some(20)
6577 );
6578 }
6579
6580 #[tokio::test]
6581 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
6582 let store = Arc::new(CountingConsoleLogStore::new());
6583 let aggregator = MobKitConsoleAggregator::new(store.clone());
6584 let runtime = build_single_member_runtime().await;
6585 aggregator
6586 .inner
6587 .runtimes
6588 .write()
6589 .expect("runtime registry")
6590 .insert(
6591 "runtime-a".to_string(),
6592 runtime_entry_for_test("runtime-a", &runtime),
6593 );
6594 let inserted = store
6595 .append_if_absent(NewConsoleFrame {
6596 id: None,
6597 dedupe_key: "event-1".to_string(),
6598 timestamp_ms: 1,
6599 runtime_key: "runtime-a".to_string(),
6600 identity: "agent-a".to_string(),
6601 conversation_id: Some("agent-a".to_string()),
6602 session_id: None,
6603 kind: "text_delta".to_string(),
6604 status: ConsoleFrameStatus::Delivered,
6605 payload: json!({ "delta": "hello" }),
6606 source: ConsoleFrameSource {
6607 kind: ConsoleFrameSourceKind::ConsoleEvent,
6608 source_cursor: None,
6609 },
6610 source_event_id: Some("event-1".to_string()),
6611 interaction_id: None,
6612 turn_id: None,
6613 run_id: None,
6614 parent_frame_id: None,
6615 caused_by_frame_id: None,
6616 })
6617 .await
6618 .expect("append frame");
6619
6620 let page = aggregator
6621 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6622 identity: Some("agent-a".to_string()),
6623 mode: ConsoleTimelineMode::Since,
6624 after: Some(inserted.frame.cursor),
6625 limit: 10,
6626 ..ConsoleTimelineWindowQuery::default()
6627 })
6628 .await
6629 .expect("query continuation");
6630
6631 assert!(page.frames.is_empty());
6632 assert_eq!(
6633 store.source_watermark_calls(),
6634 0,
6635 "empty since continuation must not synchronously force session-history backfill"
6636 );
6637 let _ = runtime.mob_handle().stop().await;
6638 }
6639
6640 #[test]
6641 fn legacy_timeline_struct_literals_remain_source_compatible() {
6642 let query = ConsoleTimelineQuery {
6643 identity: Some("agent-a".to_string()),
6644 conversation_id: None,
6645 after: Some(ConsoleCursor::from("console:1")),
6646 limit: 10,
6647 };
6648 let page = ConsoleTimelinePage {
6649 frames: Vec::new(),
6650 next_cursor: query.after.clone(),
6651 };
6652
6653 assert_eq!(query.limit, 10);
6654 assert_eq!(
6655 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6656 Some(1)
6657 );
6658 }
6659
6660 #[tokio::test]
6661 async fn query_timeline_is_store_local_for_registered_runtimes() {
6662 let store = Arc::new(CountingConsoleLogStore::new());
6663 let aggregator = MobKitConsoleAggregator::new(store.clone());
6664 let runtime = build_single_member_runtime().await;
6665 aggregator
6666 .inner
6667 .runtimes
6668 .write()
6669 .expect("runtime registry")
6670 .insert(
6671 "runtime-a".to_string(),
6672 runtime_entry_for_test("runtime-a", &runtime),
6673 );
6674 store
6675 .append_if_absent(NewConsoleFrame {
6676 id: None,
6677 dedupe_key: "event-1".to_string(),
6678 timestamp_ms: 1,
6679 runtime_key: "runtime-a".to_string(),
6680 identity: "agent-a".to_string(),
6681 conversation_id: Some("agent-a".to_string()),
6682 session_id: None,
6683 kind: "text_delta".to_string(),
6684 status: ConsoleFrameStatus::Delivered,
6685 payload: json!({ "delta": "hello" }),
6686 source: ConsoleFrameSource {
6687 kind: ConsoleFrameSourceKind::ConsoleEvent,
6688 source_cursor: None,
6689 },
6690 source_event_id: Some("event-1".to_string()),
6691 interaction_id: None,
6692 turn_id: None,
6693 run_id: None,
6694 parent_frame_id: None,
6695 caused_by_frame_id: None,
6696 })
6697 .await
6698 .expect("append frame");
6699
6700 let page = tokio::time::timeout(
6701 Duration::from_millis(250),
6702 aggregator.query_timeline(ConsoleTimelineQuery {
6703 identity: Some("agent-a".to_string()),
6704 limit: 10,
6705 ..ConsoleTimelineQuery::default()
6706 }),
6707 )
6708 .await
6709 .expect("timeline query should not wait for session history")
6710 .expect("timeline query succeeds");
6711
6712 assert_eq!(page.frames.len(), 1);
6713 assert_eq!(
6714 store.source_watermark_calls(),
6715 0,
6716 "query_timeline must not synchronously touch session-history watermarks"
6717 );
6718 let _ = runtime.mob_handle().stop().await;
6719 }
6720
6721 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6722 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
6723 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
6724 delay: Duration::from_secs(2),
6725 }))
6726 .await;
6727 let aggregator = MobKitConsoleAggregator::in_memory();
6728 aggregator
6729 .inner
6730 .runtimes
6731 .write()
6732 .expect("runtime registry")
6733 .insert(
6734 "runtime-a".to_string(),
6735 runtime_entry_for_test("runtime-a", &runtime),
6736 );
6737
6738 let start = Instant::now();
6739 let accepted = tokio::time::timeout(
6740 Duration::from_millis(300),
6741 aggregator.send(ConsoleSendRequest {
6742 identity: "test/agent-a".to_string(),
6743 content: json!("hello slow agent"),
6744 origin: "console:test".to_string(),
6745 idempotency_key: "nonblocking-send".to_string(),
6746 handling_mode: Some("queue".to_string()),
6747 }),
6748 )
6749 .await
6750 .expect("console send should return once the input is accepted")
6751 .expect("send succeeds");
6752
6753 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
6754 assert!(
6755 start.elapsed() < Duration::from_secs(1),
6756 "console send should not wait for the delayed LLM turn"
6757 );
6758
6759 wait_for_session_history_text(
6760 &aggregator,
6761 "test/agent-a",
6762 "slow ok",
6763 Duration::from_secs(5),
6764 )
6765 .await
6766 .expect("background dispatch should still complete and project history");
6767 let _ = runtime.mob_handle().stop().await;
6768 }
6769
6770 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6771 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
6772 {
6773 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
6774 let aggregator = MobKitConsoleAggregator::in_memory();
6775 aggregator.register_runtime(ConsoleRuntimeRegistration {
6776 runtime_key: "runtime-late".to_string(),
6777 runtime: runtime.clone(),
6778 identity_namespace: "late".to_string(),
6779 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6780 });
6781
6782 runtime
6783 .spawn(SpawnMemberSpec::from_wire(
6784 "worker".to_string(),
6785 "agent-late".to_string(),
6786 Some("You are agent-late.".into()),
6787 None,
6788 None,
6789 ))
6790 .await
6791 .expect("late member spawns");
6792 let session_id = send_message_on_mob_with_mode(
6793 &runtime.mob_handle(),
6794 "agent-late",
6795 ContentInput::Text("hello after registration".to_string()),
6796 meerkat_core::types::HandlingMode::Queue,
6797 )
6798 .await
6799 .expect("direct member send succeeds");
6800 wait_for_identity_record(
6801 &aggregator,
6802 "late/agent-late",
6803 Some(session_id.as_str()),
6804 Duration::from_secs(5),
6805 )
6806 .await?;
6807
6808 wait_for_session_history_text(
6809 &aggregator,
6810 "late/agent-late",
6811 "You are agent-late.",
6812 Duration::from_secs(5),
6813 )
6814 .await?;
6815 let _ = runtime.mob_handle().stop().await;
6816 Ok(())
6817 }
6818
6819 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6820 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
6821 -> Result<(), String> {
6822 let runtime = build_single_member_runtime().await;
6823 let entry = runtime_entry_for_test("runtime-a", &runtime);
6824 let resolved = member_sources_for_entry(&entry)
6825 .await
6826 .into_iter()
6827 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6828 .expect("agent-a member exists");
6829 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6830 .await
6831 .expect("identity record exists");
6832 let session_id = record.session_id.expect("agent-a has a session");
6833 wait_for_runtime_session_history_text(
6834 &runtime,
6835 &session_id,
6836 "You are agent-a.",
6837 Duration::from_secs(5),
6838 )
6839 .await?;
6840
6841 let aggregator = MobKitConsoleAggregator::in_memory();
6842 aggregator
6843 .inner
6844 .runtimes
6845 .write()
6846 .expect("runtime registry")
6847 .insert("runtime-a".to_string(), entry);
6848 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
6849 aggregator
6850 .store()
6851 .record_source_watermark(
6852 &watermark_key,
6853 ConsoleFrameSourceKind::SessionHistory,
6854 &format_session_history_watermark(&session_id, 0, current_time_ms()),
6855 )
6856 .await
6857 .expect("record fresh empty watermark");
6858
6859 let page = tokio::time::timeout(
6860 Duration::from_secs(2),
6861 aggregator.query_timeline(ConsoleTimelineQuery {
6862 identity: Some("test/agent-a".to_string()),
6863 limit: 20,
6864 ..ConsoleTimelineQuery::default()
6865 }),
6866 )
6867 .await
6868 .expect("explicit identity query should not stall")
6869 .expect("query succeeds");
6870
6871 assert!(
6872 page.frames.is_empty(),
6873 "empty fresh-watermark query should return promptly before async backfill; frames: {:#?}",
6874 page.frames
6875 );
6876 wait_for_session_history_text(
6877 &aggregator,
6878 "test/agent-a",
6879 "You are agent-a.",
6880 Duration::from_secs(5),
6881 )
6882 .await?;
6883 let _ = runtime.mob_handle().stop().await;
6884 Ok(())
6885 }
6886
6887 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6888 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
6889 {
6890 let store = Arc::new(CountingConsoleLogStore::new());
6891 let aggregator = MobKitConsoleAggregator::new(store.clone());
6892 let runtime = build_single_member_runtime().await;
6893 let entry = runtime_entry_for_test("runtime-a", &runtime);
6894 let resolved = member_sources_for_entry(&entry)
6895 .await
6896 .into_iter()
6897 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6898 .expect("agent-a member exists");
6899 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6900 .await
6901 .expect("identity record exists");
6902 let session_id = record.session_id.expect("agent-a has a session");
6903 wait_for_runtime_session_history_text(
6904 &runtime,
6905 &session_id,
6906 "You are agent-a.",
6907 Duration::from_secs(5),
6908 )
6909 .await?;
6910 aggregator
6911 .inner
6912 .runtimes
6913 .write()
6914 .expect("runtime registry")
6915 .insert("runtime-a".to_string(), entry);
6916
6917 store
6918 .append_if_absent(NewConsoleFrame {
6919 id: None,
6920 dedupe_key: "stale-session-history-agent-a".to_string(),
6921 timestamp_ms: 10,
6922 runtime_key: "runtime-a".to_string(),
6923 identity: "test/agent-a".to_string(),
6924 conversation_id: Some("test/agent-a".to_string()),
6925 session_id: Some(session_id.clone()),
6926 kind: "user_input".to_string(),
6927 status: ConsoleFrameStatus::Delivered,
6928 payload: json!({
6929 "text": "stale projected history",
6930 "type": "user_input",
6931 }),
6932 source: ConsoleFrameSource {
6933 kind: ConsoleFrameSourceKind::SessionHistory,
6934 source_cursor: Some("stale-session-history-agent-a".to_string()),
6935 },
6936 source_event_id: Some("stale-session-history-agent-a".to_string()),
6937 interaction_id: None,
6938 turn_id: None,
6939 run_id: None,
6940 parent_frame_id: None,
6941 caused_by_frame_id: None,
6942 })
6943 .await
6944 .expect("append stale history frame");
6945
6946 let fresh_prompt = "fresh prompt after stale history";
6947 let sent_session_id = send_message_on_mob_with_mode(
6948 &runtime.mob_handle(),
6949 "agent-a",
6950 ContentInput::Text(fresh_prompt.to_string()),
6951 meerkat_core::types::HandlingMode::Queue,
6952 )
6953 .await
6954 .expect("direct member send succeeds");
6955 assert_eq!(sent_session_id, session_id);
6956 wait_for_runtime_session_history_text(
6957 &runtime,
6958 &session_id,
6959 fresh_prompt,
6960 Duration::from_secs(5),
6961 )
6962 .await?;
6963
6964 let page = aggregator
6965 .query_timeline(ConsoleTimelineQuery {
6966 identity: Some("test/agent-a".to_string()),
6967 limit: 50,
6968 ..ConsoleTimelineQuery::default()
6969 })
6970 .await
6971 .expect("query child timeline");
6972
6973 assert!(
6974 page.frames.iter().any(|frame| {
6975 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
6976 && session_history_content_text(frame).as_deref()
6977 == Some("stale projected history")
6978 }),
6979 "explicit identity query should return existing history before async refresh; frames: {:#?}",
6980 page.frames
6981 );
6982 wait_for_session_history_text(
6983 &aggregator,
6984 "test/agent-a",
6985 fresh_prompt,
6986 Duration::from_secs(5),
6987 )
6988 .await?;
6989 assert!(
6990 store.source_watermark_calls() > 0,
6991 "stale existing session history should force a targeted source refresh"
6992 );
6993 let _ = runtime.mob_handle().stop().await;
6994 Ok(())
6995 }
6996
6997 async fn wait_for_session_history_text(
6998 aggregator: &MobKitConsoleAggregator,
6999 identity: &str,
7000 expected: &str,
7001 timeout: Duration,
7002 ) -> Result<(), String> {
7003 let deadline = Instant::now() + timeout;
7004 let mut observed = Vec::new();
7005 while Instant::now() < deadline {
7006 let _ = aggregator.list_identities().await;
7007 let page = aggregator
7008 .query_timeline(ConsoleTimelineQuery {
7009 identity: Some(identity.to_string()),
7010 limit: 20,
7011 ..ConsoleTimelineQuery::default()
7012 })
7013 .await
7014 .expect("query timeline");
7015 observed = page.frames;
7016 if observed.iter().any(|frame| {
7017 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7018 && session_history_content_text(frame).as_deref() == Some(expected)
7019 }) {
7020 return Ok(());
7021 }
7022 tokio::time::sleep(Duration::from_millis(25)).await;
7023 }
7024
7025 Err(format!(
7026 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7027 ))
7028 }
7029
7030 async fn wait_for_session_backfill_target(
7031 aggregator: &MobKitConsoleAggregator,
7032 identity: &str,
7033 timeout: Duration,
7034 ) -> Result<String, String> {
7035 let deadline = Instant::now() + timeout;
7036 while Instant::now() < deadline {
7037 if let Some(target) =
7038 session_backfill_target_for_identity(&aggregator.inner, identity).await
7039 {
7040 return Ok(target.session_id);
7041 }
7042 tokio::time::sleep(Duration::from_millis(25)).await;
7043 }
7044 Err(format!(
7045 "session backfill target for {identity:?} was not resolvable"
7046 ))
7047 }
7048
7049 async fn wait_for_runtime_session_history_text(
7050 runtime: &UnifiedRuntime,
7051 session_id: &str,
7052 expected: &str,
7053 timeout: Duration,
7054 ) -> Result<(), String> {
7055 let deadline = Instant::now() + timeout;
7056 let mut observed = Vec::new();
7057 while Instant::now() < deadline {
7058 let page = runtime
7059 .mob_runtime()
7060 .read_session_history(session_id, 0, Some(20))
7061 .await
7062 .map_err(|err| err.to_string())?;
7063 observed = page.messages;
7064 if observed.iter().enumerate().any(|(idx, message)| {
7065 let Some(message) = serde_json::to_value(message).ok() else {
7066 return false;
7067 };
7068 frames_from_session_history_message(
7069 "runtime-a",
7070 "test/agent-a",
7071 session_id,
7072 idx,
7073 message,
7074 )
7075 .iter()
7076 .any(|frame| {
7077 matches!(
7078 frame.kind.as_str(),
7079 "user_input" | "system_notice" | "interaction_complete"
7080 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7081 })
7082 }) {
7083 return Ok(());
7084 }
7085 tokio::time::sleep(Duration::from_millis(25)).await;
7086 }
7087
7088 Err(format!(
7089 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7090 ))
7091 }
7092
7093 async fn wait_for_identity_record(
7094 aggregator: &MobKitConsoleAggregator,
7095 identity: &str,
7096 session_id: Option<&str>,
7097 timeout: Duration,
7098 ) -> Result<(), String> {
7099 let deadline = Instant::now() + timeout;
7100 let mut observed = Vec::new();
7101 while Instant::now() < deadline {
7102 observed = aggregator
7103 .list_identities()
7104 .await
7105 .map_err(|err| err.to_string())?;
7106 if observed.iter().any(|record| {
7107 record.identity == identity && record.session_id.as_deref() == session_id
7108 }) {
7109 return Ok(());
7110 }
7111 tokio::time::sleep(Duration::from_millis(25)).await;
7112 }
7113
7114 Err(format!(
7115 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7116 ))
7117 }
7118
7119 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7120 session_history_payload_text(&frame.payload)
7121 }
7122
7123 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7124 session_history_payload_text(&frame.payload)
7125 }
7126
7127 fn session_history_payload_text(payload: &Value) -> Option<String> {
7128 if let Some(text) = payload.get("text").and_then(Value::as_str) {
7129 return Some(text.to_string());
7130 }
7131 if let Some(text) = payload.get("result").and_then(Value::as_str) {
7132 return Some(text.to_string());
7133 }
7134 if let Some(text) = payload.get("body").and_then(Value::as_str) {
7135 return Some(text.to_string());
7136 }
7137 match payload.get("content")? {
7138 Value::String(text) => Some(text.clone()),
7139 Value::Array(blocks) => Some(
7140 blocks
7141 .iter()
7142 .filter_map(|block| block.get("text").and_then(Value::as_str))
7143 .collect::<Vec<_>>()
7144 .join(""),
7145 ),
7146 _ => None,
7147 }
7148 }
7149
7150 #[tokio::test]
7151 async fn query_timeline_handles_large_store_without_backfill_calls() {
7152 let store = Arc::new(CountingConsoleLogStore::new());
7153 let aggregator = MobKitConsoleAggregator::new(store.clone());
7154 for idx in 0..5_000 {
7155 store
7156 .append_if_absent(NewConsoleFrame {
7157 id: None,
7158 dedupe_key: format!("event-{idx}"),
7159 timestamp_ms: idx,
7160 runtime_key: "runtime-a".to_string(),
7161 identity: "agent-a".to_string(),
7162 conversation_id: Some("agent-a".to_string()),
7163 session_id: Some("session-a".to_string()),
7164 kind: "text_delta".to_string(),
7165 status: ConsoleFrameStatus::Delivered,
7166 payload: json!({ "delta": idx }),
7167 source: ConsoleFrameSource {
7168 kind: ConsoleFrameSourceKind::ConsoleEvent,
7169 source_cursor: None,
7170 },
7171 source_event_id: Some(format!("event-{idx}")),
7172 interaction_id: None,
7173 turn_id: None,
7174 run_id: None,
7175 parent_frame_id: None,
7176 caused_by_frame_id: None,
7177 })
7178 .await
7179 .expect("append frame");
7180 }
7181
7182 let page = aggregator
7183 .query_timeline(ConsoleTimelineQuery {
7184 identity: Some("agent-a".to_string()),
7185 limit: 1_000,
7186 ..ConsoleTimelineQuery::default()
7187 })
7188 .await
7189 .expect("large query");
7190
7191 assert_eq!(page.frames.len(), 1_000);
7192 assert_eq!(store.source_watermark_calls(), 0);
7193 }
7194
7195 #[tokio::test]
7196 async fn query_timeline_rejects_future_cursor_after_store_reset() {
7197 let aggregator = MobKitConsoleAggregator::in_memory();
7198 let err = aggregator
7199 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7200 after: Some(ConsoleCursor::from("console:99")),
7201 limit: 10,
7202 ..ConsoleTimelineWindowQuery::default()
7203 })
7204 .await
7205 .expect_err("future cursor on empty/reset store must be replay-unavailable");
7206
7207 assert!(
7208 err.to_string()
7209 .contains("beyond the current store frontier"),
7210 "unexpected error: {err}"
7211 );
7212 }
7213
7214 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7215 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7216 let store = Arc::new(CountingConsoleLogStore::new());
7217 let aggregator = MobKitConsoleAggregator::new(store.clone());
7218 let runtime = build_single_member_runtime().await;
7219 aggregator
7220 .inner
7221 .runtimes
7222 .write()
7223 .expect("runtime registry")
7224 .insert(
7225 "runtime-a".to_string(),
7226 runtime_entry_for_test("runtime-a", &runtime),
7227 );
7228 let session_id =
7229 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
7230 .await
7231 .expect("spawned member is resolvable for targeted backfill");
7232 wait_for_runtime_session_history_text(
7233 &runtime,
7234 &session_id,
7235 "You are agent-a.",
7236 Duration::from_secs(5),
7237 )
7238 .await
7239 .expect("spawned member kickoff is readable before live-frame race");
7240 store
7241 .append_if_absent(NewConsoleFrame {
7242 id: None,
7243 dedupe_key: "live-tool-before-history".to_string(),
7244 timestamp_ms: 10,
7245 runtime_key: "runtime-a".to_string(),
7246 identity: "test/agent-a".to_string(),
7247 conversation_id: Some("test/agent-a".to_string()),
7248 session_id: None,
7249 kind: "tool_execution_started".to_string(),
7250 status: ConsoleFrameStatus::Delivered,
7251 payload: json!({
7252 "id": "call-live",
7253 "name": "king_search",
7254 "source_event_type": "tool_execution_started",
7255 "type": "tool_execution_started",
7256 }),
7257 source: ConsoleFrameSource {
7258 kind: ConsoleFrameSourceKind::ConsoleEvent,
7259 source_cursor: Some("live-tool-before-history".to_string()),
7260 },
7261 source_event_id: Some("live-tool-before-history".to_string()),
7262 interaction_id: None,
7263 turn_id: None,
7264 run_id: None,
7265 parent_frame_id: None,
7266 caused_by_frame_id: None,
7267 })
7268 .await
7269 .expect("append live tool frame");
7270
7271 let page = aggregator
7272 .query_timeline(ConsoleTimelineQuery {
7273 identity: Some("test/agent-a".to_string()),
7274 limit: 20,
7275 ..ConsoleTimelineQuery::default()
7276 })
7277 .await
7278 .expect("query child timeline");
7279
7280 assert!(
7281 page.frames
7282 .iter()
7283 .any(|frame| frame.kind == "tool_execution_started"),
7284 "initial explicit child timeline should return existing store frames without waiting for session history"
7285 );
7286
7287 let mut observed_backfill = false;
7288 for _ in 0..80 {
7289 let page = aggregator
7290 .query_timeline(ConsoleTimelineQuery {
7291 identity: Some("test/agent-a".to_string()),
7292 limit: 20,
7293 ..ConsoleTimelineQuery::default()
7294 })
7295 .await
7296 .expect("query child timeline after scheduled backfill");
7297 if page.frames.iter().any(|frame| {
7298 frame.kind == "user_input"
7299 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7300 && frame.payload.to_string().contains("You are agent-a.")
7301 }) {
7302 observed_backfill = true;
7303 break;
7304 }
7305 tokio::time::sleep(Duration::from_millis(25)).await;
7306 }
7307 assert!(
7308 observed_backfill,
7309 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
7310 );
7311 let _ = runtime.mob_handle().stop().await;
7312 }
7313
7314 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7315 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
7316 let store = Arc::new(CountingConsoleLogStore::new());
7317 let aggregator = MobKitConsoleAggregator::new(store.clone());
7318 let (_temp, runtime, _delayed_service) =
7319 build_stress_runtime(64, Duration::from_millis(0)).await;
7320 aggregator
7321 .inner
7322 .runtimes
7323 .write()
7324 .expect("runtime registry")
7325 .insert(
7326 "runtime-a".to_string(),
7327 runtime_entry_for_test("runtime-a", runtime.as_ref()),
7328 );
7329 for idx in 0..1_000 {
7330 store
7331 .append_if_absent(NewConsoleFrame {
7332 id: None,
7333 dedupe_key: format!("event-{idx}"),
7334 timestamp_ms: idx,
7335 runtime_key: "runtime-a".to_string(),
7336 identity: "test/agent-0".to_string(),
7337 conversation_id: Some("test/agent-0".to_string()),
7338 session_id: Some("session-a".to_string()),
7339 kind: "text_delta".to_string(),
7340 status: ConsoleFrameStatus::Delivered,
7341 payload: json!({ "delta": idx }),
7342 source: ConsoleFrameSource {
7343 kind: ConsoleFrameSourceKind::ConsoleEvent,
7344 source_cursor: None,
7345 },
7346 source_event_id: Some(format!("event-{idx}")),
7347 interaction_id: None,
7348 turn_id: None,
7349 run_id: None,
7350 parent_frame_id: None,
7351 caused_by_frame_id: None,
7352 })
7353 .await
7354 .expect("append frame");
7355 }
7356
7357 let page = tokio::time::timeout(
7358 Duration::from_secs(2),
7359 aggregator.query_timeline(ConsoleTimelineQuery {
7360 identity: Some("test/agent-0".to_string()),
7361 limit: 1_000,
7362 ..ConsoleTimelineQuery::default()
7363 }),
7364 )
7365 .await
7366 .expect("identity timeline query should not rediscover the roster per frame")
7367 .expect("timeline query succeeds");
7368
7369 assert_eq!(page.frames.len(), 1_000);
7370 assert_eq!(store.source_watermark_calls(), 0);
7371 let _ = runtime.mob_handle().stop().await;
7372 }
7373
7374 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7375 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
7376 const MEMBER_COUNT: usize = 32;
7377 let (_temp, runtime, delayed_service) =
7378 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
7379 let aggregator = MobKitConsoleAggregator::in_memory();
7380 aggregator
7381 .inner
7382 .runtimes
7383 .write()
7384 .expect("runtime registry")
7385 .insert(
7386 "runtime-stress".to_string(),
7387 runtime_entry_for_test("runtime-stress", &runtime),
7388 );
7389
7390 let started = Instant::now();
7391 aggregator
7392 .refresh_session_history()
7393 .await
7394 .expect("stress refresh");
7395 let elapsed = started.elapsed();
7396
7397 assert!(
7398 delayed_service.read_calls() >= MEMBER_COUNT,
7399 "expected at least one history read per member, saw {}",
7400 delayed_service.read_calls()
7401 );
7402 assert!(
7403 delayed_service.max_active_reads() > 1,
7404 "session history backfill should fan out instead of reading members serially"
7405 );
7406 assert!(
7407 delayed_service.max_active_reads()
7408 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
7409 "session history backfill should respect the default concurrency limit"
7410 );
7411 assert!(
7412 elapsed < Duration::from_millis(600),
7413 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
7414 MEMBER_COUNT * 40
7415 );
7416 let _ = runtime.mob_handle().stop().await;
7417 }
7418
7419 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7420 async fn session_history_backfill_respects_configured_concurrency_limit() {
7421 const MEMBER_COUNT: usize = 16;
7422 let (_temp, runtime, delayed_service) =
7423 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
7424 let aggregator =
7425 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
7426 max_concurrent_session_backfills: 4,
7427 ..ConsoleAggregatorOptions::default()
7428 });
7429 aggregator
7430 .inner
7431 .runtimes
7432 .write()
7433 .expect("runtime registry")
7434 .insert(
7435 "runtime-stress".to_string(),
7436 runtime_entry_for_test("runtime-stress", &runtime),
7437 );
7438
7439 aggregator
7440 .refresh_session_history()
7441 .await
7442 .expect("stress refresh");
7443
7444 assert!(
7445 delayed_service.max_active_reads() <= 4,
7446 "configured concurrency limit should bound session history reads, saw {}",
7447 delayed_service.max_active_reads()
7448 );
7449 let _ = runtime.mob_handle().stop().await;
7450 }
7451
7452 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7453 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
7454 const MEMBER_COUNT: usize = 24;
7455 const LIVE_EVENT_COUNT: usize = 2_048;
7456 let (_temp, runtime, _delayed_service) =
7457 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
7458 let aggregator = MobKitConsoleAggregator::in_memory();
7459 aggregator.register_runtime(ConsoleRuntimeRegistration {
7460 runtime_key: "runtime-burst".to_string(),
7461 runtime: runtime.clone(),
7462 identity_namespace: "stress".to_string(),
7463 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7464 });
7465 let console_events = runtime.console_events();
7466 for idx in 0..LIVE_EVENT_COUNT {
7467 console_events
7468 .append(
7469 "agent-0",
7470 Some("burst-turn".to_string()),
7471 "text_delta",
7472 json!({ "delta": format!("frame-{idx}") }),
7473 )
7474 .await;
7475 }
7476
7477 let deadline = Instant::now() + Duration::from_secs(5);
7478 let mut observed = 0;
7479 while Instant::now() < deadline {
7480 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
7481 if observed >= LIVE_EVENT_COUNT {
7482 break;
7483 }
7484 tokio::time::sleep(Duration::from_millis(25)).await;
7485 }
7486
7487 assert_eq!(
7488 observed, LIVE_EVENT_COUNT,
7489 "live pump should not drop frames while slow background backfill is running"
7490 );
7491 let _ = runtime.mob_handle().stop().await;
7492 }
7493
7494 async fn count_console_event_frames(
7495 aggregator: &MobKitConsoleAggregator,
7496 identity: &str,
7497 ) -> usize {
7498 let mut after = None;
7499 let mut count = 0;
7500 loop {
7501 let page = aggregator
7502 .query_timeline(ConsoleTimelineQuery {
7503 identity: Some(identity.to_string()),
7504 after,
7505 limit: 1_000,
7506 ..ConsoleTimelineQuery::default()
7507 })
7508 .await
7509 .expect("query burst timeline");
7510 if page.frames.is_empty() {
7511 break;
7512 }
7513 count += page
7514 .frames
7515 .iter()
7516 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
7517 .count();
7518 after = page.next_cursor;
7519 if after.is_none() {
7520 break;
7521 }
7522 }
7523 count
7524 }
7525
7526 #[tokio::test]
7527 async fn status_updates_get_replayable_aggregate_cursors() {
7528 let aggregator = MobKitConsoleAggregator::in_memory();
7529 let frame = NewConsoleFrame {
7530 id: None,
7531 dedupe_key: "send-1".to_string(),
7532 timestamp_ms: 1,
7533 runtime_key: "runtime-a".to_string(),
7534 identity: "agent-a".to_string(),
7535 conversation_id: Some("agent-a".to_string()),
7536 session_id: Some("session-1".to_string()),
7537 kind: "user_input".to_string(),
7538 status: ConsoleFrameStatus::Accepted,
7539 payload: json!({ "content": "hello" }),
7540 source: ConsoleFrameSource {
7541 kind: ConsoleFrameSourceKind::Send,
7542 source_cursor: None,
7543 },
7544 source_event_id: None,
7545 interaction_id: Some("interaction-1".to_string()),
7546 turn_id: None,
7547 run_id: None,
7548 parent_frame_id: None,
7549 caused_by_frame_id: None,
7550 };
7551 let inserted = aggregator
7552 .store()
7553 .append_if_absent(frame)
7554 .await
7555 .expect("append frame");
7556
7557 update_frame_status_and_emit(
7558 &aggregator.inner,
7559 &inserted.frame.id,
7560 ConsoleFrameStatus::Delivered,
7561 )
7562 .await
7563 .expect("update status");
7564
7565 let page = aggregator
7566 .query_timeline(ConsoleTimelineQuery {
7567 identity: Some("agent-a".to_string()),
7568 after: Some(inserted.frame.cursor.clone()),
7569 limit: 10,
7570 ..ConsoleTimelineQuery::default()
7571 })
7572 .await
7573 .expect("query timeline");
7574 assert_eq!(page.frames.len(), 1);
7575 assert_eq!(page.frames[0].kind, "frame_updated");
7576 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
7577 assert_eq!(
7578 page.frames[0]
7579 .payload
7580 .get("frame")
7581 .and_then(|frame| frame.get("status"))
7582 .and_then(Value::as_str),
7583 Some("delivered")
7584 );
7585 }
7586
7587 #[tokio::test]
7588 async fn steer_delivery_appends_terminal_control_frame() {
7589 let aggregator = MobKitConsoleAggregator::in_memory();
7590 let frame = NewConsoleFrame {
7591 id: None,
7592 dedupe_key: "send-steer-1".to_string(),
7593 timestamp_ms: 1,
7594 runtime_key: "runtime-a".to_string(),
7595 identity: "agent-a".to_string(),
7596 conversation_id: Some("agent-a".to_string()),
7597 session_id: Some("session-1".to_string()),
7598 kind: "user_input".to_string(),
7599 status: ConsoleFrameStatus::Delivered,
7600 payload: json!({
7601 "content": "operator steer",
7602 "handling_mode": "steer",
7603 }),
7604 source: ConsoleFrameSource {
7605 kind: ConsoleFrameSourceKind::Send,
7606 source_cursor: None,
7607 },
7608 source_event_id: None,
7609 interaction_id: Some("interaction-steer-1".to_string()),
7610 turn_id: None,
7611 run_id: None,
7612 parent_frame_id: None,
7613 caused_by_frame_id: None,
7614 };
7615 let inserted = aggregator
7616 .store()
7617 .append_if_absent(frame)
7618 .await
7619 .expect("append steer input");
7620
7621 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
7622 .await
7623 .expect("append steer terminal");
7624
7625 let page = aggregator
7626 .query_timeline(ConsoleTimelineQuery {
7627 identity: Some("agent-a".to_string()),
7628 after: Some(inserted.frame.cursor.clone()),
7629 limit: 10,
7630 ..ConsoleTimelineQuery::default()
7631 })
7632 .await
7633 .expect("query timeline");
7634
7635 assert_eq!(page.frames.len(), 1);
7636 let terminal = &page.frames[0];
7637 assert_eq!(terminal.kind, "interaction_complete");
7638 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
7639 assert_eq!(
7640 terminal.interaction_id.as_deref(),
7641 Some("interaction-steer-1")
7642 );
7643 assert_eq!(
7644 terminal.parent_frame_id.as_deref(),
7645 Some(inserted.frame.id.as_str())
7646 );
7647 assert_eq!(
7648 terminal.payload.get("reason").and_then(Value::as_str),
7649 Some("steer_delivered")
7650 );
7651 }
7652
7653 #[tokio::test]
7654 async fn history_counterpart_scan_is_not_capped_to_one_page() {
7655 let aggregator = MobKitConsoleAggregator::in_memory();
7656 for idx in 0..1_005 {
7657 aggregator
7658 .store()
7659 .append_if_absent(NewConsoleFrame {
7660 id: None,
7661 dedupe_key: format!("filler-{idx}"),
7662 timestamp_ms: idx,
7663 runtime_key: "runtime-a".to_string(),
7664 identity: "agent-a".to_string(),
7665 conversation_id: Some("agent-a".to_string()),
7666 session_id: Some("session-a".to_string()),
7667 kind: "text_delta".to_string(),
7668 status: ConsoleFrameStatus::Completed,
7669 payload: json!({ "delta": idx }),
7670 source: ConsoleFrameSource {
7671 kind: ConsoleFrameSourceKind::ConsoleEvent,
7672 source_cursor: None,
7673 },
7674 source_event_id: Some(format!("filler-{idx}")),
7675 interaction_id: None,
7676 turn_id: None,
7677 run_id: None,
7678 parent_frame_id: None,
7679 caused_by_frame_id: None,
7680 })
7681 .await
7682 .expect("append filler");
7683 }
7684 aggregator
7685 .store()
7686 .append_if_absent(NewConsoleFrame {
7687 id: None,
7688 dedupe_key: "live-user-input".to_string(),
7689 timestamp_ms: 2_000,
7690 runtime_key: "runtime-a".to_string(),
7691 identity: "agent-a".to_string(),
7692 conversation_id: Some("agent-a".to_string()),
7693 session_id: Some("session-a".to_string()),
7694 kind: "user_input".to_string(),
7695 status: ConsoleFrameStatus::Delivered,
7696 payload: json!({ "content": "already here" }),
7697 source: ConsoleFrameSource {
7698 kind: ConsoleFrameSourceKind::ConsoleEvent,
7699 source_cursor: None,
7700 },
7701 source_event_id: Some("live-user-input".to_string()),
7702 interaction_id: None,
7703 turn_id: None,
7704 run_id: None,
7705 parent_frame_id: None,
7706 caused_by_frame_id: None,
7707 })
7708 .await
7709 .expect("append live input");
7710
7711 let history = NewConsoleFrame {
7712 id: None,
7713 dedupe_key: "history-user-input".to_string(),
7714 timestamp_ms: 3_000,
7715 runtime_key: "runtime-a".to_string(),
7716 identity: "agent-a".to_string(),
7717 conversation_id: Some("agent-a".to_string()),
7718 session_id: Some("session-a".to_string()),
7719 kind: "user_input".to_string(),
7720 status: ConsoleFrameStatus::Completed,
7721 payload: json!({ "content": "already here" }),
7722 source: ConsoleFrameSource {
7723 kind: ConsoleFrameSourceKind::SessionHistory,
7724 source_cursor: Some("session-a:1006".to_string()),
7725 },
7726 source_event_id: None,
7727 interaction_id: None,
7728 turn_id: None,
7729 run_id: None,
7730 parent_frame_id: None,
7731 caused_by_frame_id: None,
7732 };
7733
7734 assert!(
7735 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7736 .await
7737 .expect("counterpart scan")
7738 );
7739 }
7740
7741 #[tokio::test]
7742 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
7743 let aggregator = MobKitConsoleAggregator::in_memory();
7744 aggregator
7745 .store()
7746 .append_if_absent(NewConsoleFrame {
7747 id: None,
7748 dedupe_key: "live-user-input".to_string(),
7749 timestamp_ms: 2_000,
7750 runtime_key: "runtime-a".to_string(),
7751 identity: "agent-a".to_string(),
7752 conversation_id: Some("agent-a".to_string()),
7753 session_id: Some("session-a".to_string()),
7754 kind: "user_input".to_string(),
7755 status: ConsoleFrameStatus::Delivered,
7756 payload: json!({ "content": "hello from operator" }),
7757 source: ConsoleFrameSource {
7758 kind: ConsoleFrameSourceKind::ConsoleEvent,
7759 source_cursor: None,
7760 },
7761 source_event_id: Some("live-user-input".to_string()),
7762 interaction_id: None,
7763 turn_id: None,
7764 run_id: None,
7765 parent_frame_id: None,
7766 caused_by_frame_id: None,
7767 })
7768 .await
7769 .expect("append live input");
7770
7771 let history = NewConsoleFrame {
7772 id: None,
7773 dedupe_key: "history-user-input".to_string(),
7774 timestamp_ms: 3_000,
7775 runtime_key: "runtime-a".to_string(),
7776 identity: "agent-a".to_string(),
7777 conversation_id: Some("agent-a".to_string()),
7778 session_id: Some("session-a".to_string()),
7779 kind: "user_input".to_string(),
7780 status: ConsoleFrameStatus::Completed,
7781 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
7782 source: ConsoleFrameSource {
7783 kind: ConsoleFrameSourceKind::SessionHistory,
7784 source_cursor: Some("session-a:2".to_string()),
7785 },
7786 source_event_id: None,
7787 interaction_id: None,
7788 turn_id: None,
7789 run_id: None,
7790 parent_frame_id: None,
7791 caused_by_frame_id: None,
7792 };
7793
7794 assert!(
7795 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7796 .await
7797 .expect("counterpart scan")
7798 );
7799 }
7800
7801 #[tokio::test]
7802 async fn history_counterpart_scan_matches_content_block_user_prompts() {
7803 let aggregator = MobKitConsoleAggregator::in_memory();
7804 aggregator
7805 .store()
7806 .append_if_absent(NewConsoleFrame {
7807 id: None,
7808 dedupe_key: "live-user-input".to_string(),
7809 timestamp_ms: 2_000,
7810 runtime_key: "runtime-a".to_string(),
7811 identity: "agent-a".to_string(),
7812 conversation_id: Some("agent-a".to_string()),
7813 session_id: Some("session-a".to_string()),
7814 kind: "user_input".to_string(),
7815 status: ConsoleFrameStatus::Delivered,
7816 payload: json!({ "content": "hello from operator" }),
7817 source: ConsoleFrameSource {
7818 kind: ConsoleFrameSourceKind::ConsoleEvent,
7819 source_cursor: None,
7820 },
7821 source_event_id: Some("live-user-input".to_string()),
7822 interaction_id: None,
7823 turn_id: None,
7824 run_id: None,
7825 parent_frame_id: None,
7826 caused_by_frame_id: None,
7827 })
7828 .await
7829 .expect("append live input");
7830
7831 let history = NewConsoleFrame {
7832 id: None,
7833 dedupe_key: "history-user-input".to_string(),
7834 timestamp_ms: 3_000,
7835 runtime_key: "runtime-a".to_string(),
7836 identity: "agent-a".to_string(),
7837 conversation_id: Some("agent-a".to_string()),
7838 session_id: Some("session-a".to_string()),
7839 kind: "user_input".to_string(),
7840 status: ConsoleFrameStatus::Completed,
7841 payload: json!({
7842 "content": [{ "type": "text", "text": "hello from operator" }]
7843 }),
7844 source: ConsoleFrameSource {
7845 kind: ConsoleFrameSourceKind::SessionHistory,
7846 source_cursor: Some("session-a:2".to_string()),
7847 },
7848 source_event_id: None,
7849 interaction_id: None,
7850 turn_id: None,
7851 run_id: None,
7852 parent_frame_id: None,
7853 caused_by_frame_id: None,
7854 };
7855
7856 assert!(
7857 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7858 .await
7859 .expect("counterpart scan")
7860 );
7861 }
7862
7863 #[tokio::test]
7864 async fn history_counterpart_scan_matches_live_tool_results() {
7865 let aggregator = MobKitConsoleAggregator::in_memory();
7866 aggregator
7867 .store()
7868 .append_if_absent(NewConsoleFrame {
7869 id: None,
7870 dedupe_key: "live-tool-result".to_string(),
7871 timestamp_ms: 2_000,
7872 runtime_key: "runtime-a".to_string(),
7873 identity: "agent-a".to_string(),
7874 conversation_id: Some("agent-a".to_string()),
7875 session_id: Some("session-a".to_string()),
7876 kind: "tool_execution_completed".to_string(),
7877 status: ConsoleFrameStatus::Delivered,
7878 payload: json!({
7879 "id": "call-1",
7880 "tool_call_id": "call-1",
7881 "result": "{ \"count\": 70 }"
7882 }),
7883 source: ConsoleFrameSource {
7884 kind: ConsoleFrameSourceKind::ConsoleEvent,
7885 source_cursor: None,
7886 },
7887 source_event_id: Some("live-tool-result".to_string()),
7888 interaction_id: None,
7889 turn_id: None,
7890 run_id: None,
7891 parent_frame_id: None,
7892 caused_by_frame_id: None,
7893 })
7894 .await
7895 .expect("append live tool result");
7896
7897 let history = NewConsoleFrame {
7898 id: None,
7899 dedupe_key: "history-tool-result".to_string(),
7900 timestamp_ms: 3_000,
7901 runtime_key: "runtime-a".to_string(),
7902 identity: "agent-a".to_string(),
7903 conversation_id: Some("agent-a".to_string()),
7904 session_id: Some("session-a".to_string()),
7905 kind: "tool_execution_completed".to_string(),
7906 status: ConsoleFrameStatus::Completed,
7907 payload: json!({
7908 "id": "call-1",
7909 "tool_call_id": "call-1",
7910 "result": "{ \"count\": 70 }",
7911 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
7912 }),
7913 source: ConsoleFrameSource {
7914 kind: ConsoleFrameSourceKind::SessionHistory,
7915 source_cursor: Some("session-a:3:0".to_string()),
7916 },
7917 source_event_id: None,
7918 interaction_id: None,
7919 turn_id: None,
7920 run_id: None,
7921 parent_frame_id: None,
7922 caused_by_frame_id: None,
7923 };
7924
7925 assert!(
7926 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7927 .await
7928 .expect("counterpart scan")
7929 );
7930 }
7931
7932 #[tokio::test]
7933 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
7934 let aggregator = MobKitConsoleAggregator::in_memory();
7935 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
7936 aggregator
7937 .store()
7938 .append_if_absent(NewConsoleFrame {
7939 id: None,
7940 dedupe_key: format!("live-delta-{idx}"),
7941 timestamp_ms: 2_000 + idx as u64,
7942 runtime_key: "runtime-a".to_string(),
7943 identity: "agent-a".to_string(),
7944 conversation_id: Some("agent-a".to_string()),
7945 session_id: Some("session-a".to_string()),
7946 kind: "text_delta".to_string(),
7947 status: ConsoleFrameStatus::Delivered,
7948 payload: json!({ "delta": delta }),
7949 source: ConsoleFrameSource {
7950 kind: ConsoleFrameSourceKind::ConsoleEvent,
7951 source_cursor: None,
7952 },
7953 source_event_id: Some(format!("live-delta-{idx}")),
7954 interaction_id: Some("turn-a".to_string()),
7955 turn_id: None,
7956 run_id: None,
7957 parent_frame_id: None,
7958 caused_by_frame_id: None,
7959 })
7960 .await
7961 .expect("append live delta");
7962 }
7963
7964 let history = NewConsoleFrame {
7965 id: None,
7966 dedupe_key: "history-assistant-complete".to_string(),
7967 timestamp_ms: 3_000,
7968 runtime_key: "runtime-a".to_string(),
7969 identity: "agent-a".to_string(),
7970 conversation_id: Some("agent-a".to_string()),
7971 session_id: Some("session-a".to_string()),
7972 kind: "interaction_complete".to_string(),
7973 status: ConsoleFrameStatus::Completed,
7974 payload: json!({ "result": "Ready and standing by." }),
7975 source: ConsoleFrameSource {
7976 kind: ConsoleFrameSourceKind::SessionHistory,
7977 source_cursor: Some("session-a:3".to_string()),
7978 },
7979 source_event_id: None,
7980 interaction_id: None,
7981 turn_id: None,
7982 run_id: None,
7983 parent_frame_id: None,
7984 caused_by_frame_id: None,
7985 };
7986
7987 assert!(
7988 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7989 .await
7990 .expect("counterpart scan")
7991 );
7992 }
7993
7994 #[test]
7995 fn session_history_watermark_key_is_session_scoped() {
7996 assert_ne!(
7997 session_history_watermark_runtime_key("runtime-a", "session-1"),
7998 session_history_watermark_runtime_key("runtime-a", "session-2")
7999 );
8000 }
8001
8002 #[test]
8003 fn session_history_watermarks_are_cursor_and_ttl_aware() {
8004 let legacy = "session:with:colon:42";
8005 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8006 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8007
8008 assert_eq!(
8009 parse_session_history_watermark(legacy, "session:with:colon"),
8010 Some(42)
8011 );
8012 assert_eq!(
8013 parse_session_history_watermark(&checked, "session:with:colon"),
8014 Some(43)
8015 );
8016 assert!(session_history_watermark_is_fresh(
8017 &checked,
8018 "session:with:colon",
8019 1_500
8020 ));
8021 assert!(!session_history_watermark_is_fresh(
8022 &checked,
8023 "session:with:colon",
8024 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8025 ));
8026 assert!(session_history_watermark_is_fresh(
8027 &empty_checked,
8028 "session:with:colon",
8029 1_500
8030 ));
8031 assert!(!session_history_watermark_is_fresh(
8032 &empty_checked,
8033 "session:with:colon",
8034 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8035 ));
8036 }
8037
8038 #[test]
8039 fn session_history_messages_project_to_renderable_frames() {
8040 let user = frame_from_session_history_message(
8041 "runtime-a",
8042 "agent-a",
8043 "session-a",
8044 0,
8045 json!({
8046 "role": "user",
8047 "content": "hello",
8048 "timestamp_ms": 10
8049 }),
8050 )
8051 .expect("user history frame");
8052 let assistant = frame_from_session_history_message(
8053 "runtime-a",
8054 "agent-a",
8055 "session-a",
8056 1,
8057 json!({
8058 "role": "assistant",
8059 "content": "hi there",
8060 "stop_reason": "end_turn",
8061 "usage": { "input_tokens": 1, "output_tokens": 1, "total_tokens": 2 },
8062 "timestamp_ms": 11
8063 }),
8064 )
8065 .expect("assistant history frame");
8066
8067 assert_eq!(user.kind, "user_input");
8068 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8069 assert_eq!(
8070 user.payload["content"],
8071 json!([{ "type": "text", "text": "hello" }])
8072 );
8073 assert_eq!(assistant.kind, "interaction_complete");
8074 assert_eq!(assistant.payload["text"], json!("hi there"));
8075 assert!(
8076 assistant
8077 .dedupe_key
8078 .starts_with("session-history:runtime-a:session-a:1:")
8079 );
8080 }
8081
8082 #[test]
8083 fn session_history_projection_filters_scaffold_user_messages() {
8084 let spawn_notice = frame_from_session_history_message(
8085 "runtime-a",
8086 "agent-a",
8087 "session-a",
8088 0,
8089 json!({
8090 "role": "user",
8091 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8092 "timestamp_ms": 10
8093 }),
8094 );
8095 let peer_update = frame_from_session_history_message(
8096 "runtime-a",
8097 "agent-a",
8098 "session-a",
8099 1,
8100 json!({
8101 "role": "user",
8102 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8103 "timestamp_ms": 11
8104 }),
8105 );
8106 let real_user = frame_from_session_history_message(
8107 "runtime-a",
8108 "agent-a",
8109 "session-a",
8110 2,
8111 json!({
8112 "role": "user",
8113 "content": "Please review the incident notes.",
8114 "timestamp_ms": 12
8115 }),
8116 );
8117
8118 assert!(spawn_notice.is_none());
8119 assert!(peer_update.is_none());
8120 assert!(real_user.is_some());
8121 }
8122
8123 #[test]
8124 fn session_history_projection_skips_non_transcript_messages() {
8125 let skipped = frame_from_session_history_message(
8126 "runtime-a",
8127 "agent-a",
8128 "session-a",
8129 0,
8130 json!({
8131 "content": "internal system prompt"
8132 }),
8133 );
8134 assert!(skipped.is_none());
8135 }
8136
8137 #[test]
8138 fn session_history_projection_extracts_assistant_blocks() {
8139 let frame = frame_from_session_history_message(
8140 "runtime-a",
8141 "agent-a",
8142 "session-a",
8143 0,
8144 json!({
8145 "role": "block_assistant",
8146 "blocks": [
8147 { "block_type": "text", "data": { "text": "hello " } },
8148 { "block_type": "text", "data": { "text": "there" } }
8149 ],
8150 "stop_reason": "end_turn"
8151 }),
8152 )
8153 .expect("assistant block history frame");
8154 assert_eq!(frame.payload["text"], json!("hello there"));
8155 }
8156
8157 #[test]
8158 fn session_history_projection_extracts_nested_text_block_data() {
8159 let frame = frame_from_session_history_message(
8160 "runtime-a",
8161 "agent-a",
8162 "session-a",
8163 0,
8164 json!({
8165 "role": "block_assistant",
8166 "blocks": [
8167 {
8168 "block_type": "text",
8169 "data": { "text": "Ready and standing by." }
8170 }
8171 ],
8172 "stop_reason": "end_turn",
8173 "created_at": "1970-01-01T00:00:00.010Z"
8174 }),
8175 )
8176 .expect("assistant block history frame");
8177
8178 assert_eq!(frame.kind, "interaction_complete");
8179 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
8180 }
8181
8182 #[test]
8183 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
8184 let frame = frame_from_session_history_message(
8185 "runtime-a",
8186 "agent-a",
8187 "session-a",
8188 0,
8189 json!({
8190 "role": "block_assistant",
8191 "blocks": [
8192 {
8193 "block_type": "reasoning",
8194 "data": { "text": "**Planning**\nI should not be rendered." }
8195 },
8196 {
8197 "block_type": "text",
8198 "data": { "text": "Visible answer." }
8199 }
8200 ],
8201 "stop_reason": "end_turn",
8202 "created_at": "1970-01-01T00:00:00.010Z"
8203 }),
8204 )
8205 .expect("assistant block history frame");
8206
8207 assert_eq!(frame.kind, "interaction_complete");
8208 assert_eq!(frame.payload["result"], json!("Visible answer."));
8209 assert_eq!(frame.payload["text"], json!("Visible answer."));
8210 }
8211
8212 #[test]
8213 fn session_history_projection_leaves_reasoning_only_result_empty() {
8214 let frame = frame_from_session_history_message(
8215 "runtime-a",
8216 "agent-a",
8217 "session-a",
8218 0,
8219 json!({
8220 "role": "block_assistant",
8221 "blocks": [
8222 {
8223 "block_type": "reasoning",
8224 "data": { "text": "Private planning text." }
8225 },
8226 {
8227 "block_type": "tool_use",
8228 "data": { "id": "toolu-1", "name": "peers", "args": {} }
8229 }
8230 ],
8231 "stop_reason": "end_turn",
8232 "created_at": "1970-01-01T00:00:00.010Z"
8233 }),
8234 )
8235 .expect("assistant block history frame");
8236
8237 assert_eq!(frame.kind, "interaction_complete");
8238 assert_eq!(frame.payload["result"], json!(""));
8239 assert_eq!(frame.payload["text"], json!(""));
8240 }
8241
8242 #[test]
8243 fn session_history_projection_preserves_tool_results() {
8244 let frames = frames_from_session_history_message(
8245 "runtime-a",
8246 "agent-a",
8247 "session-a",
8248 5,
8249 json!({
8250 "role": "tool_results",
8251 "results": [
8252 {
8253 "tool_use_id": "call-peers",
8254 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
8255 "is_error": false
8256 }
8257 ],
8258 "created_at": "1970-01-01T00:00:00.050Z"
8259 }),
8260 );
8261
8262 assert_eq!(frames.len(), 1);
8263 let frame = &frames[0];
8264 assert_eq!(frame.kind, "tool_execution_completed");
8265 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
8266 assert_eq!(
8267 frame.payload["result"],
8268 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
8269 );
8270 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
8271 assert_eq!(frame.timestamp_ms, 50);
8272 }
8273
8274 #[test]
8275 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
8276 let frames = frames_from_session_history_message(
8277 "runtime-a",
8278 "review:singleton",
8279 "session-a",
8280 7,
8281 json!({
8282 "role": "block_assistant",
8283 "blocks": [
8284 {
8285 "block_type": "tool_use",
8286 "data": {
8287 "id": "call-spawn",
8288 "name": "mob_spawn_member",
8289 "args": {
8290 "member_id": "review-worker-alpha",
8291 "profile": "review-worker",
8292 "initial_message": "Review Initiative Alpha and save the result."
8293 }
8294 }
8295 }
8296 ],
8297 "stop_reason": "tool_use",
8298 "created_at": "1970-01-01T00:00:00.070Z"
8299 }),
8300 );
8301
8302 assert_eq!(frames.len(), 2);
8303 let child = frames
8304 .iter()
8305 .find(|frame| frame.identity == "review-worker-alpha")
8306 .expect("spawned member initial message frame");
8307 assert_eq!(child.kind, "user_input");
8308 assert_eq!(child.timestamp_ms, 70);
8309 assert_eq!(
8310 child.payload["message"]["content"],
8311 "Review Initiative Alpha and save the result."
8312 );
8313 assert_eq!(child.payload["parent_identity"], "review:singleton");
8314 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
8315 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
8316 }
8317
8318 #[test]
8319 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
8320 let frames = frames_from_session_history_message(
8321 "runtime-a",
8322 "review:singleton",
8323 "session-a",
8324 8,
8325 json!({
8326 "role": "block_assistant",
8327 "blocks": [
8328 {
8329 "block_type": "tool_use",
8330 "data": {
8331 "id": "call-spawn-many",
8332 "name": "mob_spawn_member",
8333 "args": {
8334 "specs": [
8335 {
8336 "agent_identity": "review-worker-alpha",
8337 "profile": "review-worker",
8338 "initial_message": "Review Initiative Alpha."
8339 },
8340 {
8341 "agent_identity": "review-worker-beta",
8342 "profile": "review-worker",
8343 "initial_message": "Review Initiative Beta."
8344 }
8345 ]
8346 }
8347 }
8348 }
8349 ],
8350 "stop_reason": "tool_use",
8351 "created_at": "1970-01-01T00:00:00.080Z"
8352 }),
8353 );
8354
8355 assert_eq!(frames.len(), 3);
8356 let child_messages = frames
8357 .iter()
8358 .filter(|frame| frame.kind == "user_input")
8359 .map(|frame| {
8360 (
8361 frame.identity.as_str(),
8362 frame.payload["message"]["content"]
8363 .as_str()
8364 .unwrap_or_default(),
8365 )
8366 })
8367 .collect::<Vec<_>>();
8368 assert_eq!(
8369 child_messages,
8370 vec![
8371 ("review-worker-alpha", "Review Initiative Alpha."),
8372 ("review-worker-beta", "Review Initiative Beta."),
8373 ]
8374 );
8375 }
8376
8377 #[tokio::test]
8378 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
8379 let aggregator = MobKitConsoleAggregator::in_memory();
8380 let frames = frames_from_session_history_message(
8381 "runtime-a",
8382 "review:singleton",
8383 "session-a",
8384 9,
8385 json!({
8386 "role": "block_assistant",
8387 "blocks": [
8388 {
8389 "block_type": "tool_use",
8390 "data": {
8391 "id": "call-spawn-many",
8392 "name": "mob_spawn_member",
8393 "args": {
8394 "specs": [
8395 {
8396 "agent_identity": "review-worker-alpha",
8397 "profile": "review-worker",
8398 "initial_message": "Review Initiative Alpha."
8399 },
8400 {
8401 "agent_identity": "review-worker-beta",
8402 "profile": "review-worker",
8403 "initial_message": "Review Initiative Beta."
8404 }
8405 ]
8406 }
8407 }
8408 }
8409 ],
8410 "stop_reason": "tool_use",
8411 "created_at": "1970-01-01T00:00:00.090Z"
8412 }),
8413 );
8414 for frame in frames {
8415 aggregator
8416 .store()
8417 .append_if_absent(frame)
8418 .await
8419 .expect("append projected session-history frame");
8420 }
8421
8422 let child_page = aggregator
8423 .query_timeline(ConsoleTimelineQuery {
8424 identity: Some("review-worker-alpha".to_string()),
8425 limit: 20,
8426 ..ConsoleTimelineQuery::default()
8427 })
8428 .await
8429 .expect("query child timeline");
8430 assert_eq!(child_page.frames.len(), 1);
8431 assert_eq!(child_page.frames[0].kind, "user_input");
8432 assert_eq!(
8433 child_page.frames[0].payload["message"]["content"],
8434 "Review Initiative Alpha."
8435 );
8436 assert_eq!(
8437 child_page.frames[0].payload["parent_identity"],
8438 "review:singleton"
8439 );
8440
8441 let parent_page = aggregator
8442 .query_timeline(ConsoleTimelineQuery {
8443 identity: Some("review:singleton".to_string()),
8444 limit: 20,
8445 ..ConsoleTimelineQuery::default()
8446 })
8447 .await
8448 .expect("query parent timeline");
8449 assert!(
8450 parent_page
8451 .frames
8452 .iter()
8453 .all(|frame| frame.identity == "review:singleton"),
8454 "child initial messages must not leak into parent timeline: {:#?}",
8455 parent_page.frames
8456 );
8457 }
8458
8459 #[test]
8460 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
8461 let frames = frames_from_session_history_message_with_namespace(
8462 "runtime-a",
8463 "test/review:singleton",
8464 "test",
8465 "session-a",
8466 10,
8467 json!({
8468 "role": "block_assistant",
8469 "blocks": [
8470 {
8471 "block_type": "tool_use",
8472 "data": {
8473 "id": "call-spawn-many",
8474 "name": "mob_spawn_member",
8475 "args": {
8476 "specs": [
8477 {
8478 "agent_identity": "review-worker-alpha",
8479 "profile": "review-worker",
8480 "initial_message": "Review Initiative Alpha."
8481 },
8482 {
8483 "agent_identity": "test/review-worker-beta",
8484 "profile": "review-worker",
8485 "initial_message": "Review Initiative Beta."
8486 }
8487 ]
8488 }
8489 }
8490 }
8491 ],
8492 "stop_reason": "tool_use",
8493 "created_at": "1970-01-01T00:00:00.100Z"
8494 }),
8495 );
8496
8497 let child_identities = frames
8498 .iter()
8499 .filter(|frame| frame.kind == "user_input")
8500 .map(|frame| frame.identity.as_str())
8501 .collect::<Vec<_>>();
8502 assert_eq!(
8503 child_identities,
8504 vec!["test/review-worker-alpha", "test/review-worker-beta"]
8505 );
8506 }
8507
8508 #[tokio::test]
8509 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
8510 let aggregator = MobKitConsoleAggregator::in_memory();
8511 aggregator
8512 .inner
8513 .identity_read_model
8514 .replace(vec![
8515 identity_record_for_test("test/review:singleton"),
8516 identity_record_for_test("test/review-worker-alpha"),
8517 ])
8518 .await;
8519 let frames = frames_from_session_history_message_with_namespace(
8520 "runtime-a",
8521 "test/review:singleton",
8522 "test",
8523 "session-a",
8524 11,
8525 json!({
8526 "role": "block_assistant",
8527 "blocks": [
8528 {
8529 "block_type": "tool_use",
8530 "data": {
8531 "id": "call-spawn",
8532 "name": "mob_spawn_member",
8533 "args": {
8534 "agent_identity": "review-worker-alpha",
8535 "profile": "review-worker",
8536 "initial_message": "Review Initiative Alpha."
8537 }
8538 }
8539 }
8540 ],
8541 "stop_reason": "tool_use",
8542 "created_at": "1970-01-01T00:00:00.110Z"
8543 }),
8544 );
8545 for frame in frames {
8546 aggregator
8547 .store()
8548 .append_if_absent(frame)
8549 .await
8550 .expect("append projected namespaced session-history frame");
8551 }
8552
8553 let namespaced_child_page = aggregator
8554 .query_timeline(ConsoleTimelineQuery {
8555 identity: Some("test/review-worker-alpha".to_string()),
8556 limit: 20,
8557 ..ConsoleTimelineQuery::default()
8558 })
8559 .await
8560 .expect("query namespaced child timeline");
8561 assert_eq!(namespaced_child_page.frames.len(), 1);
8562 assert_eq!(
8563 namespaced_child_page.frames[0].identity,
8564 "test/review-worker-alpha"
8565 );
8566 assert_eq!(
8567 namespaced_child_page.frames[0].payload["message"]["content"],
8568 "Review Initiative Alpha."
8569 );
8570
8571 let raw_child_page = aggregator
8572 .query_timeline(ConsoleTimelineQuery {
8573 identity: Some("review-worker-alpha".to_string()),
8574 limit: 20,
8575 ..ConsoleTimelineQuery::default()
8576 })
8577 .await
8578 .expect("query raw child timeline");
8579 assert!(
8580 raw_child_page.frames.is_empty(),
8581 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
8582 raw_child_page.frames
8583 );
8584 }
8585
8586 #[test]
8587 fn namespace_helpers_preserve_namespace_named_member_identity() {
8588 assert_eq!(apply_namespace("test", "test"), "test/test");
8589 assert_eq!(
8590 strip_namespace("test/test", "test").as_deref(),
8591 Some("test")
8592 );
8593 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
8594 assert_eq!(
8595 strip_namespace("test/worker", "test").as_deref(),
8596 Some("worker")
8597 );
8598 assert_eq!(strip_namespace("test", "test"), None);
8599 }
8600
8601 #[test]
8602 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
8603 assert!(!member_id_matches_durable_identity(
8604 "rt:review:singleton:0",
8605 "review:singleton"
8606 ));
8607 assert!(!member_id_matches_durable_identity(
8608 "review:singleton:gen1",
8609 "review:singleton"
8610 ));
8611 assert!(!member_id_matches_durable_identity(
8612 "review:singleton:1",
8613 "review:singleton"
8614 ));
8615 assert!(!member_id_matches_durable_identity(
8616 "rt:review:singleton:qa:0",
8617 "review:singleton"
8618 ));
8619 assert!(!member_id_matches_durable_identity(
8620 "review:singleton:qa",
8621 "review:singleton"
8622 ));
8623 }
8624
8625 #[test]
8626 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
8627 let frame = frame_from_session_history_message(
8628 "runtime-a",
8629 "agent-a",
8630 "session-a",
8631 0,
8632 json!({
8633 "role": "user",
8634 "content": "hello",
8635 "created_at": "2026-05-12T05:00:06.564227Z"
8636 }),
8637 )
8638 .expect("user history frame");
8639
8640 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
8641 }
8642}