1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::MeerkatId;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images,
24 is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32 SourceIngestionTransition,
33};
34pub use store::{
35 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36 SqliteConsoleLogStore,
37};
38pub use types::{
39 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59 CachedOnly,
60 IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65 inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70 pub session_history_backfill_enabled: bool,
71 pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75 fn default() -> Self {
76 Self {
77 session_history_backfill_enabled: true,
78 max_concurrent_session_backfills: 16,
79 }
80 }
81}
82
83struct AggregatorInner {
84 store: Arc<dyn ConsoleLogStore>,
85 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89 session_backfill_permits: Arc<Semaphore>,
90 identity_read_model: ConsoleIdentityReadModel,
91 options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97 refresh_lock: Arc<tokio::sync::Mutex<()>>,
98 primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102 fn default() -> Self {
103 Self {
104 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106 primed: Arc::new(AtomicBool::new(false)),
107 }
108 }
109}
110
111impl ConsoleIdentityReadModel {
112 async fn snapshot(
113 &self,
114 inner: Arc<AggregatorInner>,
115 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116 if !self.primed.load(Ordering::Acquire) {
117 self.prime_now(inner).await?;
118 }
119 Ok(self.inner.read().await.clone())
120 }
121
122 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123 self.inner.read().await.clone()
124 }
125
126 async fn refresh_now_if_idle(
127 &self,
128 inner: Arc<AggregatorInner>,
129 mode: IdentityCollectionMode,
130 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132 return None;
133 };
134 let result = collect_identity_records(&inner, mode).await;
135 drop(guard);
136 match result {
137 Ok(identities) => {
138 self.replace(identities.clone()).await;
139 Some(Ok(identities))
140 }
141 Err(err) => Some(Err(err)),
142 }
143 }
144
145 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146 if self.primed.load(Ordering::Acquire) {
147 return Ok(());
148 }
149 let _guard = self.refresh_lock.clone().lock_owned().await;
150 if self.primed.load(Ordering::Acquire) {
151 return Ok(());
152 }
153 let identities =
154 collect_identity_records(&inner, IdentityCollectionMode::CachedOnly).await?;
155 *self.inner.write().await = identities;
156 self.primed.store(true, Ordering::Release);
157 Ok(())
158 }
159
160 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
161 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
162 return;
163 };
164 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
165 return;
166 };
167 let read_model = self.clone();
168 runtime_handle.spawn(async move {
169 let _guard = guard;
170 match collect_identity_records(&inner, IdentityCollectionMode::IncludeLiveMembers).await
171 {
172 Ok(identities) => {
173 *read_model.inner.write().await = identities;
174 read_model.primed.store(true, Ordering::Release);
175 }
176 Err(err) => {
177 tracing::warn!(error = %err, "console identity read-model refresh failed");
178 }
179 }
180 });
181 }
182
183 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
184 *self.inner.write().await = identities;
185 self.primed.store(true, Ordering::Release);
186 }
187}
188
189#[derive(Clone)]
190struct RuntimeEntry {
191 runtime_key: String,
192 identity_namespace: String,
193 runtime: MobRuntime,
194 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
195 console_events: ConsoleEventStore,
196 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
197}
198
199#[derive(Clone)]
200struct ResolvedConsoleMember {
201 entry: RuntimeEntry,
202 handle: MobHandle,
203 member: MobMemberListEntry,
204 source_mob_id: String,
205 runtime_identity: String,
206}
207
208fn console_member_for_resolved_member(
209 resolved: &ResolvedConsoleMember,
210 record: &ConsoleIdentityRecord,
211) -> ConsoleMember {
212 ConsoleMember {
213 agent_identity: resolved.member.agent_identity.to_string(),
214 role: resolved.member.role.to_string(),
215 state: format!("{:?}", resolved.member.state),
216 model_capabilities: Default::default(),
217 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
218 session_id: record.session_id.clone(),
219 wired_to: resolved
220 .member
221 .wired_to
222 .iter()
223 .map(ToString::to_string)
224 .collect(),
225 labels: record.labels.clone(),
226 }
227}
228
229async fn resolved_member_visible(
230 resolved: &ResolvedConsoleMember,
231 record: &ConsoleIdentityRecord,
232) -> bool {
233 if !raw_resolved_member_visible(resolved, record) {
234 return false;
235 }
236 !live_record_shadowed_by_hidden_durable_binding(resolved, record).await
237}
238
239fn raw_resolved_member_visible(
240 resolved: &ResolvedConsoleMember,
241 record: &ConsoleIdentityRecord,
242) -> bool {
243 resolved.entry.visibility_policy.identity_visible(record)
244 && resolved
245 .entry
246 .visibility_policy
247 .member_visible(&console_member_for_resolved_member(resolved, record))
248}
249
250pub trait ConsoleVisibilityPolicy: Send + Sync {
251 fn include_implicit_delegate_members(&self) -> bool {
252 true
253 }
254
255 fn member_visible(&self, _member: &ConsoleMember) -> bool {
256 true
257 }
258
259 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
260 true
261 }
262
263 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
264 true
265 }
266
267 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
268 None
269 }
270}
271
272#[derive(Debug, Default)]
273pub struct AllowAllConsoleVisibilityPolicy;
274
275impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
276
277#[derive(Debug, Default)]
278pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
279
280impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
281 fn include_implicit_delegate_members(&self) -> bool {
282 false
283 }
284
285 fn member_visible(&self, member: &ConsoleMember) -> bool {
286 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
287 }
288
289 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
290 !is_implicit_delegate_member(
291 record
292 .labels
293 .get("role")
294 .map(String::as_str)
295 .unwrap_or_default(),
296 &record.labels,
297 )
298 }
299}
300
301#[derive(Clone)]
302pub struct ConsoleRuntimeRegistration {
303 pub runtime_key: String,
304 pub runtime: Arc<UnifiedRuntime>,
305 pub identity_namespace: String,
306 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
307}
308
309impl MobKitConsoleAggregator {
310 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
311 Self::new_with_options(store, ConsoleAggregatorOptions::default())
312 }
313
314 pub fn new_with_options(
315 store: Arc<dyn ConsoleLogStore>,
316 mut options: ConsoleAggregatorOptions,
317 ) -> Self {
318 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
319 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
320 Self {
321 inner: Arc::new(AggregatorInner {
322 store,
323 runtimes: RwLock::new(BTreeMap::new()),
324 event_tx,
325 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
326 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
327 session_backfill_permits: Arc::new(Semaphore::new(
328 options.max_concurrent_session_backfills,
329 )),
330 identity_read_model: ConsoleIdentityReadModel::default(),
331 options,
332 }),
333 }
334 }
335
336 pub fn in_memory() -> Self {
337 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
338 }
339
340 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
341 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
342 }
343
344 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
345 self.inner.event_tx.subscribe()
346 }
347
348 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
349 self.inner.store.clone()
350 }
351
352 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
353 let identity_runtime = registration.runtime.identity_runtime().cloned();
354 self.register_runtime_handles_with_policy(
355 registration.runtime_key,
356 registration.identity_namespace,
357 registration.runtime.mob_runtime().clone(),
358 identity_runtime,
359 registration.runtime.console_events(),
360 registration.visibility_policy,
361 );
362 }
363
364 pub(crate) fn register_runtime_handles_with_policy(
365 &self,
366 runtime_key: impl Into<String>,
367 identity_namespace: impl Into<String>,
368 runtime: MobRuntime,
369 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
370 console_events: ConsoleEventStore,
371 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
372 ) {
373 let runtime_key = runtime_key.into();
374 let identity_namespace = identity_namespace.into();
375 let entry = RuntimeEntry {
376 runtime_key: runtime_key.clone(),
377 identity_namespace,
378 runtime,
379 identity_runtime,
380 console_events: console_events.clone(),
381 visibility_policy,
382 };
383 if let Ok(mut runtimes) = self.inner.runtimes.write() {
384 runtimes.insert(runtime_key.clone(), entry);
385 }
386 self.inner
387 .identity_read_model
388 .refresh_soon(self.inner.clone());
389 let inner = self.inner.clone();
390 let events_for_live = console_events.clone();
391 let events_for_live_recovery = console_events.clone();
392 let runtime_key_for_live = runtime_key.clone();
393 tokio::spawn(async move {
394 let mut rx = events_for_live.subscribe();
395 loop {
396 match rx.recv().await {
397 Ok(envelope) => {
398 let _ =
399 project_console_event(inner.clone(), &runtime_key_for_live, envelope)
400 .await;
401 }
402 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
403 let _ = recover_lagged_source_events(
404 inner.clone(),
405 &runtime_key_for_live,
406 &events_for_live_recovery,
407 )
408 .await;
409 }
410 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
411 }
412 }
413 });
414
415 let inner = self.inner.clone();
416 let events_for_replay = console_events;
417 let runtime_key_for_replay = runtime_key;
418 tokio::spawn(async move {
419 let mut ingestion_state = SourceIngestionState::Registered;
420 if let Ok((next, _effects)) =
421 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
422 {
423 ingestion_state = next;
424 }
425 if let Ok(events) = events_for_replay.replay_all(None).await {
426 for envelope in events {
427 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
428 .await;
429 }
430 }
431 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
432 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
433 if let Ok((next, _effects)) =
434 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
435 {
436 ingestion_state = next;
437 }
438 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
439 });
440 }
441
442 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
443 if let Some(identities) = self
444 .inner
445 .identity_read_model
446 .refresh_now_if_idle(
447 self.inner.clone(),
448 IdentityCollectionMode::IncludeLiveMembers,
449 )
450 .await
451 {
452 let identities = identities?;
453 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
454 return Ok(identities);
455 }
456 self.inner
457 .identity_read_model
458 .refresh_soon(self.inner.clone());
459 let identities = self
460 .inner
461 .identity_read_model
462 .snapshot(self.inner.clone())
463 .await?;
464 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
465 Ok(identities)
466 }
467
468 #[cfg(test)]
469 pub(crate) async fn list_identities_fresh(
470 &self,
471 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
472 let _guard = self
473 .inner
474 .identity_read_model
475 .refresh_lock
476 .clone()
477 .lock_owned()
478 .await;
479 let identities =
480 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
481 .await?;
482 self.inner
483 .identity_read_model
484 .replace(identities.clone())
485 .await;
486 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
487 Ok(identities)
488 }
489
490 pub async fn inspect_identity(
491 &self,
492 identity: &str,
493 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
494 let entries = self
495 .inner
496 .runtimes
497 .read()
498 .map_err(|_| runtime_registry_lock_error())?
499 .values()
500 .cloned()
501 .collect::<Vec<_>>();
502 let live_records = self.live_records_for_identity(identity).await;
503 let mut durable_matches = Vec::new();
504 if !requested_identity_has_runtime_member_alias(identity, &entries) {
505 for entry in &entries {
506 let Some(identity_runtime) = entry.identity_runtime.clone() else {
507 continue;
508 };
509 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
510 {
511 let Ok(parsed_identity) =
512 crate::identity_first::AgentIdentity::parse(&raw_identity)
513 else {
514 continue;
515 };
516 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
517 continue;
518 };
519 let mut record = identity_record_for_status(entry, &status);
520 let topology_peers =
521 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
522 record.topology_peers = topology_peers
523 .get(&record.identity)
524 .cloned()
525 .unwrap_or_default();
526 if console_identity_record_visible(entry, &record).await {
527 durable_matches.push((entry.clone(), record));
528 }
529 }
530 }
531 }
532 if durable_matches.len() > 1 {
533 let candidates = durable_matches
534 .iter()
535 .map(|(_, record)| record.runtime_member_id.clone())
536 .collect::<Vec<_>>()
537 .join(", ");
538 return Err(format!(
539 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
540 )
541 .into());
542 }
543 if let Some((entry, record)) = durable_matches.into_iter().next() {
544 let durable_live_records = self
545 .live_records_for_durable_record(&entry, identity, &record, &live_records)
546 .await;
547 let visible_durable_live_records =
548 visible_live_records_for_entry(&entry, &durable_live_records).await;
549 if let Some(ambiguous_error) =
550 ambiguous_live_alias_error(identity, &visible_durable_live_records)
551 {
552 return Err(ambiguous_error.into());
553 }
554 if let Some(stale_error) =
555 stale_durable_record_error(identity, &record, &durable_live_records)
556 {
557 return Err(stale_error.into());
558 }
559 if let Some(session_id) = record.session_id.clone() {
560 spawn_session_history_backfill_target(
561 self.inner.clone(),
562 SessionBackfillTarget {
563 entry,
564 record: record.clone(),
565 session_id,
566 },
567 false,
568 );
569 }
570 return Ok(Some(ConsoleIdentityInspection {
571 peers: record.topology_peers.clone(),
572 identity: record,
573 }));
574 }
575 let mut runtime_id_matches = Vec::new();
576 for entry in &entries {
577 let Some(identity_runtime) = entry.identity_runtime.clone() else {
578 continue;
579 };
580 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
581 continue;
582 };
583 let Some(status) = identity_runtime
584 .statuses()
585 .await
586 .into_iter()
587 .find(|status| {
588 status
589 .agent_runtime_id
590 .as_ref()
591 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
592 })
593 else {
594 continue;
595 };
596 let mut record = identity_record_for_status(entry, &status);
597 let topology_peers =
598 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
599 record.topology_peers = topology_peers
600 .get(&record.identity)
601 .cloned()
602 .unwrap_or_default();
603 if console_identity_record_visible(entry, &record).await {
604 runtime_id_matches.push((entry.clone(), record));
605 }
606 }
607 if runtime_id_matches.len() > 1 {
608 let candidates = runtime_id_matches
609 .iter()
610 .map(|(_, record)| record.identity.clone())
611 .collect::<Vec<_>>()
612 .join(", ");
613 return Err(format!(
614 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
615 )
616 .into());
617 }
618 if let Some((entry, record)) = runtime_id_matches.into_iter().next() {
619 let durable_live_records = self
620 .live_records_for_durable_record(&entry, identity, &record, &live_records)
621 .await;
622 let visible_durable_live_records =
623 visible_live_records_for_entry(&entry, &durable_live_records).await;
624 if let Some(ambiguous_error) =
625 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
626 {
627 return Err(ambiguous_error.into());
628 }
629 if let Some(stale_error) =
630 stale_durable_record_error(identity, &record, &durable_live_records)
631 {
632 return Err(stale_error.into());
633 }
634 if let Some(session_id) = record.session_id.clone() {
635 spawn_session_history_backfill_target(
636 self.inner.clone(),
637 SessionBackfillTarget {
638 entry: entry.clone(),
639 record: record.clone(),
640 session_id,
641 },
642 false,
643 );
644 }
645 return Ok(Some(ConsoleIdentityInspection {
646 peers: record.topology_peers.clone(),
647 identity: record,
648 }));
649 }
650
651 let live_matches = self.resolve_visible_members(identity).await;
652 if live_matches.len() > 1 {
653 let candidates = live_matches
654 .iter()
655 .map(|resolved| resolved.runtime_identity.clone())
656 .collect::<Vec<_>>()
657 .join(", ");
658 return Err(format!(
659 "ambiguous live identity alias {identity}: candidates [{candidates}]"
660 )
661 .into());
662 }
663 if let Some(resolved) = live_matches.into_iter().next() {
664 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
665 return Ok(None);
666 };
667 if let Some(stale_error) =
668 stale_live_record_binding_error(&resolved.entry, identity, &record).await
669 {
670 return Err(stale_error.into());
671 }
672 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
673 && let Ok(parsed_identity) =
674 crate::identity_first::AgentIdentity::parse(&record.identity)
675 && let Ok(status) = identity_runtime.status(&parsed_identity).await
676 {
677 let durable_record = identity_record_for_status(&resolved.entry, &status);
678 let durable_live_records = self
679 .live_records_for_durable_record(
680 &resolved.entry,
681 identity,
682 &durable_record,
683 &live_records,
684 )
685 .await;
686 let visible_durable_live_records =
687 visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
688 if let Some(ambiguous_error) =
689 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
690 {
691 return Err(ambiguous_error.into());
692 }
693 if let Some(stale_error) =
694 stale_durable_record_error(identity, &durable_record, &durable_live_records)
695 {
696 return Err(stale_error.into());
697 }
698 if status
699 .agent_runtime_id
700 .as_ref()
701 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
702 {
703 return Ok(None);
704 }
705 }
706 if !resolved_member_visible(&resolved, &record).await {
707 return Ok(None);
708 }
709 if let Some(session_id) = record.session_id.clone() {
710 spawn_session_history_backfill_target(
711 self.inner.clone(),
712 SessionBackfillTarget {
713 entry: resolved.entry.clone(),
714 record: record.clone(),
715 session_id,
716 },
717 false,
718 );
719 }
720 let peers = resolved
721 .member
722 .wired_to
723 .iter()
724 .map(ToString::to_string)
725 .collect();
726 return Ok(Some(ConsoleIdentityInspection {
727 identity: record,
728 peers,
729 }));
730 }
731
732 Ok(None)
733 }
734
735 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
736 let entries = self
737 .inner
738 .runtimes
739 .read()
740 .map_err(|_| runtime_registry_lock_error())?
741 .clone();
742 let live_records = self.live_records_for_identity(identity).await;
743 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
744 let mut durable_matches = Vec::new();
745 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
746 for entry in entries.values() {
747 let Some(identity_runtime) = entry.identity_runtime.clone() else {
748 continue;
749 };
750 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
751 {
752 let Ok(parsed_identity) =
753 crate::identity_first::AgentIdentity::parse(&raw_identity)
754 else {
755 continue;
756 };
757 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
758 continue;
759 };
760 let record = identity_record_for_status(entry, &status);
761 if !console_identity_record_visible(entry, &record).await {
762 continue;
763 }
764 durable_matches.push((
765 entry.clone(),
766 identity_runtime.clone(),
767 parsed_identity,
768 record,
769 ));
770 }
771 }
772 }
773 if durable_matches.len() > 1 {
774 let candidates = durable_matches
775 .iter()
776 .map(|(_, _, identity, _)| identity.as_str().to_string())
777 .collect::<Vec<_>>()
778 .join(", ");
779 return Err(format!(
780 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
781 )
782 .into());
783 }
784 if let Some((entry, identity_runtime, parsed_identity, record)) =
785 durable_matches.into_iter().next()
786 {
787 let durable_live_records = self
788 .live_records_for_durable_record(&entry, identity, &record, &live_records)
789 .await;
790 let visible_durable_live_records =
791 visible_live_records_for_entry(&entry, &durable_live_records).await;
792 if let Some(ambiguous_error) =
793 ambiguous_live_alias_error(identity, &visible_durable_live_records)
794 {
795 return Err(ambiguous_error.into());
796 }
797 if let Some(stale_error) =
798 stale_durable_record_error(identity, &record, &durable_live_records)
799 {
800 return Err(stale_error.into());
801 }
802 identity_runtime
803 .retire(&parsed_identity)
804 .await
805 .map_err(|err| -> ConsoleLogError {
806 format!("retire failed for {identity}: {err}").into()
807 })?;
808 {
809 let mut durable_identities =
810 namespace_match_candidates(identity, &entry.identity_namespace);
811 durable_identities.push(parsed_identity.as_str().to_string());
812 durable_identities.sort();
813 durable_identities.dedup();
814 if let Err(err) =
815 retire_stale_console_members_for_runtime_entry(&entry, &durable_identities)
816 .await
817 {
818 tracing::warn!(
819 identity,
820 error = %err,
821 "stale console member cleanup failed after durable identity retire"
822 );
823 }
824 }
825 return Ok(true);
826 }
827 let matches = self.resolve_visible_members(identity).await;
828 if matches.len() > 1 {
829 let candidates = matches
830 .iter()
831 .map(|resolved| resolved.runtime_identity.clone())
832 .collect::<Vec<_>>()
833 .join(", ");
834 return Err(format!(
835 "ambiguous live identity alias {identity}: candidates [{candidates}]"
836 )
837 .into());
838 }
839 let mut live_retired_any = false;
840 for resolved in matches {
841 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
842 continue;
843 };
844 if !resolved_member_visible(&resolved, &record).await {
845 continue;
846 }
847 if let Some(stale_error) =
848 stale_live_record_binding_error(&resolved.entry, identity, &record).await
849 {
850 return Err(stale_error.into());
851 }
852 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
853 && let Ok(parsed_identity) =
854 crate::identity_first::AgentIdentity::parse(&record.identity)
855 && let Ok(status) = identity_runtime.status(&parsed_identity).await
856 {
857 let durable_record = identity_record_for_status(&resolved.entry, &status);
858 let durable_live_records = self
859 .live_records_for_durable_record(
860 &resolved.entry,
861 identity,
862 &durable_record,
863 &live_records,
864 )
865 .await;
866 let visible_durable_live_records =
867 visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
868 if let Some(ambiguous_error) =
869 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
870 {
871 return Err(ambiguous_error.into());
872 }
873 if let Some(stale_error) =
874 stale_durable_record_error(identity, &durable_record, &durable_live_records)
875 {
876 return Err(stale_error.into());
877 }
878 if status
879 .agent_runtime_id
880 .as_ref()
881 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
882 {
883 continue;
884 }
885 }
886 resolved
887 .handle
888 .retire(MeerkatId::from(resolved.runtime_identity.as_str()))
889 .await
890 .map_err(|err| -> ConsoleLogError {
891 format!("retire failed for {identity}: {err}").into()
892 })
893 .or_else(|err| {
894 if lifecycle_archive_cleanup_completed(&err.to_string()) {
895 Ok(())
896 } else {
897 Err(err)
898 }
899 })?;
900 live_retired_any = true;
901 }
902 Ok(live_retired_any)
903 }
904
905 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
906 self.inner.store.clear_frames().await
907 }
908
909 pub async fn query_timeline(
910 &self,
911 query: ConsoleTimelineQuery,
912 ) -> ConsoleLogResult<ConsoleTimelinePage> {
913 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
914 Ok(ConsoleTimelinePage {
915 frames: page.frames,
916 next_cursor: page.next_cursor,
917 })
918 }
919
920 pub async fn query_timeline_windowed(
921 &self,
922 query: ConsoleTimelineWindowQuery,
923 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
924 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
925 .await?;
926 if query.after.is_none()
927 && query.before.is_none()
928 && let Some(identity) = query.identity.clone()
929 {
930 let mut probe_query = query.clone();
931 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
932 let page = self.inner.store.query_windowed_frames(probe_query).await?;
933 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
934 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
935 }
936 }
937 Box::pin(self.query_timeline_visible(query)).await
938 }
939
940 async fn reject_after_cursor_beyond_store_frontier(
941 &self,
942 after: Option<&ConsoleCursor>,
943 ) -> ConsoleLogResult<()> {
944 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
945 return Ok(());
946 };
947 let latest_seq = self
948 .inner
949 .store
950 .latest_cursor()
951 .await?
952 .and_then(|cursor| cursor.seq())
953 .unwrap_or(0);
954 if after_seq > latest_seq {
955 return Err(std::io::Error::other(
956 "timeline replay cursor is beyond the current store frontier",
957 )
958 .into());
959 }
960 Ok(())
961 }
962
963 async fn query_timeline_visible(
964 &self,
965 query: ConsoleTimelineWindowQuery,
966 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
967 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
968 let mut scan_query = query.clone();
969 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
970 let mut visible_frames = Vec::with_capacity(requested_limit);
971 let mut anchor_frames = Vec::new();
972 let mut identity_visibility_cache = HashMap::new();
973 let identity_records = self.inner.identity_read_model.current().await;
974 let mut next_cursor = query.after.clone();
975 let mut since_last_scanned_cursor = query.after.clone();
976 let mut since_last_delivered_cursor = None;
977 let mut since_stopped_at_visible_limit = false;
978 let mut latest_cursor = None;
979 let mut exhausted = false;
980 let mut scanned = 0usize;
981
982 loop {
983 let page = self
984 .inner
985 .store
986 .query_windowed_frames(scan_query.clone())
987 .await?;
988 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
989 if page.frames.is_empty() {
990 exhausted = true;
991 break;
992 }
993 let raw_len = page.frames.len();
994 scanned = scanned.saturating_add(raw_len);
995 match query.mode {
996 ConsoleTimelineMode::Since => {
997 if let Some(cursor) = page.next_cursor.clone() {
998 since_last_scanned_cursor = Some(cursor.clone());
999 scan_query.after = Some(cursor);
1000 }
1001 }
1002 ConsoleTimelineMode::Recent => {
1003 if let Some(first) = page.frames.first() {
1004 scan_query.before = Some(first.cursor.clone());
1005 }
1006 next_cursor = page.next_cursor.clone().or(next_cursor);
1007 }
1008 }
1009
1010 for frame in page.frames {
1011 let allow_historical_identity =
1012 query.identity.as_deref() == Some(frame.identity.as_str());
1013 if frame_is_visible_cached(
1014 &self.inner,
1015 &frame,
1016 allow_historical_identity,
1017 &mut identity_visibility_cache,
1018 &identity_records,
1019 )
1020 .await
1021 .unwrap_or(false)
1022 {
1023 if query.mode == ConsoleTimelineMode::Recent
1024 && query.identity.is_some()
1025 && is_identity_timeline_anchor_frame(&frame)
1026 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1027 {
1028 anchor_frames.push(frame.clone());
1029 }
1030 match query.mode {
1031 ConsoleTimelineMode::Since => {
1032 if visible_frames.len() >= requested_limit {
1033 since_stopped_at_visible_limit = true;
1034 break;
1035 }
1036 since_last_delivered_cursor = Some(frame.cursor.clone());
1037 visible_frames.push(frame);
1038 if visible_frames.len() >= requested_limit {
1039 since_stopped_at_visible_limit = true;
1040 exhausted = false;
1041 break;
1042 }
1043 }
1044 ConsoleTimelineMode::Recent => {
1045 visible_frames.push(frame);
1046 }
1047 }
1048 }
1049 }
1050 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1051 && query.identity.is_some()
1052 && anchor_frames.is_empty()
1053 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1054 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1055 break;
1056 }
1057 if since_stopped_at_visible_limit {
1058 break;
1059 }
1060 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1061 exhausted = true;
1062 break;
1063 }
1064 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1065 break;
1066 }
1067 }
1068
1069 if query.mode == ConsoleTimelineMode::Recent {
1070 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1071 if visible_frames.len() > requested_limit {
1072 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1073 }
1074 if !anchor_frames.is_empty() {
1075 let anchor_cursors = anchor_frames
1076 .iter()
1077 .map(|frame| frame.cursor.clone())
1078 .collect::<Vec<_>>();
1079 let mut merged = anchor_frames;
1080 for frame in visible_frames {
1081 if !merged
1082 .iter()
1083 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1084 {
1085 merged.push(frame);
1086 }
1087 }
1088 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1089 visible_frames = merged;
1090 if visible_frames.len() > requested_limit {
1091 let mut anchors = Vec::new();
1092 let mut tail = Vec::new();
1093 for frame in visible_frames {
1094 if anchor_cursors.contains(&frame.cursor) {
1095 anchors.push(frame);
1096 } else {
1097 tail.push(frame);
1098 }
1099 }
1100 visible_frames = if anchors.len() >= requested_limit {
1101 anchors.split_off(anchors.len() - requested_limit)
1102 } else {
1103 let keep_tail = requested_limit.saturating_sub(anchors.len());
1104 if tail.len() > keep_tail {
1105 tail = tail.split_off(tail.len() - keep_tail);
1106 }
1107 anchors.extend(tail);
1108 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1109 anchors
1110 };
1111 }
1112 }
1113 }
1114
1115 Ok(ConsoleTimelineWindowPage {
1116 frames: visible_frames,
1117 next_cursor: match query.mode {
1118 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1119 since_last_delivered_cursor.or(since_last_scanned_cursor)
1120 }
1121 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1122 ConsoleTimelineMode::Recent => next_cursor,
1123 },
1124 latest_cursor,
1125 exhausted,
1126 })
1127 }
1128
1129 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1130 let runtime_keys = self
1131 .inner
1132 .runtimes
1133 .read()
1134 .map_err(|_| runtime_registry_lock_error())?
1135 .keys()
1136 .cloned()
1137 .collect::<Vec<_>>();
1138 let results =
1139 join_all(runtime_keys.into_iter().map(|runtime_key| {
1140 backfill_session_history(self.inner.clone(), runtime_key, true)
1141 }))
1142 .await;
1143 for result in results {
1144 result?;
1145 }
1146 Ok(())
1147 }
1148
1149 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1150 self.inner.store.latest_cursor().await
1151 }
1152
1153 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1154 match event {
1155 ConsoleTimelineEvent::ConsoleFrame { frame }
1156 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1157 let identity_records = self.inner.identity_read_model.current().await;
1158 frame_is_visible(&self.inner, frame, false, &identity_records)
1159 .await
1160 .unwrap_or(false)
1161 }
1162 ConsoleTimelineEvent::SnapshotStarted { .. }
1163 | ConsoleTimelineEvent::SnapshotComplete { .. }
1164 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1165 }
1166 }
1167
1168 pub async fn timeline_frame_visible_for_query(
1169 &self,
1170 frame: &ConsoleFrame,
1171 identity: Option<&str>,
1172 ) -> bool {
1173 let allow_historical_identity = identity == Some(frame.identity.as_str());
1174 let identity_records = self.inner.identity_read_model.current().await;
1175 frame_is_visible(
1176 &self.inner,
1177 frame,
1178 allow_historical_identity,
1179 &identity_records,
1180 )
1181 .await
1182 .unwrap_or(false)
1183 }
1184
1185 pub async fn send(
1186 &self,
1187 request: ConsoleSendRequest,
1188 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1189 validate_send_request(&request)?;
1190 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1191 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1192 };
1193 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1194 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1195 };
1196 if !resolved_member_visible(&resolved, &record).await {
1197 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1198 }
1199 if !member_is_addressable(&resolved.member) {
1200 return Err(ConsoleSendError::NotAddressable(request.identity));
1201 }
1202 if resolved.member.state == meerkat_mob::MemberState::Retiring {
1203 return Err(ConsoleSendError::Retired(request.identity));
1204 }
1205
1206 let content = content_input_from_value(&request.content)?;
1207 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1208 assert_member_accepts_images(
1209 &resolved.handle,
1210 resolved.entry.runtime.session_service(),
1211 &resolved.runtime_identity,
1212 &content,
1213 )
1214 .await
1215 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1216
1217 let dedupe_key = send_dedupe_key(
1218 &resolved.entry.runtime_key,
1219 &request.identity,
1220 &request.origin,
1221 &request.idempotency_key,
1222 );
1223 let handling_mode_value = request
1224 .handling_mode
1225 .as_deref()
1226 .unwrap_or("queue")
1227 .to_string();
1228 let request_fingerprint =
1229 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1230 if let Some(existing) = self
1231 .inner
1232 .store
1233 .frame_by_dedupe_key(&dedupe_key)
1234 .await
1235 .map_err(ConsoleSendError::Log)?
1236 {
1237 let same_request = existing.source.source_cursor.as_deref()
1238 == Some(request_fingerprint.as_str())
1239 || existing.source.source_cursor.is_none()
1240 && existing.payload.get("origin").and_then(Value::as_str)
1241 == Some(request.origin.as_str())
1242 && existing.payload.get("content") == Some(&request.content)
1243 && existing
1244 .payload
1245 .get("handling_mode")
1246 .and_then(Value::as_str)
1247 == Some(handling_mode_value.as_str());
1248 if !same_request {
1249 return Err(ConsoleSendError::IdempotencyConflict(
1250 request.idempotency_key,
1251 ));
1252 }
1253 return Ok(accepted_from_frame(&existing));
1254 }
1255
1256 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1257 resolved
1258 .entry
1259 .console_events
1260 .reserve_interaction_value(
1261 &resolved.runtime_identity,
1262 Some(resolved.runtime_identity.as_str()),
1263 &interaction_id,
1264 &request.origin,
1265 request.content.clone(),
1266 )
1267 .await
1268 .map_err(ConsoleSendError::State)?;
1269 let session_id = resolved
1270 .handle
1271 .resolve_bridge_session_id_observation(&MeerkatId::from(
1272 resolved.runtime_identity.as_str(),
1273 ))
1274 .await
1275 .map(|sid| sid.to_string());
1276 let mut new_frame = NewConsoleFrame {
1277 id: None,
1278 dedupe_key,
1279 timestamp_ms: current_time_ms(),
1280 runtime_key: resolved.entry.runtime_key.clone(),
1281 identity: request.identity.clone(),
1282 conversation_id: Some(request.identity.clone()),
1283 session_id: session_id.clone(),
1284 kind: "user_input".to_string(),
1285 status: ConsoleFrameStatus::Accepted,
1286 payload: json!({
1287 "content": request.content,
1288 "origin": request.origin,
1289 "idempotency_key": request.idempotency_key,
1290 "handling_mode": handling_mode_value,
1291 }),
1292 source: ConsoleFrameSource {
1293 kind: ConsoleFrameSourceKind::Send,
1294 source_cursor: Some(request_fingerprint),
1295 },
1296 source_event_id: None,
1297 interaction_id: Some(interaction_id.clone()),
1298 turn_id: None,
1299 run_id: None,
1300 parent_frame_id: None,
1301 caused_by_frame_id: None,
1302 };
1303 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1304 new_frame.payload = redacted;
1305 new_frame.status = ConsoleFrameStatus::Redacted;
1306 }
1307
1308 let _ = SendState::Requested
1309 .apply(SendTransition::PersistAccepted)
1310 .map_err(ConsoleSendError::State)?;
1311 let outcome = self
1312 .inner
1313 .store
1314 .append_if_absent(new_frame)
1315 .await
1316 .map_err(ConsoleSendError::Log)?;
1317 if outcome.disposition == AppendDisposition::Existing {
1318 return Ok(accepted_from_frame(&outcome.frame));
1319 }
1320 let _ = self
1321 .inner
1322 .event_tx
1323 .send(ConsoleTimelineEvent::ConsoleFrame {
1324 frame: outcome.frame.clone(),
1325 });
1326 let accepted = accepted_from_frame(&outcome.frame);
1327
1328 let (dispatching, _effects) = SendState::AcceptedPersisted
1329 .apply(SendTransition::StartDispatch)
1330 .map_err(ConsoleSendError::State)?;
1331 update_frame_status_and_emit(
1332 &self.inner,
1333 &outcome.frame.id,
1334 ConsoleFrameStatus::Dispatching,
1335 )
1336 .await
1337 .map_err(ConsoleSendError::Log)?;
1338
1339 spawn_console_send_dispatch(
1340 self.inner.clone(),
1341 resolved,
1342 content,
1343 handling_mode,
1344 dispatching,
1345 outcome.frame,
1346 interaction_id,
1347 );
1348 Ok(accepted)
1349 }
1350
1351 pub async fn reserve_identity_first_interaction(
1352 &self,
1353 request: ConsoleSendRequest,
1354 session_id: Option<&str>,
1355 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1356 validate_send_request(&request)?;
1357 let _content = content_input_from_value(&request.content)?;
1358 let runtime_key =
1359 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1360 let handling_mode_value = request
1361 .handling_mode
1362 .as_deref()
1363 .unwrap_or("queue")
1364 .to_string();
1365 let dedupe_key = send_dedupe_key(
1366 &runtime_key,
1367 &request.identity,
1368 &request.origin,
1369 &request.idempotency_key,
1370 );
1371 let request_fingerprint =
1372 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1373 if let Some(existing) = self
1374 .inner
1375 .store
1376 .frame_by_dedupe_key(&dedupe_key)
1377 .await
1378 .map_err(ConsoleSendError::Log)?
1379 {
1380 let same_request = existing.source.source_cursor.as_deref()
1381 == Some(request_fingerprint.as_str())
1382 || existing.source.source_cursor.is_none()
1383 && existing.payload.get("origin").and_then(Value::as_str)
1384 == Some(request.origin.as_str())
1385 && existing.payload.get("content") == Some(&request.content)
1386 && existing
1387 .payload
1388 .get("handling_mode")
1389 .and_then(Value::as_str)
1390 == Some(handling_mode_value.as_str());
1391 if !same_request {
1392 return Err(ConsoleSendError::IdempotencyConflict(
1393 request.idempotency_key,
1394 ));
1395 }
1396 return Ok(accepted_from_frame(&existing));
1397 }
1398
1399 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1400 let new_frame = NewConsoleFrame {
1401 id: None,
1402 dedupe_key,
1403 timestamp_ms: current_time_ms(),
1404 runtime_key,
1405 identity: request.identity.clone(),
1406 conversation_id: Some(request.identity.clone()),
1407 session_id: session_id.map(ToString::to_string),
1408 kind: "user_input".to_string(),
1409 status: ConsoleFrameStatus::Accepted,
1410 payload: json!({
1411 "content": request.content,
1412 "origin": request.origin,
1413 "idempotency_key": request.idempotency_key,
1414 "handling_mode": handling_mode_value,
1415 }),
1416 source: ConsoleFrameSource {
1417 kind: ConsoleFrameSourceKind::Send,
1418 source_cursor: Some(request_fingerprint),
1419 },
1420 source_event_id: None,
1421 interaction_id: Some(interaction_id),
1422 turn_id: None,
1423 run_id: None,
1424 parent_frame_id: None,
1425 caused_by_frame_id: None,
1426 };
1427 let outcome = self
1428 .inner
1429 .store
1430 .append_if_absent(new_frame)
1431 .await
1432 .map_err(ConsoleSendError::Log)?;
1433 let _ = self
1434 .inner
1435 .event_tx
1436 .send(ConsoleTimelineEvent::ConsoleFrame {
1437 frame: outcome.frame.clone(),
1438 });
1439 Ok(accepted_from_frame(&outcome.frame))
1440 }
1441
1442 async fn runtime_key_for_identity_first_send(
1443 &self,
1444 identity: &str,
1445 ) -> Result<String, ConsoleSendError> {
1446 let entries = self
1447 .inner
1448 .runtimes
1449 .read()
1450 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1451 .clone();
1452 if entries.is_empty() {
1453 return Ok("identity-first".to_string());
1454 }
1455
1456 let mut identity_runtime_keys = Vec::new();
1457 let mut hidden_durable_match = false;
1458 let mut durable_matches = Vec::new();
1459 for entry in entries.values() {
1460 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1461 continue;
1462 };
1463 identity_runtime_keys.push(entry.runtime_key.clone());
1464 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1465 continue;
1466 }
1467 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1468 {
1469 let Ok(parsed_identity) =
1470 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1471 else {
1472 continue;
1473 };
1474 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1475 let record = identity_record_for_status(entry, &status);
1476 if !console_identity_record_visible(entry, &record).await {
1477 hidden_durable_match = true;
1478 continue;
1479 }
1480 durable_matches.push((entry.clone(), record));
1481 }
1482 }
1483 }
1484 if durable_matches.len() > 1 {
1485 let candidates = durable_matches
1486 .iter()
1487 .map(|(entry, record)| format!("{}@{}", record.identity, entry.runtime_key))
1488 .collect::<Vec<_>>()
1489 .join(", ");
1490 return Err(ConsoleSendError::InvalidRequest(format!(
1491 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1492 )));
1493 }
1494 if let Some((entry, record)) = durable_matches.into_iter().next() {
1495 let live_records = self.live_records_for_identity(identity).await;
1496 let durable_live_records = self
1497 .live_records_for_durable_record(&entry, identity, &record, &live_records)
1498 .await;
1499 let visible_durable_live_records =
1500 visible_live_records_for_entry(&entry, &durable_live_records).await;
1501 if let Some(ambiguous_error) =
1502 ambiguous_live_alias_error(identity, &visible_durable_live_records)
1503 {
1504 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1505 }
1506 if let Some(stale_error) =
1507 stale_durable_record_error(identity, &record, &durable_live_records)
1508 {
1509 return Err(ConsoleSendError::InvalidRequest(stale_error));
1510 }
1511 return Ok(entry.runtime_key);
1512 }
1513 if entries.values().any(|entry| {
1514 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1515 }) {
1516 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1517 }
1518 if hidden_durable_match {
1519 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1520 }
1521
1522 match identity_runtime_keys.as_slice() {
1523 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1524 [runtime_key] => Ok(runtime_key.clone()),
1525 _ => Err(ConsoleSendError::InvalidRequest(format!(
1526 "identity-first send for '{identity}' did not match exactly one registered runtime"
1527 ))),
1528 }
1529 }
1530
1531 pub async fn mark_interaction_delivery_failed(
1532 &self,
1533 input_frame_id: &str,
1534 ) -> Result<(), ConsoleSendError> {
1535 update_frame_status_and_emit(
1536 &self.inner,
1537 input_frame_id,
1538 ConsoleFrameStatus::DeliveryFailed,
1539 )
1540 .await
1541 .map_err(ConsoleSendError::Log)?;
1542 Ok(())
1543 }
1544
1545 pub async fn mark_interaction_delivered(
1546 &self,
1547 input_frame_id: &str,
1548 ) -> Result<(), ConsoleSendError> {
1549 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1550 .await
1551 .map_err(ConsoleSendError::Log)?;
1552 Ok(())
1553 }
1554
1555 pub async fn mark_steer_interaction_delivered(
1556 &self,
1557 input_frame_id: &str,
1558 interaction_id: &str,
1559 ) -> Result<(), ConsoleSendError> {
1560 let Some(updated) = update_frame_status_and_emit(
1561 &self.inner,
1562 input_frame_id,
1563 ConsoleFrameStatus::Delivered,
1564 )
1565 .await
1566 .map_err(ConsoleSendError::Log)?
1567 else {
1568 return Ok(());
1569 };
1570 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1571 .await
1572 .map_err(ConsoleSendError::Log)?;
1573 Ok(())
1574 }
1575
1576 pub async fn binary_blob_store_for_identity(
1577 &self,
1578 identity: &str,
1579 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1580 if identity.trim().is_empty() {
1581 return Err(ConsoleSendError::InvalidRequest(
1582 "identity must be non-empty".to_string(),
1583 ));
1584 }
1585 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1586 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1587 };
1588 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1589 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1590 };
1591 if !resolved_member_visible(&resolved, &record).await {
1592 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1593 }
1594 if !member_is_addressable(&resolved.member) {
1595 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1596 }
1597 if resolved.member.state == meerkat_mob::MemberState::Retiring {
1598 return Err(ConsoleSendError::Retired(identity.to_string()));
1599 }
1600 Ok(resolved.entry.runtime.binary_blob_store())
1601 }
1602
1603 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1604 self.inner
1605 .runtimes
1606 .read()
1607 .map(|entries| {
1608 entries
1609 .values()
1610 .filter_map(|entry| entry.runtime.binary_blob_store())
1611 .collect()
1612 })
1613 .unwrap_or_default()
1614 }
1615
1616 async fn resolve_send_member(
1617 &self,
1618 identity: &str,
1619 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1620 let matches = self.resolve_visible_members(identity).await;
1621 if matches.len() > 1 {
1622 let candidates = matches
1623 .iter()
1624 .map(|resolved| resolved.runtime_identity.clone())
1625 .collect::<Vec<_>>()
1626 .join(", ");
1627 return Err(ConsoleSendError::AmbiguousIdentity {
1628 identity: identity.to_string(),
1629 candidates,
1630 });
1631 }
1632 let Some(resolved) = matches.into_iter().next() else {
1633 if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1634 return Err(ConsoleSendError::InvalidRequest(stale_error));
1635 }
1636 return Ok(None);
1637 };
1638 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1639 && let Some(stale_error) =
1640 stale_live_record_binding_error(&resolved.entry, identity, &record).await
1641 {
1642 return Err(ConsoleSendError::InvalidRequest(stale_error));
1643 }
1644 if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1645 return Err(ConsoleSendError::InvalidRequest(stale_error));
1646 }
1647 Ok(Some(resolved))
1648 }
1649
1650 async fn durable_send_binding_error(
1651 &self,
1652 identity: &str,
1653 ) -> Result<Option<String>, ConsoleSendError> {
1654 let entries = self
1655 .inner
1656 .runtimes
1657 .read()
1658 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1659 .clone();
1660 if entries.values().any(|entry| {
1661 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1662 }) {
1663 return Ok(None);
1664 }
1665 let live_records = self.live_records_for_identity(identity).await;
1666 let mut durable_matches = Vec::new();
1667 for entry in entries.values() {
1668 let Some(identity_runtime) = entry.identity_runtime.clone() else {
1669 continue;
1670 };
1671 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1672 let Ok(parsed_identity) =
1673 crate::identity_first::AgentIdentity::parse(&raw_identity)
1674 else {
1675 continue;
1676 };
1677 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1678 continue;
1679 };
1680 let record = identity_record_for_status(entry, &status);
1681 if !console_identity_record_visible(entry, &record).await {
1682 continue;
1683 }
1684 durable_matches.push((entry.clone(), record));
1685 }
1686 }
1687 if durable_matches.len() > 1 {
1688 let candidates = durable_matches
1689 .iter()
1690 .map(|(_, record)| record.runtime_member_id.clone())
1691 .collect::<Vec<_>>()
1692 .join(", ");
1693 return Err(ConsoleSendError::AmbiguousIdentity {
1694 identity: identity.to_string(),
1695 candidates,
1696 });
1697 }
1698 let Some((entry, durable_record)) = durable_matches.into_iter().next() else {
1699 return Ok(None);
1700 };
1701 let durable_live_records = self
1702 .live_records_for_durable_record(&entry, identity, &durable_record, &live_records)
1703 .await;
1704 let visible_durable_live_records =
1705 visible_live_records_for_entry(&entry, &durable_live_records).await;
1706 if let Some(ambiguous_error) =
1707 ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1708 {
1709 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1710 }
1711 Ok(stale_durable_record_error(
1712 identity,
1713 &durable_record,
1714 &durable_live_records,
1715 ))
1716 }
1717
1718 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1719 let entries = self
1720 .inner
1721 .runtimes
1722 .read()
1723 .ok()
1724 .map(|entries| entries.clone())
1725 .unwrap_or_default();
1726 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1727 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1728 for entry in entries.values() {
1729 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1730 if raw_identities.is_empty() {
1731 continue;
1732 }
1733 let mids = raw_identities
1734 .iter()
1735 .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
1736 .collect::<Vec<_>>();
1737 for resolved in member_sources_for_entry(entry).await {
1738 let session_id = resolved
1739 .handle
1740 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1741 .await
1742 .map(|sid| sid.to_string())
1743 .unwrap_or_default();
1744 if mids.contains(&resolved.member.agent_identity) {
1745 exact_matches.push((session_id, resolved));
1746 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1747 {
1748 label_matches.push((session_id, resolved));
1749 }
1750 }
1751 }
1752 let mut matches = exact_matches;
1753 matches.extend(label_matches);
1754 let mut seen_members = BTreeSet::new();
1755 matches.retain(|(session_id, resolved)| {
1756 seen_members.insert((
1757 resolved.entry.runtime_key.clone(),
1758 resolved.source_mob_id.clone(),
1759 session_id.clone(),
1760 resolved.runtime_identity.clone(),
1761 ))
1762 });
1763 matches.sort_by(|left, right| right.0.cmp(&left.0));
1764 matches.into_iter().map(|(_, resolved)| resolved).collect()
1765 }
1766
1767 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1768 let mut visible = Vec::new();
1769 for resolved in self.resolve_members(identity).await {
1770 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1771 continue;
1772 };
1773 if resolved_member_visible(&resolved, &record).await {
1774 visible.push(resolved);
1775 }
1776 }
1777 visible
1778 }
1779
1780 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1781 let mut records = Vec::new();
1782 for resolved in self.resolve_members(identity).await {
1783 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1784 continue;
1785 };
1786 if resolved_member_visible(&resolved, &record).await {
1787 records.push(record);
1788 }
1789 }
1790 records
1791 }
1792
1793 async fn live_records_for_durable_record(
1794 &self,
1795 entry: &RuntimeEntry,
1796 requested_identity: &str,
1797 durable: &ConsoleIdentityRecord,
1798 requested_live_records: &[ConsoleIdentityRecord],
1799 ) -> Vec<ConsoleIdentityRecord> {
1800 let mut records = if durable.identity == requested_identity {
1801 requested_live_records.to_vec()
1802 } else {
1803 self.live_records_for_identity(&durable.identity).await
1804 };
1805 for runtime_record in
1806 live_records_for_runtime_member(entry, &durable.runtime_member_id).await
1807 {
1808 if records.iter().any(|record| {
1809 record.runtime_key == runtime_record.runtime_key
1810 && record.runtime_member_id == runtime_record.runtime_member_id
1811 && record.session_id == runtime_record.session_id
1812 && record.identity == runtime_record.identity
1813 }) {
1814 continue;
1815 }
1816 records.push(runtime_record);
1817 }
1818 records
1819 }
1820}
1821
1822async fn console_identity_record_visible(
1823 entry: &RuntimeEntry,
1824 record: &ConsoleIdentityRecord,
1825) -> bool {
1826 if !entry.visibility_policy.identity_visible(record) {
1827 return false;
1828 }
1829 let mut bound_live_member_seen = false;
1830 let mut wrong_live_projection_seen = false;
1831 for resolved in member_sources_for_entry(entry).await {
1832 if resolved.runtime_identity != record.runtime_member_id {
1833 continue;
1834 }
1835 bound_live_member_seen = true;
1836 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
1837 continue;
1838 };
1839 if live_record.identity != record.identity {
1840 wrong_live_projection_seen = true;
1841 continue;
1842 }
1843 if raw_resolved_member_visible(&resolved, &live_record) {
1844 return true;
1845 }
1846 }
1847 if wrong_live_projection_seen {
1848 return true;
1849 }
1850 !bound_live_member_seen
1851}
1852
1853async fn live_record_shadowed_by_hidden_durable_binding(
1854 resolved: &ResolvedConsoleMember,
1855 record: &ConsoleIdentityRecord,
1856) -> bool {
1857 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
1858 return false;
1859 };
1860 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
1861 return false;
1862 };
1863 let Ok(status) = identity_runtime.status(&identity).await else {
1864 return false;
1865 };
1866 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
1867 return false;
1868 };
1869 if bound_runtime_id.as_str() == resolved.runtime_identity {
1870 return false;
1871 }
1872 for candidate in member_sources_for_entry(&resolved.entry).await {
1873 if candidate.runtime_identity != bound_runtime_id.as_str() {
1874 continue;
1875 }
1876 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
1877 continue;
1878 };
1879 if bound_record.identity == record.identity
1880 && !raw_resolved_member_visible(&candidate, &bound_record)
1881 {
1882 return true;
1883 }
1884 }
1885 false
1886}
1887
1888async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
1889 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
1890 .unwrap_or_else(|| frame.identity.clone());
1891 let frame_session_id = frame.session_id.as_deref();
1892 let mut saw_hidden = false;
1893 let mut saw_visible = false;
1894 for resolved in member_sources_for_entry(entry).await {
1895 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1896 continue;
1897 };
1898 let session_matches =
1899 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
1900 let runtime_matches = record.runtime_member_id == runtime_member_id;
1901 let identity_matches = record.identity == frame.identity;
1902 if !session_matches && !runtime_matches && !identity_matches {
1903 continue;
1904 }
1905 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
1906 return true;
1907 }
1908 if resolved_member_visible(&resolved, &record).await {
1909 saw_visible = true;
1910 } else {
1911 saw_hidden = true;
1912 }
1913 }
1914 saw_hidden && !saw_visible
1915}
1916
1917async fn live_records_for_runtime_member(
1918 entry: &RuntimeEntry,
1919 runtime_member_id: &str,
1920) -> Vec<ConsoleIdentityRecord> {
1921 let mut records = Vec::new();
1922 for resolved in member_sources_for_entry(entry).await {
1923 if resolved.runtime_identity != runtime_member_id {
1924 continue;
1925 }
1926 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1927 continue;
1928 };
1929 records.push(record);
1930 }
1931 records
1932}
1933
1934async fn visible_live_records_for_entry(
1935 entry: &RuntimeEntry,
1936 records: &[ConsoleIdentityRecord],
1937) -> Vec<ConsoleIdentityRecord> {
1938 let mut visible = Vec::new();
1939 for record in records {
1940 if record.runtime_key != entry.runtime_key
1941 || console_identity_record_visible(entry, record).await
1942 {
1943 visible.push(record.clone());
1944 }
1945 }
1946 visible
1947}
1948
1949async fn stale_live_record_binding_error(
1950 entry: &RuntimeEntry,
1951 requested_identity: &str,
1952 live_record: &ConsoleIdentityRecord,
1953) -> Option<String> {
1954 let identity_runtime = entry.identity_runtime.as_ref()?;
1955 for status in identity_runtime.statuses().await {
1956 let matches_runtime = status
1957 .agent_runtime_id
1958 .as_ref()
1959 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
1960 if !matches_runtime {
1961 continue;
1962 }
1963 let durable_record = identity_record_for_status(entry, &status);
1964 if let Some(stale_error) = stale_durable_record_error(
1965 requested_identity,
1966 &durable_record,
1967 std::slice::from_ref(live_record),
1968 ) {
1969 return Some(stale_error);
1970 }
1971 }
1972 None
1973}
1974
1975fn stale_durable_record_error(
1976 requested_identity: &str,
1977 durable: &ConsoleIdentityRecord,
1978 live_records: &[ConsoleIdentityRecord],
1979) -> Option<String> {
1980 let matching_live = live_records
1981 .iter()
1982 .filter(|record| record.identity == durable.identity)
1983 .collect::<Vec<_>>();
1984 if let Some(wrong_projection) = live_records.iter().find(|record| {
1985 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
1986 }) {
1987 return Some(format!(
1988 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
1989 durable.identity, durable.runtime_member_id, wrong_projection.identity
1990 ));
1991 }
1992 if matching_live.is_empty() {
1993 return None;
1994 }
1995 if let Some(session_mismatch) = matching_live.iter().find(|record| {
1996 record.runtime_member_id == durable.runtime_member_id
1997 && durable.session_id.is_some()
1998 && record.session_id.is_some()
1999 && durable.session_id != record.session_id
2000 }) {
2001 return Some(format!(
2002 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2003 durable.identity,
2004 durable.runtime_member_id,
2005 durable.session_id.as_deref().unwrap_or("<none>"),
2006 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2007 ));
2008 }
2009 if matching_live
2010 .iter()
2011 .any(|record| record.runtime_member_id == durable.runtime_member_id)
2012 {
2013 return None;
2014 }
2015 let live_candidates = matching_live
2016 .iter()
2017 .map(|record| record.runtime_member_id.as_str())
2018 .collect::<Vec<_>>()
2019 .join(", ");
2020 Some(format!(
2021 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2022 durable.identity, durable.runtime_member_id, live_candidates
2023 ))
2024}
2025
2026fn ambiguous_live_alias_error(
2027 requested_identity: &str,
2028 live_records: &[ConsoleIdentityRecord],
2029) -> Option<String> {
2030 if live_records.len() <= 1 {
2031 return None;
2032 }
2033 let candidates = live_records
2034 .iter()
2035 .map(|record| {
2036 format!(
2037 "{}@{}",
2038 record.runtime_member_id,
2039 record
2040 .labels
2041 .get("source_mob_id")
2042 .map(String::as_str)
2043 .unwrap_or(record.runtime_key.as_str())
2044 )
2045 })
2046 .collect::<Vec<_>>()
2047 .join(", ");
2048 Some(format!(
2049 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2050 ))
2051}
2052
2053fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2054 member_id == durable_identity
2055}
2056
2057async fn retire_stale_console_members_for_identity(
2058 handle: &MobHandle,
2059 durable_identities: &[String],
2060 keep_runtime_member_id: Option<&str>,
2061) -> Result<(), String> {
2062 let stale_members = handle
2063 .list_members_including_retiring()
2064 .await
2065 .into_iter()
2066 .filter(|member| {
2067 (durable_identities.iter().any(|durable_identity| {
2068 member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2069 }) || member.labels.get("agent_identity").is_some_and(|identity| {
2070 durable_identities
2071 .iter()
2072 .any(|durable_identity| identity == durable_identity)
2073 })) && keep_runtime_member_id
2074 .map(|keep| member.agent_identity.as_str() != keep)
2075 .unwrap_or(true)
2076 })
2077 .map(|member| member.agent_identity)
2078 .collect::<Vec<_>>();
2079 for member_id in stale_members {
2080 match handle.retire(member_id).await {
2081 Ok(()) => {}
2082 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2083 Err(err) => return Err(err.to_string()),
2084 }
2085 }
2086 Ok(())
2087}
2088
2089async fn retire_stale_console_members_for_runtime_entry(
2090 entry: &RuntimeEntry,
2091 durable_identities: &[String],
2092) -> Result<(), String> {
2093 let primary_handle = entry.runtime.handle();
2094 let primary_mob_id = primary_handle.mob_id().to_string();
2095 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2096 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2097 for (mob_id, handle) in state.mob_handles_snapshot().await {
2098 if mob_id.as_str() != primary_mob_id {
2099 handles.push((mob_id.to_string(), handle));
2100 }
2101 }
2102 }
2103 let mut seen_handles = BTreeSet::new();
2104 for (mob_id, handle) in handles {
2105 if !seen_handles.insert(mob_id) {
2106 continue;
2107 }
2108 retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2109 }
2110 Ok(())
2111}
2112
2113fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2114 is_recoverable_lifecycle_cleanup_error(error)
2115}
2116
2117fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2118 if frames.is_empty() {
2119 return true;
2120 }
2121 let latest_session_history_timestamp_ms = frames
2122 .iter()
2123 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2124 .map(|frame| frame.timestamp_ms)
2125 .max();
2126 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2127 return current_time_ms().saturating_sub(latest_timestamp_ms)
2128 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2129 }
2130 frames.iter().any(|frame| {
2131 matches!(
2132 frame.kind.as_str(),
2133 "turn_started"
2134 | "run_started"
2135 | "reasoning_delta"
2136 | "reasoning_complete"
2137 | "tool_call_requested"
2138 | "tool_call"
2139 | "tool_execution_started"
2140 | "tool_execution_completed"
2141 | "tool_result_received"
2142 )
2143 })
2144}
2145
2146fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2147 match frame.kind.as_str() {
2148 "user_input" | "run_started" => true,
2149 "interaction_started" => frame
2150 .payload
2151 .get("content")
2152 .or_else(|| frame.payload.get("prompt"))
2153 .is_some(),
2154 _ => false,
2155 }
2156}
2157
2158fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2159 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2160 for record in records {
2161 by_identity
2162 .entry(identity_record_dedupe_key(&record))
2163 .and_modify(|current| {
2164 if identity_record_prefer(&record, current) {
2165 *current = record.clone();
2166 }
2167 })
2168 .or_insert(record);
2169 }
2170 by_identity.into_values().collect()
2171}
2172
2173fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2174 format!(
2175 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2176 record.identity,
2177 record.runtime_key,
2178 record
2179 .labels
2180 .get("source_mob_id")
2181 .map(String::as_str)
2182 .unwrap_or(""),
2183 record.runtime_member_id,
2184 record.session_id.as_deref().unwrap_or("")
2185 )
2186}
2187
2188fn identity_record_prefer(
2189 candidate: &ConsoleIdentityRecord,
2190 current: &ConsoleIdentityRecord,
2191) -> bool {
2192 let candidate_is_live_label_projection =
2193 candidate
2194 .labels
2195 .get("agent_identity")
2196 .is_some_and(|identity| {
2197 identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2198 || candidate.runtime_member_id != candidate.identity
2199 });
2200 let current_is_live_label_projection =
2201 current
2202 .labels
2203 .get("agent_identity")
2204 .is_some_and(|identity| {
2205 identity == &strip_namespace(¤t.identity, "").unwrap_or_default()
2206 || current.runtime_member_id != current.identity
2207 });
2208 if candidate_is_live_label_projection != current_is_live_label_projection {
2209 return candidate_is_live_label_projection;
2210 }
2211 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2212 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2213 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2214 return candidate_has_distinct_runtime_binding;
2215 }
2216 let candidate_live = candidate.addressable && candidate.health != "retired";
2217 let current_live = current.addressable && current.health != "retired";
2218 if candidate_live != current_live {
2219 return candidate_live;
2220 }
2221 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2222}
2223
2224async fn collect_identity_records(
2225 inner: &Arc<AggregatorInner>,
2226 mode: IdentityCollectionMode,
2227) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2228 let entries = inner
2229 .runtimes
2230 .read()
2231 .map_err(|_| runtime_registry_lock_error())?
2232 .clone();
2233 let mut identities = Vec::new();
2234 for entry in entries.values() {
2235 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2236 let topology_peers =
2237 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2238 for status in identity_runtime.statuses().await {
2239 let mut record = identity_record_for_status(entry, &status);
2240 record.topology_peers = topology_peers
2241 .get(&record.identity)
2242 .cloned()
2243 .unwrap_or_default();
2244 if console_identity_record_visible(entry, &record).await {
2245 identities.push(record);
2246 }
2247 }
2248 if mode == IdentityCollectionMode::CachedOnly {
2249 continue;
2250 }
2251 }
2252 let members = if entry.identity_runtime.is_some() {
2253 match tokio::time::timeout(
2254 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2255 member_sources_for_entry(entry),
2256 )
2257 .await
2258 {
2259 Ok(members) => members,
2260 Err(_) => {
2261 tracing::warn!(
2262 runtime_key = %entry.runtime_key,
2263 "identity-first live member refresh timed out; keeping cached identity read model"
2264 );
2265 Vec::new()
2266 }
2267 }
2268 } else {
2269 member_sources_for_entry(entry).await
2270 };
2271 for resolved in members {
2272 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2273 && resolved_member_visible(&resolved, &record).await
2274 {
2275 identities.push(record);
2276 }
2277 }
2278 }
2279 Ok(dedupe_identity_records(identities))
2280}
2281
2282async fn identity_runtime_topology_peers(
2283 entry: &RuntimeEntry,
2284 identity_runtime: &crate::identity_first::IdentityRuntime,
2285) -> BTreeMap<String, Vec<String>> {
2286 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2287 for edge in identity_runtime.desired_peer_edges().await {
2288 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2289 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2290 peers.entry(a.clone()).or_default().insert(b.clone());
2291 peers.entry(b).or_default().insert(a);
2292 }
2293 peers
2294 .into_iter()
2295 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2296 .collect()
2297}
2298
2299fn spawn_identity_backfills_for_records(
2300 inner: Arc<AggregatorInner>,
2301 records: &[ConsoleIdentityRecord],
2302) {
2303 if !inner.options.session_history_backfill_enabled {
2304 return;
2305 }
2306 let entries = match inner.runtimes.read() {
2307 Ok(entries) => entries.clone(),
2308 Err(_) => return,
2309 };
2310 for record in records {
2311 if record.health == "dormant" || record.health == "uninitialized" {
2312 continue;
2313 }
2314 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2315 continue;
2316 };
2317 let Some(session_id) = record.session_id.clone() else {
2318 continue;
2319 };
2320 spawn_session_history_backfill_target(
2321 inner.clone(),
2322 SessionBackfillTarget {
2323 entry,
2324 record: record.clone(),
2325 session_id,
2326 },
2327 false,
2328 );
2329 }
2330}
2331
2332async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2333 let mut resolved = Vec::new();
2334 let primary_handle = entry.runtime.handle();
2335 let primary_mob_id = primary_handle.mob_id().to_string();
2336 for member in primary_handle.list_members_observation_snapshot().await {
2337 resolved.push(ResolvedConsoleMember {
2338 entry: entry.clone(),
2339 handle: primary_handle.clone(),
2340 runtime_identity: member.agent_identity.to_string(),
2341 source_mob_id: primary_mob_id.clone(),
2342 member,
2343 });
2344 }
2345
2346 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2347 return resolved;
2348 };
2349 if !entry.visibility_policy.include_implicit_delegate_members() {
2350 return resolved;
2351 }
2352 for (mob_id, handle) in state.mob_handles_snapshot().await {
2353 if mob_id.as_str() == primary_mob_id {
2354 continue;
2355 }
2356 for member in handle.list_members_observation_snapshot().await {
2357 resolved.push(ResolvedConsoleMember {
2358 entry: entry.clone(),
2359 handle: handle.clone(),
2360 runtime_identity: member.agent_identity.to_string(),
2361 source_mob_id: mob_id.to_string(),
2362 member,
2363 });
2364 }
2365 }
2366 resolved
2367}
2368
2369async fn dispatch_message_to_resolved_member(
2370 resolved: &ResolvedConsoleMember,
2371 content: ContentInput,
2372 handling_mode: meerkat_core::types::HandlingMode,
2373) -> Result<String, String> {
2374 let mid = MeerkatId::from(resolved.runtime_identity.as_str());
2375 match send_message_on_mob_with_mode(
2376 &resolved.handle,
2377 &resolved.runtime_identity,
2378 content.clone(),
2379 handling_mode,
2380 )
2381 .await
2382 {
2383 Ok(session_id) => Ok(session_id),
2384 Err(err) if is_not_externally_addressable(&err) => {
2385 let member = resolved
2386 .handle
2387 .member(&mid)
2388 .await
2389 .map_err(|err| err.to_string())?;
2390 let _receipt = member
2391 .internal_turn(content)
2392 .await
2393 .map_err(|err| err.to_string())?;
2394 resolved
2395 .handle
2396 .resolve_bridge_session_id_observation(&mid)
2397 .await
2398 .map(|sid| sid.to_string())
2399 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2400 }
2401 Err(err) => Err(err.to_string()),
2402 }
2403}
2404
2405fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2406 matches!(
2407 err,
2408 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2409 )
2410}
2411
2412fn spawn_console_send_dispatch(
2413 inner: Arc<AggregatorInner>,
2414 resolved: ResolvedConsoleMember,
2415 content: ContentInput,
2416 handling_mode: meerkat_core::types::HandlingMode,
2417 dispatching: SendState,
2418 user_frame: ConsoleFrame,
2419 interaction_id: String,
2420) {
2421 tokio::spawn(async move {
2422 match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2423 Ok(_) => {
2424 let _ = dispatching.apply(SendTransition::MarkDelivered);
2425 if let Err(err) = update_frame_status_and_emit(
2426 &inner,
2427 &user_frame.id,
2428 ConsoleFrameStatus::Delivered,
2429 )
2430 .await
2431 {
2432 tracing::warn!(
2433 frame_id = %user_frame.id,
2434 error = %err,
2435 "failed to update console send delivery status"
2436 );
2437 }
2438 if handling_mode == HandlingMode::Steer
2439 && let Err(err) =
2440 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2441 {
2442 tracing::warn!(
2443 frame_id = %user_frame.id,
2444 interaction_id = %interaction_id,
2445 error = %err,
2446 "failed to append console steer terminal frame"
2447 );
2448 }
2449 }
2450 Err(err) => {
2451 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2452 if let Err(update_err) = update_frame_status_and_emit(
2453 &inner,
2454 &user_frame.id,
2455 ConsoleFrameStatus::DeliveryFailed,
2456 )
2457 .await
2458 {
2459 tracing::warn!(
2460 frame_id = %user_frame.id,
2461 error = %update_err,
2462 "failed to update console send failure status"
2463 );
2464 }
2465 let failure_frame = NewConsoleFrame {
2466 id: None,
2467 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2468 timestamp_ms: current_time_ms(),
2469 runtime_key: user_frame.runtime_key,
2470 identity: user_frame.identity,
2471 conversation_id: user_frame.conversation_id,
2472 session_id: user_frame.session_id,
2473 kind: "message_delivery_failed".to_string(),
2474 status: ConsoleFrameStatus::DeliveryFailed,
2475 payload: json!({ "reason": err }),
2476 source: ConsoleFrameSource {
2477 kind: ConsoleFrameSourceKind::Synthetic,
2478 source_cursor: None,
2479 },
2480 source_event_id: None,
2481 interaction_id: Some(interaction_id),
2482 turn_id: None,
2483 run_id: None,
2484 parent_frame_id: Some(user_frame.id.clone()),
2485 caused_by_frame_id: Some(user_frame.id),
2486 };
2487 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2488 tracing::warn!(
2489 error = %append_err,
2490 "failed to append console send failure frame"
2491 );
2492 }
2493 }
2494 }
2495 });
2496}
2497
2498async fn append_steer_delivery_terminal(
2499 inner: &AggregatorInner,
2500 user_frame: &ConsoleFrame,
2501 interaction_id: &str,
2502) -> ConsoleLogResult<AppendOutcome> {
2503 append_and_emit(
2504 inner,
2505 NewConsoleFrame {
2506 id: None,
2507 dedupe_key: format!("steer-delivered:{}", user_frame.id),
2508 timestamp_ms: current_time_ms(),
2509 runtime_key: user_frame.runtime_key.clone(),
2510 identity: user_frame.identity.clone(),
2511 conversation_id: user_frame.conversation_id.clone(),
2512 session_id: user_frame.session_id.clone(),
2513 kind: "interaction_complete".to_string(),
2514 status: ConsoleFrameStatus::Completed,
2515 payload: json!({
2516 "reason": "steer_delivered",
2517 "handling_mode": "steer",
2518 }),
2519 source: ConsoleFrameSource {
2520 kind: ConsoleFrameSourceKind::Synthetic,
2521 source_cursor: None,
2522 },
2523 source_event_id: None,
2524 interaction_id: Some(interaction_id.to_string()),
2525 turn_id: None,
2526 run_id: None,
2527 parent_frame_id: Some(user_frame.id.clone()),
2528 caused_by_frame_id: Some(user_frame.id.clone()),
2529 },
2530 )
2531 .await
2532}
2533
2534#[derive(Debug)]
2535pub enum ConsoleSendError {
2536 UnknownIdentity(String),
2537 AmbiguousIdentity {
2538 identity: String,
2539 candidates: String,
2540 },
2541 NotAddressable(String),
2542 Retired(String),
2543 InvalidContent(String),
2544 InvalidHandlingMode(String),
2545 InvalidRequest(String),
2546 IdempotencyConflict(String),
2547 State(&'static str),
2548 Dispatch(String),
2549 Log(ConsoleLogError),
2550}
2551
2552impl std::fmt::Display for ConsoleSendError {
2553 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2554 match self {
2555 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2556 Self::AmbiguousIdentity {
2557 identity,
2558 candidates,
2559 } => write!(
2560 f,
2561 "ambiguous live identity alias {identity}: candidates [{candidates}]"
2562 ),
2563 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2564 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2565 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2566 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2567 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2568 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2569 Self::State(message) => write!(f, "console send state error: {message}"),
2570 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2571 Self::Log(err) => write!(f, "console log error: {err}"),
2572 }
2573 }
2574}
2575
2576impl std::error::Error for ConsoleSendError {}
2577
2578async fn backfill_session_history(
2579 inner: Arc<AggregatorInner>,
2580 runtime_key: String,
2581 force_refresh: bool,
2582) -> ConsoleLogResult<()> {
2583 if !inner.options.session_history_backfill_enabled {
2584 return Ok(());
2585 }
2586 let Some(entry) = inner
2587 .runtimes
2588 .read()
2589 .ok()
2590 .and_then(|entries| entries.get(&runtime_key).cloned())
2591 else {
2592 return Ok(());
2593 };
2594 let members = member_sources_for_entry(&entry).await;
2595 let mut targets = Vec::new();
2596 for resolved in members {
2597 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2598 continue;
2599 };
2600 if !resolved_member_visible(&resolved, &record).await {
2601 continue;
2602 }
2603 let Some(session_id) = record.session_id.clone() else {
2604 continue;
2605 };
2606 targets.push(SessionBackfillTarget {
2607 entry: entry.clone(),
2608 record,
2609 session_id,
2610 });
2611 }
2612 backfill_session_history_targets(inner, targets, force_refresh).await
2613}
2614
2615#[derive(Clone)]
2616struct SessionBackfillTarget {
2617 entry: RuntimeEntry,
2618 record: ConsoleIdentityRecord,
2619 session_id: String,
2620}
2621
2622async fn backfill_session_history_targets(
2623 inner: Arc<AggregatorInner>,
2624 targets: Vec<SessionBackfillTarget>,
2625 force_refresh: bool,
2626) -> ConsoleLogResult<()> {
2627 let mut tasks = tokio::task::JoinSet::new();
2628 for target in targets {
2629 tasks.spawn(backfill_one_session_history(
2630 inner.clone(),
2631 target,
2632 force_refresh,
2633 ));
2634 }
2635 let mut first_error = None;
2636 while let Some(result) = tasks.join_next().await {
2637 match result {
2638 Ok(Ok(())) => {}
2639 Ok(Err(err)) => {
2640 if first_error.is_none() {
2641 first_error = Some(err);
2642 }
2643 }
2644 Err(err) => {
2645 if first_error.is_none() {
2646 first_error = Some(Box::new(std::io::Error::other(format!(
2647 "session backfill task failed: {err}"
2648 ))) as ConsoleLogError);
2649 }
2650 }
2651 }
2652 }
2653 if let Some(err) = first_error {
2654 Err(err)
2655 } else {
2656 Ok(())
2657 }
2658}
2659
2660async fn backfill_one_session_history(
2661 inner: Arc<AggregatorInner>,
2662 target: SessionBackfillTarget,
2663 force_refresh: bool,
2664) -> ConsoleLogResult<()> {
2665 let _permit = inner
2666 .session_backfill_permits
2667 .clone()
2668 .acquire_owned()
2669 .await
2670 .map_err(|err| -> ConsoleLogError {
2671 Box::new(std::io::Error::other(format!(
2672 "session backfill limiter closed: {err}"
2673 )))
2674 })?;
2675 let SessionBackfillTarget {
2676 entry,
2677 record,
2678 session_id,
2679 } = target;
2680 let watermark_runtime_key =
2681 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2682 let watermark = inner
2683 .store
2684 .source_watermark(
2685 &watermark_runtime_key,
2686 ConsoleFrameSourceKind::SessionHistory,
2687 )
2688 .await?;
2689 let now_ms = current_time_ms();
2690 let mut offset = watermark
2691 .as_deref()
2692 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2693 .unwrap_or(0);
2694 if !force_refresh
2695 && watermark.as_deref().is_some_and(|watermark| {
2696 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2697 })
2698 {
2699 return Ok(());
2700 }
2701 loop {
2702 let page = match entry
2703 .runtime
2704 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2705 .await
2706 {
2707 Ok(page) => page,
2708 Err(err) => {
2709 append_backfill_gap(
2710 &inner,
2711 &entry.runtime_key,
2712 &record.identity,
2713 err.to_string(),
2714 )
2715 .await?;
2716 break;
2717 }
2718 };
2719 let page_value = match serde_json::to_value(page) {
2720 Ok(value) => value,
2721 Err(err) => {
2722 append_backfill_gap(
2723 &inner,
2724 &entry.runtime_key,
2725 &record.identity,
2726 err.to_string(),
2727 )
2728 .await?;
2729 break;
2730 }
2731 };
2732 let base_offset = page_value
2733 .get("offset")
2734 .and_then(Value::as_u64)
2735 .unwrap_or(offset as u64) as usize;
2736 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
2737 append_backfill_gap(
2738 &inner,
2739 &entry.runtime_key,
2740 &record.identity,
2741 "session history page missing messages".to_string(),
2742 )
2743 .await?;
2744 break;
2745 };
2746 if messages.is_empty() {
2747 if offset > 0 {
2748 record_session_history_watermark(
2749 &inner,
2750 &watermark_runtime_key,
2751 &session_id,
2752 offset,
2753 )
2754 .await?;
2755 }
2756 break;
2757 }
2758 for (idx, message) in messages.iter().enumerate() {
2759 let absolute_offset = base_offset + idx;
2760 let frames = frames_from_session_history_message_with_namespace(
2761 &entry.runtime_key,
2762 &record.identity,
2763 &entry.identity_namespace,
2764 &session_id,
2765 absolute_offset,
2766 message.clone(),
2767 );
2768 for mut frame in frames {
2769 if history_frame_has_existing_counterpart(&inner, &frame).await? {
2770 continue;
2771 }
2772 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
2773 frame.payload = redacted;
2774 frame.status = ConsoleFrameStatus::Redacted;
2775 }
2776 append_and_emit(&inner, frame).await?;
2777 }
2778 }
2779 offset = base_offset + messages.len();
2780 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
2781 .await?;
2782 let has_more = page_value
2783 .get("has_more")
2784 .and_then(Value::as_bool)
2785 .unwrap_or(false);
2786 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
2787 break;
2788 }
2789 }
2790 Ok(())
2791}
2792
2793async fn record_session_history_watermark(
2794 inner: &AggregatorInner,
2795 watermark_runtime_key: &str,
2796 session_id: &str,
2797 offset: usize,
2798) -> ConsoleLogResult<()> {
2799 inner
2800 .store
2801 .record_source_watermark(
2802 watermark_runtime_key,
2803 ConsoleFrameSourceKind::SessionHistory,
2804 &format_session_history_watermark(session_id, offset, current_time_ms()),
2805 )
2806 .await
2807}
2808
2809fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
2810 if !inner.options.session_history_backfill_enabled {
2811 return;
2812 }
2813 tokio::spawn(async move {
2814 {
2815 let mut active = inner.active_session_backfills.lock().await;
2816 if !active.insert(runtime_key.clone()) {
2817 return;
2818 }
2819 }
2820 let result = backfill_session_history(inner.clone(), runtime_key.clone(), false).await;
2821 let mut active = inner.active_session_backfills.lock().await;
2822 active.remove(&runtime_key);
2823 drop(active);
2824 if let Err(err) = result {
2825 tracing::warn!(
2826 runtime_key = %runtime_key,
2827 error = %err,
2828 "console session-history backfill failed"
2829 );
2830 }
2831 });
2832}
2833
2834fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
2835 if !inner.options.session_history_backfill_enabled {
2836 return;
2837 }
2838 tokio::spawn(async move {
2839 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
2840 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2841 loop {
2842 interval.tick().await;
2843 let runtime_still_registered = inner
2844 .runtimes
2845 .read()
2846 .ok()
2847 .is_some_and(|entries| entries.contains_key(&runtime_key));
2848 if !runtime_still_registered {
2849 break;
2850 }
2851 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
2852 }
2853 });
2854}
2855
2856fn spawn_session_history_backfill_target(
2857 inner: Arc<AggregatorInner>,
2858 target: SessionBackfillTarget,
2859 force_refresh: bool,
2860) {
2861 if !inner.options.session_history_backfill_enabled {
2862 return;
2863 }
2864 tokio::spawn(async move {
2865 let active_key = targeted_session_history_active_key(&target, force_refresh);
2866 let result =
2867 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
2868 if let Err(err) = result {
2869 tracing::warn!(
2870 active_key = %active_key,
2871 error = %err,
2872 "console targeted session-history backfill failed"
2873 );
2874 }
2875 });
2876}
2877
2878async fn run_targeted_session_history_backfill(
2879 inner: Arc<AggregatorInner>,
2880 target: SessionBackfillTarget,
2881 force_refresh: bool,
2882) -> ConsoleLogResult<()> {
2883 let active_key = targeted_session_history_active_key(&target, force_refresh);
2884 {
2885 let mut active = inner.active_session_backfills.lock().await;
2886 if !active.insert(active_key.clone()) {
2887 return Ok(());
2888 }
2889 }
2890 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
2891 let mut active = inner.active_session_backfills.lock().await;
2892 active.remove(&active_key);
2893 result
2894}
2895
2896fn targeted_session_history_active_key(
2897 target: &SessionBackfillTarget,
2898 force_refresh: bool,
2899) -> String {
2900 let mode = if force_refresh { "force" } else { "refresh" };
2901 format!(
2902 "{}:session-history:{}:{mode}",
2903 target.entry.runtime_key, target.session_id
2904 )
2905}
2906
2907fn spawn_session_history_backfill_for_identity(
2908 inner: Arc<AggregatorInner>,
2909 identity: String,
2910 force_refresh: bool,
2911) {
2912 if !inner.options.session_history_backfill_enabled {
2913 return;
2914 }
2915 tokio::spawn(async move {
2916 for target in session_backfill_targets_for_identity(&inner, &identity).await {
2917 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
2918 }
2919 });
2920}
2921
2922fn spawn_opportunistic_session_history_backfill_for_identity(
2923 inner: Arc<AggregatorInner>,
2924 identity: String,
2925) {
2926 if !inner.options.session_history_backfill_enabled {
2927 return;
2928 }
2929 tokio::spawn(async move {
2930 for target in session_backfill_targets_for_identity(&inner, &identity).await {
2931 let active_key = format!(
2932 "{}:session-history:{}",
2933 target.entry.runtime_key, target.session_id
2934 );
2935 {
2936 let mut seen = inner.opportunistic_session_backfills.lock().await;
2937 if !seen.insert(active_key) {
2938 continue;
2939 }
2940 }
2941 spawn_session_history_backfill_target(inner.clone(), target, false);
2942 }
2943 });
2944}
2945
2946#[cfg(test)]
2947async fn session_backfill_target_for_identity(
2948 inner: &AggregatorInner,
2949 identity: &str,
2950) -> Option<SessionBackfillTarget> {
2951 session_backfill_targets_for_identity(inner, identity)
2952 .await
2953 .into_iter()
2954 .next()
2955}
2956
2957async fn session_backfill_targets_for_identity(
2958 inner: &AggregatorInner,
2959 identity: &str,
2960) -> Vec<SessionBackfillTarget> {
2961 let entries = inner
2962 .runtimes
2963 .read()
2964 .ok()
2965 .map(|entries| entries.clone())
2966 .unwrap_or_default();
2967 let mut targets = Vec::new();
2968 for entry in entries.values() {
2969 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2970 if raw_identities.is_empty() {
2971 continue;
2972 }
2973 let mids = raw_identities
2974 .iter()
2975 .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
2976 .collect::<Vec<_>>();
2977 for resolved in member_sources_for_entry(entry)
2978 .await
2979 .into_iter()
2980 .filter(|candidate| {
2981 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
2982 })
2983 {
2984 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2985 continue;
2986 };
2987 if !resolved_member_visible(&resolved, &record).await {
2988 continue;
2989 }
2990 let Some(session_id) = record.session_id.clone() else {
2991 continue;
2992 };
2993 targets.push(SessionBackfillTarget {
2994 entry: entry.clone(),
2995 record,
2996 session_id,
2997 });
2998 }
2999 }
3000 targets
3001}
3002
3003fn resolved_member_matches_raw_identities(
3004 resolved: &ResolvedConsoleMember,
3005 raw_identities: &[String],
3006 mids: &[MeerkatId],
3007) -> bool {
3008 mids.contains(&resolved.member.agent_identity)
3009 || resolved
3010 .member
3011 .labels
3012 .get("agent_identity")
3013 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3014}
3015
3016async fn recover_lagged_source_events(
3017 inner: Arc<AggregatorInner>,
3018 runtime_key: &str,
3019 console_events: &ConsoleEventStore,
3020) -> ConsoleLogResult<()> {
3021 let watermark = inner
3022 .store
3023 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3024 .await?;
3025 match console_events.replay_all(watermark.as_deref()).await {
3026 Ok(events) => {
3027 for envelope in events {
3028 project_console_event(inner.clone(), runtime_key, envelope).await?;
3029 }
3030 }
3031 Err(err) => {
3032 append_source_gap(
3033 &inner,
3034 runtime_key,
3035 format!(
3036 "{}:{}:{}",
3037 err.error, err.stream, err.requested_last_event_id
3038 ),
3039 )
3040 .await?;
3041 }
3042 }
3043 Ok(())
3044}
3045
3046async fn append_source_gap(
3047 inner: &AggregatorInner,
3048 runtime_key: &str,
3049 reason: String,
3050) -> ConsoleLogResult<()> {
3051 append_and_emit(
3052 inner,
3053 NewConsoleFrame {
3054 id: None,
3055 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3056 timestamp_ms: current_time_ms(),
3057 runtime_key: runtime_key.to_string(),
3058 identity: "__console__".to_string(),
3059 conversation_id: None,
3060 session_id: None,
3061 kind: "replay_unavailable".to_string(),
3062 status: ConsoleFrameStatus::DeliveryFailed,
3063 payload: json!({
3064 "reason": reason,
3065 "source_kind": "console_event",
3066 }),
3067 source: ConsoleFrameSource {
3068 kind: ConsoleFrameSourceKind::Synthetic,
3069 source_cursor: None,
3070 },
3071 source_event_id: None,
3072 interaction_id: None,
3073 turn_id: None,
3074 run_id: None,
3075 parent_frame_id: None,
3076 caused_by_frame_id: None,
3077 },
3078 )
3079 .await?;
3080 let _ = inner
3081 .event_tx
3082 .send(ConsoleTimelineEvent::ReplayUnavailable {
3083 requested_cursor: format!("source-gap:{runtime_key}"),
3084 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3085 });
3086 Ok(())
3087}
3088
3089async fn append_backfill_gap(
3090 inner: &AggregatorInner,
3091 runtime_key: &str,
3092 identity: &str,
3093 reason: String,
3094) -> ConsoleLogResult<()> {
3095 append_and_emit(
3096 inner,
3097 NewConsoleFrame {
3098 id: None,
3099 dedupe_key: format!(
3100 "session-backfill-gap:{runtime_key}:{identity}:{}",
3101 current_time_ms()
3102 ),
3103 timestamp_ms: current_time_ms(),
3104 runtime_key: runtime_key.to_string(),
3105 identity: identity.to_string(),
3106 conversation_id: Some(identity.to_string()),
3107 session_id: None,
3108 kind: "replay_unavailable".to_string(),
3109 status: ConsoleFrameStatus::DeliveryFailed,
3110 payload: json!({
3111 "reason": reason,
3112 "source_kind": "session_history",
3113 }),
3114 source: ConsoleFrameSource {
3115 kind: ConsoleFrameSourceKind::Synthetic,
3116 source_cursor: None,
3117 },
3118 source_event_id: None,
3119 interaction_id: None,
3120 turn_id: None,
3121 run_id: None,
3122 parent_frame_id: None,
3123 caused_by_frame_id: None,
3124 },
3125 )
3126 .await?;
3127 Ok(())
3128}
3129
3130async fn project_console_event(
3131 inner: Arc<AggregatorInner>,
3132 runtime_key: &str,
3133 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3134) -> ConsoleLogResult<()> {
3135 let Some(entry) = inner
3136 .runtimes
3137 .read()
3138 .ok()
3139 .and_then(|entries| entries.get(runtime_key).cloned())
3140 else {
3141 return Ok(());
3142 };
3143 let mut frame = frame_from_console_event(&entry, envelope);
3144 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3145 frame.payload = redacted;
3146 frame.status = ConsoleFrameStatus::Redacted;
3147 }
3148 let source_cursor = frame
3149 .source_event_id
3150 .clone()
3151 .unwrap_or_else(|| frame.dedupe_key.clone());
3152 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3153 Some(frame.identity.clone())
3154 } else {
3155 None
3156 };
3157 let opportunistic_refresh_identity = if refresh_identity.is_none()
3158 && console_event_should_start_session_history_backfill(&frame)
3159 {
3160 Some(frame.identity.clone())
3161 } else {
3162 None
3163 };
3164 append_and_emit(&inner, frame).await?;
3165 inner
3166 .store
3167 .record_source_watermark(
3168 &entry.runtime_key,
3169 ConsoleFrameSourceKind::ConsoleEvent,
3170 &source_cursor,
3171 )
3172 .await?;
3173 if let Some(identity) = refresh_identity {
3174 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3175 } else if let Some(identity) = opportunistic_refresh_identity {
3176 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3177 }
3178 Ok(())
3179}
3180
3181fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3182 matches!(
3183 frame.kind.as_str(),
3184 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3185 ) || frame.session_id.is_some()
3186}
3187
3188fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3189 if frame.identity == SYSTEM_EVENT_IDENTITY {
3190 return false;
3191 }
3192 matches!(
3193 frame.kind.as_str(),
3194 "turn_started"
3195 | "run_started"
3196 | "reasoning_complete"
3197 | "tool_call_requested"
3198 | "tool_call"
3199 | "tool_execution_started"
3200 | "text_delta"
3201 | "system_notice"
3202 )
3203}
3204
3205async fn append_and_emit(
3206 inner: &AggregatorInner,
3207 frame: NewConsoleFrame,
3208) -> ConsoleLogResult<AppendOutcome> {
3209 let outcome = inner.store.append_if_absent(frame).await?;
3210 if outcome.disposition == AppendDisposition::Inserted {
3211 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3212 frame: outcome.frame.clone(),
3213 });
3214 }
3215 Ok(outcome)
3216}
3217
3218async fn update_frame_status_and_emit(
3219 inner: &AggregatorInner,
3220 frame_id: &str,
3221 status: ConsoleFrameStatus,
3222) -> ConsoleLogResult<Option<ConsoleFrame>> {
3223 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3224 return Ok(None);
3225 };
3226 let update_marker = NewConsoleFrame {
3227 id: None,
3228 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3229 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3230 runtime_key: updated.runtime_key.clone(),
3231 identity: updated.identity.clone(),
3232 conversation_id: updated.conversation_id.clone(),
3233 session_id: updated.session_id.clone(),
3234 kind: "frame_updated".to_string(),
3235 status: updated.status,
3236 payload: json!({ "frame": updated.clone() }),
3237 source: ConsoleFrameSource {
3238 kind: ConsoleFrameSourceKind::Synthetic,
3239 source_cursor: None,
3240 },
3241 source_event_id: None,
3242 interaction_id: updated.interaction_id.clone(),
3243 turn_id: updated.turn_id.clone(),
3244 run_id: updated.run_id.clone(),
3245 parent_frame_id: Some(updated.id.clone()),
3246 caused_by_frame_id: Some(updated.id.clone()),
3247 };
3248 let outcome = inner.store.append_if_absent(update_marker).await?;
3249 if outcome.disposition == AppendDisposition::Inserted {
3250 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3251 frame: outcome.frame,
3252 });
3253 }
3254 Ok(Some(updated))
3255}
3256
3257fn frame_from_console_event(
3258 entry: &RuntimeEntry,
3259 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3260) -> NewConsoleFrame {
3261 let turn_id = envelope
3262 .data
3263 .get("turn_id")
3264 .and_then(Value::as_str)
3265 .map(ToString::to_string);
3266 let run_id = envelope
3267 .data
3268 .get("run_id")
3269 .and_then(Value::as_str)
3270 .map(ToString::to_string);
3271 let status = match envelope.event_type.as_str() {
3272 "interaction_started" => ConsoleFrameStatus::Accepted,
3273 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3274 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3275 _ => ConsoleFrameStatus::Delivered,
3276 };
3277 let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3278 NewConsoleFrame {
3279 id: Some(envelope.event_id.clone()),
3280 dedupe_key: format!("console-event:{}:{}", entry.runtime_key, envelope.event_id),
3281 timestamp_ms: envelope.timestamp_ms,
3282 runtime_key: entry.runtime_key.clone(),
3283 identity: identity.clone(),
3284 conversation_id: Some(identity),
3285 session_id: envelope
3286 .data
3287 .get("session_id")
3288 .and_then(Value::as_str)
3289 .map(ToString::to_string),
3290 kind: envelope.event_type,
3291 status,
3292 payload: envelope.data,
3293 source: ConsoleFrameSource {
3294 kind: ConsoleFrameSourceKind::ConsoleEvent,
3295 source_cursor: None,
3296 },
3297 source_event_id: Some(envelope.event_id),
3298 interaction_id: envelope.interaction_id,
3299 turn_id,
3300 run_id,
3301 parent_frame_id: None,
3302 caused_by_frame_id: None,
3303 }
3304}
3305
3306#[cfg(test)]
3307fn frame_from_session_history_message(
3308 runtime_key: &str,
3309 identity: &str,
3310 session_id: &str,
3311 offset: usize,
3312 message: Value,
3313) -> Option<NewConsoleFrame> {
3314 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3315 .into_iter()
3316 .next()
3317}
3318
3319#[cfg(test)]
3320fn frames_from_session_history_message(
3321 runtime_key: &str,
3322 identity: &str,
3323 session_id: &str,
3324 offset: usize,
3325 message: Value,
3326) -> Vec<NewConsoleFrame> {
3327 frames_from_session_history_message_with_namespace(
3328 runtime_key,
3329 identity,
3330 "",
3331 session_id,
3332 offset,
3333 message,
3334 )
3335}
3336
3337fn frames_from_session_history_message_with_namespace(
3338 runtime_key: &str,
3339 identity: &str,
3340 identity_namespace: &str,
3341 session_id: &str,
3342 offset: usize,
3343 message: Value,
3344) -> Vec<NewConsoleFrame> {
3345 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3346 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3347 return Vec::new();
3348 };
3349 if let Message::ToolResults {
3350 results,
3351 created_at,
3352 } = parsed
3353 {
3354 return results
3355 .into_iter()
3356 .enumerate()
3357 .map(|(idx, result)| {
3358 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3359 let result_text = result.text_content();
3360 let tool_use_id = result.tool_use_id.clone();
3361 NewConsoleFrame {
3362 id: None,
3363 dedupe_key: format!(
3364 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3365 ),
3366 timestamp_ms: created_at.timestamp_millis().max(0) as u64,
3367 runtime_key: runtime_key.to_string(),
3368 identity: identity.to_string(),
3369 conversation_id: Some(identity.to_string()),
3370 session_id: Some(session_id.to_string()),
3371 kind: "tool_execution_completed".to_string(),
3372 status: ConsoleFrameStatus::Completed,
3373 payload: json!({
3374 "id": tool_use_id,
3375 "tool_call_id": tool_use_id,
3376 "result": result_text,
3377 "content": content,
3378 "is_error": result.is_error,
3379 "source_event_type": "session_history",
3380 "type": "session_history",
3381 }),
3382 source: ConsoleFrameSource {
3383 kind: ConsoleFrameSourceKind::SessionHistory,
3384 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3385 },
3386 source_event_id: None,
3387 interaction_id: None,
3388 turn_id: None,
3389 run_id: None,
3390 parent_frame_id: None,
3391 caused_by_frame_id: None,
3392 }
3393 })
3394 .collect();
3395 }
3396 let (kind, timestamp_ms, payload) = match &parsed {
3397 Message::User(user) => {
3398 if session_history_user_message_is_scaffold(&message) {
3399 return Vec::new();
3400 }
3401 (
3402 "user_input",
3403 user.created_at.timestamp_millis().max(0) as u64,
3404 json!({
3405 "content": user.content,
3406 "message": message,
3407 }),
3408 )
3409 }
3410 Message::Assistant(assistant) => (
3411 "interaction_complete",
3412 assistant.created_at.timestamp_millis().max(0) as u64,
3413 json!({
3414 "result": assistant.content,
3415 "text": assistant.content,
3416 "message": message,
3417 "source_event_type": "session_history",
3418 "type": "session_history",
3419 }),
3420 ),
3421 Message::BlockAssistant(assistant) => {
3422 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3423 (
3424 "interaction_complete",
3425 assistant.created_at.timestamp_millis().max(0) as u64,
3426 json!({
3427 "result": text,
3428 "text": text,
3429 "message": message,
3430 "source_event_type": "session_history",
3431 "type": "session_history",
3432 }),
3433 )
3434 }
3435 Message::SystemNotice(notice) => (
3436 "system_notice",
3437 notice.created_at.timestamp_millis().max(0) as u64,
3438 json!({
3439 "message": message,
3440 "kind": notice.kind,
3441 "render_class": notice.kind.render_class(),
3442 "body": notice.body,
3443 "blocks": notice.blocks,
3444 "source_event_type": "session_history",
3445 "type": "session_history",
3446 }),
3447 ),
3448 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3449 };
3450 let mut frames = vec![NewConsoleFrame {
3451 id: None,
3452 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3453 timestamp_ms,
3454 runtime_key: runtime_key.to_string(),
3455 identity: identity.to_string(),
3456 conversation_id: Some(identity.to_string()),
3457 session_id: Some(session_id.to_string()),
3458 kind: kind.to_string(),
3459 status: ConsoleFrameStatus::Completed,
3460 payload,
3461 source: ConsoleFrameSource {
3462 kind: ConsoleFrameSourceKind::SessionHistory,
3463 source_cursor: Some(format!("{session_id}:{offset}")),
3464 },
3465 source_event_id: None,
3466 interaction_id: None,
3467 turn_id: None,
3468 run_id: None,
3469 parent_frame_id: None,
3470 caused_by_frame_id: None,
3471 }];
3472 if let Message::BlockAssistant(assistant) = &parsed {
3473 frames.extend(spawn_initial_message_frames_from_assistant(
3474 runtime_key,
3475 identity,
3476 identity_namespace,
3477 session_id,
3478 offset,
3479 assistant,
3480 &payload_hash,
3481 ));
3482 }
3483 frames
3484}
3485
3486fn spawn_initial_message_frames_from_assistant(
3487 runtime_key: &str,
3488 parent_identity: &str,
3489 identity_namespace: &str,
3490 session_id: &str,
3491 offset: usize,
3492 assistant: &meerkat_core::types::BlockAssistantMessage,
3493 payload_hash: &str,
3494) -> Vec<NewConsoleFrame> {
3495 let mut frames = Vec::new();
3496 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3497 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3498 continue;
3499 }
3500 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3501 continue;
3502 };
3503 for (spawn_idx, (target_identity, initial_message)) in
3504 spawn_initial_messages_from_tool_args(&args)
3505 .into_iter()
3506 .enumerate()
3507 {
3508 let target_identity = apply_namespace(&target_identity, identity_namespace);
3509 let message_content = match &initial_message {
3510 Value::String(text) => json!([
3511 {
3512 "type": "text",
3513 "text": text,
3514 }
3515 ]),
3516 other => other.clone(),
3517 };
3518 let message = match &initial_message {
3519 Value::String(text) => json!({
3520 "role": "user",
3521 "content": text,
3522 "created_at": assistant.created_at,
3523 }),
3524 other => json!({
3525 "role": "user",
3526 "content": other,
3527 "created_at": assistant.created_at,
3528 }),
3529 };
3530 frames.push(NewConsoleFrame {
3531 id: None,
3532 dedupe_key: format!(
3533 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3534 ),
3535 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3536 runtime_key: runtime_key.to_string(),
3537 identity: target_identity.clone(),
3538 conversation_id: Some(target_identity),
3539 session_id: None,
3540 kind: "user_input".to_string(),
3541 status: ConsoleFrameStatus::Completed,
3542 payload: json!({
3543 "content": message_content,
3544 "message": message,
3545 "source_event_type": "session_history_spawn_initial_message",
3546 "type": "session_history",
3547 "tool_call_id": tool.id,
3548 "parent_identity": parent_identity,
3549 "via_tool": tool.name,
3550 }),
3551 source: ConsoleFrameSource {
3552 kind: ConsoleFrameSourceKind::SessionHistory,
3553 source_cursor: Some(format!(
3554 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
3555 )),
3556 },
3557 source_event_id: None,
3558 interaction_id: None,
3559 turn_id: None,
3560 run_id: None,
3561 parent_frame_id: None,
3562 caused_by_frame_id: None,
3563 });
3564 }
3565 }
3566 frames
3567}
3568
3569fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
3570 let mut messages = Vec::new();
3571 if let Some(message) = spawn_initial_message_from_record(args) {
3572 messages.push(message);
3573 }
3574 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
3575 for spec in specs {
3576 if let Some(message) = spawn_initial_message_from_record(spec) {
3577 messages.push(message);
3578 }
3579 }
3580 }
3581 messages
3582}
3583
3584fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
3585 let target_identity = record
3586 .get("member_id")
3587 .or_else(|| record.get("agent_identity"))
3588 .and_then(Value::as_str)?
3589 .trim();
3590 if target_identity.is_empty() {
3591 return None;
3592 }
3593 let initial_message = record
3594 .get("initial_message")
3595 .or_else(|| record.get("task"))?
3596 .clone();
3597 Some((target_identity.to_string(), initial_message))
3598}
3599
3600fn session_history_user_message_is_scaffold(message: &Value) -> bool {
3601 message
3602 .get("content")
3603 .is_some_and(scaffold_message_content_is_noise)
3604}
3605
3606fn scaffold_message_content_is_noise(value: &Value) -> bool {
3607 match value {
3608 Value::String(text) => scaffold_message_text_is_noise(text),
3609 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
3610 Value::Object(map) => ["text", "content", "message"]
3611 .iter()
3612 .filter_map(|key| map.get(*key))
3613 .any(scaffold_message_content_is_noise),
3614 _ => false,
3615 }
3616}
3617
3618fn scaffold_message_text_is_noise(text: &str) -> bool {
3619 let trimmed = text.trim_start();
3620 trimmed.starts_with("[PEER UPDATE]")
3621 || trimmed
3622 .to_ascii_lowercase()
3623 .starts_with("you have been spawned")
3624}
3625
3626fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
3627 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
3628 rest.split(':').next()?.parse().ok()
3629}
3630
3631fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
3632 format!("{session_id}:{offset}:{checked_at_ms}")
3633}
3634
3635fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
3636 let Some(checked_at_ms) = watermark
3637 .rsplit_once(':')
3638 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
3639 else {
3640 return false;
3641 };
3642 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
3643 let ttl_ms = if offset > 0 {
3644 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
3645 } else {
3646 SESSION_HISTORY_REFRESH_TTL_MS
3647 };
3648 now_ms.saturating_sub(checked_at_ms) < ttl_ms
3649}
3650
3651async fn history_frame_has_existing_counterpart(
3652 inner: &AggregatorInner,
3653 frame: &NewConsoleFrame,
3654) -> ConsoleLogResult<bool> {
3655 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
3656 let Some(fingerprint) = fingerprint else {
3657 return Ok(false);
3658 };
3659 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
3660 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
3661 let mut after = None;
3662 loop {
3663 let page = inner
3664 .store
3665 .query_frames(ConsoleTimelineQuery {
3666 identity: Some(frame.identity.clone()),
3667 conversation_id: frame.conversation_id.clone(),
3668 after,
3669 limit: 1_000,
3670 })
3671 .await?;
3672 for existing in &page.frames {
3673 let same_session = existing.session_id == frame.session_id
3674 || existing.session_id.is_none()
3675 || frame.session_id.is_none();
3676 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
3677 continue;
3678 }
3679 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
3680 == Some(&fingerprint)
3681 {
3682 return Ok(true);
3683 }
3684 if assistant_terminal
3685 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
3686 {
3687 let turn_key = existing
3688 .interaction_id
3689 .as_deref()
3690 .or(existing.turn_id.as_deref())
3691 .or(existing.run_id.as_deref())
3692 .unwrap_or("session");
3693 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
3694 aggregated.push_str(delta);
3695 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
3696 return Ok(true);
3697 }
3698 }
3699 }
3700 if page.frames.is_empty() || page.next_cursor.is_none() {
3701 return Ok(false);
3702 }
3703 after = page.next_cursor;
3704 }
3705}
3706
3707fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
3708 format!("{runtime_key}:session-history:{session_id}")
3709}
3710
3711fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3712 match kind {
3713 "user_input" | "interaction_started" => payload
3714 .get("content")
3715 .map(stable_value_fingerprint)
3716 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
3717 "tool_execution_completed" => tool_result_fingerprint(payload),
3718 "text_delta" => {
3719 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
3720 }
3721 "text_complete" | "interaction_complete" | "run_completed" => {
3722 assistant_terminal_fingerprint(kind, payload)
3723 }
3724 _ => None,
3725 }
3726}
3727
3728fn tool_result_fingerprint(payload: &Value) -> Option<String> {
3729 let id = payload
3730 .get("tool_call_id")
3731 .or_else(|| payload.get("id"))
3732 .and_then(Value::as_str)
3733 .unwrap_or("");
3734 let result = payload
3735 .get("result")
3736 .or_else(|| payload.get("content"))
3737 .map(stable_value_fingerprint)?;
3738 Some(format!("{id}:{result}"))
3739}
3740
3741fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3742 match kind {
3743 "text_complete" | "interaction_complete" | "run_completed" => payload
3744 .get("text")
3745 .or_else(|| payload.get("result"))
3746 .or_else(|| payload.get("content"))
3747 .map(stable_value_fingerprint),
3748 _ => None,
3749 }
3750}
3751
3752fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
3753 if kind != "text_delta" {
3754 return None;
3755 }
3756 payload
3757 .get("delta")
3758 .or_else(|| payload.get("text"))
3759 .or_else(|| payload.get("content"))
3760 .and_then(Value::as_str)
3761 .or_else(|| payload.as_str())
3762}
3763
3764fn stable_value_fingerprint(value: &Value) -> String {
3765 if let Some(text) = content_value_text(value) {
3766 return normalize_transcript_fingerprint_text(&text);
3767 }
3768 match value {
3769 Value::String(text) => normalize_transcript_fingerprint_text(text),
3770 other => serde_json::to_string(other).unwrap_or_default(),
3771 }
3772}
3773
3774fn content_value_text(value: &Value) -> Option<String> {
3775 match value {
3776 Value::String(text) => Some(text.clone()),
3777 Value::Array(items) => {
3778 let text = items
3779 .iter()
3780 .filter_map(content_value_text)
3781 .collect::<String>();
3782 (!text.is_empty()).then_some(text)
3783 }
3784 Value::Object(map) => ["text", "content", "message", "blocks"]
3785 .iter()
3786 .filter_map(|key| map.get(*key))
3787 .find_map(content_value_text),
3788 _ => None,
3789 }
3790}
3791
3792fn normalize_transcript_fingerprint_text(text: &str) -> String {
3793 let trimmed = text.trim();
3794 trimmed
3795 .strip_prefix("[EVENT via rpc] ")
3796 .unwrap_or(trimmed)
3797 .trim()
3798 .to_string()
3799}
3800
3801#[derive(Debug, Clone, Copy)]
3802enum CachedIdentityVisibility {
3803 Visible,
3804 Hidden,
3805 Missing,
3806}
3807
3808async fn frame_is_visible(
3809 inner: &AggregatorInner,
3810 frame: &ConsoleFrame,
3811 allow_historical_identity: bool,
3812 identity_records: &[ConsoleIdentityRecord],
3813) -> ConsoleLogResult<bool> {
3814 let mut identity_visibility_cache = HashMap::new();
3815 frame_is_visible_cached(
3816 inner,
3817 frame,
3818 allow_historical_identity,
3819 &mut identity_visibility_cache,
3820 identity_records,
3821 )
3822 .await
3823}
3824
3825async fn frame_is_visible_cached(
3826 inner: &AggregatorInner,
3827 frame: &ConsoleFrame,
3828 allow_historical_identity: bool,
3829 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
3830 identity_records: &[ConsoleIdentityRecord],
3831) -> ConsoleLogResult<bool> {
3832 let entry = {
3833 let entries = inner
3834 .runtimes
3835 .read()
3836 .map_err(|_| runtime_registry_lock_error())?;
3837 if entries.is_empty() {
3838 return Ok(true);
3839 }
3840 let Some(entry) = entries.get(&frame.runtime_key) else {
3841 return Ok(false);
3842 };
3843 entry.clone()
3844 };
3845 if frame.identity != "__console__" {
3846 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
3847 let identity_visibility =
3848 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
3849 cached
3850 } else {
3851 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
3852 .unwrap_or_else(|| frame.identity.clone());
3853 let visibility = match identity_records.iter().find(|record| {
3854 record.runtime_key == frame.runtime_key
3855 && (record.identity == frame.identity
3856 || record.runtime_member_id == runtime_member_id)
3857 }) {
3858 Some(record) => {
3859 if console_identity_record_visible(&entry, record).await {
3860 CachedIdentityVisibility::Visible
3861 } else {
3862 CachedIdentityVisibility::Hidden
3863 }
3864 }
3865 None => CachedIdentityVisibility::Missing,
3866 };
3867 identity_visibility_cache.insert(cache_key, visibility);
3868 visibility
3869 };
3870 match identity_visibility {
3871 CachedIdentityVisibility::Visible => {
3872 if frame_matches_hidden_member(&entry, frame).await {
3873 return Ok(false);
3874 }
3875 }
3876 CachedIdentityVisibility::Hidden => return Ok(false),
3877 CachedIdentityVisibility::Missing => {
3878 if frame_matches_hidden_member(&entry, frame).await {
3879 return Ok(false);
3880 }
3881 return Ok(
3882 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
3883 );
3884 }
3885 }
3886 }
3887 Ok(entry.visibility_policy.frame_visible(frame))
3888}
3889
3890async fn identity_record_for_member(
3891 entry: &RuntimeEntry,
3892 handle: &MobHandle,
3893 member: &MobMemberListEntry,
3894) -> Option<ConsoleIdentityRecord> {
3895 let runtime_member_id = member.agent_identity.to_string();
3896 let durable_identity = member
3897 .labels
3898 .get("agent_identity")
3899 .filter(|value| !value.trim().is_empty())
3900 .map_or(runtime_member_id.as_str(), String::as_str);
3901 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
3902 let addressable = member_is_addressable(member);
3903 let visibility = if member.state == meerkat_mob::MemberState::Retiring {
3904 ConsoleVisibility::RetiredReadable
3905 } else if addressable {
3906 ConsoleVisibility::Addressable
3907 } else {
3908 ConsoleVisibility::Hidden
3909 };
3910 let session_id = handle
3911 .resolve_bridge_session_id_observation(&member.agent_identity)
3912 .await
3913 .map(|sid| sid.to_string());
3914 let display_name = member
3915 .labels
3916 .get("display_name")
3917 .cloned()
3918 .unwrap_or_else(|| runtime_member_id.clone());
3919 let mut labels = member.labels.clone();
3920 labels
3921 .entry("role".to_string())
3922 .or_insert_with(|| member.role.to_string());
3923 Some(ConsoleIdentityRecord {
3924 identity,
3925 display_name,
3926 runtime_key: entry.runtime_key.clone(),
3927 runtime_member_id,
3928 session_id,
3929 visibility,
3930 addressable,
3931 health: if addressable {
3932 "ready"
3933 } else {
3934 "hidden_by_policy"
3935 }
3936 .to_string(),
3937 topology_peers: member.wired_to.iter().map(ToString::to_string).collect(),
3938 labels,
3939 })
3940}
3941
3942async fn identity_record_for_resolved_member(
3943 resolved: &ResolvedConsoleMember,
3944) -> Option<ConsoleIdentityRecord> {
3945 let mut record =
3946 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
3947 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
3948 record
3949 .labels
3950 .entry("source_mob_id".to_string())
3951 .or_insert_with(|| resolved.source_mob_id.clone());
3952 }
3953 Some(record)
3954}
3955
3956fn identity_record_for_status(
3957 entry: &RuntimeEntry,
3958 status: &crate::identity_first::IdentityStatus,
3959) -> ConsoleIdentityRecord {
3960 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
3961 let runtime_member_id = status
3962 .agent_runtime_id
3963 .as_ref()
3964 .map(crate::identity_first::AgentRuntimeId::as_str)
3965 .unwrap_or_else(|| status.identity.as_str())
3966 .to_string();
3967 let addressable = status.addressability
3968 == crate::identity_first::AgentAddressability::Addressable
3969 && matches!(
3970 status.state,
3971 crate::identity_first::IdentityLifecycleState::Active
3972 | crate::identity_first::IdentityLifecycleState::Dormant
3973 | crate::identity_first::IdentityLifecycleState::Uninitialized
3974 );
3975 let visibility = match status.state {
3976 crate::identity_first::IdentityLifecycleState::Retiring => {
3977 ConsoleVisibility::RetiredReadable
3978 }
3979 crate::identity_first::IdentityLifecycleState::Broken
3980 | crate::identity_first::IdentityLifecycleState::Suspended => {
3981 ConsoleVisibility::Unreachable
3982 }
3983 _ if addressable => ConsoleVisibility::Addressable,
3984 _ => ConsoleVisibility::Hidden,
3985 };
3986 let health = match status.state {
3987 crate::identity_first::IdentityLifecycleState::Active => "ready",
3988 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
3989 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
3990 crate::identity_first::IdentityLifecycleState::Broken => "broken",
3991 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
3992 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
3993 }
3994 .to_string();
3995 let display_name = status
3996 .display_name
3997 .as_ref()
3998 .map(crate::identity_first::DisplayName::as_str)
3999 .unwrap_or_else(|| status.identity.as_str())
4000 .to_string();
4001 ConsoleIdentityRecord {
4002 identity,
4003 display_name,
4004 runtime_key: entry.runtime_key.clone(),
4005 runtime_member_id,
4006 session_id: status.session_id.as_ref().map(ToString::to_string),
4007 visibility,
4008 addressable,
4009 health,
4010 topology_peers: Vec::new(),
4011 labels: status.labels.clone(),
4012 }
4013}
4014
4015pub(crate) fn is_implicit_delegate_member(
4016 role: &str,
4017 labels: &std::collections::BTreeMap<String, String>,
4018) -> bool {
4019 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4020}
4021
4022fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4023 member
4024 .labels
4025 .get("addressable")
4026 .map(|value| !value.eq_ignore_ascii_case("false"))
4027 .unwrap_or(true)
4028}
4029
4030fn apply_namespace(identity: &str, namespace: &str) -> String {
4031 let namespace = namespace.trim().trim_matches('/');
4032 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4033 identity.to_string()
4034 } else {
4035 format!("{namespace}/{identity}")
4036 }
4037}
4038
4039fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4040 let namespace = namespace.trim().trim_matches('/');
4041 if namespace.is_empty() {
4042 return Some(identity.to_string());
4043 }
4044 identity
4045 .strip_prefix(namespace)
4046 .and_then(|rest| rest.strip_prefix('/'))
4047 .map(ToString::to_string)
4048}
4049
4050fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4051 let namespace = namespace.trim().trim_matches('/');
4052 if namespace.is_empty() {
4053 return vec![identity.to_string()];
4054 }
4055 let mut candidates = Vec::new();
4056 if let Some(stripped) = strip_namespace(identity, namespace) {
4057 candidates.push(stripped);
4058 candidates.push(identity.to_string());
4059 }
4060 candidates.sort();
4061 candidates.dedup();
4062 candidates
4063}
4064
4065fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4066 namespace_match_candidates(identity, namespace)
4067 .iter()
4068 .any(|candidate| candidate.starts_with("rt:"))
4069}
4070
4071fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4072 entries.iter().any(|entry| {
4073 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4074 })
4075}
4076
4077fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4078 if request.identity.trim().is_empty() {
4079 return Err(ConsoleSendError::InvalidRequest(
4080 "identity must be non-empty".to_string(),
4081 ));
4082 }
4083 if request.origin.trim().is_empty() {
4084 return Err(ConsoleSendError::InvalidRequest(
4085 "origin must be non-empty".to_string(),
4086 ));
4087 }
4088 if request.idempotency_key.trim().is_empty() {
4089 return Err(ConsoleSendError::InvalidRequest(
4090 "idempotency_key must be non-empty".to_string(),
4091 ));
4092 }
4093 Ok(())
4094}
4095
4096fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4097 let content: ContentInput = serde_json::from_value(value.clone())
4098 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4099 match &content {
4100 ContentInput::Text(text) if text.trim().is_empty() => Err(
4101 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4102 ),
4103 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4104 "content blocks must be non-empty".to_string(),
4105 )),
4106 _ => Ok(content),
4107 }
4108}
4109
4110fn parse_handling_mode(
4111 value: Option<&str>,
4112) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4113 match value.unwrap_or("queue") {
4114 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4115 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4116 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4117 }
4118}
4119
4120fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4121 ConsoleInteractionAccepted {
4122 interaction_id: frame
4123 .interaction_id
4124 .clone()
4125 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4126 identity: frame.identity.clone(),
4127 conversation_id: frame.conversation_id.clone(),
4128 session_id: frame.session_id.clone(),
4129 input_frame_id: frame.id.clone(),
4130 cursor: frame.cursor.clone(),
4131 status: frame.status,
4132 }
4133}
4134
4135fn send_dedupe_key(
4136 runtime_key: &str,
4137 identity: &str,
4138 origin: &str,
4139 idempotency_key: &str,
4140) -> String {
4141 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4142}
4143
4144fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4145 let content_json = serde_json::to_string(content).unwrap_or_default();
4146 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4147}
4148
4149fn hash_short(value: &str) -> String {
4150 let mut hasher = Sha256::new();
4151 hasher.update(value.as_bytes());
4152 let digest = hasher.finalize();
4153 to_hex(&digest[..8])
4154}
4155
4156fn to_hex(bytes: &[u8]) -> String {
4157 const HEX: &[u8; 16] = b"0123456789abcdef";
4158 let mut out = String::with_capacity(bytes.len() * 2);
4159 for byte in bytes {
4160 out.push(HEX[(byte >> 4) as usize] as char);
4161 out.push(HEX[(byte & 0x0f) as usize] as char);
4162 }
4163 out
4164}
4165
4166fn current_time_ms() -> u64 {
4167 match SystemTime::now().duration_since(UNIX_EPOCH) {
4168 Ok(duration) => duration.as_millis() as u64,
4169 Err(_) => 0,
4170 }
4171}
4172
4173fn runtime_registry_lock_error() -> ConsoleLogError {
4174 Box::new(std::io::Error::other(
4175 "console runtime registry lock poisoned",
4176 ))
4177}
4178
4179#[cfg(test)]
4180#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4181mod tests {
4182 use std::sync::Arc;
4183 use std::sync::atomic::{AtomicUsize, Ordering};
4184 use std::time::Duration;
4185 use std::time::Instant;
4186
4187 use futures::StreamExt;
4188 use meerkat::{AgentFactory, Config, build_ephemeral_service};
4189 use meerkat_client::types::LlmStream;
4190 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4191 use meerkat_core::{
4192 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4193 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4194 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4195 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4196 StreamError,
4197 };
4198 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4199 use serde_json::json;
4200
4201 use super::*;
4202 use crate::mob_handle_runtime::MobBootstrapSpec;
4203
4204 #[derive(Debug)]
4205 struct HideHiddenNoiseFrames;
4206
4207 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4208 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4209 frame.kind != "hidden_noise"
4210 }
4211 }
4212
4213 #[derive(Debug)]
4214 struct HideRuntimeMemberIdentity(&'static str);
4215
4216 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4217 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4218 record.runtime_member_id != self.0
4219 }
4220 }
4221
4222 #[derive(Debug)]
4223 struct HideRuntimeMemberOnly(&'static str);
4224
4225 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4226 fn member_visible(&self, member: &ConsoleMember) -> bool {
4227 member.agent_identity != self.0
4228 }
4229 }
4230
4231 #[derive(Debug)]
4232 struct HideConsoleIdentity(&'static str);
4233
4234 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4235 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4236 record.identity != self.0
4237 }
4238 }
4239
4240 struct CountingConsoleLogStore {
4241 inner: InMemoryConsoleLogStore,
4242 source_watermark_calls: AtomicUsize,
4243 record_watermark_calls: AtomicUsize,
4244 }
4245
4246 impl CountingConsoleLogStore {
4247 fn new() -> Self {
4248 Self {
4249 inner: InMemoryConsoleLogStore::new(),
4250 source_watermark_calls: AtomicUsize::new(0),
4251 record_watermark_calls: AtomicUsize::new(0),
4252 }
4253 }
4254
4255 fn source_watermark_calls(&self) -> usize {
4256 self.source_watermark_calls.load(Ordering::SeqCst)
4257 }
4258 }
4259
4260 struct SlowTestClient {
4261 delay: Duration,
4262 }
4263
4264 #[async_trait::async_trait]
4265 impl LlmClient for SlowTestClient {
4266 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4267 Ok(messages.to_vec())
4268 }
4269
4270 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4271 let delay = self.delay;
4272 let delayed_text = futures::stream::once(async move {
4273 tokio::time::sleep(delay).await;
4274 Ok(LlmEvent::TextDelta {
4275 delta: "slow ok".to_string(),
4276 meta: None,
4277 })
4278 });
4279 let done = futures::stream::once(async {
4280 Ok(LlmEvent::Done {
4281 outcome: LlmDoneOutcome::Success {
4282 stop_reason: StopReason::EndTurn,
4283 },
4284 })
4285 });
4286 Box::pin(delayed_text.chain(done))
4287 }
4288
4289 fn provider(&self) -> &'static str {
4290 "slow-test"
4291 }
4292
4293 async fn health_check(&self) -> Result<(), LlmError> {
4294 Ok(())
4295 }
4296 }
4297
4298 #[derive(Clone)]
4299 struct DelayedHistorySessionService {
4300 inner: Arc<dyn MobSessionService>,
4301 delay: Duration,
4302 read_calls: Arc<AtomicUsize>,
4303 active_reads: Arc<AtomicUsize>,
4304 max_active_reads: Arc<AtomicUsize>,
4305 }
4306
4307 impl DelayedHistorySessionService {
4308 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4309 Self {
4310 inner,
4311 delay,
4312 read_calls: Arc::new(AtomicUsize::new(0)),
4313 active_reads: Arc::new(AtomicUsize::new(0)),
4314 max_active_reads: Arc::new(AtomicUsize::new(0)),
4315 }
4316 }
4317
4318 fn read_calls(&self) -> usize {
4319 self.read_calls.load(Ordering::SeqCst)
4320 }
4321
4322 fn max_active_reads(&self) -> usize {
4323 self.max_active_reads.load(Ordering::SeqCst)
4324 }
4325 }
4326
4327 #[async_trait::async_trait]
4328 impl SessionService for DelayedHistorySessionService {
4329 async fn create_session(
4330 &self,
4331 req: meerkat_core::CreateSessionRequest,
4332 ) -> Result<RunResult, SessionError> {
4333 self.inner.create_session(req).await
4334 }
4335
4336 async fn start_turn(
4337 &self,
4338 id: &SessionId,
4339 req: StartTurnRequest,
4340 ) -> Result<RunResult, SessionError> {
4341 self.inner.start_turn(id, req).await
4342 }
4343
4344 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4345 self.inner.interrupt(id).await
4346 }
4347
4348 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4349 self.inner.read(id).await
4350 }
4351
4352 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4353 self.inner.list(query).await
4354 }
4355
4356 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4357 self.inner.archive(id).await
4358 }
4359
4360 async fn subscribe_session_events(
4361 &self,
4362 id: &SessionId,
4363 ) -> Result<EventStream, StreamError> {
4364 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4365 }
4366 }
4367
4368 #[async_trait::async_trait]
4369 impl SessionServiceCommsExt for DelayedHistorySessionService {
4370 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4371 self.inner.comms_runtime(session_id).await
4372 }
4373 }
4374
4375 #[async_trait::async_trait]
4376 impl SessionServiceControlExt for DelayedHistorySessionService {
4377 async fn append_system_context(
4378 &self,
4379 id: &SessionId,
4380 req: AppendSystemContextRequest,
4381 ) -> Result<AppendSystemContextResult, SessionControlError> {
4382 self.inner.append_system_context(id, req).await
4383 }
4384 }
4385
4386 #[async_trait::async_trait]
4387 impl SessionServiceHistoryExt for DelayedHistorySessionService {
4388 async fn read_history(
4389 &self,
4390 id: &SessionId,
4391 query: SessionHistoryQuery,
4392 ) -> Result<SessionHistoryPage, SessionError> {
4393 self.read_calls.fetch_add(1, Ordering::SeqCst);
4394 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4395 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4396 tokio::time::sleep(self.delay).await;
4397 let result = self.inner.read_history(id, query).await;
4398 self.active_reads.fetch_sub(1, Ordering::SeqCst);
4399 result
4400 }
4401 }
4402
4403 #[async_trait::async_trait]
4404 impl MobSessionService for DelayedHistorySessionService {
4405 fn supports_persistent_sessions(&self) -> bool {
4406 self.inner.supports_persistent_sessions()
4407 }
4408
4409 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4410 self.inner.runtime_adapter()
4411 }
4412
4413 async fn session_belongs_to_mob(
4414 &self,
4415 session_id: &SessionId,
4416 mob_id: &meerkat_mob::MobId,
4417 ) -> bool {
4418 self.inner.session_belongs_to_mob(session_id, mob_id).await
4419 }
4420
4421 async fn cancel_all_checkpointers(&self) {
4422 self.inner.cancel_all_checkpointers().await;
4423 }
4424
4425 async fn rearm_all_checkpointers(&self) {
4426 self.inner.rearm_all_checkpointers().await;
4427 }
4428 }
4429
4430 #[async_trait::async_trait]
4431 impl ConsoleLogStore for CountingConsoleLogStore {
4432 async fn append_if_absent(
4433 &self,
4434 frame: NewConsoleFrame,
4435 ) -> ConsoleLogResult<AppendOutcome> {
4436 self.inner.append_if_absent(frame).await
4437 }
4438
4439 async fn update_frame_status(
4440 &self,
4441 frame_id: &str,
4442 status: ConsoleFrameStatus,
4443 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4444 self.inner.update_frame_status(frame_id, status).await
4445 }
4446
4447 async fn query_frames(
4448 &self,
4449 query: ConsoleTimelineQuery,
4450 ) -> ConsoleLogResult<ConsoleTimelinePage> {
4451 self.inner.query_frames(query).await
4452 }
4453
4454 async fn query_windowed_frames(
4455 &self,
4456 query: ConsoleTimelineWindowQuery,
4457 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4458 self.inner.query_windowed_frames(query).await
4459 }
4460
4461 async fn frame_by_dedupe_key(
4462 &self,
4463 dedupe_key: &str,
4464 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4465 self.inner.frame_by_dedupe_key(dedupe_key).await
4466 }
4467
4468 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4469 self.inner.latest_cursor().await
4470 }
4471
4472 async fn clear_frames(&self) -> ConsoleLogResult<()> {
4473 self.inner.clear_frames().await
4474 }
4475
4476 async fn record_source_watermark(
4477 &self,
4478 runtime_key: &str,
4479 source_kind: ConsoleFrameSourceKind,
4480 source_cursor: &str,
4481 ) -> ConsoleLogResult<()> {
4482 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
4483 self.inner
4484 .record_source_watermark(runtime_key, source_kind, source_cursor)
4485 .await
4486 }
4487
4488 async fn source_watermark(
4489 &self,
4490 runtime_key: &str,
4491 source_kind: ConsoleFrameSourceKind,
4492 ) -> ConsoleLogResult<Option<String>> {
4493 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
4494 self.inner.source_watermark(runtime_key, source_kind).await
4495 }
4496 }
4497
4498 async fn build_single_member_runtime() -> UnifiedRuntime {
4499 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
4500 }
4501
4502 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
4503 let definition = MobDefinition::from_toml(
4504 r#"
4505[mob]
4506id = "console-aggregator-perf-test"
4507
4508[profiles.worker]
4509model = "gpt-5.5"
4510external_addressable = true
4511
4512[profiles.worker.tools]
4513comms = true
4514"#,
4515 )
4516 .expect("definition parses");
4517 let runtime = UnifiedRuntime::builder()
4518 .definition(definition)
4519 .default_llm_client(client)
4520 .build()
4521 .await
4522 .expect("runtime builds");
4523 runtime
4524 .spawn(SpawnMemberSpec::from_wire(
4525 "worker".to_string(),
4526 "agent-a".to_string(),
4527 Some("You are agent-a.".into()),
4528 None,
4529 None,
4530 ))
4531 .await
4532 .expect("member spawns");
4533 runtime
4534 }
4535
4536 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
4537 let definition = MobDefinition::from_toml(&format!(
4538 r#"
4539[mob]
4540id = "{mob_id}"
4541
4542[profiles.worker]
4543model = "gpt-5.5"
4544external_addressable = true
4545
4546[profiles.worker.tools]
4547comms = true
4548"#
4549 ))
4550 .expect("definition parses");
4551 UnifiedRuntime::builder()
4552 .definition(definition)
4553 .default_llm_client(Arc::new(TestClient::default()))
4554 .build()
4555 .await
4556 .expect("runtime builds")
4557 }
4558
4559 async fn build_stress_runtime(
4560 member_count: usize,
4561 history_delay: Duration,
4562 ) -> (
4563 tempfile::TempDir,
4564 Arc<UnifiedRuntime>,
4565 DelayedHistorySessionService,
4566 ) {
4567 let temp_dir = tempfile::tempdir().expect("temp dir");
4568 let session_path = temp_dir.path().join("sessions");
4569 std::fs::create_dir_all(&session_path).expect("session path");
4570 let factory = AgentFactory::new(&session_path).comms(true);
4571 let base_service = Arc::new(build_ephemeral_service(
4572 factory,
4573 Config::default(),
4574 member_count + 8,
4575 ));
4576 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
4577 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
4578 let definition = MobDefinition::from_toml(
4579 r#"
4580[mob]
4581id = "console-aggregator-stress-test"
4582
4583[profiles.worker]
4584model = "gpt-5.5"
4585external_addressable = true
4586
4587[profiles.worker.tools]
4588comms = true
4589"#,
4590 )
4591 .expect("definition parses");
4592 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
4593 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
4594 allow_ephemeral_sessions: true,
4595 notify_orchestrator_on_resume: true,
4596 default_llm_client: Some(Arc::new(TestClient::default())),
4597 });
4598 let runtime = Arc::new(
4599 UnifiedRuntime::bootstrap(
4600 spec,
4601 crate::types::MobKitConfig {
4602 modules: Vec::new(),
4603 discovery: crate::types::DiscoverySpec {
4604 namespace: "stress".to_string(),
4605 modules: Vec::new(),
4606 },
4607 pre_spawn: Vec::new(),
4608 },
4609 Duration::from_secs(5),
4610 )
4611 .await
4612 .expect("runtime boots"),
4613 );
4614 for idx in 0..member_count {
4615 runtime
4616 .spawn(SpawnMemberSpec::from_wire(
4617 "worker".to_string(),
4618 format!("agent-{idx}"),
4619 Some(format!("You are agent-{idx}.").into()),
4620 None,
4621 None,
4622 ))
4623 .await
4624 .expect("member spawns");
4625 }
4626 (temp_dir, runtime, delayed_service)
4627 }
4628
4629 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
4630 RuntimeEntry {
4631 runtime_key: runtime_key.to_string(),
4632 identity_namespace: "test".to_string(),
4633 runtime: runtime.mob_runtime().clone(),
4634 identity_runtime: runtime.identity_runtime().cloned(),
4635 console_events: runtime.console_events(),
4636 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4637 }
4638 }
4639
4640 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
4641 ConsoleIdentityRecord {
4642 identity: identity.to_string(),
4643 display_name: identity.to_string(),
4644 runtime_key: "runtime-cache".to_string(),
4645 runtime_member_id: identity.to_string(),
4646 session_id: Some(format!("session-{identity}")),
4647 visibility: ConsoleVisibility::Addressable,
4648 addressable: true,
4649 health: "ready".to_string(),
4650 topology_peers: Vec::new(),
4651 labels: BTreeMap::new(),
4652 }
4653 }
4654
4655 #[test]
4656 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
4657 let mut durable = identity_record_for_test("test/review:singleton");
4658 durable.runtime_member_id = "rt:review:singleton:1".to_string();
4659 durable.session_id = Some("session-stale".to_string());
4660 durable
4661 .labels
4662 .insert("agent_identity".to_string(), "review:singleton".to_string());
4663
4664 let mut live = identity_record_for_test("test/review:singleton");
4665 live.runtime_member_id = "rt:review:singleton:0".to_string();
4666 live.session_id = Some("session-live".to_string());
4667 live.labels
4668 .insert("agent_identity".to_string(), "review:singleton".to_string());
4669
4670 let records = dedupe_identity_records(vec![durable, live]);
4671
4672 assert_eq!(
4673 records.len(),
4674 2,
4675 "list projection must expose stale durable/live alias split-brain instead of picking one record"
4676 );
4677 assert!(
4678 records
4679 .iter()
4680 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
4681 );
4682 assert!(
4683 records
4684 .iter()
4685 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
4686 );
4687 }
4688
4689 #[test]
4690 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
4691 let mut first = identity_record_for_test("test/review:singleton");
4692 first.runtime_member_id = "rt:review:singleton:0".to_string();
4693 first
4694 .labels
4695 .insert("agent_identity".to_string(), "review:singleton".to_string());
4696
4697 let mut second = identity_record_for_test("test/review:singleton");
4698 second.runtime_member_id = "rt:review:singleton:1".to_string();
4699 second
4700 .labels
4701 .insert("agent_identity".to_string(), "review:singleton".to_string());
4702
4703 let records = dedupe_identity_records(vec![first, second]);
4704
4705 assert_eq!(
4706 records.len(),
4707 2,
4708 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
4709 );
4710 }
4711
4712 #[test]
4713 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
4714 let mut first = identity_record_for_test("test/review:singleton");
4715 first.runtime_member_id = "rt:review:singleton:0".to_string();
4716 first.session_id = None;
4717 first
4718 .labels
4719 .insert("agent_identity".to_string(), "review:singleton".to_string());
4720 first
4721 .labels
4722 .insert("source_mob_id".to_string(), "mob-a".to_string());
4723
4724 let mut second = first.clone();
4725 second
4726 .labels
4727 .insert("source_mob_id".to_string(), "mob-b".to_string());
4728
4729 let records = dedupe_identity_records(vec![first, second]);
4730
4731 assert_eq!(
4732 records.len(),
4733 2,
4734 "same member/session aliases from different source mobs must remain visible"
4735 );
4736 }
4737
4738 #[test]
4739 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
4740 let durable = identity_record_for_test("test/review:singleton");
4741 let live = durable.clone();
4742
4743 let records = dedupe_identity_records(vec![durable, live]);
4744
4745 assert_eq!(
4746 records.len(),
4747 1,
4748 "healthy durable and primary live projections should not duplicate the roster"
4749 );
4750 }
4751
4752 #[test]
4753 fn stale_durable_record_error_detects_session_split_brain() {
4754 let mut durable = identity_record_for_test("test/review:singleton");
4755 durable.runtime_member_id = "rt:review:singleton:0".to_string();
4756 durable.session_id = Some("session-durable".to_string());
4757
4758 let mut live = durable.clone();
4759 live.session_id = Some("session-live".to_string());
4760
4761 let err = stale_durable_record_error("review:singleton", &durable, &[live])
4762 .expect("session mismatch must be stale split-brain");
4763
4764 assert!(
4765 err.contains("stale durable identity alias"),
4766 "unexpected error: {err}"
4767 );
4768 assert!(
4769 err.contains("session-durable") && err.contains("session-live"),
4770 "error should name both sessions: {err}"
4771 );
4772 }
4773
4774 async fn identity_runtime_for_test(
4775 identities: &[&str],
4776 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
4777 {
4778 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
4779 crate::identity_first::IdentityRuntimeConfig {
4780 continuity_store: Arc::new(
4781 crate::identity_first::LocalContinuityStore::in_memory()?
4782 ),
4783 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
4784 runtime_instance_id: "console-aggregator-identity-test".to_string(),
4785 has_runtime_store: true,
4786 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
4787 bridge: None,
4788 default_timeout: None,
4789 },
4790 ));
4791 for identity in identities {
4792 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
4793 let record = crate::identity_first::ContinuityRecord {
4794 identity: identity.clone(),
4795 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
4796 "rt:{}:0",
4797 identity.as_str()
4798 ))?,
4799 session_id: SessionId::new(),
4800 generation: crate::identity_first::ContinuityGeneration::new(0),
4801 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
4802 };
4803 runtime
4804 .register(
4805 crate::identity_first::DurableAgentSpec {
4806 identity: identity.clone(),
4807 profile: meerkat_mob::ProfileName::from("worker"),
4808 addressability: crate::identity_first::AgentAddressability::Addressable,
4809 display_name: None,
4810 labels: BTreeMap::new(),
4811 context: None,
4812 additional_instructions: Vec::new(),
4813 initial_message: None,
4814 runtime_mode_override: None,
4815 },
4816 crate::identity_first::IdentityLifecycleState::Active,
4817 Some(record),
4818 Some(crate::identity_first::LeaseGrant {
4819 identity,
4820 fencing_token: crate::identity_first::FencingToken::new(1),
4821 ttl: Duration::from_mins(5),
4822 }),
4823 )
4824 .await;
4825 }
4826 Ok(runtime)
4827 }
4828
4829 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4830 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
4831 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4832 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
4833 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
4834 let aggregator = MobKitConsoleAggregator::in_memory();
4835 aggregator.register_runtime_handles_with_policy(
4836 "runtime-a",
4837 "",
4838 runtime.mob_runtime().clone(),
4839 Some(identity_runtime),
4840 runtime.console_events(),
4841 Arc::new(AllowAllConsoleVisibilityPolicy),
4842 );
4843
4844 assert!(
4845 aggregator
4846 .inspect_identity("rt:review:singleton:0")
4847 .await?
4848 .is_none(),
4849 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
4850 );
4851 assert!(
4852 !aggregator.retire_identity("rt:review:singleton:0").await?,
4853 "runtime-member-id shaped requests must not retire a durable rt:* identity"
4854 );
4855 let reserve_err = aggregator
4856 .reserve_identity_first_interaction(
4857 ConsoleSendRequest {
4858 identity: "rt:review:singleton:0".to_string(),
4859 content: json!("hello"),
4860 origin: "console:test".to_string(),
4861 idempotency_key: "rt-shadow-reserve".to_string(),
4862 handling_mode: Some("queue".to_string()),
4863 },
4864 None,
4865 )
4866 .await
4867 .expect_err(
4868 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
4869 );
4870 assert!(
4871 reserve_err.to_string().contains("unknown identity"),
4872 "unexpected reserve error: {reserve_err}"
4873 );
4874 let send_err = aggregator
4875 .send(ConsoleSendRequest {
4876 identity: "rt:review:singleton:0".to_string(),
4877 content: json!("hello"),
4878 origin: "console:test".to_string(),
4879 idempotency_key: "rt-shadow-send".to_string(),
4880 handling_mode: Some("queue".to_string()),
4881 })
4882 .await
4883 .expect_err(
4884 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
4885 );
4886 assert!(
4887 send_err.to_string().contains("unknown identity"),
4888 "unexpected send error: {send_err}"
4889 );
4890 let blob_err = match aggregator
4891 .binary_blob_store_for_identity("rt:review:singleton:0")
4892 .await
4893 {
4894 Ok(_) => panic!(
4895 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
4896 ),
4897 Err(err) => err,
4898 };
4899 assert!(
4900 blob_err.to_string().contains("unknown identity"),
4901 "unexpected blob error: {blob_err}"
4902 );
4903
4904 let _ = runtime.mob_handle().stop().await;
4905 Ok(())
4906 }
4907
4908 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4909 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
4910 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4911 let runtime = build_empty_runtime("identity-label-test").await;
4912 let mut labels = BTreeMap::new();
4913 labels.insert(
4914 "agent_identity".to_string(),
4915 "channel:C0SMOKEOB3".to_string(),
4916 );
4917 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
4918 runtime
4919 .spawn(
4920 SpawnMemberSpec::from_wire(
4921 "worker".to_string(),
4922 "rt:channel:C0SMOKEOB3:0".to_string(),
4923 Some("You are C0SMOKEOB3.".into()),
4924 None,
4925 None,
4926 )
4927 .with_labels(labels),
4928 )
4929 .await
4930 .expect("member spawns");
4931
4932 let entry = RuntimeEntry {
4933 runtime_key: "runtime-a".to_string(),
4934 identity_namespace: String::new(),
4935 runtime: runtime.mob_runtime().clone(),
4936 identity_runtime: runtime.identity_runtime().cloned(),
4937 console_events: runtime.console_events(),
4938 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4939 };
4940 let aggregator = MobKitConsoleAggregator::in_memory();
4941 aggregator
4942 .inner
4943 .runtimes
4944 .write()
4945 .expect("runtime registry")
4946 .insert("runtime-a".to_string(), entry);
4947
4948 let records = aggregator.list_identities_fresh().await?;
4949 let record = records
4950 .iter()
4951 .find(|record| record.identity == "channel:C0SMOKEOB3")
4952 .expect("durable identity is exposed");
4953 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
4954
4955 let inspection = aggregator
4956 .inspect_identity("channel:C0SMOKEOB3")
4957 .await?
4958 .expect("durable identity resolves back to runtime member");
4959 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
4960 assert_eq!(
4961 inspection.identity.runtime_member_id,
4962 "rt:channel:C0SMOKEOB3:0"
4963 );
4964
4965 let _ = runtime.mob_handle().stop().await;
4966 Ok(())
4967 }
4968
4969 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4970 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
4971 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4972 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
4973 let mut labels = BTreeMap::new();
4974 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4975 runtime
4976 .spawn(
4977 SpawnMemberSpec::from_wire(
4978 "worker".to_string(),
4979 "rt:review:singleton:0".to_string(),
4980 Some("You are the Review Agent.".into()),
4981 None,
4982 None,
4983 )
4984 .with_labels(labels),
4985 )
4986 .await?;
4987 let hidden_session_id = runtime
4988 .mob_handle()
4989 .resolve_bridge_session_id_observation(&meerkat_mob::ids::MeerkatId::from(
4990 "rt:review:singleton:0",
4991 ))
4992 .await
4993 .map(|sid| sid.to_string());
4994 let mut duplicate_labels = BTreeMap::new();
4995 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4996 runtime
4997 .spawn(
4998 SpawnMemberSpec::from_wire(
4999 "worker".to_string(),
5000 "rt:review:singleton:1".to_string(),
5001 Some("You are a visible duplicate Review Agent.".into()),
5002 None,
5003 None,
5004 )
5005 .with_labels(duplicate_labels),
5006 )
5007 .await?;
5008
5009 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5010 let aggregator = MobKitConsoleAggregator::in_memory();
5011 aggregator.register_runtime_handles_with_policy(
5012 "runtime-a",
5013 "",
5014 runtime.mob_runtime().clone(),
5015 Some(identity_runtime),
5016 runtime.console_events(),
5017 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5018 );
5019
5020 assert!(
5021 aggregator
5022 .inspect_identity("review:singleton")
5023 .await?
5024 .is_none(),
5025 "member-hidden live alias must not inspect through aggregator"
5026 );
5027 let records = aggregator.list_identities_fresh().await?;
5028 assert!(
5029 records
5030 .iter()
5031 .all(|record| record.identity != "review:singleton"),
5032 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5033 );
5034 let appended = aggregator
5035 .store()
5036 .append_if_absent(NewConsoleFrame {
5037 id: None,
5038 dedupe_key: "member-hidden-frame".to_string(),
5039 timestamp_ms: 1,
5040 runtime_key: "runtime-a".to_string(),
5041 identity: "review:singleton".to_string(),
5042 conversation_id: Some("review:singleton".to_string()),
5043 session_id: hidden_session_id,
5044 kind: "text_delta".to_string(),
5045 status: ConsoleFrameStatus::Delivered,
5046 payload: json!({ "delta": "hidden" }),
5047 source: ConsoleFrameSource {
5048 kind: ConsoleFrameSourceKind::ConsoleEvent,
5049 source_cursor: None,
5050 },
5051 source_event_id: Some("member-hidden-frame".to_string()),
5052 interaction_id: None,
5053 turn_id: None,
5054 run_id: None,
5055 parent_frame_id: None,
5056 caused_by_frame_id: None,
5057 })
5058 .await?;
5059 assert!(
5060 !aggregator
5061 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5062 frame: appended.frame.clone()
5063 })
5064 .await,
5065 "member-hidden live alias must not pass timeline event visibility"
5066 );
5067 let identity_only = aggregator
5068 .store()
5069 .append_if_absent(NewConsoleFrame {
5070 id: None,
5071 dedupe_key: "member-hidden-identity-only-frame".to_string(),
5072 timestamp_ms: 2,
5073 runtime_key: "runtime-a".to_string(),
5074 identity: "review:singleton".to_string(),
5075 conversation_id: Some("review:singleton".to_string()),
5076 session_id: None,
5077 kind: "text_delta".to_string(),
5078 status: ConsoleFrameStatus::Delivered,
5079 payload: json!({ "delta": "identity-only-hidden" }),
5080 source: ConsoleFrameSource {
5081 kind: ConsoleFrameSourceKind::ConsoleEvent,
5082 source_cursor: None,
5083 },
5084 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5085 interaction_id: None,
5086 turn_id: None,
5087 run_id: None,
5088 parent_frame_id: None,
5089 caused_by_frame_id: None,
5090 })
5091 .await?;
5092 assert!(
5093 !aggregator
5094 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5095 frame: identity_only.frame.clone()
5096 })
5097 .await,
5098 "member-hidden durable alias must not leak identity-only timeline frames"
5099 );
5100 let page = aggregator
5101 .query_timeline(ConsoleTimelineQuery {
5102 identity: Some("review:singleton".to_string()),
5103 limit: 10,
5104 ..ConsoleTimelineQuery::default()
5105 })
5106 .await?;
5107 assert!(
5108 page.frames.is_empty(),
5109 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5110 );
5111 assert!(
5112 !aggregator.retire_identity("review:singleton").await?,
5113 "member-hidden live alias must not retire through aggregator"
5114 );
5115 let reserve_err = aggregator
5116 .reserve_identity_first_interaction(
5117 ConsoleSendRequest {
5118 identity: "review:singleton".to_string(),
5119 content: json!("hello"),
5120 origin: "console:test".to_string(),
5121 idempotency_key: "member-hidden-reserve".to_string(),
5122 handling_mode: Some("queue".to_string()),
5123 },
5124 None,
5125 )
5126 .await
5127 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5128 assert!(
5129 reserve_err.to_string().contains("unknown identity"),
5130 "unexpected reserve error: {reserve_err}"
5131 );
5132 let send_err = aggregator
5133 .send(ConsoleSendRequest {
5134 identity: "review:singleton".to_string(),
5135 content: json!("hello"),
5136 origin: "console:test".to_string(),
5137 idempotency_key: "member-hidden-send".to_string(),
5138 handling_mode: Some("queue".to_string()),
5139 })
5140 .await
5141 .expect_err("member-hidden live alias must not receive sends");
5142 assert!(
5143 send_err.to_string().contains("unknown identity"),
5144 "unexpected send error: {send_err}"
5145 );
5146 let blob_result = aggregator
5147 .binary_blob_store_for_identity("review:singleton")
5148 .await;
5149 let Err(blob_err) = blob_result else {
5150 panic!("member-hidden live alias must not expose blob store");
5151 };
5152 assert!(
5153 blob_err.to_string().contains("unknown identity"),
5154 "unexpected blob error: {blob_err}"
5155 );
5156
5157 let _ = runtime.mob_handle().stop().await;
5158 Ok(())
5159 }
5160
5161 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5162 async fn explicit_session_backfill_target_resolves_durable_identity_label()
5163 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5164 let runtime = build_empty_runtime("identity-label-backfill-test").await;
5165 let mut labels = BTreeMap::new();
5166 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5167 runtime
5168 .spawn(
5169 SpawnMemberSpec::from_wire(
5170 "worker".to_string(),
5171 "rt:review:singleton:0".to_string(),
5172 Some("You are the Review Agent.".into()),
5173 None,
5174 None,
5175 )
5176 .with_labels(labels),
5177 )
5178 .await
5179 .expect("member spawns");
5180
5181 let entry = RuntimeEntry {
5182 runtime_key: "runtime-a".to_string(),
5183 identity_namespace: String::new(),
5184 runtime: runtime.mob_runtime().clone(),
5185 identity_runtime: runtime.identity_runtime().cloned(),
5186 console_events: runtime.console_events(),
5187 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5188 };
5189 let aggregator = MobKitConsoleAggregator::in_memory();
5190 aggregator
5191 .inner
5192 .runtimes
5193 .write()
5194 .expect("runtime registry")
5195 .insert("runtime-a".to_string(), entry);
5196
5197 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5198 .await
5199 .expect("durable label should resolve targeted session backfill");
5200 assert_eq!(target.record.identity, "review:singleton");
5201 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5202
5203 let _ = runtime.mob_handle().stop().await;
5204 Ok(())
5205 }
5206
5207 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5208 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5209 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5210 let runtime = build_empty_runtime("identity-stale-durable-test").await;
5211 let mut labels = BTreeMap::new();
5212 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5213 runtime
5214 .spawn(
5215 SpawnMemberSpec::from_wire(
5216 "worker".to_string(),
5217 "rt:review:singleton:0".to_string(),
5218 Some("You are the live Review Agent.".into()),
5219 None,
5220 None,
5221 )
5222 .with_labels(labels),
5223 )
5224 .await
5225 .expect("member spawns");
5226
5227 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5228 crate::identity_first::IdentityRuntimeConfig {
5229 continuity_store: Arc::new(
5230 crate::identity_first::LocalContinuityStore::in_memory()?
5231 ),
5232 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5233 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5234 has_runtime_store: true,
5235 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5236 bridge: None,
5237 default_timeout: None,
5238 },
5239 ));
5240 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5241 let record = crate::identity_first::ContinuityRecord {
5242 identity: identity.clone(),
5243 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5244 "rt:review:singleton:1",
5245 )?,
5246 session_id: SessionId::new(),
5247 generation: crate::identity_first::ContinuityGeneration::new(1),
5248 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5249 };
5250 identity_runtime
5251 .register(
5252 crate::identity_first::DurableAgentSpec {
5253 identity: identity.clone(),
5254 profile: meerkat_mob::ProfileName::from("worker"),
5255 addressability: crate::identity_first::AgentAddressability::Addressable,
5256 display_name: None,
5257 labels: BTreeMap::new(),
5258 context: None,
5259 additional_instructions: Vec::new(),
5260 initial_message: None,
5261 runtime_mode_override: None,
5262 },
5263 crate::identity_first::IdentityLifecycleState::Active,
5264 Some(record),
5265 Some(crate::identity_first::LeaseGrant {
5266 identity,
5267 fencing_token: crate::identity_first::FencingToken::new(7),
5268 ttl: Duration::from_mins(5),
5269 }),
5270 )
5271 .await;
5272
5273 let aggregator = MobKitConsoleAggregator::in_memory();
5274 aggregator
5275 .inner
5276 .runtimes
5277 .write()
5278 .expect("runtime registry")
5279 .insert(
5280 "runtime-a".to_string(),
5281 RuntimeEntry {
5282 runtime_key: "runtime-a".to_string(),
5283 identity_namespace: String::new(),
5284 runtime: runtime.mob_runtime().clone(),
5285 identity_runtime: Some(identity_runtime),
5286 console_events: runtime.console_events(),
5287 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5288 },
5289 );
5290
5291 let err = aggregator
5292 .inspect_identity("review:singleton")
5293 .await
5294 .expect_err("stale durable binding must not inspect as healthy");
5295 assert!(
5296 err.to_string().contains("stale durable identity alias"),
5297 "unexpected error: {err}"
5298 );
5299 let retire_err = aggregator
5300 .retire_identity("review:singleton")
5301 .await
5302 .expect_err("stale durable binding must not retire as healthy");
5303 assert!(
5304 retire_err
5305 .to_string()
5306 .contains("stale durable identity alias"),
5307 "unexpected retire error: {retire_err}"
5308 );
5309 let send_err = aggregator
5310 .send(ConsoleSendRequest {
5311 identity: "review:singleton".to_string(),
5312 content: json!("hello"),
5313 origin: "console:test".to_string(),
5314 idempotency_key: "stale-durable-send".to_string(),
5315 handling_mode: Some("queue".to_string()),
5316 })
5317 .await
5318 .expect_err("stale durable binding must not send to the wrong live member");
5319 assert!(
5320 send_err
5321 .to_string()
5322 .contains("stale durable identity alias"),
5323 "unexpected send error: {send_err}"
5324 );
5325 let reserve_err = aggregator
5326 .reserve_identity_first_interaction(
5327 ConsoleSendRequest {
5328 identity: "review:singleton".to_string(),
5329 content: json!("hello"),
5330 origin: "console:test".to_string(),
5331 idempotency_key: "stale-durable-reserve".to_string(),
5332 handling_mode: Some("queue".to_string()),
5333 },
5334 None,
5335 )
5336 .await
5337 .expect_err("identity-first reserve must not bypass stale durable binding");
5338 assert!(
5339 reserve_err
5340 .to_string()
5341 .contains("stale durable identity alias"),
5342 "unexpected reserve error: {reserve_err}"
5343 );
5344 let blob_err = match aggregator
5345 .binary_blob_store_for_identity("review:singleton")
5346 .await
5347 {
5348 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5349 Err(err) => err,
5350 };
5351 assert!(
5352 blob_err
5353 .to_string()
5354 .contains("stale durable identity alias"),
5355 "unexpected blob error: {blob_err}"
5356 );
5357 let records = aggregator.list_identities_fresh().await?;
5358 assert!(
5359 records
5360 .iter()
5361 .any(|record| record.identity == "review:singleton"
5362 && record.runtime_member_id == "rt:review:singleton:0"),
5363 "live alias should still be listed; records: {records:#?}"
5364 );
5365 assert!(
5366 records
5367 .iter()
5368 .any(|record| record.identity == "review:singleton"
5369 && record.runtime_member_id == "rt:review:singleton:1"),
5370 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5371 );
5372
5373 let _ = runtime.mob_handle().stop().await;
5374 Ok(())
5375 }
5376
5377 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5378 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
5379 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5380 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
5381 let mut labels = BTreeMap::new();
5382 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5383 runtime
5384 .spawn(
5385 SpawnMemberSpec::from_wire(
5386 "worker".to_string(),
5387 "rt:review:singleton:0".to_string(),
5388 Some("You are the mislabeled Review Agent.".into()),
5389 None,
5390 None,
5391 )
5392 .with_labels(labels),
5393 )
5394 .await
5395 .expect("member spawns");
5396
5397 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5398 crate::identity_first::IdentityRuntimeConfig {
5399 continuity_store: Arc::new(
5400 crate::identity_first::LocalContinuityStore::in_memory()?
5401 ),
5402 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5403 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
5404 has_runtime_store: true,
5405 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5406 bridge: None,
5407 default_timeout: None,
5408 },
5409 ));
5410 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5411 let record = crate::identity_first::ContinuityRecord {
5412 identity: identity.clone(),
5413 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5414 "rt:review:singleton:0",
5415 )?,
5416 session_id: SessionId::new(),
5417 generation: crate::identity_first::ContinuityGeneration::new(1),
5418 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5419 };
5420 identity_runtime
5421 .register(
5422 crate::identity_first::DurableAgentSpec {
5423 identity: identity.clone(),
5424 profile: meerkat_mob::ProfileName::from("worker"),
5425 addressability: crate::identity_first::AgentAddressability::Addressable,
5426 display_name: None,
5427 labels: BTreeMap::new(),
5428 context: None,
5429 additional_instructions: Vec::new(),
5430 initial_message: None,
5431 runtime_mode_override: None,
5432 },
5433 crate::identity_first::IdentityLifecycleState::Active,
5434 Some(record),
5435 Some(crate::identity_first::LeaseGrant {
5436 identity,
5437 fencing_token: crate::identity_first::FencingToken::new(7),
5438 ttl: Duration::from_mins(5),
5439 }),
5440 )
5441 .await;
5442
5443 let aggregator = MobKitConsoleAggregator::in_memory();
5444 aggregator
5445 .inner
5446 .runtimes
5447 .write()
5448 .expect("runtime registry")
5449 .insert(
5450 "runtime-a".to_string(),
5451 RuntimeEntry {
5452 runtime_key: "runtime-a".to_string(),
5453 identity_namespace: String::new(),
5454 runtime: runtime.mob_runtime().clone(),
5455 identity_runtime: Some(identity_runtime),
5456 console_events: runtime.console_events(),
5457 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5458 },
5459 );
5460
5461 let err = aggregator
5462 .inspect_identity("review:singleton")
5463 .await
5464 .expect_err("wrong projected live identity must not inspect as healthy");
5465 assert!(
5466 err.to_string()
5467 .contains("projects identity other:singleton"),
5468 "unexpected error: {err}"
5469 );
5470 let retire_err = aggregator
5471 .retire_identity("review:singleton")
5472 .await
5473 .expect_err("wrong projected live identity must not retire as healthy");
5474 assert!(
5475 retire_err
5476 .to_string()
5477 .contains("projects identity other:singleton"),
5478 "unexpected retire error: {retire_err}"
5479 );
5480 let send_err = aggregator
5481 .send(ConsoleSendRequest {
5482 identity: "review:singleton".to_string(),
5483 content: json!("hello"),
5484 origin: "console:test".to_string(),
5485 idempotency_key: "wrong-projection-send".to_string(),
5486 handling_mode: Some("queue".to_string()),
5487 })
5488 .await
5489 .expect_err("wrong projected live identity must not send as unknown");
5490 assert!(
5491 send_err
5492 .to_string()
5493 .contains("projects identity other:singleton"),
5494 "unexpected send error: {send_err}"
5495 );
5496 let reserve_err = aggregator
5497 .reserve_identity_first_interaction(
5498 ConsoleSendRequest {
5499 identity: "review:singleton".to_string(),
5500 content: json!("hello"),
5501 origin: "console:test".to_string(),
5502 idempotency_key: "wrong-projection-reserve".to_string(),
5503 handling_mode: Some("queue".to_string()),
5504 },
5505 None,
5506 )
5507 .await
5508 .expect_err("identity-first reserve must not bypass wrong live projection");
5509 assert!(
5510 reserve_err
5511 .to_string()
5512 .contains("projects identity other:singleton"),
5513 "unexpected reserve error: {reserve_err}"
5514 );
5515 let blob_err = match aggregator
5516 .binary_blob_store_for_identity("review:singleton")
5517 .await
5518 {
5519 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
5520 Err(err) => err,
5521 };
5522 assert!(
5523 blob_err
5524 .to_string()
5525 .contains("projects identity other:singleton"),
5526 "unexpected blob error: {blob_err}"
5527 );
5528 let projected_err = aggregator
5529 .inspect_identity("other:singleton")
5530 .await
5531 .expect_err("wrong projected alias must not inspect through live-only fallback");
5532 assert!(
5533 projected_err
5534 .to_string()
5535 .contains("identity runtime binding for review:singleton"),
5536 "unexpected projected-alias error: {projected_err}"
5537 );
5538 let projected_retire_err = aggregator
5539 .retire_identity("other:singleton")
5540 .await
5541 .expect_err("wrong projected alias must not retire through live-only fallback");
5542 assert!(
5543 projected_retire_err
5544 .to_string()
5545 .contains("identity runtime binding for review:singleton"),
5546 "unexpected projected-alias retire error: {projected_retire_err}"
5547 );
5548
5549 let _ = runtime.mob_handle().stop().await;
5550 Ok(())
5551 }
5552
5553 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5554 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
5555 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5556 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
5557 let mut labels = BTreeMap::new();
5558 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5559 runtime
5560 .spawn(
5561 SpawnMemberSpec::from_wire(
5562 "worker".to_string(),
5563 "rt:review:singleton:0".to_string(),
5564 Some("You are a hidden wrong projection.".into()),
5565 None,
5566 None,
5567 )
5568 .with_labels(labels),
5569 )
5570 .await
5571 .expect("member spawns");
5572
5573 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5574 crate::identity_first::IdentityRuntimeConfig {
5575 continuity_store: Arc::new(
5576 crate::identity_first::LocalContinuityStore::in_memory()?
5577 ),
5578 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5579 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
5580 has_runtime_store: true,
5581 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5582 bridge: None,
5583 default_timeout: None,
5584 },
5585 ));
5586 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5587 identity_runtime
5588 .register(
5589 crate::identity_first::DurableAgentSpec {
5590 identity: identity.clone(),
5591 profile: meerkat_mob::ProfileName::from("worker"),
5592 addressability: crate::identity_first::AgentAddressability::Addressable,
5593 display_name: None,
5594 labels: BTreeMap::new(),
5595 context: None,
5596 additional_instructions: Vec::new(),
5597 initial_message: None,
5598 runtime_mode_override: None,
5599 },
5600 crate::identity_first::IdentityLifecycleState::Active,
5601 Some(crate::identity_first::ContinuityRecord {
5602 identity: identity.clone(),
5603 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5604 "rt:review:singleton:0",
5605 )?,
5606 session_id: SessionId::new(),
5607 generation: crate::identity_first::ContinuityGeneration::new(1),
5608 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5609 }),
5610 Some(crate::identity_first::LeaseGrant {
5611 identity,
5612 fencing_token: crate::identity_first::FencingToken::new(7),
5613 ttl: Duration::from_mins(5),
5614 }),
5615 )
5616 .await;
5617
5618 let aggregator = MobKitConsoleAggregator::in_memory();
5619 aggregator
5620 .inner
5621 .runtimes
5622 .write()
5623 .expect("runtime registry")
5624 .insert(
5625 "runtime-a".to_string(),
5626 RuntimeEntry {
5627 runtime_key: "runtime-a".to_string(),
5628 identity_namespace: String::new(),
5629 runtime: runtime.mob_runtime().clone(),
5630 identity_runtime: Some(identity_runtime),
5631 console_events: runtime.console_events(),
5632 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
5633 },
5634 );
5635
5636 let err = aggregator
5637 .inspect_identity("review:singleton")
5638 .await
5639 .expect_err("hidden wrong projection must still fail durable binding validation");
5640 assert!(
5641 err.to_string()
5642 .contains("projects identity other:singleton"),
5643 "unexpected hidden wrong-projection error: {err}"
5644 );
5645 let reserve_err = aggregator
5646 .reserve_identity_first_interaction(
5647 ConsoleSendRequest {
5648 identity: "review:singleton".to_string(),
5649 content: json!("hello"),
5650 origin: "console:test".to_string(),
5651 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
5652 handling_mode: Some("queue".to_string()),
5653 },
5654 None,
5655 )
5656 .await
5657 .expect_err("reserve must not hide wrong projection from durable validation");
5658 assert!(
5659 reserve_err
5660 .to_string()
5661 .contains("projects identity other:singleton"),
5662 "unexpected reserve error: {reserve_err}"
5663 );
5664
5665 let _ = runtime.mob_handle().stop().await;
5666 Ok(())
5667 }
5668
5669 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5670 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
5671 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5672 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
5673 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
5674 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5675 crate::identity_first::IdentityRuntimeConfig {
5676 continuity_store: Arc::new(
5677 crate::identity_first::LocalContinuityStore::in_memory()?
5678 ),
5679 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5680 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
5681 has_runtime_store: true,
5682 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5683 bridge: None,
5684 default_timeout: None,
5685 },
5686 ));
5687 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5688 crate::identity_first::IdentityRuntimeConfig {
5689 continuity_store: Arc::new(
5690 crate::identity_first::LocalContinuityStore::in_memory()?
5691 ),
5692 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5693 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
5694 has_runtime_store: true,
5695 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5696 bridge: None,
5697 default_timeout: None,
5698 },
5699 ));
5700 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
5701 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5702 identity_runtime
5703 .register(
5704 crate::identity_first::DurableAgentSpec {
5705 identity: identity.clone(),
5706 profile: meerkat_mob::ProfileName::from("worker"),
5707 addressability: crate::identity_first::AgentAddressability::Addressable,
5708 display_name: None,
5709 labels: BTreeMap::new(),
5710 context: None,
5711 additional_instructions: Vec::new(),
5712 initial_message: None,
5713 runtime_mode_override: None,
5714 },
5715 crate::identity_first::IdentityLifecycleState::Active,
5716 Some(crate::identity_first::ContinuityRecord {
5717 identity: identity.clone(),
5718 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5719 "rt:review:singleton:0",
5720 )?,
5721 session_id: SessionId::new(),
5722 generation: crate::identity_first::ContinuityGeneration::new(1),
5723 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5724 }),
5725 Some(crate::identity_first::LeaseGrant {
5726 identity,
5727 fencing_token: crate::identity_first::FencingToken::new(7),
5728 ttl: Duration::from_mins(5),
5729 }),
5730 )
5731 .await;
5732 }
5733
5734 let aggregator = MobKitConsoleAggregator::in_memory();
5735 {
5736 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
5737 runtimes.insert(
5738 "a-hidden".to_string(),
5739 RuntimeEntry {
5740 runtime_key: "a-hidden".to_string(),
5741 identity_namespace: String::new(),
5742 runtime: hidden_runtime.mob_runtime().clone(),
5743 identity_runtime: Some(hidden_identity_runtime),
5744 console_events: hidden_runtime.console_events(),
5745 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
5746 },
5747 );
5748 runtimes.insert(
5749 "b-visible".to_string(),
5750 RuntimeEntry {
5751 runtime_key: "b-visible".to_string(),
5752 identity_namespace: String::new(),
5753 runtime: visible_runtime.mob_runtime().clone(),
5754 identity_runtime: Some(visible_identity_runtime),
5755 console_events: visible_runtime.console_events(),
5756 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5757 },
5758 );
5759 }
5760
5761 let inspection = aggregator
5762 .inspect_identity("rt:review:singleton:0")
5763 .await?
5764 .expect("visible runtime-id match should be returned");
5765 assert_eq!(inspection.identity.runtime_key, "b-visible");
5766
5767 let _ = hidden_runtime.mob_handle().stop().await;
5768 let _ = visible_runtime.mob_handle().stop().await;
5769 Ok(())
5770 }
5771
5772 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5773 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
5774 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5775 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
5776 let mut primary_labels = BTreeMap::new();
5777 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5778 runtime
5779 .spawn(
5780 SpawnMemberSpec::from_wire(
5781 "worker".to_string(),
5782 "rt:review:singleton:0".to_string(),
5783 Some("You are the primary Review Agent.".into()),
5784 None,
5785 None,
5786 )
5787 .with_labels(primary_labels),
5788 )
5789 .await
5790 .expect("primary member spawns");
5791
5792 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
5793 let mut delegate_labels = BTreeMap::new();
5794 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5795 delegate_runtime
5796 .spawn(
5797 SpawnMemberSpec::from_wire(
5798 "worker".to_string(),
5799 "rt:review:singleton:0".to_string(),
5800 Some("You are the wrong projected Review Agent.".into()),
5801 None,
5802 None,
5803 )
5804 .with_labels(delegate_labels),
5805 )
5806 .await
5807 .expect("delegate member spawns");
5808 runtime
5809 .mob_runtime()
5810 .agent_mob_mcp_state()
5811 .expect("runtime should expose mob MCP state")
5812 .mob_insert_handle(
5813 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
5814 delegate_runtime.mob_handle().clone(),
5815 )
5816 .await;
5817
5818 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5819 crate::identity_first::IdentityRuntimeConfig {
5820 continuity_store: Arc::new(
5821 crate::identity_first::LocalContinuityStore::in_memory()?
5822 ),
5823 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5824 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
5825 has_runtime_store: true,
5826 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5827 bridge: None,
5828 default_timeout: None,
5829 },
5830 ));
5831 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5832 let record = crate::identity_first::ContinuityRecord {
5833 identity: identity.clone(),
5834 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5835 "rt:review:singleton:0",
5836 )?,
5837 session_id: SessionId::new(),
5838 generation: crate::identity_first::ContinuityGeneration::new(1),
5839 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5840 };
5841 identity_runtime
5842 .register(
5843 crate::identity_first::DurableAgentSpec {
5844 identity: identity.clone(),
5845 profile: meerkat_mob::ProfileName::from("worker"),
5846 addressability: crate::identity_first::AgentAddressability::Addressable,
5847 display_name: None,
5848 labels: BTreeMap::new(),
5849 context: None,
5850 additional_instructions: Vec::new(),
5851 initial_message: None,
5852 runtime_mode_override: None,
5853 },
5854 crate::identity_first::IdentityLifecycleState::Active,
5855 Some(record),
5856 Some(crate::identity_first::LeaseGrant {
5857 identity,
5858 fencing_token: crate::identity_first::FencingToken::new(7),
5859 ttl: Duration::from_mins(5),
5860 }),
5861 )
5862 .await;
5863
5864 let aggregator = MobKitConsoleAggregator::in_memory();
5865 aggregator
5866 .inner
5867 .runtimes
5868 .write()
5869 .expect("runtime registry")
5870 .insert(
5871 "runtime-a".to_string(),
5872 RuntimeEntry {
5873 runtime_key: "runtime-a".to_string(),
5874 identity_namespace: String::new(),
5875 runtime: runtime.mob_runtime().clone(),
5876 identity_runtime: Some(identity_runtime),
5877 console_events: runtime.console_events(),
5878 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5879 },
5880 );
5881
5882 let err = aggregator
5883 .inspect_identity("review:singleton")
5884 .await
5885 .expect_err("cross-source wrong projection must not be hidden by primary source");
5886 assert!(
5887 err.to_string()
5888 .contains("projects identity other:singleton")
5889 || err.to_string().contains("ambiguous live identity alias"),
5890 "unexpected error: {err}"
5891 );
5892
5893 let _ = delegate_runtime.mob_handle().stop().await;
5894 let _ = runtime.mob_handle().stop().await;
5895 Ok(())
5896 }
5897
5898 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5899 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
5900 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5901 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
5902 let mut labels = BTreeMap::new();
5903 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5904 runtime
5905 .spawn(
5906 SpawnMemberSpec::from_wire(
5907 "worker".to_string(),
5908 "rt:review:singleton:0".to_string(),
5909 Some("You are the first Review Agent.".into()),
5910 None,
5911 None,
5912 )
5913 .with_labels(labels.clone()),
5914 )
5915 .await
5916 .expect("first member spawns");
5917 runtime
5918 .spawn(
5919 SpawnMemberSpec::from_wire(
5920 "worker".to_string(),
5921 "rt:review:singleton:1".to_string(),
5922 Some("You are the second Review Agent.".into()),
5923 None,
5924 None,
5925 )
5926 .with_labels(labels),
5927 )
5928 .await
5929 .expect("second member spawns");
5930
5931 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5932 crate::identity_first::IdentityRuntimeConfig {
5933 continuity_store: Arc::new(
5934 crate::identity_first::LocalContinuityStore::in_memory()?
5935 ),
5936 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5937 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
5938 has_runtime_store: true,
5939 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5940 bridge: None,
5941 default_timeout: None,
5942 },
5943 ));
5944 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5945 identity_runtime
5946 .register(
5947 crate::identity_first::DurableAgentSpec {
5948 identity: identity.clone(),
5949 profile: meerkat_mob::ProfileName::from("worker"),
5950 addressability: crate::identity_first::AgentAddressability::Addressable,
5951 display_name: None,
5952 labels: BTreeMap::new(),
5953 context: None,
5954 additional_instructions: Vec::new(),
5955 initial_message: None,
5956 runtime_mode_override: None,
5957 },
5958 crate::identity_first::IdentityLifecycleState::Active,
5959 Some(crate::identity_first::ContinuityRecord {
5960 identity: identity.clone(),
5961 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5962 "rt:review:singleton:0",
5963 )?,
5964 session_id: SessionId::new(),
5965 generation: crate::identity_first::ContinuityGeneration::new(1),
5966 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5967 }),
5968 Some(crate::identity_first::LeaseGrant {
5969 identity,
5970 fencing_token: crate::identity_first::FencingToken::new(8),
5971 ttl: Duration::from_mins(5),
5972 }),
5973 )
5974 .await;
5975
5976 let aggregator = MobKitConsoleAggregator::in_memory();
5977 aggregator
5978 .inner
5979 .runtimes
5980 .write()
5981 .expect("runtime registry")
5982 .insert(
5983 "runtime-a".to_string(),
5984 RuntimeEntry {
5985 runtime_key: "runtime-a".to_string(),
5986 identity_namespace: String::new(),
5987 runtime: runtime.mob_runtime().clone(),
5988 identity_runtime: Some(identity_runtime),
5989 console_events: runtime.console_events(),
5990 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5991 },
5992 );
5993
5994 let err = aggregator
5995 .inspect_identity("review:singleton")
5996 .await
5997 .expect_err("duplicate live alias must not inspect via durable fast path");
5998 assert!(
5999 err.to_string().contains("ambiguous live identity alias"),
6000 "unexpected error: {err}"
6001 );
6002 let retire_err = aggregator
6003 .retire_identity("review:singleton")
6004 .await
6005 .expect_err("duplicate live alias must not retire via durable fast path");
6006 assert!(
6007 retire_err
6008 .to_string()
6009 .contains("ambiguous live identity alias"),
6010 "unexpected retire error: {retire_err}"
6011 );
6012 let runtime_id_err = aggregator
6013 .inspect_identity("rt:review:singleton:0")
6014 .await
6015 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6016 assert!(
6017 runtime_id_err
6018 .to_string()
6019 .contains("ambiguous live identity alias"),
6020 "unexpected runtime-id inspect error: {runtime_id_err}"
6021 );
6022 let runtime_id_retire_err = aggregator
6023 .retire_identity("rt:review:singleton:0")
6024 .await
6025 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6026 assert!(
6027 runtime_id_retire_err
6028 .to_string()
6029 .contains("ambiguous live identity alias")
6030 || runtime_id_retire_err
6031 .to_string()
6032 .contains("stale durable identity alias"),
6033 "unexpected runtime-id retire error: {runtime_id_retire_err}"
6034 );
6035
6036 let _ = runtime.mob_handle().stop().await;
6037 Ok(())
6038 }
6039
6040 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6041 async fn inspect_identity_ignores_hidden_live_alias_candidates()
6042 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6043 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6044 let mut labels = BTreeMap::new();
6045 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6046 runtime
6047 .spawn(
6048 SpawnMemberSpec::from_wire(
6049 "worker".to_string(),
6050 "rt:review:singleton:0".to_string(),
6051 Some("You are the visible Review Agent.".into()),
6052 None,
6053 None,
6054 )
6055 .with_labels(labels.clone()),
6056 )
6057 .await
6058 .expect("visible member spawns");
6059 runtime
6060 .spawn(
6061 SpawnMemberSpec::from_wire(
6062 "worker".to_string(),
6063 "rt:review:singleton:1".to_string(),
6064 Some("You are hidden maintenance noise.".into()),
6065 None,
6066 None,
6067 )
6068 .with_labels(labels),
6069 )
6070 .await
6071 .expect("hidden member spawns");
6072
6073 let aggregator = MobKitConsoleAggregator::in_memory();
6074 aggregator.register_runtime_handles_with_policy(
6075 "runtime-a",
6076 "",
6077 runtime.mob_runtime().clone(),
6078 None,
6079 runtime.console_events(),
6080 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6081 );
6082
6083 let inspection = aggregator
6084 .inspect_identity("review:singleton")
6085 .await?
6086 .expect("visible live alias should inspect without hidden ambiguity");
6087 assert_eq!(inspection.identity.identity, "review:singleton");
6088 assert_eq!(
6089 inspection.identity.runtime_member_id,
6090 "rt:review:singleton:0"
6091 );
6092
6093 let retired = aggregator.retire_identity("review:singleton").await?;
6094 assert!(
6095 retired,
6096 "visible live alias should retire without hidden ambiguity"
6097 );
6098
6099 let _ = runtime.mob_handle().stop().await;
6100 Ok(())
6101 }
6102
6103 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6104 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6105 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6106 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6107 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6108 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6109 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6110
6111 let aggregator = MobKitConsoleAggregator::in_memory();
6112 aggregator.register_runtime_handles_with_policy(
6113 "a-hidden",
6114 "",
6115 hidden_runtime.mob_runtime().clone(),
6116 Some(hidden_identity_runtime),
6117 hidden_runtime.console_events(),
6118 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6119 );
6120 aggregator.register_runtime_handles_with_policy(
6121 "b-visible",
6122 "",
6123 visible_runtime.mob_runtime().clone(),
6124 Some(visible_identity_runtime),
6125 visible_runtime.console_events(),
6126 Arc::new(AllowAllConsoleVisibilityPolicy),
6127 );
6128
6129 let accepted = aggregator
6130 .reserve_identity_first_interaction(
6131 ConsoleSendRequest {
6132 identity: "review:singleton".to_string(),
6133 content: json!("hello"),
6134 origin: "console:test".to_string(),
6135 idempotency_key: "visible-after-hidden-reserve".to_string(),
6136 handling_mode: Some("queue".to_string()),
6137 },
6138 None,
6139 )
6140 .await?;
6141 let page = aggregator
6142 .query_timeline(ConsoleTimelineQuery {
6143 identity: Some("review:singleton".to_string()),
6144 ..ConsoleTimelineQuery::default()
6145 })
6146 .await?;
6147 let frame = page
6148 .frames
6149 .iter()
6150 .find(|frame| frame.id == accepted.input_frame_id)
6151 .ok_or("accepted frame missing from timeline")?;
6152 assert_eq!(frame.runtime_key, "b-visible");
6153
6154 let _ = hidden_runtime.mob_handle().stop().await;
6155 let _ = visible_runtime.mob_handle().stop().await;
6156 Ok(())
6157 }
6158
6159 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6160 async fn identity_first_list_identities_projects_cached_topology_peers()
6161 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6162 let runtime = build_empty_runtime("identity-topology-cache-test").await;
6163 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6164 identity_runtime
6165 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6166 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6167 crate::identity_first::AgentIdentity::parse("agent:beta")?,
6168 )?])
6169 .await;
6170
6171 let aggregator = MobKitConsoleAggregator::in_memory();
6172 aggregator.register_runtime_handles_with_policy(
6173 "identity-first",
6174 "",
6175 runtime.mob_runtime().clone(),
6176 Some(identity_runtime),
6177 runtime.console_events(),
6178 Arc::new(AllowAllConsoleVisibilityPolicy),
6179 );
6180
6181 let identities = aggregator.list_identities().await?;
6182 let alpha = identities
6183 .iter()
6184 .find(|record| record.identity == "agent:alpha")
6185 .ok_or("agent:alpha identity missing")?;
6186 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6187 let inspection = aggregator
6188 .inspect_identity("agent:alpha")
6189 .await?
6190 .ok_or("agent:alpha inspection missing")?;
6191 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6192
6193 let _ = runtime.mob_handle().stop().await;
6194 Ok(())
6195 }
6196
6197 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6198 async fn identity_first_list_identities_includes_member_only_spawned_workers()
6199 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6200 let runtime = build_empty_runtime("identity-member-only-test").await;
6201 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6202 let aggregator = MobKitConsoleAggregator::in_memory();
6203 aggregator.register_runtime_handles_with_policy(
6204 "identity-first",
6205 "",
6206 runtime.mob_runtime().clone(),
6207 Some(identity_runtime),
6208 runtime.console_events(),
6209 Arc::new(AllowAllConsoleVisibilityPolicy),
6210 );
6211
6212 runtime
6213 .spawn(SpawnMemberSpec::from_wire(
6214 "worker".to_string(),
6215 "agent:beta".to_string(),
6216 Some("You are beta.".into()),
6217 None,
6218 None,
6219 ))
6220 .await?;
6221
6222 let identities = aggregator.list_identities().await?;
6223 assert!(
6224 identities
6225 .iter()
6226 .any(|record| record.identity == "agent:beta"),
6227 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6228 );
6229
6230 let _ = runtime.mob_handle().stop().await;
6231 Ok(())
6232 }
6233
6234 #[tokio::test]
6235 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6236 let aggregator = MobKitConsoleAggregator::in_memory();
6237 let record = identity_record_for_test("agent-cached");
6238 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6239 aggregator
6240 .inner
6241 .identity_read_model
6242 .primed
6243 .store(true, Ordering::Release);
6244 let _guard = aggregator
6245 .inner
6246 .identity_read_model
6247 .refresh_lock
6248 .clone()
6249 .lock_owned()
6250 .await;
6251
6252 let identities =
6253 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6254 .await
6255 .expect("hot identity list should not wait for refresh lock")
6256 .expect("identity list succeeds");
6257
6258 assert_eq!(identities, vec![record]);
6259 }
6260
6261 #[tokio::test]
6262 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6263 let aggregator = MobKitConsoleAggregator::in_memory();
6264 let guard = aggregator
6265 .inner
6266 .identity_read_model
6267 .refresh_lock
6268 .clone()
6269 .lock_owned()
6270 .await;
6271 let waiter = tokio::spawn({
6272 let aggregator = aggregator.clone();
6273 async move { aggregator.list_identities().await }
6274 });
6275
6276 tokio::time::sleep(Duration::from_millis(20)).await;
6277 assert!(
6278 !waiter.is_finished(),
6279 "cold identity list should wait for the in-flight refresh to finish"
6280 );
6281
6282 let record = identity_record_for_test("agent-primed");
6283 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6284 aggregator
6285 .inner
6286 .identity_read_model
6287 .primed
6288 .store(true, Ordering::Release);
6289 drop(guard);
6290
6291 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6292 .await
6293 .expect("cold identity list waiter should resume")
6294 .expect("waiter joins")
6295 .expect("identity list succeeds");
6296 assert_eq!(identities, vec![record]);
6297 }
6298
6299 #[tokio::test]
6300 async fn query_timeline_reads_from_aggregate_store() {
6301 let aggregator = MobKitConsoleAggregator::in_memory();
6302 let frame = NewConsoleFrame {
6303 id: None,
6304 dedupe_key: "event-1".to_string(),
6305 timestamp_ms: 1,
6306 runtime_key: "runtime-a".to_string(),
6307 identity: "agent-a".to_string(),
6308 conversation_id: Some("agent-a".to_string()),
6309 session_id: None,
6310 kind: "text_delta".to_string(),
6311 status: ConsoleFrameStatus::Delivered,
6312 payload: json!({ "delta": "hello" }),
6313 source: ConsoleFrameSource {
6314 kind: ConsoleFrameSourceKind::ConsoleEvent,
6315 source_cursor: None,
6316 },
6317 source_event_id: Some("event-1".to_string()),
6318 interaction_id: None,
6319 turn_id: None,
6320 run_id: None,
6321 parent_frame_id: None,
6322 caused_by_frame_id: None,
6323 };
6324 aggregator
6325 .store()
6326 .append_if_absent(frame)
6327 .await
6328 .expect("append frame");
6329
6330 let page = aggregator
6331 .query_timeline(ConsoleTimelineQuery {
6332 identity: Some("agent-a".to_string()),
6333 limit: 10,
6334 ..ConsoleTimelineQuery::default()
6335 })
6336 .await
6337 .expect("query timeline");
6338 assert_eq!(page.frames.len(), 1);
6339 assert_eq!(page.frames[0].kind, "text_delta");
6340 }
6341
6342 #[tokio::test]
6343 async fn identity_recent_anchor_respects_query_limit() {
6344 let aggregator = MobKitConsoleAggregator::in_memory();
6345 aggregator
6346 .store()
6347 .append_if_absent(NewConsoleFrame {
6348 id: None,
6349 dedupe_key: "anchored-user-input".to_string(),
6350 timestamp_ms: 1,
6351 runtime_key: "runtime-a".to_string(),
6352 identity: "agent-a".to_string(),
6353 conversation_id: Some("agent-a".to_string()),
6354 session_id: None,
6355 kind: "user_input".to_string(),
6356 status: ConsoleFrameStatus::Delivered,
6357 payload: json!({ "content": "anchor me" }),
6358 source: ConsoleFrameSource {
6359 kind: ConsoleFrameSourceKind::Synthetic,
6360 source_cursor: None,
6361 },
6362 source_event_id: Some("anchored-user-input".to_string()),
6363 interaction_id: Some("turn-a".to_string()),
6364 turn_id: None,
6365 run_id: None,
6366 parent_frame_id: None,
6367 caused_by_frame_id: None,
6368 })
6369 .await
6370 .expect("append anchor");
6371 for idx in 2..=40 {
6372 aggregator
6373 .store()
6374 .append_if_absent(NewConsoleFrame {
6375 id: None,
6376 dedupe_key: format!("noisy-tail-{idx}"),
6377 timestamp_ms: idx,
6378 runtime_key: "runtime-a".to_string(),
6379 identity: "agent-a".to_string(),
6380 conversation_id: Some("agent-a".to_string()),
6381 session_id: None,
6382 kind: "reasoning_delta".to_string(),
6383 status: ConsoleFrameStatus::Delivered,
6384 payload: json!({ "delta": idx }),
6385 source: ConsoleFrameSource {
6386 kind: ConsoleFrameSourceKind::ConsoleEvent,
6387 source_cursor: None,
6388 },
6389 source_event_id: Some(format!("noisy-tail-{idx}")),
6390 interaction_id: Some("turn-a".to_string()),
6391 turn_id: None,
6392 run_id: None,
6393 parent_frame_id: None,
6394 caused_by_frame_id: None,
6395 })
6396 .await
6397 .expect("append noisy tail");
6398 }
6399
6400 let page = aggregator
6401 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6402 identity: Some("agent-a".to_string()),
6403 mode: ConsoleTimelineMode::Recent,
6404 limit: 5,
6405 ..ConsoleTimelineWindowQuery::default()
6406 })
6407 .await
6408 .expect("query recent identity timeline");
6409
6410 assert_eq!(
6411 page.frames.len(),
6412 5,
6413 "identity anchor merge must not exceed the requested limit"
6414 );
6415 assert!(
6416 page.frames
6417 .iter()
6418 .any(|frame| frame.dedupe_key == "anchored-user-input"),
6419 "the bounded result should still retain the useful turn anchor: {:#?}",
6420 page.frames
6421 );
6422 assert_eq!(
6423 page.frames.last().and_then(|frame| frame.cursor.seq()),
6424 Some(40)
6425 );
6426 }
6427
6428 #[tokio::test]
6429 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
6430 let aggregator = MobKitConsoleAggregator::in_memory();
6431 let runtime = build_single_member_runtime().await;
6432 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
6433 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
6434 aggregator
6435 .inner
6436 .runtimes
6437 .write()
6438 .expect("runtime registry")
6439 .insert("runtime-a".to_string(), entry);
6440
6441 for idx in 0..1_500 {
6442 aggregator
6443 .store()
6444 .append_if_absent(NewConsoleFrame {
6445 id: None,
6446 dedupe_key: format!("hidden-gap-{idx}"),
6447 timestamp_ms: idx,
6448 runtime_key: "runtime-a".to_string(),
6449 identity: "agent-a".to_string(),
6450 conversation_id: Some("agent-a".to_string()),
6451 session_id: None,
6452 kind: "hidden_noise".to_string(),
6453 status: ConsoleFrameStatus::Delivered,
6454 payload: json!({ "delta": idx }),
6455 source: ConsoleFrameSource {
6456 kind: ConsoleFrameSourceKind::ConsoleEvent,
6457 source_cursor: None,
6458 },
6459 source_event_id: Some(format!("hidden-gap-{idx}")),
6460 interaction_id: None,
6461 turn_id: None,
6462 run_id: None,
6463 parent_frame_id: None,
6464 caused_by_frame_id: None,
6465 })
6466 .await
6467 .expect("append hidden frame");
6468 }
6469 aggregator
6470 .store()
6471 .append_if_absent(NewConsoleFrame {
6472 id: None,
6473 dedupe_key: "visible-after-gap".to_string(),
6474 timestamp_ms: 1_501,
6475 runtime_key: "runtime-a".to_string(),
6476 identity: "agent-a".to_string(),
6477 conversation_id: Some("agent-a".to_string()),
6478 session_id: None,
6479 kind: "interaction_complete".to_string(),
6480 status: ConsoleFrameStatus::Delivered,
6481 payload: json!({ "text": "visible after hidden gap" }),
6482 source: ConsoleFrameSource {
6483 kind: ConsoleFrameSourceKind::ConsoleEvent,
6484 source_cursor: None,
6485 },
6486 source_event_id: Some("visible-after-gap".to_string()),
6487 interaction_id: Some("turn-visible".to_string()),
6488 turn_id: None,
6489 run_id: None,
6490 parent_frame_id: None,
6491 caused_by_frame_id: None,
6492 })
6493 .await
6494 .expect("append visible frame");
6495
6496 let page = aggregator
6497 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6498 identity: Some("agent-a".to_string()),
6499 mode: ConsoleTimelineMode::Since,
6500 limit: 1,
6501 ..ConsoleTimelineWindowQuery::default()
6502 })
6503 .await
6504 .expect("query timeline");
6505 assert_eq!(page.frames.len(), 1);
6506 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
6507 assert_eq!(
6508 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6509 Some(1_501)
6510 );
6511 let _ = runtime.mob_handle().stop().await;
6512 }
6513
6514 #[tokio::test]
6515 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
6516 let aggregator = MobKitConsoleAggregator::in_memory();
6517 for idx in 1..=300 {
6518 aggregator
6519 .store()
6520 .append_if_absent(NewConsoleFrame {
6521 id: None,
6522 dedupe_key: format!("visible-{idx}"),
6523 timestamp_ms: idx,
6524 runtime_key: "runtime-a".to_string(),
6525 identity: "agent-a".to_string(),
6526 conversation_id: Some("agent-a".to_string()),
6527 session_id: None,
6528 kind: "interaction_complete".to_string(),
6529 status: ConsoleFrameStatus::Delivered,
6530 payload: json!({ "text": format!("visible {idx}") }),
6531 source: ConsoleFrameSource {
6532 kind: ConsoleFrameSourceKind::ConsoleEvent,
6533 source_cursor: None,
6534 },
6535 source_event_id: Some(format!("visible-{idx}")),
6536 interaction_id: Some(format!("turn-{idx}")),
6537 turn_id: None,
6538 run_id: None,
6539 parent_frame_id: None,
6540 caused_by_frame_id: None,
6541 })
6542 .await
6543 .expect("append visible frame");
6544 }
6545
6546 let first = aggregator
6547 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6548 identity: Some("agent-a".to_string()),
6549 mode: ConsoleTimelineMode::Since,
6550 limit: 10,
6551 ..ConsoleTimelineWindowQuery::default()
6552 })
6553 .await
6554 .expect("query first page");
6555 assert_eq!(first.frames.len(), 10);
6556 assert_eq!(
6557 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6558 Some(10)
6559 );
6560
6561 let second = aggregator
6562 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6563 identity: Some("agent-a".to_string()),
6564 mode: ConsoleTimelineMode::Since,
6565 after: first.next_cursor,
6566 limit: 10,
6567 ..ConsoleTimelineWindowQuery::default()
6568 })
6569 .await
6570 .expect("query second page");
6571 assert_eq!(second.frames.len(), 10);
6572 assert_eq!(second.frames[0].dedupe_key, "visible-11");
6573 assert_eq!(
6574 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6575 Some(20)
6576 );
6577 }
6578
6579 #[tokio::test]
6580 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
6581 let store = Arc::new(CountingConsoleLogStore::new());
6582 let aggregator = MobKitConsoleAggregator::new(store.clone());
6583 let runtime = build_single_member_runtime().await;
6584 aggregator
6585 .inner
6586 .runtimes
6587 .write()
6588 .expect("runtime registry")
6589 .insert(
6590 "runtime-a".to_string(),
6591 runtime_entry_for_test("runtime-a", &runtime),
6592 );
6593 let inserted = store
6594 .append_if_absent(NewConsoleFrame {
6595 id: None,
6596 dedupe_key: "event-1".to_string(),
6597 timestamp_ms: 1,
6598 runtime_key: "runtime-a".to_string(),
6599 identity: "agent-a".to_string(),
6600 conversation_id: Some("agent-a".to_string()),
6601 session_id: None,
6602 kind: "text_delta".to_string(),
6603 status: ConsoleFrameStatus::Delivered,
6604 payload: json!({ "delta": "hello" }),
6605 source: ConsoleFrameSource {
6606 kind: ConsoleFrameSourceKind::ConsoleEvent,
6607 source_cursor: None,
6608 },
6609 source_event_id: Some("event-1".to_string()),
6610 interaction_id: None,
6611 turn_id: None,
6612 run_id: None,
6613 parent_frame_id: None,
6614 caused_by_frame_id: None,
6615 })
6616 .await
6617 .expect("append frame");
6618
6619 let page = aggregator
6620 .query_timeline_windowed(ConsoleTimelineWindowQuery {
6621 identity: Some("agent-a".to_string()),
6622 mode: ConsoleTimelineMode::Since,
6623 after: Some(inserted.frame.cursor),
6624 limit: 10,
6625 ..ConsoleTimelineWindowQuery::default()
6626 })
6627 .await
6628 .expect("query continuation");
6629
6630 assert!(page.frames.is_empty());
6631 assert_eq!(
6632 store.source_watermark_calls(),
6633 0,
6634 "empty since continuation must not synchronously force session-history backfill"
6635 );
6636 let _ = runtime.mob_handle().stop().await;
6637 }
6638
6639 #[test]
6640 fn legacy_timeline_struct_literals_remain_source_compatible() {
6641 let query = ConsoleTimelineQuery {
6642 identity: Some("agent-a".to_string()),
6643 conversation_id: None,
6644 after: Some(ConsoleCursor::from("console:1")),
6645 limit: 10,
6646 };
6647 let page = ConsoleTimelinePage {
6648 frames: Vec::new(),
6649 next_cursor: query.after.clone(),
6650 };
6651
6652 assert_eq!(query.limit, 10);
6653 assert_eq!(
6654 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6655 Some(1)
6656 );
6657 }
6658
6659 #[tokio::test]
6660 async fn query_timeline_is_store_local_for_registered_runtimes() {
6661 let store = Arc::new(CountingConsoleLogStore::new());
6662 let aggregator = MobKitConsoleAggregator::new(store.clone());
6663 let runtime = build_single_member_runtime().await;
6664 aggregator
6665 .inner
6666 .runtimes
6667 .write()
6668 .expect("runtime registry")
6669 .insert(
6670 "runtime-a".to_string(),
6671 runtime_entry_for_test("runtime-a", &runtime),
6672 );
6673 store
6674 .append_if_absent(NewConsoleFrame {
6675 id: None,
6676 dedupe_key: "event-1".to_string(),
6677 timestamp_ms: 1,
6678 runtime_key: "runtime-a".to_string(),
6679 identity: "agent-a".to_string(),
6680 conversation_id: Some("agent-a".to_string()),
6681 session_id: None,
6682 kind: "text_delta".to_string(),
6683 status: ConsoleFrameStatus::Delivered,
6684 payload: json!({ "delta": "hello" }),
6685 source: ConsoleFrameSource {
6686 kind: ConsoleFrameSourceKind::ConsoleEvent,
6687 source_cursor: None,
6688 },
6689 source_event_id: Some("event-1".to_string()),
6690 interaction_id: None,
6691 turn_id: None,
6692 run_id: None,
6693 parent_frame_id: None,
6694 caused_by_frame_id: None,
6695 })
6696 .await
6697 .expect("append frame");
6698
6699 let page = tokio::time::timeout(
6700 Duration::from_millis(250),
6701 aggregator.query_timeline(ConsoleTimelineQuery {
6702 identity: Some("agent-a".to_string()),
6703 limit: 10,
6704 ..ConsoleTimelineQuery::default()
6705 }),
6706 )
6707 .await
6708 .expect("timeline query should not wait for session history")
6709 .expect("timeline query succeeds");
6710
6711 assert_eq!(page.frames.len(), 1);
6712 assert_eq!(
6713 store.source_watermark_calls(),
6714 0,
6715 "query_timeline must not synchronously touch session-history watermarks"
6716 );
6717 let _ = runtime.mob_handle().stop().await;
6718 }
6719
6720 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6721 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
6722 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
6723 delay: Duration::from_secs(2),
6724 }))
6725 .await;
6726 let aggregator = MobKitConsoleAggregator::in_memory();
6727 aggregator
6728 .inner
6729 .runtimes
6730 .write()
6731 .expect("runtime registry")
6732 .insert(
6733 "runtime-a".to_string(),
6734 runtime_entry_for_test("runtime-a", &runtime),
6735 );
6736
6737 let start = Instant::now();
6738 let accepted = tokio::time::timeout(
6739 Duration::from_millis(300),
6740 aggregator.send(ConsoleSendRequest {
6741 identity: "test/agent-a".to_string(),
6742 content: json!("hello slow agent"),
6743 origin: "console:test".to_string(),
6744 idempotency_key: "nonblocking-send".to_string(),
6745 handling_mode: Some("queue".to_string()),
6746 }),
6747 )
6748 .await
6749 .expect("console send should return once the input is accepted")
6750 .expect("send succeeds");
6751
6752 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
6753 assert!(
6754 start.elapsed() < Duration::from_secs(1),
6755 "console send should not wait for the delayed LLM turn"
6756 );
6757
6758 wait_for_session_history_text(
6759 &aggregator,
6760 "test/agent-a",
6761 "slow ok",
6762 Duration::from_secs(5),
6763 )
6764 .await
6765 .expect("background dispatch should still complete and project history");
6766 let _ = runtime.mob_handle().stop().await;
6767 }
6768
6769 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6770 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
6771 {
6772 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
6773 let aggregator = MobKitConsoleAggregator::in_memory();
6774 aggregator.register_runtime(ConsoleRuntimeRegistration {
6775 runtime_key: "runtime-late".to_string(),
6776 runtime: runtime.clone(),
6777 identity_namespace: "late".to_string(),
6778 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6779 });
6780
6781 runtime
6782 .spawn(SpawnMemberSpec::from_wire(
6783 "worker".to_string(),
6784 "agent-late".to_string(),
6785 Some("You are agent-late.".into()),
6786 None,
6787 None,
6788 ))
6789 .await
6790 .expect("late member spawns");
6791 let session_id = send_message_on_mob_with_mode(
6792 &runtime.mob_handle(),
6793 "agent-late",
6794 ContentInput::Text("hello after registration".to_string()),
6795 meerkat_core::types::HandlingMode::Queue,
6796 )
6797 .await
6798 .expect("direct member send succeeds");
6799 wait_for_identity_record(
6800 &aggregator,
6801 "late/agent-late",
6802 Some(session_id.as_str()),
6803 Duration::from_secs(5),
6804 )
6805 .await?;
6806
6807 wait_for_session_history_text(
6808 &aggregator,
6809 "late/agent-late",
6810 "You are agent-late.",
6811 Duration::from_secs(5),
6812 )
6813 .await?;
6814 let _ = runtime.mob_handle().stop().await;
6815 Ok(())
6816 }
6817
6818 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6819 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
6820 -> Result<(), String> {
6821 let runtime = build_single_member_runtime().await;
6822 let entry = runtime_entry_for_test("runtime-a", &runtime);
6823 let resolved = member_sources_for_entry(&entry)
6824 .await
6825 .into_iter()
6826 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6827 .expect("agent-a member exists");
6828 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6829 .await
6830 .expect("identity record exists");
6831 let session_id = record.session_id.expect("agent-a has a session");
6832 wait_for_runtime_session_history_text(
6833 &runtime,
6834 &session_id,
6835 "You are agent-a.",
6836 Duration::from_secs(5),
6837 )
6838 .await?;
6839
6840 let aggregator = MobKitConsoleAggregator::in_memory();
6841 aggregator
6842 .inner
6843 .runtimes
6844 .write()
6845 .expect("runtime registry")
6846 .insert("runtime-a".to_string(), entry);
6847 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
6848 aggregator
6849 .store()
6850 .record_source_watermark(
6851 &watermark_key,
6852 ConsoleFrameSourceKind::SessionHistory,
6853 &format_session_history_watermark(&session_id, 0, current_time_ms()),
6854 )
6855 .await
6856 .expect("record fresh empty watermark");
6857
6858 let page = tokio::time::timeout(
6859 Duration::from_secs(2),
6860 aggregator.query_timeline(ConsoleTimelineQuery {
6861 identity: Some("test/agent-a".to_string()),
6862 limit: 20,
6863 ..ConsoleTimelineQuery::default()
6864 }),
6865 )
6866 .await
6867 .expect("explicit identity query should not stall")
6868 .expect("query succeeds");
6869
6870 assert!(
6871 page.frames.is_empty(),
6872 "empty fresh-watermark query should return promptly before async backfill; frames: {:#?}",
6873 page.frames
6874 );
6875 wait_for_session_history_text(
6876 &aggregator,
6877 "test/agent-a",
6878 "You are agent-a.",
6879 Duration::from_secs(5),
6880 )
6881 .await?;
6882 let _ = runtime.mob_handle().stop().await;
6883 Ok(())
6884 }
6885
6886 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
6888 {
6889 let store = Arc::new(CountingConsoleLogStore::new());
6890 let aggregator = MobKitConsoleAggregator::new(store.clone());
6891 let runtime = build_single_member_runtime().await;
6892 let entry = runtime_entry_for_test("runtime-a", &runtime);
6893 let resolved = member_sources_for_entry(&entry)
6894 .await
6895 .into_iter()
6896 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6897 .expect("agent-a member exists");
6898 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6899 .await
6900 .expect("identity record exists");
6901 let session_id = record.session_id.expect("agent-a has a session");
6902 wait_for_runtime_session_history_text(
6903 &runtime,
6904 &session_id,
6905 "You are agent-a.",
6906 Duration::from_secs(5),
6907 )
6908 .await?;
6909 aggregator
6910 .inner
6911 .runtimes
6912 .write()
6913 .expect("runtime registry")
6914 .insert("runtime-a".to_string(), entry);
6915
6916 store
6917 .append_if_absent(NewConsoleFrame {
6918 id: None,
6919 dedupe_key: "stale-session-history-agent-a".to_string(),
6920 timestamp_ms: 10,
6921 runtime_key: "runtime-a".to_string(),
6922 identity: "test/agent-a".to_string(),
6923 conversation_id: Some("test/agent-a".to_string()),
6924 session_id: Some(session_id.clone()),
6925 kind: "user_input".to_string(),
6926 status: ConsoleFrameStatus::Delivered,
6927 payload: json!({
6928 "text": "stale projected history",
6929 "type": "user_input",
6930 }),
6931 source: ConsoleFrameSource {
6932 kind: ConsoleFrameSourceKind::SessionHistory,
6933 source_cursor: Some("stale-session-history-agent-a".to_string()),
6934 },
6935 source_event_id: Some("stale-session-history-agent-a".to_string()),
6936 interaction_id: None,
6937 turn_id: None,
6938 run_id: None,
6939 parent_frame_id: None,
6940 caused_by_frame_id: None,
6941 })
6942 .await
6943 .expect("append stale history frame");
6944
6945 let fresh_prompt = "fresh prompt after stale history";
6946 let sent_session_id = send_message_on_mob_with_mode(
6947 &runtime.mob_handle(),
6948 "agent-a",
6949 ContentInput::Text(fresh_prompt.to_string()),
6950 meerkat_core::types::HandlingMode::Queue,
6951 )
6952 .await
6953 .expect("direct member send succeeds");
6954 assert_eq!(sent_session_id, session_id);
6955 wait_for_runtime_session_history_text(
6956 &runtime,
6957 &session_id,
6958 fresh_prompt,
6959 Duration::from_secs(5),
6960 )
6961 .await?;
6962
6963 let page = aggregator
6964 .query_timeline(ConsoleTimelineQuery {
6965 identity: Some("test/agent-a".to_string()),
6966 limit: 50,
6967 ..ConsoleTimelineQuery::default()
6968 })
6969 .await
6970 .expect("query child timeline");
6971
6972 assert!(
6973 page.frames.iter().any(|frame| {
6974 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
6975 && session_history_content_text(frame).as_deref()
6976 == Some("stale projected history")
6977 }),
6978 "explicit identity query should return existing history before async refresh; frames: {:#?}",
6979 page.frames
6980 );
6981 wait_for_session_history_text(
6982 &aggregator,
6983 "test/agent-a",
6984 fresh_prompt,
6985 Duration::from_secs(5),
6986 )
6987 .await?;
6988 assert!(
6989 store.source_watermark_calls() > 0,
6990 "stale existing session history should force a targeted source refresh"
6991 );
6992 let _ = runtime.mob_handle().stop().await;
6993 Ok(())
6994 }
6995
6996 async fn wait_for_session_history_text(
6997 aggregator: &MobKitConsoleAggregator,
6998 identity: &str,
6999 expected: &str,
7000 timeout: Duration,
7001 ) -> Result<(), String> {
7002 let deadline = Instant::now() + timeout;
7003 let mut observed = Vec::new();
7004 while Instant::now() < deadline {
7005 let _ = aggregator.list_identities().await;
7006 let page = aggregator
7007 .query_timeline(ConsoleTimelineQuery {
7008 identity: Some(identity.to_string()),
7009 limit: 20,
7010 ..ConsoleTimelineQuery::default()
7011 })
7012 .await
7013 .expect("query timeline");
7014 observed = page.frames;
7015 if observed.iter().any(|frame| {
7016 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7017 && session_history_content_text(frame).as_deref() == Some(expected)
7018 }) {
7019 return Ok(());
7020 }
7021 tokio::time::sleep(Duration::from_millis(25)).await;
7022 }
7023
7024 Err(format!(
7025 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7026 ))
7027 }
7028
7029 async fn wait_for_session_backfill_target(
7030 aggregator: &MobKitConsoleAggregator,
7031 identity: &str,
7032 timeout: Duration,
7033 ) -> Result<String, String> {
7034 let deadline = Instant::now() + timeout;
7035 while Instant::now() < deadline {
7036 if let Some(target) =
7037 session_backfill_target_for_identity(&aggregator.inner, identity).await
7038 {
7039 return Ok(target.session_id);
7040 }
7041 tokio::time::sleep(Duration::from_millis(25)).await;
7042 }
7043 Err(format!(
7044 "session backfill target for {identity:?} was not resolvable"
7045 ))
7046 }
7047
7048 async fn wait_for_runtime_session_history_text(
7049 runtime: &UnifiedRuntime,
7050 session_id: &str,
7051 expected: &str,
7052 timeout: Duration,
7053 ) -> Result<(), String> {
7054 let deadline = Instant::now() + timeout;
7055 let mut observed = Vec::new();
7056 while Instant::now() < deadline {
7057 let page = runtime
7058 .mob_runtime()
7059 .read_session_history(session_id, 0, Some(20))
7060 .await
7061 .map_err(|err| err.to_string())?;
7062 observed = page.messages;
7063 if observed.iter().enumerate().any(|(idx, message)| {
7064 let Some(message) = serde_json::to_value(message).ok() else {
7065 return false;
7066 };
7067 frames_from_session_history_message(
7068 "runtime-a",
7069 "test/agent-a",
7070 session_id,
7071 idx,
7072 message,
7073 )
7074 .iter()
7075 .any(|frame| {
7076 matches!(
7077 frame.kind.as_str(),
7078 "user_input" | "system_notice" | "interaction_complete"
7079 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7080 })
7081 }) {
7082 return Ok(());
7083 }
7084 tokio::time::sleep(Duration::from_millis(25)).await;
7085 }
7086
7087 Err(format!(
7088 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7089 ))
7090 }
7091
7092 async fn wait_for_identity_record(
7093 aggregator: &MobKitConsoleAggregator,
7094 identity: &str,
7095 session_id: Option<&str>,
7096 timeout: Duration,
7097 ) -> Result<(), String> {
7098 let deadline = Instant::now() + timeout;
7099 let mut observed = Vec::new();
7100 while Instant::now() < deadline {
7101 observed = aggregator
7102 .list_identities()
7103 .await
7104 .map_err(|err| err.to_string())?;
7105 if observed.iter().any(|record| {
7106 record.identity == identity && record.session_id.as_deref() == session_id
7107 }) {
7108 return Ok(());
7109 }
7110 tokio::time::sleep(Duration::from_millis(25)).await;
7111 }
7112
7113 Err(format!(
7114 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7115 ))
7116 }
7117
7118 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7119 session_history_payload_text(&frame.payload)
7120 }
7121
7122 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7123 session_history_payload_text(&frame.payload)
7124 }
7125
7126 fn session_history_payload_text(payload: &Value) -> Option<String> {
7127 if let Some(text) = payload.get("text").and_then(Value::as_str) {
7128 return Some(text.to_string());
7129 }
7130 if let Some(text) = payload.get("result").and_then(Value::as_str) {
7131 return Some(text.to_string());
7132 }
7133 if let Some(text) = payload.get("body").and_then(Value::as_str) {
7134 return Some(text.to_string());
7135 }
7136 match payload.get("content")? {
7137 Value::String(text) => Some(text.clone()),
7138 Value::Array(blocks) => Some(
7139 blocks
7140 .iter()
7141 .filter_map(|block| block.get("text").and_then(Value::as_str))
7142 .collect::<Vec<_>>()
7143 .join(""),
7144 ),
7145 _ => None,
7146 }
7147 }
7148
7149 #[tokio::test]
7150 async fn query_timeline_handles_large_store_without_backfill_calls() {
7151 let store = Arc::new(CountingConsoleLogStore::new());
7152 let aggregator = MobKitConsoleAggregator::new(store.clone());
7153 for idx in 0..5_000 {
7154 store
7155 .append_if_absent(NewConsoleFrame {
7156 id: None,
7157 dedupe_key: format!("event-{idx}"),
7158 timestamp_ms: idx,
7159 runtime_key: "runtime-a".to_string(),
7160 identity: "agent-a".to_string(),
7161 conversation_id: Some("agent-a".to_string()),
7162 session_id: Some("session-a".to_string()),
7163 kind: "text_delta".to_string(),
7164 status: ConsoleFrameStatus::Delivered,
7165 payload: json!({ "delta": idx }),
7166 source: ConsoleFrameSource {
7167 kind: ConsoleFrameSourceKind::ConsoleEvent,
7168 source_cursor: None,
7169 },
7170 source_event_id: Some(format!("event-{idx}")),
7171 interaction_id: None,
7172 turn_id: None,
7173 run_id: None,
7174 parent_frame_id: None,
7175 caused_by_frame_id: None,
7176 })
7177 .await
7178 .expect("append frame");
7179 }
7180
7181 let page = aggregator
7182 .query_timeline(ConsoleTimelineQuery {
7183 identity: Some("agent-a".to_string()),
7184 limit: 1_000,
7185 ..ConsoleTimelineQuery::default()
7186 })
7187 .await
7188 .expect("large query");
7189
7190 assert_eq!(page.frames.len(), 1_000);
7191 assert_eq!(store.source_watermark_calls(), 0);
7192 }
7193
7194 #[tokio::test]
7195 async fn query_timeline_rejects_future_cursor_after_store_reset() {
7196 let aggregator = MobKitConsoleAggregator::in_memory();
7197 let err = aggregator
7198 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7199 after: Some(ConsoleCursor::from("console:99")),
7200 limit: 10,
7201 ..ConsoleTimelineWindowQuery::default()
7202 })
7203 .await
7204 .expect_err("future cursor on empty/reset store must be replay-unavailable");
7205
7206 assert!(
7207 err.to_string()
7208 .contains("beyond the current store frontier"),
7209 "unexpected error: {err}"
7210 );
7211 }
7212
7213 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7214 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7215 let store = Arc::new(CountingConsoleLogStore::new());
7216 let aggregator = MobKitConsoleAggregator::new(store.clone());
7217 let runtime = build_single_member_runtime().await;
7218 aggregator
7219 .inner
7220 .runtimes
7221 .write()
7222 .expect("runtime registry")
7223 .insert(
7224 "runtime-a".to_string(),
7225 runtime_entry_for_test("runtime-a", &runtime),
7226 );
7227 let session_id =
7228 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
7229 .await
7230 .expect("spawned member is resolvable for targeted backfill");
7231 wait_for_runtime_session_history_text(
7232 &runtime,
7233 &session_id,
7234 "You are agent-a.",
7235 Duration::from_secs(5),
7236 )
7237 .await
7238 .expect("spawned member kickoff is readable before live-frame race");
7239 store
7240 .append_if_absent(NewConsoleFrame {
7241 id: None,
7242 dedupe_key: "live-tool-before-history".to_string(),
7243 timestamp_ms: 10,
7244 runtime_key: "runtime-a".to_string(),
7245 identity: "test/agent-a".to_string(),
7246 conversation_id: Some("test/agent-a".to_string()),
7247 session_id: None,
7248 kind: "tool_execution_started".to_string(),
7249 status: ConsoleFrameStatus::Delivered,
7250 payload: json!({
7251 "id": "call-live",
7252 "name": "king_search",
7253 "source_event_type": "tool_execution_started",
7254 "type": "tool_execution_started",
7255 }),
7256 source: ConsoleFrameSource {
7257 kind: ConsoleFrameSourceKind::ConsoleEvent,
7258 source_cursor: Some("live-tool-before-history".to_string()),
7259 },
7260 source_event_id: Some("live-tool-before-history".to_string()),
7261 interaction_id: None,
7262 turn_id: None,
7263 run_id: None,
7264 parent_frame_id: None,
7265 caused_by_frame_id: None,
7266 })
7267 .await
7268 .expect("append live tool frame");
7269
7270 let page = aggregator
7271 .query_timeline(ConsoleTimelineQuery {
7272 identity: Some("test/agent-a".to_string()),
7273 limit: 20,
7274 ..ConsoleTimelineQuery::default()
7275 })
7276 .await
7277 .expect("query child timeline");
7278
7279 assert!(
7280 page.frames
7281 .iter()
7282 .any(|frame| frame.kind == "tool_execution_started"),
7283 "initial explicit child timeline should return existing store frames without waiting for session history"
7284 );
7285
7286 let mut observed_backfill = false;
7287 for _ in 0..80 {
7288 let page = aggregator
7289 .query_timeline(ConsoleTimelineQuery {
7290 identity: Some("test/agent-a".to_string()),
7291 limit: 20,
7292 ..ConsoleTimelineQuery::default()
7293 })
7294 .await
7295 .expect("query child timeline after scheduled backfill");
7296 if page.frames.iter().any(|frame| {
7297 frame.kind == "user_input"
7298 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7299 && frame.payload.to_string().contains("You are agent-a.")
7300 }) {
7301 observed_backfill = true;
7302 break;
7303 }
7304 tokio::time::sleep(Duration::from_millis(25)).await;
7305 }
7306 assert!(
7307 observed_backfill,
7308 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
7309 );
7310 let _ = runtime.mob_handle().stop().await;
7311 }
7312
7313 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7314 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
7315 let store = Arc::new(CountingConsoleLogStore::new());
7316 let aggregator = MobKitConsoleAggregator::new(store.clone());
7317 let (_temp, runtime, _delayed_service) =
7318 build_stress_runtime(64, Duration::from_millis(0)).await;
7319 aggregator
7320 .inner
7321 .runtimes
7322 .write()
7323 .expect("runtime registry")
7324 .insert(
7325 "runtime-a".to_string(),
7326 runtime_entry_for_test("runtime-a", runtime.as_ref()),
7327 );
7328 for idx in 0..1_000 {
7329 store
7330 .append_if_absent(NewConsoleFrame {
7331 id: None,
7332 dedupe_key: format!("event-{idx}"),
7333 timestamp_ms: idx,
7334 runtime_key: "runtime-a".to_string(),
7335 identity: "test/agent-0".to_string(),
7336 conversation_id: Some("test/agent-0".to_string()),
7337 session_id: Some("session-a".to_string()),
7338 kind: "text_delta".to_string(),
7339 status: ConsoleFrameStatus::Delivered,
7340 payload: json!({ "delta": idx }),
7341 source: ConsoleFrameSource {
7342 kind: ConsoleFrameSourceKind::ConsoleEvent,
7343 source_cursor: None,
7344 },
7345 source_event_id: Some(format!("event-{idx}")),
7346 interaction_id: None,
7347 turn_id: None,
7348 run_id: None,
7349 parent_frame_id: None,
7350 caused_by_frame_id: None,
7351 })
7352 .await
7353 .expect("append frame");
7354 }
7355
7356 let page = tokio::time::timeout(
7357 Duration::from_secs(2),
7358 aggregator.query_timeline(ConsoleTimelineQuery {
7359 identity: Some("test/agent-0".to_string()),
7360 limit: 1_000,
7361 ..ConsoleTimelineQuery::default()
7362 }),
7363 )
7364 .await
7365 .expect("identity timeline query should not rediscover the roster per frame")
7366 .expect("timeline query succeeds");
7367
7368 assert_eq!(page.frames.len(), 1_000);
7369 assert_eq!(store.source_watermark_calls(), 0);
7370 let _ = runtime.mob_handle().stop().await;
7371 }
7372
7373 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7374 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
7375 const MEMBER_COUNT: usize = 32;
7376 let (_temp, runtime, delayed_service) =
7377 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
7378 let aggregator = MobKitConsoleAggregator::in_memory();
7379 aggregator
7380 .inner
7381 .runtimes
7382 .write()
7383 .expect("runtime registry")
7384 .insert(
7385 "runtime-stress".to_string(),
7386 runtime_entry_for_test("runtime-stress", &runtime),
7387 );
7388
7389 let started = Instant::now();
7390 aggregator
7391 .refresh_session_history()
7392 .await
7393 .expect("stress refresh");
7394 let elapsed = started.elapsed();
7395
7396 assert!(
7397 delayed_service.read_calls() >= MEMBER_COUNT,
7398 "expected at least one history read per member, saw {}",
7399 delayed_service.read_calls()
7400 );
7401 assert!(
7402 delayed_service.max_active_reads() > 1,
7403 "session history backfill should fan out instead of reading members serially"
7404 );
7405 assert!(
7406 delayed_service.max_active_reads()
7407 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
7408 "session history backfill should respect the default concurrency limit"
7409 );
7410 assert!(
7411 elapsed < Duration::from_millis(600),
7412 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
7413 MEMBER_COUNT * 40
7414 );
7415 let _ = runtime.mob_handle().stop().await;
7416 }
7417
7418 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7419 async fn session_history_backfill_respects_configured_concurrency_limit() {
7420 const MEMBER_COUNT: usize = 16;
7421 let (_temp, runtime, delayed_service) =
7422 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
7423 let aggregator =
7424 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
7425 max_concurrent_session_backfills: 4,
7426 ..ConsoleAggregatorOptions::default()
7427 });
7428 aggregator
7429 .inner
7430 .runtimes
7431 .write()
7432 .expect("runtime registry")
7433 .insert(
7434 "runtime-stress".to_string(),
7435 runtime_entry_for_test("runtime-stress", &runtime),
7436 );
7437
7438 aggregator
7439 .refresh_session_history()
7440 .await
7441 .expect("stress refresh");
7442
7443 assert!(
7444 delayed_service.max_active_reads() <= 4,
7445 "configured concurrency limit should bound session history reads, saw {}",
7446 delayed_service.max_active_reads()
7447 );
7448 let _ = runtime.mob_handle().stop().await;
7449 }
7450
7451 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7452 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
7453 const MEMBER_COUNT: usize = 24;
7454 const LIVE_EVENT_COUNT: usize = 2_048;
7455 let (_temp, runtime, _delayed_service) =
7456 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
7457 let aggregator = MobKitConsoleAggregator::in_memory();
7458 aggregator.register_runtime(ConsoleRuntimeRegistration {
7459 runtime_key: "runtime-burst".to_string(),
7460 runtime: runtime.clone(),
7461 identity_namespace: "stress".to_string(),
7462 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7463 });
7464 let console_events = runtime.console_events();
7465 for idx in 0..LIVE_EVENT_COUNT {
7466 console_events
7467 .append(
7468 "agent-0",
7469 Some("burst-turn".to_string()),
7470 "text_delta",
7471 json!({ "delta": format!("frame-{idx}") }),
7472 )
7473 .await;
7474 }
7475
7476 let deadline = Instant::now() + Duration::from_secs(5);
7477 let mut observed = 0;
7478 while Instant::now() < deadline {
7479 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
7480 if observed >= LIVE_EVENT_COUNT {
7481 break;
7482 }
7483 tokio::time::sleep(Duration::from_millis(25)).await;
7484 }
7485
7486 assert_eq!(
7487 observed, LIVE_EVENT_COUNT,
7488 "live pump should not drop frames while slow background backfill is running"
7489 );
7490 let _ = runtime.mob_handle().stop().await;
7491 }
7492
7493 async fn count_console_event_frames(
7494 aggregator: &MobKitConsoleAggregator,
7495 identity: &str,
7496 ) -> usize {
7497 let mut after = None;
7498 let mut count = 0;
7499 loop {
7500 let page = aggregator
7501 .query_timeline(ConsoleTimelineQuery {
7502 identity: Some(identity.to_string()),
7503 after,
7504 limit: 1_000,
7505 ..ConsoleTimelineQuery::default()
7506 })
7507 .await
7508 .expect("query burst timeline");
7509 if page.frames.is_empty() {
7510 break;
7511 }
7512 count += page
7513 .frames
7514 .iter()
7515 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
7516 .count();
7517 after = page.next_cursor;
7518 if after.is_none() {
7519 break;
7520 }
7521 }
7522 count
7523 }
7524
7525 #[tokio::test]
7526 async fn status_updates_get_replayable_aggregate_cursors() {
7527 let aggregator = MobKitConsoleAggregator::in_memory();
7528 let frame = NewConsoleFrame {
7529 id: None,
7530 dedupe_key: "send-1".to_string(),
7531 timestamp_ms: 1,
7532 runtime_key: "runtime-a".to_string(),
7533 identity: "agent-a".to_string(),
7534 conversation_id: Some("agent-a".to_string()),
7535 session_id: Some("session-1".to_string()),
7536 kind: "user_input".to_string(),
7537 status: ConsoleFrameStatus::Accepted,
7538 payload: json!({ "content": "hello" }),
7539 source: ConsoleFrameSource {
7540 kind: ConsoleFrameSourceKind::Send,
7541 source_cursor: None,
7542 },
7543 source_event_id: None,
7544 interaction_id: Some("interaction-1".to_string()),
7545 turn_id: None,
7546 run_id: None,
7547 parent_frame_id: None,
7548 caused_by_frame_id: None,
7549 };
7550 let inserted = aggregator
7551 .store()
7552 .append_if_absent(frame)
7553 .await
7554 .expect("append frame");
7555
7556 update_frame_status_and_emit(
7557 &aggregator.inner,
7558 &inserted.frame.id,
7559 ConsoleFrameStatus::Delivered,
7560 )
7561 .await
7562 .expect("update status");
7563
7564 let page = aggregator
7565 .query_timeline(ConsoleTimelineQuery {
7566 identity: Some("agent-a".to_string()),
7567 after: Some(inserted.frame.cursor.clone()),
7568 limit: 10,
7569 ..ConsoleTimelineQuery::default()
7570 })
7571 .await
7572 .expect("query timeline");
7573 assert_eq!(page.frames.len(), 1);
7574 assert_eq!(page.frames[0].kind, "frame_updated");
7575 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
7576 assert_eq!(
7577 page.frames[0]
7578 .payload
7579 .get("frame")
7580 .and_then(|frame| frame.get("status"))
7581 .and_then(Value::as_str),
7582 Some("delivered")
7583 );
7584 }
7585
7586 #[tokio::test]
7587 async fn steer_delivery_appends_terminal_control_frame() {
7588 let aggregator = MobKitConsoleAggregator::in_memory();
7589 let frame = NewConsoleFrame {
7590 id: None,
7591 dedupe_key: "send-steer-1".to_string(),
7592 timestamp_ms: 1,
7593 runtime_key: "runtime-a".to_string(),
7594 identity: "agent-a".to_string(),
7595 conversation_id: Some("agent-a".to_string()),
7596 session_id: Some("session-1".to_string()),
7597 kind: "user_input".to_string(),
7598 status: ConsoleFrameStatus::Delivered,
7599 payload: json!({
7600 "content": "operator steer",
7601 "handling_mode": "steer",
7602 }),
7603 source: ConsoleFrameSource {
7604 kind: ConsoleFrameSourceKind::Send,
7605 source_cursor: None,
7606 },
7607 source_event_id: None,
7608 interaction_id: Some("interaction-steer-1".to_string()),
7609 turn_id: None,
7610 run_id: None,
7611 parent_frame_id: None,
7612 caused_by_frame_id: None,
7613 };
7614 let inserted = aggregator
7615 .store()
7616 .append_if_absent(frame)
7617 .await
7618 .expect("append steer input");
7619
7620 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
7621 .await
7622 .expect("append steer terminal");
7623
7624 let page = aggregator
7625 .query_timeline(ConsoleTimelineQuery {
7626 identity: Some("agent-a".to_string()),
7627 after: Some(inserted.frame.cursor.clone()),
7628 limit: 10,
7629 ..ConsoleTimelineQuery::default()
7630 })
7631 .await
7632 .expect("query timeline");
7633
7634 assert_eq!(page.frames.len(), 1);
7635 let terminal = &page.frames[0];
7636 assert_eq!(terminal.kind, "interaction_complete");
7637 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
7638 assert_eq!(
7639 terminal.interaction_id.as_deref(),
7640 Some("interaction-steer-1")
7641 );
7642 assert_eq!(
7643 terminal.parent_frame_id.as_deref(),
7644 Some(inserted.frame.id.as_str())
7645 );
7646 assert_eq!(
7647 terminal.payload.get("reason").and_then(Value::as_str),
7648 Some("steer_delivered")
7649 );
7650 }
7651
7652 #[tokio::test]
7653 async fn history_counterpart_scan_is_not_capped_to_one_page() {
7654 let aggregator = MobKitConsoleAggregator::in_memory();
7655 for idx in 0..1_005 {
7656 aggregator
7657 .store()
7658 .append_if_absent(NewConsoleFrame {
7659 id: None,
7660 dedupe_key: format!("filler-{idx}"),
7661 timestamp_ms: idx,
7662 runtime_key: "runtime-a".to_string(),
7663 identity: "agent-a".to_string(),
7664 conversation_id: Some("agent-a".to_string()),
7665 session_id: Some("session-a".to_string()),
7666 kind: "text_delta".to_string(),
7667 status: ConsoleFrameStatus::Completed,
7668 payload: json!({ "delta": idx }),
7669 source: ConsoleFrameSource {
7670 kind: ConsoleFrameSourceKind::ConsoleEvent,
7671 source_cursor: None,
7672 },
7673 source_event_id: Some(format!("filler-{idx}")),
7674 interaction_id: None,
7675 turn_id: None,
7676 run_id: None,
7677 parent_frame_id: None,
7678 caused_by_frame_id: None,
7679 })
7680 .await
7681 .expect("append filler");
7682 }
7683 aggregator
7684 .store()
7685 .append_if_absent(NewConsoleFrame {
7686 id: None,
7687 dedupe_key: "live-user-input".to_string(),
7688 timestamp_ms: 2_000,
7689 runtime_key: "runtime-a".to_string(),
7690 identity: "agent-a".to_string(),
7691 conversation_id: Some("agent-a".to_string()),
7692 session_id: Some("session-a".to_string()),
7693 kind: "user_input".to_string(),
7694 status: ConsoleFrameStatus::Delivered,
7695 payload: json!({ "content": "already here" }),
7696 source: ConsoleFrameSource {
7697 kind: ConsoleFrameSourceKind::ConsoleEvent,
7698 source_cursor: None,
7699 },
7700 source_event_id: Some("live-user-input".to_string()),
7701 interaction_id: None,
7702 turn_id: None,
7703 run_id: None,
7704 parent_frame_id: None,
7705 caused_by_frame_id: None,
7706 })
7707 .await
7708 .expect("append live input");
7709
7710 let history = NewConsoleFrame {
7711 id: None,
7712 dedupe_key: "history-user-input".to_string(),
7713 timestamp_ms: 3_000,
7714 runtime_key: "runtime-a".to_string(),
7715 identity: "agent-a".to_string(),
7716 conversation_id: Some("agent-a".to_string()),
7717 session_id: Some("session-a".to_string()),
7718 kind: "user_input".to_string(),
7719 status: ConsoleFrameStatus::Completed,
7720 payload: json!({ "content": "already here" }),
7721 source: ConsoleFrameSource {
7722 kind: ConsoleFrameSourceKind::SessionHistory,
7723 source_cursor: Some("session-a:1006".to_string()),
7724 },
7725 source_event_id: None,
7726 interaction_id: None,
7727 turn_id: None,
7728 run_id: None,
7729 parent_frame_id: None,
7730 caused_by_frame_id: None,
7731 };
7732
7733 assert!(
7734 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7735 .await
7736 .expect("counterpart scan")
7737 );
7738 }
7739
7740 #[tokio::test]
7741 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
7742 let aggregator = MobKitConsoleAggregator::in_memory();
7743 aggregator
7744 .store()
7745 .append_if_absent(NewConsoleFrame {
7746 id: None,
7747 dedupe_key: "live-user-input".to_string(),
7748 timestamp_ms: 2_000,
7749 runtime_key: "runtime-a".to_string(),
7750 identity: "agent-a".to_string(),
7751 conversation_id: Some("agent-a".to_string()),
7752 session_id: Some("session-a".to_string()),
7753 kind: "user_input".to_string(),
7754 status: ConsoleFrameStatus::Delivered,
7755 payload: json!({ "content": "hello from operator" }),
7756 source: ConsoleFrameSource {
7757 kind: ConsoleFrameSourceKind::ConsoleEvent,
7758 source_cursor: None,
7759 },
7760 source_event_id: Some("live-user-input".to_string()),
7761 interaction_id: None,
7762 turn_id: None,
7763 run_id: None,
7764 parent_frame_id: None,
7765 caused_by_frame_id: None,
7766 })
7767 .await
7768 .expect("append live input");
7769
7770 let history = NewConsoleFrame {
7771 id: None,
7772 dedupe_key: "history-user-input".to_string(),
7773 timestamp_ms: 3_000,
7774 runtime_key: "runtime-a".to_string(),
7775 identity: "agent-a".to_string(),
7776 conversation_id: Some("agent-a".to_string()),
7777 session_id: Some("session-a".to_string()),
7778 kind: "user_input".to_string(),
7779 status: ConsoleFrameStatus::Completed,
7780 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
7781 source: ConsoleFrameSource {
7782 kind: ConsoleFrameSourceKind::SessionHistory,
7783 source_cursor: Some("session-a:2".to_string()),
7784 },
7785 source_event_id: None,
7786 interaction_id: None,
7787 turn_id: None,
7788 run_id: None,
7789 parent_frame_id: None,
7790 caused_by_frame_id: None,
7791 };
7792
7793 assert!(
7794 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7795 .await
7796 .expect("counterpart scan")
7797 );
7798 }
7799
7800 #[tokio::test]
7801 async fn history_counterpart_scan_matches_content_block_user_prompts() {
7802 let aggregator = MobKitConsoleAggregator::in_memory();
7803 aggregator
7804 .store()
7805 .append_if_absent(NewConsoleFrame {
7806 id: None,
7807 dedupe_key: "live-user-input".to_string(),
7808 timestamp_ms: 2_000,
7809 runtime_key: "runtime-a".to_string(),
7810 identity: "agent-a".to_string(),
7811 conversation_id: Some("agent-a".to_string()),
7812 session_id: Some("session-a".to_string()),
7813 kind: "user_input".to_string(),
7814 status: ConsoleFrameStatus::Delivered,
7815 payload: json!({ "content": "hello from operator" }),
7816 source: ConsoleFrameSource {
7817 kind: ConsoleFrameSourceKind::ConsoleEvent,
7818 source_cursor: None,
7819 },
7820 source_event_id: Some("live-user-input".to_string()),
7821 interaction_id: None,
7822 turn_id: None,
7823 run_id: None,
7824 parent_frame_id: None,
7825 caused_by_frame_id: None,
7826 })
7827 .await
7828 .expect("append live input");
7829
7830 let history = NewConsoleFrame {
7831 id: None,
7832 dedupe_key: "history-user-input".to_string(),
7833 timestamp_ms: 3_000,
7834 runtime_key: "runtime-a".to_string(),
7835 identity: "agent-a".to_string(),
7836 conversation_id: Some("agent-a".to_string()),
7837 session_id: Some("session-a".to_string()),
7838 kind: "user_input".to_string(),
7839 status: ConsoleFrameStatus::Completed,
7840 payload: json!({
7841 "content": [{ "type": "text", "text": "hello from operator" }]
7842 }),
7843 source: ConsoleFrameSource {
7844 kind: ConsoleFrameSourceKind::SessionHistory,
7845 source_cursor: Some("session-a:2".to_string()),
7846 },
7847 source_event_id: None,
7848 interaction_id: None,
7849 turn_id: None,
7850 run_id: None,
7851 parent_frame_id: None,
7852 caused_by_frame_id: None,
7853 };
7854
7855 assert!(
7856 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7857 .await
7858 .expect("counterpart scan")
7859 );
7860 }
7861
7862 #[tokio::test]
7863 async fn history_counterpart_scan_matches_live_tool_results() {
7864 let aggregator = MobKitConsoleAggregator::in_memory();
7865 aggregator
7866 .store()
7867 .append_if_absent(NewConsoleFrame {
7868 id: None,
7869 dedupe_key: "live-tool-result".to_string(),
7870 timestamp_ms: 2_000,
7871 runtime_key: "runtime-a".to_string(),
7872 identity: "agent-a".to_string(),
7873 conversation_id: Some("agent-a".to_string()),
7874 session_id: Some("session-a".to_string()),
7875 kind: "tool_execution_completed".to_string(),
7876 status: ConsoleFrameStatus::Delivered,
7877 payload: json!({
7878 "id": "call-1",
7879 "tool_call_id": "call-1",
7880 "result": "{ \"count\": 70 }"
7881 }),
7882 source: ConsoleFrameSource {
7883 kind: ConsoleFrameSourceKind::ConsoleEvent,
7884 source_cursor: None,
7885 },
7886 source_event_id: Some("live-tool-result".to_string()),
7887 interaction_id: None,
7888 turn_id: None,
7889 run_id: None,
7890 parent_frame_id: None,
7891 caused_by_frame_id: None,
7892 })
7893 .await
7894 .expect("append live tool result");
7895
7896 let history = NewConsoleFrame {
7897 id: None,
7898 dedupe_key: "history-tool-result".to_string(),
7899 timestamp_ms: 3_000,
7900 runtime_key: "runtime-a".to_string(),
7901 identity: "agent-a".to_string(),
7902 conversation_id: Some("agent-a".to_string()),
7903 session_id: Some("session-a".to_string()),
7904 kind: "tool_execution_completed".to_string(),
7905 status: ConsoleFrameStatus::Completed,
7906 payload: json!({
7907 "id": "call-1",
7908 "tool_call_id": "call-1",
7909 "result": "{ \"count\": 70 }",
7910 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
7911 }),
7912 source: ConsoleFrameSource {
7913 kind: ConsoleFrameSourceKind::SessionHistory,
7914 source_cursor: Some("session-a:3:0".to_string()),
7915 },
7916 source_event_id: None,
7917 interaction_id: None,
7918 turn_id: None,
7919 run_id: None,
7920 parent_frame_id: None,
7921 caused_by_frame_id: None,
7922 };
7923
7924 assert!(
7925 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7926 .await
7927 .expect("counterpart scan")
7928 );
7929 }
7930
7931 #[tokio::test]
7932 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
7933 let aggregator = MobKitConsoleAggregator::in_memory();
7934 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
7935 aggregator
7936 .store()
7937 .append_if_absent(NewConsoleFrame {
7938 id: None,
7939 dedupe_key: format!("live-delta-{idx}"),
7940 timestamp_ms: 2_000 + idx as u64,
7941 runtime_key: "runtime-a".to_string(),
7942 identity: "agent-a".to_string(),
7943 conversation_id: Some("agent-a".to_string()),
7944 session_id: Some("session-a".to_string()),
7945 kind: "text_delta".to_string(),
7946 status: ConsoleFrameStatus::Delivered,
7947 payload: json!({ "delta": delta }),
7948 source: ConsoleFrameSource {
7949 kind: ConsoleFrameSourceKind::ConsoleEvent,
7950 source_cursor: None,
7951 },
7952 source_event_id: Some(format!("live-delta-{idx}")),
7953 interaction_id: Some("turn-a".to_string()),
7954 turn_id: None,
7955 run_id: None,
7956 parent_frame_id: None,
7957 caused_by_frame_id: None,
7958 })
7959 .await
7960 .expect("append live delta");
7961 }
7962
7963 let history = NewConsoleFrame {
7964 id: None,
7965 dedupe_key: "history-assistant-complete".to_string(),
7966 timestamp_ms: 3_000,
7967 runtime_key: "runtime-a".to_string(),
7968 identity: "agent-a".to_string(),
7969 conversation_id: Some("agent-a".to_string()),
7970 session_id: Some("session-a".to_string()),
7971 kind: "interaction_complete".to_string(),
7972 status: ConsoleFrameStatus::Completed,
7973 payload: json!({ "result": "Ready and standing by." }),
7974 source: ConsoleFrameSource {
7975 kind: ConsoleFrameSourceKind::SessionHistory,
7976 source_cursor: Some("session-a:3".to_string()),
7977 },
7978 source_event_id: None,
7979 interaction_id: None,
7980 turn_id: None,
7981 run_id: None,
7982 parent_frame_id: None,
7983 caused_by_frame_id: None,
7984 };
7985
7986 assert!(
7987 history_frame_has_existing_counterpart(&aggregator.inner, &history)
7988 .await
7989 .expect("counterpart scan")
7990 );
7991 }
7992
7993 #[test]
7994 fn session_history_watermark_key_is_session_scoped() {
7995 assert_ne!(
7996 session_history_watermark_runtime_key("runtime-a", "session-1"),
7997 session_history_watermark_runtime_key("runtime-a", "session-2")
7998 );
7999 }
8000
8001 #[test]
8002 fn session_history_watermarks_are_cursor_and_ttl_aware() {
8003 let legacy = "session:with:colon:42";
8004 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8005 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8006
8007 assert_eq!(
8008 parse_session_history_watermark(legacy, "session:with:colon"),
8009 Some(42)
8010 );
8011 assert_eq!(
8012 parse_session_history_watermark(&checked, "session:with:colon"),
8013 Some(43)
8014 );
8015 assert!(session_history_watermark_is_fresh(
8016 &checked,
8017 "session:with:colon",
8018 1_500
8019 ));
8020 assert!(!session_history_watermark_is_fresh(
8021 &checked,
8022 "session:with:colon",
8023 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8024 ));
8025 assert!(session_history_watermark_is_fresh(
8026 &empty_checked,
8027 "session:with:colon",
8028 1_500
8029 ));
8030 assert!(!session_history_watermark_is_fresh(
8031 &empty_checked,
8032 "session:with:colon",
8033 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8034 ));
8035 }
8036
8037 #[test]
8038 fn session_history_messages_project_to_renderable_frames() {
8039 let user = frame_from_session_history_message(
8040 "runtime-a",
8041 "agent-a",
8042 "session-a",
8043 0,
8044 json!({
8045 "role": "user",
8046 "content": "hello",
8047 "timestamp_ms": 10
8048 }),
8049 )
8050 .expect("user history frame");
8051 let assistant = frame_from_session_history_message(
8052 "runtime-a",
8053 "agent-a",
8054 "session-a",
8055 1,
8056 json!({
8057 "role": "assistant",
8058 "content": "hi there",
8059 "stop_reason": "end_turn",
8060 "usage": { "input_tokens": 1, "output_tokens": 1, "total_tokens": 2 },
8061 "timestamp_ms": 11
8062 }),
8063 )
8064 .expect("assistant history frame");
8065
8066 assert_eq!(user.kind, "user_input");
8067 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8068 assert_eq!(
8069 user.payload["content"],
8070 json!([{ "type": "text", "text": "hello" }])
8071 );
8072 assert_eq!(assistant.kind, "interaction_complete");
8073 assert_eq!(assistant.payload["text"], json!("hi there"));
8074 assert!(
8075 assistant
8076 .dedupe_key
8077 .starts_with("session-history:runtime-a:session-a:1:")
8078 );
8079 }
8080
8081 #[test]
8082 fn session_history_projection_filters_scaffold_user_messages() {
8083 let spawn_notice = frame_from_session_history_message(
8084 "runtime-a",
8085 "agent-a",
8086 "session-a",
8087 0,
8088 json!({
8089 "role": "user",
8090 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8091 "timestamp_ms": 10
8092 }),
8093 );
8094 let peer_update = frame_from_session_history_message(
8095 "runtime-a",
8096 "agent-a",
8097 "session-a",
8098 1,
8099 json!({
8100 "role": "user",
8101 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8102 "timestamp_ms": 11
8103 }),
8104 );
8105 let real_user = frame_from_session_history_message(
8106 "runtime-a",
8107 "agent-a",
8108 "session-a",
8109 2,
8110 json!({
8111 "role": "user",
8112 "content": "Please review the incident notes.",
8113 "timestamp_ms": 12
8114 }),
8115 );
8116
8117 assert!(spawn_notice.is_none());
8118 assert!(peer_update.is_none());
8119 assert!(real_user.is_some());
8120 }
8121
8122 #[test]
8123 fn session_history_projection_skips_non_transcript_messages() {
8124 let skipped = frame_from_session_history_message(
8125 "runtime-a",
8126 "agent-a",
8127 "session-a",
8128 0,
8129 json!({
8130 "content": "internal system prompt"
8131 }),
8132 );
8133 assert!(skipped.is_none());
8134 }
8135
8136 #[test]
8137 fn session_history_projection_extracts_assistant_blocks() {
8138 let frame = frame_from_session_history_message(
8139 "runtime-a",
8140 "agent-a",
8141 "session-a",
8142 0,
8143 json!({
8144 "role": "block_assistant",
8145 "blocks": [
8146 { "block_type": "text", "data": { "text": "hello " } },
8147 { "block_type": "text", "data": { "text": "there" } }
8148 ],
8149 "stop_reason": "end_turn"
8150 }),
8151 )
8152 .expect("assistant block history frame");
8153 assert_eq!(frame.payload["text"], json!("hello there"));
8154 }
8155
8156 #[test]
8157 fn session_history_projection_extracts_nested_text_block_data() {
8158 let frame = frame_from_session_history_message(
8159 "runtime-a",
8160 "agent-a",
8161 "session-a",
8162 0,
8163 json!({
8164 "role": "block_assistant",
8165 "blocks": [
8166 {
8167 "block_type": "text",
8168 "data": { "text": "Ready and standing by." }
8169 }
8170 ],
8171 "stop_reason": "end_turn",
8172 "created_at": "1970-01-01T00:00:00.010Z"
8173 }),
8174 )
8175 .expect("assistant block history frame");
8176
8177 assert_eq!(frame.kind, "interaction_complete");
8178 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
8179 }
8180
8181 #[test]
8182 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
8183 let frame = frame_from_session_history_message(
8184 "runtime-a",
8185 "agent-a",
8186 "session-a",
8187 0,
8188 json!({
8189 "role": "block_assistant",
8190 "blocks": [
8191 {
8192 "block_type": "reasoning",
8193 "data": { "text": "**Planning**\nI should not be rendered." }
8194 },
8195 {
8196 "block_type": "text",
8197 "data": { "text": "Visible answer." }
8198 }
8199 ],
8200 "stop_reason": "end_turn",
8201 "created_at": "1970-01-01T00:00:00.010Z"
8202 }),
8203 )
8204 .expect("assistant block history frame");
8205
8206 assert_eq!(frame.kind, "interaction_complete");
8207 assert_eq!(frame.payload["result"], json!("Visible answer."));
8208 assert_eq!(frame.payload["text"], json!("Visible answer."));
8209 }
8210
8211 #[test]
8212 fn session_history_projection_leaves_reasoning_only_result_empty() {
8213 let frame = frame_from_session_history_message(
8214 "runtime-a",
8215 "agent-a",
8216 "session-a",
8217 0,
8218 json!({
8219 "role": "block_assistant",
8220 "blocks": [
8221 {
8222 "block_type": "reasoning",
8223 "data": { "text": "Private planning text." }
8224 },
8225 {
8226 "block_type": "tool_use",
8227 "data": { "id": "toolu-1", "name": "peers", "args": {} }
8228 }
8229 ],
8230 "stop_reason": "end_turn",
8231 "created_at": "1970-01-01T00:00:00.010Z"
8232 }),
8233 )
8234 .expect("assistant block history frame");
8235
8236 assert_eq!(frame.kind, "interaction_complete");
8237 assert_eq!(frame.payload["result"], json!(""));
8238 assert_eq!(frame.payload["text"], json!(""));
8239 }
8240
8241 #[test]
8242 fn session_history_projection_preserves_tool_results() {
8243 let frames = frames_from_session_history_message(
8244 "runtime-a",
8245 "agent-a",
8246 "session-a",
8247 5,
8248 json!({
8249 "role": "tool_results",
8250 "results": [
8251 {
8252 "tool_use_id": "call-peers",
8253 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
8254 "is_error": false
8255 }
8256 ],
8257 "created_at": "1970-01-01T00:00:00.050Z"
8258 }),
8259 );
8260
8261 assert_eq!(frames.len(), 1);
8262 let frame = &frames[0];
8263 assert_eq!(frame.kind, "tool_execution_completed");
8264 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
8265 assert_eq!(
8266 frame.payload["result"],
8267 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
8268 );
8269 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
8270 assert_eq!(frame.timestamp_ms, 50);
8271 }
8272
8273 #[test]
8274 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
8275 let frames = frames_from_session_history_message(
8276 "runtime-a",
8277 "review:singleton",
8278 "session-a",
8279 7,
8280 json!({
8281 "role": "block_assistant",
8282 "blocks": [
8283 {
8284 "block_type": "tool_use",
8285 "data": {
8286 "id": "call-spawn",
8287 "name": "mob_spawn_member",
8288 "args": {
8289 "member_id": "review-worker-alpha",
8290 "profile": "review-worker",
8291 "initial_message": "Review Initiative Alpha and save the result."
8292 }
8293 }
8294 }
8295 ],
8296 "stop_reason": "tool_use",
8297 "created_at": "1970-01-01T00:00:00.070Z"
8298 }),
8299 );
8300
8301 assert_eq!(frames.len(), 2);
8302 let child = frames
8303 .iter()
8304 .find(|frame| frame.identity == "review-worker-alpha")
8305 .expect("spawned member initial message frame");
8306 assert_eq!(child.kind, "user_input");
8307 assert_eq!(child.timestamp_ms, 70);
8308 assert_eq!(
8309 child.payload["message"]["content"],
8310 "Review Initiative Alpha and save the result."
8311 );
8312 assert_eq!(child.payload["parent_identity"], "review:singleton");
8313 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
8314 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
8315 }
8316
8317 #[test]
8318 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
8319 let frames = frames_from_session_history_message(
8320 "runtime-a",
8321 "review:singleton",
8322 "session-a",
8323 8,
8324 json!({
8325 "role": "block_assistant",
8326 "blocks": [
8327 {
8328 "block_type": "tool_use",
8329 "data": {
8330 "id": "call-spawn-many",
8331 "name": "mob_spawn_member",
8332 "args": {
8333 "specs": [
8334 {
8335 "agent_identity": "review-worker-alpha",
8336 "profile": "review-worker",
8337 "initial_message": "Review Initiative Alpha."
8338 },
8339 {
8340 "agent_identity": "review-worker-beta",
8341 "profile": "review-worker",
8342 "initial_message": "Review Initiative Beta."
8343 }
8344 ]
8345 }
8346 }
8347 }
8348 ],
8349 "stop_reason": "tool_use",
8350 "created_at": "1970-01-01T00:00:00.080Z"
8351 }),
8352 );
8353
8354 assert_eq!(frames.len(), 3);
8355 let child_messages = frames
8356 .iter()
8357 .filter(|frame| frame.kind == "user_input")
8358 .map(|frame| {
8359 (
8360 frame.identity.as_str(),
8361 frame.payload["message"]["content"]
8362 .as_str()
8363 .unwrap_or_default(),
8364 )
8365 })
8366 .collect::<Vec<_>>();
8367 assert_eq!(
8368 child_messages,
8369 vec![
8370 ("review-worker-alpha", "Review Initiative Alpha."),
8371 ("review-worker-beta", "Review Initiative Beta."),
8372 ]
8373 );
8374 }
8375
8376 #[tokio::test]
8377 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
8378 let aggregator = MobKitConsoleAggregator::in_memory();
8379 let frames = frames_from_session_history_message(
8380 "runtime-a",
8381 "review:singleton",
8382 "session-a",
8383 9,
8384 json!({
8385 "role": "block_assistant",
8386 "blocks": [
8387 {
8388 "block_type": "tool_use",
8389 "data": {
8390 "id": "call-spawn-many",
8391 "name": "mob_spawn_member",
8392 "args": {
8393 "specs": [
8394 {
8395 "agent_identity": "review-worker-alpha",
8396 "profile": "review-worker",
8397 "initial_message": "Review Initiative Alpha."
8398 },
8399 {
8400 "agent_identity": "review-worker-beta",
8401 "profile": "review-worker",
8402 "initial_message": "Review Initiative Beta."
8403 }
8404 ]
8405 }
8406 }
8407 }
8408 ],
8409 "stop_reason": "tool_use",
8410 "created_at": "1970-01-01T00:00:00.090Z"
8411 }),
8412 );
8413 for frame in frames {
8414 aggregator
8415 .store()
8416 .append_if_absent(frame)
8417 .await
8418 .expect("append projected session-history frame");
8419 }
8420
8421 let child_page = aggregator
8422 .query_timeline(ConsoleTimelineQuery {
8423 identity: Some("review-worker-alpha".to_string()),
8424 limit: 20,
8425 ..ConsoleTimelineQuery::default()
8426 })
8427 .await
8428 .expect("query child timeline");
8429 assert_eq!(child_page.frames.len(), 1);
8430 assert_eq!(child_page.frames[0].kind, "user_input");
8431 assert_eq!(
8432 child_page.frames[0].payload["message"]["content"],
8433 "Review Initiative Alpha."
8434 );
8435 assert_eq!(
8436 child_page.frames[0].payload["parent_identity"],
8437 "review:singleton"
8438 );
8439
8440 let parent_page = aggregator
8441 .query_timeline(ConsoleTimelineQuery {
8442 identity: Some("review:singleton".to_string()),
8443 limit: 20,
8444 ..ConsoleTimelineQuery::default()
8445 })
8446 .await
8447 .expect("query parent timeline");
8448 assert!(
8449 parent_page
8450 .frames
8451 .iter()
8452 .all(|frame| frame.identity == "review:singleton"),
8453 "child initial messages must not leak into parent timeline: {:#?}",
8454 parent_page.frames
8455 );
8456 }
8457
8458 #[test]
8459 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
8460 let frames = frames_from_session_history_message_with_namespace(
8461 "runtime-a",
8462 "test/review:singleton",
8463 "test",
8464 "session-a",
8465 10,
8466 json!({
8467 "role": "block_assistant",
8468 "blocks": [
8469 {
8470 "block_type": "tool_use",
8471 "data": {
8472 "id": "call-spawn-many",
8473 "name": "mob_spawn_member",
8474 "args": {
8475 "specs": [
8476 {
8477 "agent_identity": "review-worker-alpha",
8478 "profile": "review-worker",
8479 "initial_message": "Review Initiative Alpha."
8480 },
8481 {
8482 "agent_identity": "test/review-worker-beta",
8483 "profile": "review-worker",
8484 "initial_message": "Review Initiative Beta."
8485 }
8486 ]
8487 }
8488 }
8489 }
8490 ],
8491 "stop_reason": "tool_use",
8492 "created_at": "1970-01-01T00:00:00.100Z"
8493 }),
8494 );
8495
8496 let child_identities = frames
8497 .iter()
8498 .filter(|frame| frame.kind == "user_input")
8499 .map(|frame| frame.identity.as_str())
8500 .collect::<Vec<_>>();
8501 assert_eq!(
8502 child_identities,
8503 vec!["test/review-worker-alpha", "test/review-worker-beta"]
8504 );
8505 }
8506
8507 #[tokio::test]
8508 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
8509 let aggregator = MobKitConsoleAggregator::in_memory();
8510 aggregator
8511 .inner
8512 .identity_read_model
8513 .replace(vec![
8514 identity_record_for_test("test/review:singleton"),
8515 identity_record_for_test("test/review-worker-alpha"),
8516 ])
8517 .await;
8518 let frames = frames_from_session_history_message_with_namespace(
8519 "runtime-a",
8520 "test/review:singleton",
8521 "test",
8522 "session-a",
8523 11,
8524 json!({
8525 "role": "block_assistant",
8526 "blocks": [
8527 {
8528 "block_type": "tool_use",
8529 "data": {
8530 "id": "call-spawn",
8531 "name": "mob_spawn_member",
8532 "args": {
8533 "agent_identity": "review-worker-alpha",
8534 "profile": "review-worker",
8535 "initial_message": "Review Initiative Alpha."
8536 }
8537 }
8538 }
8539 ],
8540 "stop_reason": "tool_use",
8541 "created_at": "1970-01-01T00:00:00.110Z"
8542 }),
8543 );
8544 for frame in frames {
8545 aggregator
8546 .store()
8547 .append_if_absent(frame)
8548 .await
8549 .expect("append projected namespaced session-history frame");
8550 }
8551
8552 let namespaced_child_page = aggregator
8553 .query_timeline(ConsoleTimelineQuery {
8554 identity: Some("test/review-worker-alpha".to_string()),
8555 limit: 20,
8556 ..ConsoleTimelineQuery::default()
8557 })
8558 .await
8559 .expect("query namespaced child timeline");
8560 assert_eq!(namespaced_child_page.frames.len(), 1);
8561 assert_eq!(
8562 namespaced_child_page.frames[0].identity,
8563 "test/review-worker-alpha"
8564 );
8565 assert_eq!(
8566 namespaced_child_page.frames[0].payload["message"]["content"],
8567 "Review Initiative Alpha."
8568 );
8569
8570 let raw_child_page = aggregator
8571 .query_timeline(ConsoleTimelineQuery {
8572 identity: Some("review-worker-alpha".to_string()),
8573 limit: 20,
8574 ..ConsoleTimelineQuery::default()
8575 })
8576 .await
8577 .expect("query raw child timeline");
8578 assert!(
8579 raw_child_page.frames.is_empty(),
8580 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
8581 raw_child_page.frames
8582 );
8583 }
8584
8585 #[test]
8586 fn namespace_helpers_preserve_namespace_named_member_identity() {
8587 assert_eq!(apply_namespace("test", "test"), "test/test");
8588 assert_eq!(
8589 strip_namespace("test/test", "test").as_deref(),
8590 Some("test")
8591 );
8592 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
8593 assert_eq!(
8594 strip_namespace("test/worker", "test").as_deref(),
8595 Some("worker")
8596 );
8597 assert_eq!(strip_namespace("test", "test"), None);
8598 }
8599
8600 #[test]
8601 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
8602 assert!(!member_id_matches_durable_identity(
8603 "rt:review:singleton:0",
8604 "review:singleton"
8605 ));
8606 assert!(!member_id_matches_durable_identity(
8607 "review:singleton:gen1",
8608 "review:singleton"
8609 ));
8610 assert!(!member_id_matches_durable_identity(
8611 "review:singleton:1",
8612 "review:singleton"
8613 ));
8614 assert!(!member_id_matches_durable_identity(
8615 "rt:review:singleton:qa:0",
8616 "review:singleton"
8617 ));
8618 assert!(!member_id_matches_durable_identity(
8619 "review:singleton:qa",
8620 "review:singleton"
8621 ));
8622 }
8623
8624 #[test]
8625 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
8626 let frame = frame_from_session_history_message(
8627 "runtime-a",
8628 "agent-a",
8629 "session-a",
8630 0,
8631 json!({
8632 "role": "user",
8633 "content": "hello",
8634 "created_at": "2026-05-12T05:00:06.564227Z"
8635 }),
8636 )
8637 .expect("user history frame");
8638
8639 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
8640 }
8641}