1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images,
24 is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32 SourceIngestionTransition,
33};
34pub use store::{
35 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36 SqliteConsoleLogStore,
37};
38pub use types::{
39 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59 CachedOnly,
60 IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65 inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70 pub session_history_backfill_enabled: bool,
71 pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75 fn default() -> Self {
76 Self {
77 session_history_backfill_enabled: true,
78 max_concurrent_session_backfills: 16,
79 }
80 }
81}
82
83struct AggregatorInner {
84 store: Arc<dyn ConsoleLogStore>,
85 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89 session_backfill_permits: Arc<Semaphore>,
90 session_backfill_epochs: std::sync::Mutex<BTreeMap<String, u64>>,
98 identity_read_model: ConsoleIdentityReadModel,
99 options: ConsoleAggregatorOptions,
100 live_projection_shutdowns: std::sync::Mutex<BTreeMap<String, tokio::sync::watch::Sender<bool>>>,
106}
107
108#[derive(Clone)]
109struct ConsoleIdentityReadModel {
110 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
111 refresh_lock: Arc<tokio::sync::Mutex<()>>,
112 primed: Arc<AtomicBool>,
113}
114
115impl Default for ConsoleIdentityReadModel {
116 fn default() -> Self {
117 Self {
118 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
119 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
120 primed: Arc::new(AtomicBool::new(false)),
121 }
122 }
123}
124
125impl ConsoleIdentityReadModel {
126 async fn snapshot(
127 &self,
128 inner: Arc<AggregatorInner>,
129 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
130 if !self.primed.load(Ordering::Acquire) {
131 Box::pin(self.prime_now(inner)).await?;
132 }
133 Ok(self.inner.read().await.clone())
134 }
135
136 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
137 self.inner.read().await.clone()
138 }
139
140 async fn refresh_now_if_idle(
141 &self,
142 inner: Arc<AggregatorInner>,
143 mode: IdentityCollectionMode,
144 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
145 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
146 return None;
147 };
148 let result = Box::pin(collect_identity_records(&inner, mode)).await;
149 drop(guard);
150 match result {
151 Ok(identities) => {
152 self.replace(identities.clone()).await;
153 Some(Ok(identities))
154 }
155 Err(err) => Some(Err(err)),
156 }
157 }
158
159 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
160 if self.primed.load(Ordering::Acquire) {
161 return Ok(());
162 }
163 let _guard = self.refresh_lock.clone().lock_owned().await;
164 if self.primed.load(Ordering::Acquire) {
165 return Ok(());
166 }
167 let identities = Box::pin(collect_identity_records(
168 &inner,
169 IdentityCollectionMode::CachedOnly,
170 ))
171 .await?;
172 *self.inner.write().await = identities;
173 self.primed.store(true, Ordering::Release);
174 Ok(())
175 }
176
177 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
178 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
179 return;
180 };
181 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
182 return;
183 };
184 let read_model = self.clone();
185 runtime_handle.spawn(async move {
186 let _guard = guard;
187 match Box::pin(collect_identity_records(
188 &inner,
189 IdentityCollectionMode::IncludeLiveMembers,
190 ))
191 .await
192 {
193 Ok(identities) => {
194 *read_model.inner.write().await = identities;
195 read_model.primed.store(true, Ordering::Release);
196 }
197 Err(err) => {
198 tracing::warn!(error = %err, "console identity read-model refresh failed");
199 }
200 }
201 });
202 }
203
204 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
205 *self.inner.write().await = identities;
206 self.primed.store(true, Ordering::Release);
207 }
208}
209
210#[derive(Clone)]
211struct RuntimeEntry {
212 runtime_key: String,
213 identity_namespace: String,
214 runtime: MobRuntime,
215 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
216 console_events: ConsoleEventStore,
217 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
218}
219
220#[derive(Clone)]
221struct ResolvedConsoleMember {
222 entry: RuntimeEntry,
223 handle: MobHandle,
224 member: MobMemberListEntry,
225 source_mob_id: String,
226 runtime_identity: String,
227}
228
229fn console_member_for_resolved_member(
230 resolved: &ResolvedConsoleMember,
231 record: &ConsoleIdentityRecord,
232) -> ConsoleMember {
233 ConsoleMember {
234 agent_identity: crate::member_comms_id::runtime_alias_str(
235 resolved.member.agent_identity.as_str(),
236 )
237 .into_owned(),
238 role: resolved.member.role.to_string(),
239 state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
240 model_capabilities: Default::default(),
241 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
242 session_id: record.session_id.clone(),
243 wired_to: resolved
244 .member
245 .wired_to
246 .iter()
247 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
248 .collect(),
249 labels: record.labels.clone(),
250 progress: None,
251 }
252}
253
254async fn resolved_member_visible(
255 resolved: &ResolvedConsoleMember,
256 record: &ConsoleIdentityRecord,
257) -> bool {
258 if !raw_resolved_member_visible(resolved, record) {
259 return false;
260 }
261 !Box::pin(live_record_shadowed_by_hidden_durable_binding(
262 resolved, record,
263 ))
264 .await
265}
266
267fn raw_resolved_member_visible(
268 resolved: &ResolvedConsoleMember,
269 record: &ConsoleIdentityRecord,
270) -> bool {
271 resolved.entry.visibility_policy.identity_visible(record)
272 && resolved
273 .entry
274 .visibility_policy
275 .member_visible(&console_member_for_resolved_member(resolved, record))
276}
277
278pub trait ConsoleVisibilityPolicy: Send + Sync {
279 fn include_implicit_delegate_members(&self) -> bool {
280 true
281 }
282
283 fn member_visible(&self, _member: &ConsoleMember) -> bool {
284 true
285 }
286
287 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
288 true
289 }
290
291 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
292 true
293 }
294
295 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
296 None
297 }
298}
299
300#[derive(Debug, Default)]
301pub struct AllowAllConsoleVisibilityPolicy;
302
303impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
304
305#[derive(Debug, Default)]
306pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
307
308impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
309 fn include_implicit_delegate_members(&self) -> bool {
310 false
311 }
312
313 fn member_visible(&self, member: &ConsoleMember) -> bool {
314 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
315 }
316
317 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
318 !is_implicit_delegate_member(
319 record
320 .labels
321 .get("role")
322 .map(String::as_str)
323 .unwrap_or_default(),
324 &record.labels,
325 )
326 }
327}
328
329#[derive(Clone)]
330pub struct ConsoleRuntimeRegistration {
331 pub runtime_key: String,
332 pub runtime: Arc<UnifiedRuntime>,
333 pub identity_namespace: String,
334 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
335}
336
337impl MobKitConsoleAggregator {
338 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
339 Self::new_with_options(store, ConsoleAggregatorOptions::default())
340 }
341
342 pub fn new_with_options(
343 store: Arc<dyn ConsoleLogStore>,
344 mut options: ConsoleAggregatorOptions,
345 ) -> Self {
346 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
347 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
348 Self {
349 inner: Arc::new(AggregatorInner {
350 store,
351 runtimes: RwLock::new(BTreeMap::new()),
352 event_tx,
353 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
354 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
355 session_backfill_permits: Arc::new(Semaphore::new(
356 options.max_concurrent_session_backfills,
357 )),
358 session_backfill_epochs: std::sync::Mutex::new(BTreeMap::new()),
359 identity_read_model: ConsoleIdentityReadModel::default(),
360 options,
361 live_projection_shutdowns: std::sync::Mutex::new(BTreeMap::new()),
362 }),
363 }
364 }
365
366 pub fn in_memory() -> Self {
367 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
368 }
369
370 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
371 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
372 }
373
374 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
375 self.inner.event_tx.subscribe()
376 }
377
378 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
379 self.inner.store.clone()
380 }
381
382 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
383 let identity_runtime = registration.runtime.identity_runtime().cloned();
384 self.register_runtime_handles_with_policy(
385 registration.runtime_key,
386 registration.identity_namespace,
387 registration.runtime.mob_runtime().clone(),
388 identity_runtime,
389 registration.runtime.console_events(),
390 registration.visibility_policy,
391 );
392 }
393
394 pub(crate) fn register_runtime_handles_with_policy(
395 &self,
396 runtime_key: impl Into<String>,
397 identity_namespace: impl Into<String>,
398 runtime: MobRuntime,
399 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
400 console_events: ConsoleEventStore,
401 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
402 ) {
403 let runtime_key = runtime_key.into();
404 let identity_namespace = identity_namespace.into();
405 let entry = RuntimeEntry {
406 runtime_key: runtime_key.clone(),
407 identity_namespace,
408 runtime,
409 identity_runtime,
410 console_events: console_events.clone(),
411 visibility_policy,
412 };
413 if let Ok(mut runtimes) = self.inner.runtimes.write() {
414 runtimes.insert(runtime_key.clone(), entry);
415 }
416 {
422 let prefix = format!("{runtime_key}:session-history:");
423 self.inner
424 .session_backfill_epochs
425 .lock()
426 .unwrap_or_else(std::sync::PoisonError::into_inner)
427 .retain(|key, _| !key.starts_with(&prefix));
428 }
429 let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false);
432 if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
433 && let Some(previous) = shutdowns.insert(runtime_key.clone(), shutdown_tx)
434 {
435 let _ = previous.send(true);
436 }
437 self.inner
438 .identity_read_model
439 .refresh_soon(self.inner.clone());
440 let inner = self.inner.clone();
441 let events_for_live = console_events.clone();
442 let events_for_live_recovery = console_events.clone();
443 let runtime_key_for_live = runtime_key.clone();
444 tokio::spawn(async move {
445 let mut rx = events_for_live.subscribe();
446 loop {
447 tokio::select! {
448 changed = shutdown_rx.changed() => {
449 if changed.is_err() || *shutdown_rx.borrow() {
451 break;
452 }
453 }
454 received = rx.recv() => match received {
455 Ok(envelope) => {
456 let _ = project_console_event(
457 inner.clone(),
458 &runtime_key_for_live,
459 envelope,
460 )
461 .await;
462 }
463 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
464 let _ = recover_lagged_source_events(
465 inner.clone(),
466 &runtime_key_for_live,
467 &events_for_live_recovery,
468 )
469 .await;
470 }
471 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
472 },
473 }
474 }
475 });
476
477 let inner = self.inner.clone();
478 let events_for_replay = console_events;
479 let runtime_key_for_replay = runtime_key;
480 tokio::spawn(async move {
481 let mut ingestion_state = SourceIngestionState::Registered;
482 if let Ok((next, _effects)) =
483 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
484 {
485 ingestion_state = next;
486 }
487 if let Ok(events) = events_for_replay.replay_all(None).await {
488 for envelope in events {
489 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
490 .await;
491 }
492 }
493 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
494 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
495 if let Ok((next, _effects)) =
496 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
497 {
498 ingestion_state = next;
499 }
500 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
501 });
502 }
503
504 pub fn unregister_runtime(&self, runtime_key: &str) {
514 let removed = self
515 .inner
516 .runtimes
517 .write()
518 .ok()
519 .and_then(|mut runtimes| runtimes.remove(runtime_key))
520 .is_some();
521 if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
522 && let Some(shutdown) = shutdowns.remove(runtime_key)
523 {
524 let _ = shutdown.send(true);
525 }
526 {
530 let prefix = format!("{runtime_key}:session-history:");
531 self.inner
532 .session_backfill_epochs
533 .lock()
534 .unwrap_or_else(std::sync::PoisonError::into_inner)
535 .retain(|key, _| !key.starts_with(&prefix));
536 }
537 if removed {
538 self.inner
539 .identity_read_model
540 .refresh_soon(self.inner.clone());
541 }
542 }
543
544 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
545 if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
546 self.inner.clone(),
547 IdentityCollectionMode::IncludeLiveMembers,
548 ))
549 .await
550 {
551 let identities = identities?;
552 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
553 return Ok(identities);
554 }
555 self.inner
556 .identity_read_model
557 .refresh_soon(self.inner.clone());
558 let identities =
559 Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
560 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
561 Ok(identities)
562 }
563
564 #[cfg(test)]
565 pub(crate) async fn list_identities_fresh(
566 &self,
567 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
568 let _guard = self
569 .inner
570 .identity_read_model
571 .refresh_lock
572 .clone()
573 .lock_owned()
574 .await;
575 let identities =
576 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
577 .await?;
578 self.inner
579 .identity_read_model
580 .replace(identities.clone())
581 .await;
582 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
583 Ok(identities)
584 }
585
586 pub async fn inspect_identity(
587 &self,
588 identity: &str,
589 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
590 let entries = self
591 .inner
592 .runtimes
593 .read()
594 .map_err(|_| runtime_registry_lock_error())?
595 .values()
596 .cloned()
597 .collect::<Vec<_>>();
598 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
599 let mut durable_matches = Vec::new();
600 if !requested_identity_has_runtime_member_alias(identity, &entries) {
601 for entry in &entries {
602 let Some(identity_runtime) = entry.identity_runtime.clone() else {
603 continue;
604 };
605 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
606 {
607 let Ok(parsed_identity) =
608 crate::identity_first::AgentIdentity::parse(&raw_identity)
609 else {
610 continue;
611 };
612 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
613 continue;
614 };
615 let mut record = identity_record_for_status(entry, &status);
616 let topology_peers =
617 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
618 record.topology_peers = topology_peers
619 .get(&record.identity)
620 .cloned()
621 .unwrap_or_default();
622 if Box::pin(console_identity_record_visible(entry, &record)).await {
623 durable_matches.push((
624 entry.clone(),
625 identity_runtime.clone(),
626 parsed_identity,
627 record,
628 ));
629 }
630 }
631 }
632 }
633 if durable_matches.len() > 1 {
634 let candidates = durable_matches
635 .iter()
636 .map(|(_, _, _, record)| record.runtime_member_id.clone())
637 .collect::<Vec<_>>()
638 .join(", ");
639 return Err(format!(
640 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
641 )
642 .into());
643 }
644 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
645 durable_matches.into_iter().next()
646 {
647 let durable_live_records = Box::pin(self.live_records_for_durable_record(
648 &entry,
649 identity,
650 &record,
651 &live_records,
652 ))
653 .await;
654 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
655 &entry,
656 &durable_live_records,
657 ))
658 .await;
659 if let Some(ambiguous_error) =
660 ambiguous_live_alias_error(identity, &visible_durable_live_records)
661 {
662 return Err(ambiguous_error.into());
663 }
664 if let Some(rebound_record) = self_heal_durable_session_mismatch(
665 &entry,
666 &identity_runtime,
667 &parsed_identity,
668 identity,
669 &record,
670 &durable_live_records,
671 )
672 .await
673 .map_err(ConsoleLogError::from)?
674 {
675 record = rebound_record;
676 }
677 if let Some(stale_error) =
678 stale_durable_record_error(identity, &record, &durable_live_records)
679 {
680 return Err(stale_error.into());
681 }
682 if let Some(session_id) = record.session_id.clone() {
683 spawn_session_history_backfill_target(
684 self.inner.clone(),
685 SessionBackfillTarget {
686 entry,
687 record: record.clone(),
688 session_id,
689 },
690 false,
691 );
692 }
693 return Ok(Some(ConsoleIdentityInspection {
694 peers: record.topology_peers.clone(),
695 identity: record,
696 }));
697 }
698 let mut runtime_id_matches = Vec::new();
699 for entry in &entries {
700 let Some(identity_runtime) = entry.identity_runtime.clone() else {
701 continue;
702 };
703 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
704 continue;
705 };
706 let Some(status) = identity_runtime
707 .statuses()
708 .await
709 .into_iter()
710 .find(|status| {
711 status
712 .agent_runtime_id
713 .as_ref()
714 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
715 })
716 else {
717 continue;
718 };
719 let mut record = identity_record_for_status(entry, &status);
720 let topology_peers =
721 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
722 record.topology_peers = topology_peers
723 .get(&record.identity)
724 .cloned()
725 .unwrap_or_default();
726 if Box::pin(console_identity_record_visible(entry, &record)).await {
727 runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
728 }
729 }
730 if runtime_id_matches.len() > 1 {
731 let candidates = runtime_id_matches
732 .iter()
733 .map(|(_, _, _, record)| record.identity.clone())
734 .collect::<Vec<_>>()
735 .join(", ");
736 return Err(format!(
737 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
738 )
739 .into());
740 }
741 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
742 runtime_id_matches.into_iter().next()
743 {
744 let durable_live_records = Box::pin(self.live_records_for_durable_record(
745 &entry,
746 identity,
747 &record,
748 &live_records,
749 ))
750 .await;
751 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
752 &entry,
753 &durable_live_records,
754 ))
755 .await;
756 if let Some(ambiguous_error) =
757 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
758 {
759 return Err(ambiguous_error.into());
760 }
761 if let Some(rebound_record) = self_heal_durable_session_mismatch(
762 &entry,
763 &identity_runtime,
764 &parsed_identity,
765 identity,
766 &record,
767 &durable_live_records,
768 )
769 .await
770 .map_err(ConsoleLogError::from)?
771 {
772 record = rebound_record;
773 }
774 if let Some(stale_error) =
775 stale_durable_record_error(identity, &record, &durable_live_records)
776 {
777 return Err(stale_error.into());
778 }
779 if let Some(session_id) = record.session_id.clone() {
780 spawn_session_history_backfill_target(
781 self.inner.clone(),
782 SessionBackfillTarget {
783 entry: entry.clone(),
784 record: record.clone(),
785 session_id,
786 },
787 false,
788 );
789 }
790 return Ok(Some(ConsoleIdentityInspection {
791 peers: record.topology_peers.clone(),
792 identity: record,
793 }));
794 }
795
796 let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
797 if live_matches.len() > 1 {
798 let candidates = live_matches
799 .iter()
800 .map(|resolved| resolved.runtime_identity.clone())
801 .collect::<Vec<_>>()
802 .join(", ");
803 return Err(format!(
804 "ambiguous live identity alias {identity}: candidates [{candidates}]"
805 )
806 .into());
807 }
808 if let Some(resolved) = live_matches.into_iter().next() {
809 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
810 return Ok(None);
811 };
812 if let Some(stale_error) =
813 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
814 {
815 return Err(stale_error.into());
816 }
817 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
818 && let Some(parsed_identity) = identity_runtime
819 .owned_identity_for_member_alias(&resolved.runtime_identity)
820 .await
821 && let Ok(status) = identity_runtime.status(&parsed_identity).await
822 {
823 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
824 let durable_live_records = Box::pin(self.live_records_for_durable_record(
825 &resolved.entry,
826 identity,
827 &durable_record,
828 &live_records,
829 ))
830 .await;
831 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
832 &resolved.entry,
833 &durable_live_records,
834 ))
835 .await;
836 if let Some(ambiguous_error) =
837 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
838 {
839 return Err(ambiguous_error.into());
840 }
841 if let Some(rebound_record) = self_heal_durable_session_mismatch(
842 &resolved.entry,
843 identity_runtime,
844 &parsed_identity,
845 identity,
846 &durable_record,
847 &durable_live_records,
848 )
849 .await
850 .map_err(ConsoleLogError::from)?
851 {
852 durable_record = rebound_record;
853 }
854 if let Some(stale_error) =
855 stale_durable_record_error(identity, &durable_record, &durable_live_records)
856 {
857 return Err(stale_error.into());
858 }
859 if status
860 .agent_runtime_id
861 .as_ref()
862 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
863 {
864 return Ok(None);
865 }
866 }
867 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
868 return Ok(None);
869 }
870 if let Some(session_id) = record.session_id.clone() {
871 spawn_session_history_backfill_target(
872 self.inner.clone(),
873 SessionBackfillTarget {
874 entry: resolved.entry.clone(),
875 record: record.clone(),
876 session_id,
877 },
878 false,
879 );
880 }
881 let peers = resolved
882 .member
883 .wired_to
884 .iter()
885 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
886 .collect();
887 return Ok(Some(ConsoleIdentityInspection {
888 identity: record,
889 peers,
890 }));
891 }
892
893 Ok(None)
894 }
895
896 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
897 let entries = self
898 .inner
899 .runtimes
900 .read()
901 .map_err(|_| runtime_registry_lock_error())?
902 .clone();
903 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
904 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
905 let mut durable_matches = Vec::new();
906 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
907 for entry in entries.values() {
908 let Some(identity_runtime) = entry.identity_runtime.clone() else {
909 continue;
910 };
911 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
912 {
913 let Ok(parsed_identity) =
914 crate::identity_first::AgentIdentity::parse(&raw_identity)
915 else {
916 continue;
917 };
918 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
919 continue;
920 };
921 let record = identity_record_for_status(entry, &status);
922 if !Box::pin(console_identity_record_visible(entry, &record)).await {
923 continue;
924 }
925 durable_matches.push((
926 entry.clone(),
927 identity_runtime.clone(),
928 parsed_identity,
929 record,
930 ));
931 }
932 }
933 }
934 if durable_matches.len() > 1 {
935 let candidates = durable_matches
936 .iter()
937 .map(|(_, _, identity, _)| identity.as_str().to_string())
938 .collect::<Vec<_>>()
939 .join(", ");
940 return Err(format!(
941 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
942 )
943 .into());
944 }
945 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
946 durable_matches.into_iter().next()
947 {
948 let durable_live_records = Box::pin(self.live_records_for_durable_record(
949 &entry,
950 identity,
951 &record,
952 &live_records,
953 ))
954 .await;
955 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
956 &entry,
957 &durable_live_records,
958 ))
959 .await;
960 if let Some(ambiguous_error) =
961 ambiguous_live_alias_error(identity, &visible_durable_live_records)
962 {
963 return Err(ambiguous_error.into());
964 }
965 if let Some(rebound_record) = self_heal_durable_session_mismatch(
966 &entry,
967 &identity_runtime,
968 &parsed_identity,
969 identity,
970 &record,
971 &durable_live_records,
972 )
973 .await
974 .map_err(ConsoleLogError::from)?
975 {
976 record = rebound_record;
977 }
978 if let Some(stale_error) =
979 stale_durable_record_error(identity, &record, &durable_live_records)
980 {
981 return Err(stale_error.into());
982 }
983 let cleanup_entry = entry.clone();
984 let include_current = !identity_runtime.has_session_bridge();
985 let (_, cleanup_error) = identity_runtime
986 .retire_and_cleanup_live_members_tracked(
987 &parsed_identity,
988 None,
989 move |retired_alias| async move {
990 let retired_alias = retired_alias?;
991 let stale_members = Box::pin(stale_generated_members_for_runtime_entry(
992 &cleanup_entry,
993 retired_alias.as_str(),
994 include_current,
995 ))
996 .await;
997 Box::pin(retire_captured_console_members(stale_members))
998 .await
999 .err()
1000 },
1001 )
1002 .await
1003 .map_err(|err| -> ConsoleLogError {
1004 format!("retire failed for {identity}: {err}").into()
1005 })?;
1006 if let Some(err) = cleanup_error {
1007 tracing::warn!(
1008 identity,
1009 error = %err,
1010 "stale console member cleanup failed after durable identity retire"
1011 );
1012 }
1013 return Ok(true);
1014 }
1015 let matches = Box::pin(self.resolve_visible_members(identity)).await;
1016 if matches.len() > 1 {
1017 let candidates = matches
1018 .iter()
1019 .map(|resolved| resolved.runtime_identity.clone())
1020 .collect::<Vec<_>>()
1021 .join(", ");
1022 return Err(format!(
1023 "ambiguous live identity alias {identity}: candidates [{candidates}]"
1024 )
1025 .into());
1026 }
1027 let mut live_retired_any = false;
1028 for resolved in matches {
1029 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1030 continue;
1031 };
1032 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1033 continue;
1034 }
1035 if let Some(stale_error) =
1036 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1037 {
1038 return Err(stale_error.into());
1039 }
1040 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
1041 && let Some(parsed_identity) = identity_runtime
1042 .identity_for_member_mutation(&resolved.runtime_identity)
1043 .await
1044 {
1045 let status = identity_runtime.status(&parsed_identity).await.map_err(
1046 |err| -> ConsoleLogError {
1047 format!("retire failed for {identity}: {err}").into()
1048 },
1049 )?;
1050 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
1051 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1052 &resolved.entry,
1053 identity,
1054 &durable_record,
1055 &live_records,
1056 ))
1057 .await;
1058 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1059 &resolved.entry,
1060 &durable_live_records,
1061 ))
1062 .await;
1063 if let Some(ambiguous_error) =
1064 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
1065 {
1066 return Err(ambiguous_error.into());
1067 }
1068 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1069 &resolved.entry,
1070 identity_runtime,
1071 &parsed_identity,
1072 identity,
1073 &durable_record,
1074 &durable_live_records,
1075 )
1076 .await
1077 .map_err(ConsoleLogError::from)?
1078 {
1079 durable_record = rebound_record;
1080 }
1081 if let Some(stale_error) =
1082 stale_durable_record_error(identity, &durable_record, &durable_live_records)
1083 {
1084 return Err(stale_error.into());
1085 }
1086 if status
1087 .agent_runtime_id
1088 .as_ref()
1089 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
1090 {
1091 continue;
1092 }
1093 identity_runtime
1094 .retire_member_alias_tracked(&parsed_identity, &resolved.runtime_identity)
1095 .await
1096 .map_err(|err| -> ConsoleLogError {
1097 format!("retire failed for {identity}: {err}").into()
1098 })?;
1099 live_retired_any = true;
1100 continue;
1101 }
1102 resolved
1103 .handle
1104 .retire(crate::member_comms_id::mob_member_id(
1105 resolved.runtime_identity.as_str(),
1106 ))
1107 .await
1108 .map_err(|err| -> ConsoleLogError {
1109 format!("retire failed for {identity}: {err}").into()
1110 })
1111 .or_else(|err| {
1112 if lifecycle_archive_cleanup_completed(&err.to_string()) {
1113 Ok(())
1114 } else {
1115 Err(err)
1116 }
1117 })?;
1118 live_retired_any = true;
1119 }
1120 Ok(live_retired_any)
1121 }
1122
1123 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1124 self.inner.store.clear_frames().await
1125 }
1126
1127 pub async fn query_timeline(
1128 &self,
1129 query: ConsoleTimelineQuery,
1130 ) -> ConsoleLogResult<ConsoleTimelinePage> {
1131 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1132 Ok(ConsoleTimelinePage {
1133 frames: page.frames,
1134 next_cursor: page.next_cursor,
1135 })
1136 }
1137
1138 pub async fn query_timeline_windowed(
1139 &self,
1140 query: ConsoleTimelineWindowQuery,
1141 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1142 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1143 .await?;
1144 if query.after.is_none()
1145 && query.before.is_none()
1146 && let Some(identity) = query.identity.clone()
1147 {
1148 let mut probe_query = query.clone();
1149 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1150 let page = self.inner.store.query_windowed_frames(probe_query).await?;
1151 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1152 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1153 }
1154 }
1155 Box::pin(self.query_timeline_visible(query)).await
1156 }
1157
1158 async fn reject_after_cursor_beyond_store_frontier(
1159 &self,
1160 after: Option<&ConsoleCursor>,
1161 ) -> ConsoleLogResult<()> {
1162 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1163 return Ok(());
1164 };
1165 let latest_seq = self
1166 .inner
1167 .store
1168 .latest_cursor()
1169 .await?
1170 .and_then(|cursor| cursor.seq())
1171 .unwrap_or(0);
1172 if after_seq > latest_seq {
1173 return Err(std::io::Error::other(
1174 "timeline replay cursor is beyond the current store frontier",
1175 )
1176 .into());
1177 }
1178 Ok(())
1179 }
1180
1181 async fn query_timeline_visible(
1182 &self,
1183 query: ConsoleTimelineWindowQuery,
1184 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1185 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1186 let mut scan_query = query.clone();
1187 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1188 let mut visible_frames = Vec::with_capacity(requested_limit);
1189 let mut anchor_frames = Vec::new();
1190 let mut identity_visibility_cache = HashMap::new();
1191 let identity_records = self.inner.identity_read_model.current().await;
1192 let mut next_cursor = query.after.clone();
1193 let mut since_last_scanned_cursor = query.after.clone();
1194 let mut since_last_delivered_cursor = None;
1195 let mut since_stopped_at_visible_limit = false;
1196 let mut latest_cursor = None;
1197 let mut exhausted = false;
1198 let mut scanned = 0usize;
1199
1200 loop {
1201 let page = self
1202 .inner
1203 .store
1204 .query_windowed_frames(scan_query.clone())
1205 .await?;
1206 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1207 if page.frames.is_empty() {
1208 exhausted = true;
1209 break;
1210 }
1211 let raw_len = page.frames.len();
1212 scanned = scanned.saturating_add(raw_len);
1213 match query.mode {
1214 ConsoleTimelineMode::Since => {
1215 if let Some(cursor) = page.next_cursor.clone() {
1216 since_last_scanned_cursor = Some(cursor.clone());
1217 scan_query.after = Some(cursor);
1218 }
1219 }
1220 ConsoleTimelineMode::Recent => {
1221 if let Some(first) = page.frames.first() {
1222 scan_query.before = Some(first.cursor.clone());
1223 }
1224 next_cursor = page.next_cursor.clone().or(next_cursor);
1225 }
1226 }
1227
1228 for frame in page.frames {
1229 let allow_historical_identity =
1230 query.identity.as_deref() == Some(frame.identity.as_str());
1231 if Box::pin(frame_is_visible_cached(
1232 &self.inner,
1233 &frame,
1234 allow_historical_identity,
1235 &mut identity_visibility_cache,
1236 &identity_records,
1237 ))
1238 .await
1239 .unwrap_or(false)
1240 {
1241 if query.mode == ConsoleTimelineMode::Recent
1242 && query.identity.is_some()
1243 && is_identity_timeline_anchor_frame(&frame)
1244 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1245 {
1246 anchor_frames.push(frame.clone());
1247 }
1248 match query.mode {
1249 ConsoleTimelineMode::Since => {
1250 if visible_frames.len() >= requested_limit {
1251 since_stopped_at_visible_limit = true;
1252 break;
1253 }
1254 since_last_delivered_cursor = Some(frame.cursor.clone());
1255 visible_frames.push(frame);
1256 if visible_frames.len() >= requested_limit {
1257 since_stopped_at_visible_limit = true;
1258 exhausted = false;
1259 break;
1260 }
1261 }
1262 ConsoleTimelineMode::Recent => {
1263 visible_frames.push(frame);
1264 }
1265 }
1266 }
1267 }
1268 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1269 && query.identity.is_some()
1270 && anchor_frames.is_empty()
1271 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1272 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1273 break;
1274 }
1275 if since_stopped_at_visible_limit {
1276 break;
1277 }
1278 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1279 exhausted = true;
1280 break;
1281 }
1282 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1283 break;
1284 }
1285 }
1286
1287 if query.mode == ConsoleTimelineMode::Recent {
1288 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1289 if visible_frames.len() > requested_limit {
1290 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1291 }
1292 if !anchor_frames.is_empty() {
1293 let anchor_cursors = anchor_frames
1294 .iter()
1295 .map(|frame| frame.cursor.clone())
1296 .collect::<Vec<_>>();
1297 let mut merged = anchor_frames;
1298 for frame in visible_frames {
1299 if !merged
1300 .iter()
1301 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1302 {
1303 merged.push(frame);
1304 }
1305 }
1306 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1307 visible_frames = merged;
1308 if visible_frames.len() > requested_limit {
1309 let mut anchors = Vec::new();
1310 let mut tail = Vec::new();
1311 for frame in visible_frames {
1312 if anchor_cursors.contains(&frame.cursor) {
1313 anchors.push(frame);
1314 } else {
1315 tail.push(frame);
1316 }
1317 }
1318 visible_frames = if anchors.len() >= requested_limit {
1319 anchors.split_off(anchors.len() - requested_limit)
1320 } else {
1321 let keep_tail = requested_limit.saturating_sub(anchors.len());
1322 if tail.len() > keep_tail {
1323 tail = tail.split_off(tail.len() - keep_tail);
1324 }
1325 anchors.extend(tail);
1326 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1327 anchors
1328 };
1329 }
1330 }
1331 }
1332
1333 Ok(ConsoleTimelineWindowPage {
1334 frames: visible_frames,
1335 next_cursor: match query.mode {
1336 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1337 since_last_delivered_cursor.or(since_last_scanned_cursor)
1338 }
1339 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1340 ConsoleTimelineMode::Recent => next_cursor,
1341 },
1342 latest_cursor,
1343 exhausted,
1344 })
1345 }
1346
1347 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1348 let runtime_keys = self
1349 .inner
1350 .runtimes
1351 .read()
1352 .map_err(|_| runtime_registry_lock_error())?
1353 .keys()
1354 .cloned()
1355 .collect::<Vec<_>>();
1356 let results =
1357 join_all(runtime_keys.into_iter().map(|runtime_key| {
1358 backfill_session_history(self.inner.clone(), runtime_key, true)
1359 }))
1360 .await;
1361 for result in results {
1362 result?;
1363 }
1364 Ok(())
1365 }
1366
1367 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1368 self.inner.store.latest_cursor().await
1369 }
1370
1371 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1372 self.timeline_event_visible_for_subscriber(event, None)
1373 .await
1374 }
1375
1376 pub async fn timeline_event_visible_for_subscriber(
1394 &self,
1395 event: &ConsoleTimelineEvent,
1396 subscriber_identity: Option<&str>,
1397 ) -> bool {
1398 match event {
1399 ConsoleTimelineEvent::ConsoleFrame { frame }
1400 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1401 let identity_records = self.inner.identity_read_model.current().await;
1402 let identity_known = identity_records.iter().any(|record| {
1403 record.runtime_key == frame.runtime_key
1404 && (record.identity == frame.identity
1405 || record.runtime_member_id == frame.identity)
1406 });
1407 if !identity_known
1408 && frame.identity != "__console__"
1409 && frame.identity != SYSTEM_EVENT_IDENTITY
1410 {
1411 self.inner
1416 .identity_read_model
1417 .refresh_soon(self.inner.clone());
1418 }
1419 let allow_historical_identity =
1420 subscriber_identity == Some(frame.identity.as_str());
1421 Box::pin(frame_is_visible(
1422 &self.inner,
1423 frame,
1424 allow_historical_identity,
1425 &identity_records,
1426 ))
1427 .await
1428 .unwrap_or(false)
1429 }
1430 ConsoleTimelineEvent::SnapshotStarted { .. }
1431 | ConsoleTimelineEvent::SnapshotComplete { .. }
1432 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1433 }
1434 }
1435
1436 pub async fn timeline_frame_visible_for_query(
1437 &self,
1438 frame: &ConsoleFrame,
1439 identity: Option<&str>,
1440 ) -> bool {
1441 let allow_historical_identity = identity == Some(frame.identity.as_str());
1442 let identity_records = self.inner.identity_read_model.current().await;
1443 Box::pin(frame_is_visible(
1444 &self.inner,
1445 frame,
1446 allow_historical_identity,
1447 &identity_records,
1448 ))
1449 .await
1450 .unwrap_or(false)
1451 }
1452
1453 pub async fn send(
1454 &self,
1455 request: ConsoleSendRequest,
1456 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1457 validate_send_request(&request)?;
1458 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1459 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1460 };
1461 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1462 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1463 };
1464 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1465 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1466 }
1467 if !member_is_addressable(&resolved.member) {
1468 return Err(ConsoleSendError::NotAddressable(request.identity));
1469 }
1470 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1471 return Err(ConsoleSendError::Retired(request.identity));
1472 }
1473
1474 let content = content_input_from_value(&request.content)?;
1475 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1476 assert_member_accepts_images(
1477 &resolved.handle,
1478 resolved.entry.runtime.session_service(),
1479 &resolved.runtime_identity,
1480 &content,
1481 )
1482 .await
1483 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1484
1485 let dedupe_key = send_dedupe_key(
1486 &resolved.entry.runtime_key,
1487 &request.identity,
1488 &request.origin,
1489 &request.idempotency_key,
1490 );
1491 let handling_mode_value = request
1492 .handling_mode
1493 .as_deref()
1494 .unwrap_or("queue")
1495 .to_string();
1496 let request_fingerprint =
1497 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1498 if let Some(existing) = self
1499 .inner
1500 .store
1501 .frame_by_dedupe_key(&dedupe_key)
1502 .await
1503 .map_err(ConsoleSendError::Log)?
1504 {
1505 let same_request = existing.source.source_cursor.as_deref()
1506 == Some(request_fingerprint.as_str())
1507 || existing.source.source_cursor.is_none()
1508 && existing.payload.get("origin").and_then(Value::as_str)
1509 == Some(request.origin.as_str())
1510 && existing.payload.get("content") == Some(&request.content)
1511 && existing
1512 .payload
1513 .get("handling_mode")
1514 .and_then(Value::as_str)
1515 == Some(handling_mode_value.as_str());
1516 if !same_request {
1517 return Err(ConsoleSendError::IdempotencyConflict(
1518 request.idempotency_key,
1519 ));
1520 }
1521 return Ok(accepted_from_frame(&existing));
1522 }
1523
1524 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1525 resolved
1526 .entry
1527 .console_events
1528 .reserve_interaction_value(
1529 &resolved.runtime_identity,
1530 Some(resolved.runtime_identity.as_str()),
1531 &interaction_id,
1532 &request.origin,
1533 request.content.clone(),
1534 )
1535 .await
1536 .map_err(ConsoleSendError::State)?;
1537 let session_id = resolved
1538 .handle
1539 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1540 resolved.runtime_identity.as_str(),
1541 ))
1542 .await
1543 .map(|sid| sid.to_string());
1544 let mut new_frame = NewConsoleFrame {
1545 id: None,
1546 dedupe_key,
1547 timestamp_ms: current_time_ms(),
1548 runtime_key: resolved.entry.runtime_key.clone(),
1549 identity: request.identity.clone(),
1550 conversation_id: Some(request.identity.clone()),
1551 session_id: session_id.clone(),
1552 kind: "user_input".to_string(),
1553 status: ConsoleFrameStatus::Accepted,
1554 payload: json!({
1555 "content": request.content,
1556 "origin": request.origin,
1557 "idempotency_key": request.idempotency_key,
1558 "handling_mode": handling_mode_value,
1559 }),
1560 source: ConsoleFrameSource {
1561 kind: ConsoleFrameSourceKind::Send,
1562 source_cursor: Some(request_fingerprint),
1563 },
1564 source_event_id: None,
1565 interaction_id: Some(interaction_id.clone()),
1566 turn_id: None,
1567 run_id: None,
1568 parent_frame_id: None,
1569 caused_by_frame_id: None,
1570 };
1571 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1572 new_frame.payload = redacted;
1573 new_frame.status = ConsoleFrameStatus::Redacted;
1574 }
1575
1576 let _ = SendState::Requested
1577 .apply(SendTransition::PersistAccepted)
1578 .map_err(ConsoleSendError::State)?;
1579 let outcome = self
1580 .inner
1581 .store
1582 .append_if_absent(new_frame)
1583 .await
1584 .map_err(ConsoleSendError::Log)?;
1585 if outcome.disposition == AppendDisposition::Existing {
1586 return Ok(accepted_from_frame(&outcome.frame));
1587 }
1588 let _ = self
1589 .inner
1590 .event_tx
1591 .send(ConsoleTimelineEvent::ConsoleFrame {
1592 frame: outcome.frame.clone(),
1593 });
1594 let accepted = accepted_from_frame(&outcome.frame);
1595
1596 let (dispatching, _effects) = SendState::AcceptedPersisted
1597 .apply(SendTransition::StartDispatch)
1598 .map_err(ConsoleSendError::State)?;
1599 update_frame_status_and_emit(
1600 &self.inner,
1601 &outcome.frame.id,
1602 ConsoleFrameStatus::Dispatching,
1603 )
1604 .await
1605 .map_err(ConsoleSendError::Log)?;
1606
1607 spawn_console_send_dispatch(
1608 self.inner.clone(),
1609 resolved,
1610 content,
1611 handling_mode,
1612 dispatching,
1613 outcome.frame,
1614 interaction_id,
1615 );
1616 Ok(accepted)
1617 }
1618
1619 pub async fn reserve_identity_first_interaction(
1620 &self,
1621 request: ConsoleSendRequest,
1622 session_id: Option<&str>,
1623 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1624 validate_send_request(&request)?;
1625 let _content = content_input_from_value(&request.content)?;
1626 let runtime_key =
1627 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1628 let handling_mode_value = request
1629 .handling_mode
1630 .as_deref()
1631 .unwrap_or("queue")
1632 .to_string();
1633 let dedupe_key = send_dedupe_key(
1634 &runtime_key,
1635 &request.identity,
1636 &request.origin,
1637 &request.idempotency_key,
1638 );
1639 let request_fingerprint =
1640 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1641 if let Some(existing) = self
1642 .inner
1643 .store
1644 .frame_by_dedupe_key(&dedupe_key)
1645 .await
1646 .map_err(ConsoleSendError::Log)?
1647 {
1648 let same_request = existing.source.source_cursor.as_deref()
1649 == Some(request_fingerprint.as_str())
1650 || existing.source.source_cursor.is_none()
1651 && existing.payload.get("origin").and_then(Value::as_str)
1652 == Some(request.origin.as_str())
1653 && existing.payload.get("content") == Some(&request.content)
1654 && existing
1655 .payload
1656 .get("handling_mode")
1657 .and_then(Value::as_str)
1658 == Some(handling_mode_value.as_str());
1659 if !same_request {
1660 return Err(ConsoleSendError::IdempotencyConflict(
1661 request.idempotency_key,
1662 ));
1663 }
1664 return Ok(accepted_from_frame(&existing));
1665 }
1666
1667 let interaction_id = identity_first_interaction_uuid(&dedupe_key);
1668 let new_frame = NewConsoleFrame {
1669 id: None,
1670 dedupe_key,
1671 timestamp_ms: current_time_ms(),
1672 runtime_key,
1673 identity: request.identity.clone(),
1674 conversation_id: Some(request.identity.clone()),
1675 session_id: session_id.map(ToString::to_string),
1676 kind: "user_input".to_string(),
1677 status: ConsoleFrameStatus::Accepted,
1678 payload: json!({
1679 "content": request.content,
1680 "origin": request.origin,
1681 "idempotency_key": request.idempotency_key,
1682 "handling_mode": handling_mode_value,
1683 }),
1684 source: ConsoleFrameSource {
1685 kind: ConsoleFrameSourceKind::Send,
1686 source_cursor: Some(request_fingerprint),
1687 },
1688 source_event_id: None,
1689 interaction_id: Some(interaction_id),
1690 turn_id: None,
1691 run_id: None,
1692 parent_frame_id: None,
1693 caused_by_frame_id: None,
1694 };
1695 let outcome = self
1696 .inner
1697 .store
1698 .append_if_absent(new_frame)
1699 .await
1700 .map_err(ConsoleSendError::Log)?;
1701 let _ = self
1702 .inner
1703 .event_tx
1704 .send(ConsoleTimelineEvent::ConsoleFrame {
1705 frame: outcome.frame.clone(),
1706 });
1707 Ok(accepted_from_frame(&outcome.frame))
1708 }
1709
1710 async fn runtime_key_for_identity_first_send(
1711 &self,
1712 identity: &str,
1713 ) -> Result<String, ConsoleSendError> {
1714 let entries = self
1715 .inner
1716 .runtimes
1717 .read()
1718 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1719 .clone();
1720 if entries.is_empty() {
1721 return Ok("identity-first".to_string());
1722 }
1723
1724 let mut identity_runtime_keys = Vec::new();
1725 let mut hidden_durable_match = false;
1726 let mut durable_matches = Vec::new();
1727 for entry in entries.values() {
1728 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1729 continue;
1730 };
1731 identity_runtime_keys.push(entry.runtime_key.clone());
1732 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1733 continue;
1734 }
1735 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1736 {
1737 let Ok(parsed_identity) =
1738 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1739 else {
1740 continue;
1741 };
1742 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1743 let record = identity_record_for_status(entry, &status);
1744 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1745 hidden_durable_match = true;
1746 continue;
1747 }
1748 durable_matches.push((
1749 entry.clone(),
1750 identity_runtime.clone(),
1751 parsed_identity,
1752 record,
1753 ));
1754 }
1755 }
1756 }
1757 if durable_matches.len() > 1 {
1758 let candidates = durable_matches
1759 .iter()
1760 .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1761 .collect::<Vec<_>>()
1762 .join(", ");
1763 return Err(ConsoleSendError::InvalidRequest(format!(
1764 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1765 )));
1766 }
1767 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1768 durable_matches.into_iter().next()
1769 {
1770 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1771 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1772 &entry,
1773 identity,
1774 &record,
1775 &live_records,
1776 ))
1777 .await;
1778 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1779 &entry,
1780 &identity_runtime,
1781 &parsed_identity,
1782 identity,
1783 &record,
1784 &durable_live_records,
1785 )
1786 .await
1787 .map_err(ConsoleSendError::InvalidRequest)?
1788 {
1789 record = rebound_record;
1790 }
1791 if let Some(stale_error) =
1792 stale_durable_record_error(identity, &record, &durable_live_records)
1793 {
1794 return Err(ConsoleSendError::InvalidRequest(stale_error));
1795 }
1796 return Ok(entry.runtime_key);
1797 }
1798 if entries.values().any(|entry| {
1799 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1800 }) {
1801 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1802 }
1803 if hidden_durable_match {
1804 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1805 }
1806
1807 match identity_runtime_keys.as_slice() {
1808 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1809 [runtime_key] => Ok(runtime_key.clone()),
1810 _ => Err(ConsoleSendError::InvalidRequest(format!(
1811 "identity-first send for '{identity}' did not match exactly one registered runtime"
1812 ))),
1813 }
1814 }
1815
1816 pub async fn mark_interaction_delivery_failed(
1817 &self,
1818 input_frame_id: &str,
1819 ) -> Result<(), ConsoleSendError> {
1820 update_frame_status_and_emit(
1821 &self.inner,
1822 input_frame_id,
1823 ConsoleFrameStatus::DeliveryFailed,
1824 )
1825 .await
1826 .map_err(ConsoleSendError::Log)?;
1827 Ok(())
1828 }
1829
1830 pub async fn mark_interaction_delivered(
1831 &self,
1832 input_frame_id: &str,
1833 ) -> Result<(), ConsoleSendError> {
1834 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1835 .await
1836 .map_err(ConsoleSendError::Log)?;
1837 Ok(())
1838 }
1839
1840 pub async fn mark_steer_interaction_delivered(
1841 &self,
1842 input_frame_id: &str,
1843 interaction_id: &str,
1844 ) -> Result<(), ConsoleSendError> {
1845 let Some(updated) = update_frame_status_and_emit(
1846 &self.inner,
1847 input_frame_id,
1848 ConsoleFrameStatus::Delivered,
1849 )
1850 .await
1851 .map_err(ConsoleSendError::Log)?
1852 else {
1853 return Ok(());
1854 };
1855 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1856 .await
1857 .map_err(ConsoleSendError::Log)?;
1858 Ok(())
1859 }
1860
1861 pub async fn binary_blob_store_for_identity(
1862 &self,
1863 identity: &str,
1864 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1865 if identity.trim().is_empty() {
1866 return Err(ConsoleSendError::InvalidRequest(
1867 "identity must be non-empty".to_string(),
1868 ));
1869 }
1870 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1871 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1872 };
1873 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1874 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1875 };
1876 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1877 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1878 }
1879 if !member_is_addressable(&resolved.member) {
1880 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1881 }
1882 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1883 return Err(ConsoleSendError::Retired(identity.to_string()));
1884 }
1885 Ok(resolved.entry.runtime.binary_blob_store())
1886 }
1887
1888 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1889 self.inner
1890 .runtimes
1891 .read()
1892 .map(|entries| {
1893 entries
1894 .values()
1895 .filter_map(|entry| entry.runtime.binary_blob_store())
1896 .collect()
1897 })
1898 .unwrap_or_default()
1899 }
1900
1901 async fn resolve_send_member(
1902 &self,
1903 identity: &str,
1904 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1905 let mut matches = Box::pin(self.resolve_visible_members(identity)).await;
1906 let sendable_indices = sendable_resolved_member_indices(&matches);
1915 if sendable_indices.len() > 1 {
1916 if let Some(index) = Box::pin(authoritative_durable_sendable_member_index(
1917 identity,
1918 &matches,
1919 &sendable_indices,
1920 ))
1921 .await?
1922 {
1923 let resolved = matches.swap_remove(index);
1924 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1925 && let Some(stale_error) =
1926 reconcile_stale_live_record_binding(&resolved.entry, identity, &record)
1927 .await
1928 {
1929 return Err(ConsoleSendError::InvalidRequest(stale_error));
1930 }
1931 if let Some(stale_error) =
1932 Box::pin(self.durable_send_binding_error(identity)).await?
1933 {
1934 return Err(ConsoleSendError::InvalidRequest(stale_error));
1935 }
1936 return Ok(Some(resolved));
1937 }
1938 let candidates = sendable_indices
1939 .iter()
1940 .map(|index| matches[*index].runtime_identity.clone())
1941 .collect::<Vec<_>>()
1942 .join(", ");
1943 return Err(ConsoleSendError::AmbiguousIdentity {
1944 identity: identity.to_string(),
1945 candidates,
1946 });
1947 }
1948 if let Some(index) = sendable_indices.into_iter().next() {
1949 let resolved = matches.swap_remove(index);
1950 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1951 && let Some(stale_error) =
1952 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1953 {
1954 return Err(ConsoleSendError::InvalidRequest(stale_error));
1955 }
1956 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1957 return Err(ConsoleSendError::InvalidRequest(stale_error));
1958 }
1959 return Ok(Some(resolved));
1960 }
1961 if matches.len() > 1 {
1962 let candidates = matches
1963 .iter()
1964 .map(|resolved| resolved.runtime_identity.clone())
1965 .collect::<Vec<_>>()
1966 .join(", ");
1967 return Err(ConsoleSendError::AmbiguousIdentity {
1968 identity: identity.to_string(),
1969 candidates,
1970 });
1971 }
1972 let Some(resolved) = matches.into_iter().next() else {
1973 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1974 return Err(ConsoleSendError::InvalidRequest(stale_error));
1975 }
1976 return Ok(None);
1977 };
1978 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1979 && let Some(stale_error) =
1980 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1981 {
1982 return Err(ConsoleSendError::InvalidRequest(stale_error));
1983 }
1984 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1985 return Err(ConsoleSendError::InvalidRequest(stale_error));
1986 }
1987 Ok(Some(resolved))
1988 }
1989
1990 async fn durable_send_binding_error(
1991 &self,
1992 identity: &str,
1993 ) -> Result<Option<String>, ConsoleSendError> {
1994 let entries = self
1995 .inner
1996 .runtimes
1997 .read()
1998 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1999 .clone();
2000 if entries.values().any(|entry| {
2001 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
2002 }) {
2003 return Ok(None);
2004 }
2005 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
2006 let mut durable_matches = Vec::new();
2007 for entry in entries.values() {
2008 let Some(identity_runtime) = entry.identity_runtime.clone() else {
2009 continue;
2010 };
2011 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
2012 let Ok(parsed_identity) =
2013 crate::identity_first::AgentIdentity::parse(&raw_identity)
2014 else {
2015 continue;
2016 };
2017 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
2018 continue;
2019 };
2020 let record = identity_record_for_status(entry, &status);
2021 if !Box::pin(console_identity_record_visible(entry, &record)).await {
2022 continue;
2023 }
2024 durable_matches.push((
2025 entry.clone(),
2026 identity_runtime.clone(),
2027 parsed_identity,
2028 record,
2029 ));
2030 }
2031 }
2032 if durable_matches.len() > 1 {
2033 let candidates = durable_matches
2034 .iter()
2035 .map(|(_, _, _, record)| record.runtime_member_id.clone())
2036 .collect::<Vec<_>>()
2037 .join(", ");
2038 return Err(ConsoleSendError::AmbiguousIdentity {
2039 identity: identity.to_string(),
2040 candidates,
2041 });
2042 }
2043 let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
2044 durable_matches.into_iter().next()
2045 else {
2046 return Ok(None);
2047 };
2048 let durable_live_records = Box::pin(self.live_records_for_durable_record(
2049 &entry,
2050 identity,
2051 &durable_record,
2052 &live_records,
2053 ))
2054 .await;
2055 if let Some(rebound_record) = self_heal_durable_session_mismatch(
2056 &entry,
2057 &identity_runtime,
2058 &parsed_identity,
2059 identity,
2060 &durable_record,
2061 &durable_live_records,
2062 )
2063 .await
2064 .map_err(ConsoleSendError::InvalidRequest)?
2065 {
2066 durable_record = rebound_record;
2067 }
2068 Ok(stale_durable_record_error(
2069 identity,
2070 &durable_record,
2071 &durable_live_records,
2072 ))
2073 }
2074
2075 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2076 let entries = self
2077 .inner
2078 .runtimes
2079 .read()
2080 .ok()
2081 .map(|entries| entries.clone())
2082 .unwrap_or_default();
2083 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2084 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2085 for entry in entries.values() {
2086 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2087 if raw_identities.is_empty() {
2088 continue;
2089 }
2090 let mids = raw_identities
2091 .iter()
2092 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
2093 .collect::<Vec<_>>();
2094 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2095 let session_id = resolved
2096 .handle
2097 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
2098 .await
2099 .map(|sid| sid.to_string())
2100 .unwrap_or_default();
2101 if mids.contains(&resolved.member.agent_identity) {
2102 exact_matches.push((session_id, resolved));
2103 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
2104 {
2105 label_matches.push((session_id, resolved));
2106 }
2107 }
2108 }
2109 let mut matches = exact_matches;
2110 matches.extend(label_matches);
2111 let mut seen_members = BTreeSet::new();
2112 matches.retain(|(session_id, resolved)| {
2113 seen_members.insert((
2114 resolved.entry.runtime_key.clone(),
2115 resolved.source_mob_id.clone(),
2116 session_id.clone(),
2117 resolved.runtime_identity.clone(),
2118 ))
2119 });
2120 matches.sort_by(|left, right| right.0.cmp(&left.0));
2121 matches.into_iter().map(|(_, resolved)| resolved).collect()
2122 }
2123
2124 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2125 let mut visible = Vec::new();
2126 for resolved in Box::pin(self.resolve_members(identity)).await {
2127 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2128 continue;
2129 };
2130 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2131 visible.push(resolved);
2132 }
2133 }
2134 visible
2135 }
2136
2137 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
2138 let mut records = Vec::new();
2139 for resolved in Box::pin(self.resolve_members(identity)).await {
2140 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2141 continue;
2142 };
2143 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2144 records.push(record);
2145 }
2146 }
2147 records
2148 }
2149
2150 async fn live_records_for_durable_record(
2151 &self,
2152 entry: &RuntimeEntry,
2153 requested_identity: &str,
2154 durable: &ConsoleIdentityRecord,
2155 requested_live_records: &[ConsoleIdentityRecord],
2156 ) -> Vec<ConsoleIdentityRecord> {
2157 let mut records = if durable.identity == requested_identity {
2158 requested_live_records.to_vec()
2159 } else {
2160 Box::pin(self.live_records_for_identity(&durable.identity)).await
2161 };
2162 for runtime_record in Box::pin(live_records_for_runtime_member(
2163 entry,
2164 &durable.runtime_member_id,
2165 ))
2166 .await
2167 {
2168 if records.iter().any(|record| {
2169 record.runtime_key == runtime_record.runtime_key
2170 && record.runtime_member_id == runtime_record.runtime_member_id
2171 && record.session_id == runtime_record.session_id
2172 && record.identity == runtime_record.identity
2173 }) {
2174 continue;
2175 }
2176 records.push(runtime_record);
2177 }
2178 records
2179 }
2180}
2181
2182async fn console_identity_record_visible(
2183 entry: &RuntimeEntry,
2184 record: &ConsoleIdentityRecord,
2185) -> bool {
2186 if !entry.visibility_policy.identity_visible(record) {
2187 return false;
2188 }
2189 let mut bound_live_member_seen = false;
2190 let mut wrong_live_projection_seen = false;
2191 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2192 if resolved.runtime_identity != record.runtime_member_id {
2193 continue;
2194 }
2195 bound_live_member_seen = true;
2196 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2197 continue;
2198 };
2199 if live_record.identity != record.identity {
2200 wrong_live_projection_seen = true;
2201 continue;
2202 }
2203 if raw_resolved_member_visible(&resolved, &live_record) {
2204 return true;
2205 }
2206 }
2207 if wrong_live_projection_seen {
2208 return true;
2209 }
2210 !bound_live_member_seen
2211}
2212
2213async fn live_record_shadowed_by_hidden_durable_binding(
2214 resolved: &ResolvedConsoleMember,
2215 record: &ConsoleIdentityRecord,
2216) -> bool {
2217 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2218 return false;
2219 };
2220 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2221 return false;
2222 };
2223 let Ok(status) = identity_runtime.status(&identity).await else {
2224 return false;
2225 };
2226 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2227 return false;
2228 };
2229 if bound_runtime_id.as_str() == resolved.runtime_identity {
2230 return false;
2231 }
2232 for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2233 if candidate.runtime_identity != bound_runtime_id.as_str() {
2234 continue;
2235 }
2236 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2237 continue;
2238 };
2239 if bound_record.identity == record.identity
2240 && !raw_resolved_member_visible(&candidate, &bound_record)
2241 {
2242 return true;
2243 }
2244 }
2245 false
2246}
2247
2248async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2249 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2250 .unwrap_or_else(|| frame.identity.clone());
2251 let frame_session_id = frame.session_id.as_deref();
2252 let mut saw_hidden = false;
2253 let mut saw_visible = false;
2254 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2255 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2256 continue;
2257 };
2258 let session_matches =
2259 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2260 let runtime_matches = record.runtime_member_id == runtime_member_id;
2261 let identity_matches = record.identity == frame.identity;
2262 if !session_matches && !runtime_matches && !identity_matches {
2263 continue;
2264 }
2265 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2266 return true;
2267 }
2268 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2269 saw_visible = true;
2270 } else {
2271 saw_hidden = true;
2272 }
2273 }
2274 saw_hidden && !saw_visible
2275}
2276
2277async fn live_records_for_runtime_member(
2278 entry: &RuntimeEntry,
2279 runtime_member_id: &str,
2280) -> Vec<ConsoleIdentityRecord> {
2281 let mut records = Vec::new();
2282 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2283 if resolved.runtime_identity != runtime_member_id {
2284 continue;
2285 }
2286 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2287 continue;
2288 };
2289 records.push(record);
2290 }
2291 records
2292}
2293
2294async fn visible_live_records_for_entry(
2295 entry: &RuntimeEntry,
2296 records: &[ConsoleIdentityRecord],
2297) -> Vec<ConsoleIdentityRecord> {
2298 let mut visible = Vec::new();
2299 for record in records {
2300 if record.runtime_key != entry.runtime_key
2301 || Box::pin(console_identity_record_visible(entry, record)).await
2302 {
2303 visible.push(record.clone());
2304 }
2305 }
2306 visible
2307}
2308
2309async fn reconcile_stale_live_record_binding(
2310 entry: &RuntimeEntry,
2311 requested_identity: &str,
2312 live_record: &ConsoleIdentityRecord,
2313) -> Option<String> {
2314 let identity_runtime = entry.identity_runtime.as_ref()?;
2315 for status in identity_runtime.statuses().await {
2316 let matches_runtime = status
2317 .agent_runtime_id
2318 .as_ref()
2319 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2320 if !matches_runtime {
2321 continue;
2322 }
2323 let durable_record = identity_record_for_status(entry, &status);
2324 if let Some(session_mismatch) =
2325 stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2326 {
2327 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2328 continue;
2329 };
2330 let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2331 return Some(format!(
2332 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2333 ));
2334 };
2335 return match identity_runtime
2336 .rebind_session_after_live_respawn_member_alias_tracked(
2337 &status.identity,
2338 durable_record.runtime_member_id.as_str(),
2339 session_id,
2340 )
2341 .await
2342 {
2343 Ok(_) => {
2344 tracing::warn!(
2345 requested_identity,
2346 identity = %durable_record.identity,
2347 runtime_member_id = %durable_record.runtime_member_id,
2348 old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2349 new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2350 "self-healed stale durable identity alias session mismatch"
2351 );
2352 None
2353 }
2354 Err(err) => Some(format!(
2355 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2356 durable_record.identity,
2357 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2358 )),
2359 };
2360 }
2361 if let Some(stale_error) = stale_durable_record_error(
2362 requested_identity,
2363 &durable_record,
2364 std::slice::from_ref(live_record),
2365 ) {
2366 return Some(stale_error);
2367 }
2368 }
2369 None
2370}
2371
2372async fn self_heal_durable_session_mismatch(
2373 entry: &RuntimeEntry,
2374 identity_runtime: &Arc<crate::identity_first::IdentityRuntime>,
2375 parsed_identity: &crate::identity_first::AgentIdentity,
2376 requested_identity: &str,
2377 durable: &ConsoleIdentityRecord,
2378 live_records: &[ConsoleIdentityRecord],
2379) -> Result<Option<ConsoleIdentityRecord>, String> {
2380 let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2381 return Ok(None);
2382 };
2383 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2384 return Ok(None);
2385 };
2386 let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2387 format!(
2388 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2389 )
2390 })?;
2391 identity_runtime
2392 .rebind_session_after_live_respawn_member_alias_tracked(
2393 parsed_identity,
2394 durable.runtime_member_id.as_str(),
2395 session_id,
2396 )
2397 .await
2398 .map_err(|err| {
2399 format!(
2400 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2401 durable.identity, live_session_id
2402 )
2403 })?;
2404 tracing::warn!(
2405 requested_identity,
2406 identity = %durable.identity,
2407 runtime_member_id = %durable.runtime_member_id,
2408 old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2409 new_session_id = live_session_id,
2410 "self-healed stale durable identity alias session mismatch"
2411 );
2412 let status = identity_runtime
2413 .status(parsed_identity)
2414 .await
2415 .map_err(|err| {
2416 format!(
2417 "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2418 durable.identity
2419 )
2420 })?;
2421 Ok(Some(identity_record_for_status(entry, &status)))
2422}
2423
2424fn stale_durable_session_mismatch<'a>(
2425 durable: &ConsoleIdentityRecord,
2426 live_records: &'a [ConsoleIdentityRecord],
2427) -> Option<&'a ConsoleIdentityRecord> {
2428 live_records.iter().find(|record| {
2429 live_record_realizes_durable_binding(durable, record)
2430 && durable.session_id.is_some()
2431 && record.session_id.is_some()
2432 && durable.session_id != record.session_id
2433 })
2434}
2435
2436fn live_record_realizes_durable_binding(
2447 durable: &ConsoleIdentityRecord,
2448 live: &ConsoleIdentityRecord,
2449) -> bool {
2450 if live.identity != durable.identity {
2451 return false;
2452 }
2453 if live.runtime_member_id == durable.runtime_member_id {
2454 return true;
2455 }
2456 crate::member_comms_id::durable_identity_label(&live.labels)
2457 .is_some_and(|identity| live.runtime_member_id == identity)
2458}
2459
2460fn stale_durable_record_error(
2461 requested_identity: &str,
2462 durable: &ConsoleIdentityRecord,
2463 live_records: &[ConsoleIdentityRecord],
2464) -> Option<String> {
2465 let matching_live = live_records
2466 .iter()
2467 .filter(|record| record.identity == durable.identity)
2468 .collect::<Vec<_>>();
2469 if let Some(wrong_projection) = live_records.iter().find(|record| {
2470 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2471 }) {
2472 return Some(format!(
2473 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2474 durable.identity, durable.runtime_member_id, wrong_projection.identity
2475 ));
2476 }
2477 if matching_live.is_empty() {
2478 return None;
2479 }
2480 if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2481 return Some(format!(
2482 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2483 durable.identity,
2484 durable.runtime_member_id,
2485 durable.session_id.as_deref().unwrap_or("<none>"),
2486 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2487 ));
2488 }
2489 if matching_live
2490 .iter()
2491 .any(|record| live_record_realizes_durable_binding(durable, record))
2492 {
2493 return None;
2494 }
2495 let live_candidates = matching_live
2496 .iter()
2497 .map(|record| record.runtime_member_id.as_str())
2498 .collect::<Vec<_>>()
2499 .join(", ");
2500 Some(format!(
2501 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2502 durable.identity, durable.runtime_member_id, live_candidates
2503 ))
2504}
2505
2506fn ambiguous_live_alias_error(
2507 requested_identity: &str,
2508 live_records: &[ConsoleIdentityRecord],
2509) -> Option<String> {
2510 if live_records.len() <= 1 {
2511 return None;
2512 }
2513 let candidates = live_records
2514 .iter()
2515 .map(|record| {
2516 format!(
2517 "{}@{}",
2518 record.runtime_member_id,
2519 record
2520 .labels
2521 .get("source_mob_id")
2522 .map(String::as_str)
2523 .unwrap_or(record.runtime_key.as_str())
2524 )
2525 })
2526 .collect::<Vec<_>>()
2527 .join(", ");
2528 Some(format!(
2529 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2530 ))
2531}
2532
2533#[cfg(test)]
2534fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2535 crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2538}
2539
2540fn generated_runtime_alias_parts(alias: &str) -> Option<(&str, u64)> {
2541 let rest = alias.strip_prefix("rt:")?;
2542 let (identity, generation) = rest.rsplit_once(':')?;
2543 Some((identity, generation.parse().ok()?))
2544}
2545
2546async fn stale_generated_members_for_runtime_entry(
2547 entry: &RuntimeEntry,
2548 current_runtime_alias: &str,
2549 include_current: bool,
2550) -> Vec<(MobHandle, meerkat_mob::ids::AgentIdentity)> {
2551 let Some((current_identity, current_generation)) =
2552 generated_runtime_alias_parts(current_runtime_alias)
2553 else {
2554 return Vec::new();
2555 };
2556 let primary_handle = entry.runtime.handle();
2557 let primary_mob_id = primary_handle.mob_id().to_string();
2558 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2559 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2560 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2561 .await
2562 .unwrap_or_default()
2563 {
2564 if mob_id.as_str() != primary_mob_id {
2565 handles.push((mob_id.to_string(), handle));
2566 }
2567 }
2568 }
2569 let mut seen_handles = BTreeSet::new();
2570 let mut stale_members = Vec::new();
2571 for (mob_id, handle) in handles {
2572 if !seen_handles.insert(mob_id) {
2573 continue;
2574 }
2575 for member in handle.list_members_including_retiring().await {
2576 let public_alias =
2577 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str());
2578 if generated_runtime_alias_parts(public_alias.as_ref()).is_some_and(
2579 |(identity, generation)| {
2580 identity == current_identity
2581 && (generation < current_generation
2582 || (include_current && generation == current_generation))
2583 },
2584 ) {
2585 stale_members.push((handle.clone(), member.agent_identity));
2586 }
2587 }
2588 }
2589 stale_members
2590}
2591
2592async fn retire_captured_console_members(
2593 stale_members: Vec<(MobHandle, meerkat_mob::ids::AgentIdentity)>,
2594) -> Result<(), String> {
2595 for (handle, member_id) in stale_members {
2596 match handle.retire(member_id).await {
2597 Ok(()) => {}
2598 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2599 Err(err) => return Err(err.to_string()),
2600 }
2601 }
2602 Ok(())
2603}
2604
2605fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2606 is_recoverable_lifecycle_cleanup_error(error)
2607}
2608
2609fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2610 if frames.is_empty() {
2611 return true;
2612 }
2613 let latest_session_history_timestamp_ms = frames
2614 .iter()
2615 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2616 .map(|frame| frame.timestamp_ms)
2617 .max();
2618 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2619 return current_time_ms().saturating_sub(latest_timestamp_ms)
2620 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2621 }
2622 frames.iter().any(|frame| {
2623 matches!(
2624 frame.kind.as_str(),
2625 "turn_started"
2626 | "run_started"
2627 | "reasoning_delta"
2628 | "reasoning_complete"
2629 | "tool_call_requested"
2630 | "tool_call"
2631 | "tool_execution_started"
2632 | "tool_execution_completed"
2633 | "tool_result_received"
2634 )
2635 })
2636}
2637
2638fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2639 match frame.kind.as_str() {
2640 "user_input" | "run_started" => true,
2641 "interaction_started" => frame
2642 .payload
2643 .get("content")
2644 .or_else(|| frame.payload.get("prompt"))
2645 .is_some(),
2646 _ => false,
2647 }
2648}
2649
2650fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2651 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2652 for record in records {
2653 by_identity
2654 .entry(identity_record_dedupe_key(&record))
2655 .and_modify(|current| {
2656 if identity_record_prefer(&record, current) {
2657 *current = record.clone();
2658 }
2659 })
2660 .or_insert(record);
2661 }
2662 by_identity.into_values().collect()
2663}
2664
2665fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2666 format!(
2667 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2668 record.identity,
2669 record.runtime_key,
2670 record
2671 .labels
2672 .get("source_mob_id")
2673 .map(String::as_str)
2674 .unwrap_or(""),
2675 record.runtime_member_id,
2676 record.session_id.as_deref().unwrap_or("")
2677 )
2678}
2679
2680fn identity_record_prefer(
2681 candidate: &ConsoleIdentityRecord,
2682 current: &ConsoleIdentityRecord,
2683) -> bool {
2684 let candidate_is_live_label_projection =
2685 crate::member_comms_id::durable_identity_label(&candidate.labels).is_some_and(|identity| {
2686 identity == strip_namespace(&candidate.identity, "").unwrap_or_default()
2687 || candidate.runtime_member_id != candidate.identity
2688 });
2689 let current_is_live_label_projection =
2690 crate::member_comms_id::durable_identity_label(¤t.labels).is_some_and(|identity| {
2691 identity == strip_namespace(¤t.identity, "").unwrap_or_default()
2692 || current.runtime_member_id != current.identity
2693 });
2694 if candidate_is_live_label_projection != current_is_live_label_projection {
2695 return candidate_is_live_label_projection;
2696 }
2697 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2698 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2699 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2700 return candidate_has_distinct_runtime_binding;
2701 }
2702 let candidate_live = candidate.addressable && candidate.health != "retired";
2703 let current_live = current.addressable && current.health != "retired";
2704 if candidate_live != current_live {
2705 return candidate_live;
2706 }
2707 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2708}
2709
2710async fn collect_identity_records(
2711 inner: &Arc<AggregatorInner>,
2712 mode: IdentityCollectionMode,
2713) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2714 let entries = inner
2715 .runtimes
2716 .read()
2717 .map_err(|_| runtime_registry_lock_error())?
2718 .clone();
2719 let mut identities = Vec::new();
2720 for entry in entries.values() {
2721 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2722 let topology_peers =
2723 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2724 for status in identity_runtime.statuses().await {
2725 let mut record = identity_record_for_status(entry, &status);
2726 record.topology_peers = topology_peers
2727 .get(&record.identity)
2728 .cloned()
2729 .unwrap_or_default();
2730 if Box::pin(console_identity_record_visible(entry, &record)).await {
2731 identities.push(record);
2732 }
2733 }
2734 if mode == IdentityCollectionMode::CachedOnly {
2735 continue;
2736 }
2737 }
2738 let members = if entry.identity_runtime.is_some() {
2739 match Box::pin(tokio::time::timeout(
2740 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2741 member_sources_for_entry(entry),
2742 ))
2743 .await
2744 {
2745 Ok(members) => members,
2746 Err(_) => {
2747 tracing::warn!(
2748 runtime_key = %entry.runtime_key,
2749 "identity-first live member refresh timed out; keeping cached identity read model"
2750 );
2751 Vec::new()
2752 }
2753 }
2754 } else {
2755 Box::pin(member_sources_for_entry(entry)).await
2756 };
2757 for resolved in members {
2758 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2759 && Box::pin(resolved_member_visible(&resolved, &record)).await
2760 {
2761 identities.push(record);
2762 }
2763 }
2764 }
2765 Ok(dedupe_identity_records(identities))
2766}
2767
2768async fn identity_runtime_topology_peers(
2769 entry: &RuntimeEntry,
2770 identity_runtime: &crate::identity_first::IdentityRuntime,
2771) -> BTreeMap<String, Vec<String>> {
2772 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2773 for edge in identity_runtime.desired_peer_edges().await {
2774 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2775 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2776 peers.entry(a.clone()).or_default().insert(b.clone());
2777 peers.entry(b).or_default().insert(a);
2778 }
2779 peers
2780 .into_iter()
2781 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2782 .collect()
2783}
2784
2785fn spawn_identity_backfills_for_records(
2786 inner: Arc<AggregatorInner>,
2787 records: &[ConsoleIdentityRecord],
2788) {
2789 if !inner.options.session_history_backfill_enabled {
2790 return;
2791 }
2792 let entries = match inner.runtimes.read() {
2793 Ok(entries) => entries.clone(),
2794 Err(_) => return,
2795 };
2796 for record in records {
2797 if record.health == "dormant" || record.health == "uninitialized" {
2798 continue;
2799 }
2800 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2801 continue;
2802 };
2803 let Some(session_id) = record.session_id.clone() else {
2804 continue;
2805 };
2806 spawn_session_history_backfill_target(
2807 inner.clone(),
2808 SessionBackfillTarget {
2809 entry,
2810 record: record.clone(),
2811 session_id,
2812 },
2813 false,
2814 );
2815 }
2816}
2817
2818async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2819 let mut resolved = Vec::new();
2820 let primary_handle = entry.runtime.handle();
2821 let primary_mob_id = primary_handle.mob_id().to_string();
2822 for member in primary_handle.list_members_observation_snapshot().await {
2823 resolved.push(ResolvedConsoleMember {
2824 entry: entry.clone(),
2825 handle: primary_handle.clone(),
2826 runtime_identity: crate::member_comms_id::runtime_alias_str(
2829 member.agent_identity.as_str(),
2830 )
2831 .into_owned(),
2832 source_mob_id: primary_mob_id.clone(),
2833 member,
2834 });
2835 }
2836
2837 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2838 return resolved;
2839 };
2840 if !entry.visibility_policy.include_implicit_delegate_members() {
2841 return resolved;
2842 }
2843 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2844 .await
2845 .unwrap_or_default()
2846 {
2847 if mob_id.as_str() == primary_mob_id {
2848 continue;
2849 }
2850 for member in handle.list_members_observation_snapshot().await {
2851 resolved.push(ResolvedConsoleMember {
2852 entry: entry.clone(),
2853 handle: handle.clone(),
2854 runtime_identity: crate::member_comms_id::runtime_alias_str(
2857 member.agent_identity.as_str(),
2858 )
2859 .into_owned(),
2860 source_mob_id: mob_id.to_string(),
2861 member,
2862 });
2863 }
2864 }
2865 resolved
2866}
2867
2868async fn dispatch_message_to_resolved_member(
2869 resolved: &ResolvedConsoleMember,
2870 content: ContentInput,
2871 handling_mode: meerkat_core::types::HandlingMode,
2872 interaction_id: &str,
2873) -> Result<String, String> {
2874 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
2875 && let Some(identity) = identity_runtime
2876 .identity_for_member_mutation(&resolved.runtime_identity)
2877 .await
2878 {
2879 identity_runtime
2880 .send_with_mode_and_interaction_member_alias_tracked(
2881 &identity,
2882 &resolved.runtime_identity,
2883 &content,
2884 handling_mode,
2885 Some(interaction_id),
2886 )
2887 .await
2888 .map_err(|error| error.to_string())?;
2889 return identity_runtime
2890 .status(&identity)
2891 .await
2892 .map_err(|error| error.to_string())?
2893 .session_id
2894 .map(|session_id| session_id.to_string())
2895 .ok_or_else(|| "identity has no bridge session after delivery".to_string());
2896 }
2897 let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2898 match send_message_on_mob_with_mode(
2899 &resolved.handle,
2900 &resolved.runtime_identity,
2901 content.clone(),
2902 handling_mode,
2903 )
2904 .await
2905 {
2906 Ok(session_id) => Ok(session_id),
2907 Err(err) if is_not_externally_addressable(&err) => {
2908 let member = resolved
2909 .handle
2910 .member(&mid)
2911 .await
2912 .map_err(|err| err.to_string())?;
2913 let _receipt = member
2914 .internal_turn(content)
2915 .await
2916 .map_err(|err| err.to_string())?;
2917 resolved
2918 .handle
2919 .resolve_bridge_session_id_observation(&mid)
2920 .await
2921 .map(|sid| sid.to_string())
2922 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2923 }
2924 Err(err) => Err(err.to_string()),
2925 }
2926}
2927
2928fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2929 matches!(
2930 err,
2931 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2932 )
2933}
2934
2935fn spawn_console_send_dispatch(
2936 inner: Arc<AggregatorInner>,
2937 resolved: ResolvedConsoleMember,
2938 content: ContentInput,
2939 handling_mode: meerkat_core::types::HandlingMode,
2940 dispatching: SendState,
2941 user_frame: ConsoleFrame,
2942 interaction_id: String,
2943) {
2944 tokio::spawn(async move {
2945 match dispatch_message_to_resolved_member(
2946 &resolved,
2947 content,
2948 handling_mode,
2949 &interaction_id,
2950 )
2951 .await
2952 {
2953 Ok(_) => {
2954 let _ = dispatching.apply(SendTransition::MarkDelivered);
2955 if let Err(err) = update_frame_status_and_emit(
2956 &inner,
2957 &user_frame.id,
2958 ConsoleFrameStatus::Delivered,
2959 )
2960 .await
2961 {
2962 tracing::warn!(
2963 frame_id = %user_frame.id,
2964 error = %err,
2965 "failed to update console send delivery status"
2966 );
2967 }
2968 if handling_mode == HandlingMode::Steer
2969 && let Err(err) =
2970 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2971 {
2972 tracing::warn!(
2973 frame_id = %user_frame.id,
2974 interaction_id = %interaction_id,
2975 error = %err,
2976 "failed to append console steer terminal frame"
2977 );
2978 }
2979 }
2980 Err(err) => {
2981 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2982 if let Err(update_err) = update_frame_status_and_emit(
2983 &inner,
2984 &user_frame.id,
2985 ConsoleFrameStatus::DeliveryFailed,
2986 )
2987 .await
2988 {
2989 tracing::warn!(
2990 frame_id = %user_frame.id,
2991 error = %update_err,
2992 "failed to update console send failure status"
2993 );
2994 }
2995 let failure_frame = NewConsoleFrame {
2996 id: None,
2997 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2998 timestamp_ms: current_time_ms(),
2999 runtime_key: user_frame.runtime_key,
3000 identity: user_frame.identity,
3001 conversation_id: user_frame.conversation_id,
3002 session_id: user_frame.session_id,
3003 kind: "message_delivery_failed".to_string(),
3004 status: ConsoleFrameStatus::DeliveryFailed,
3005 payload: json!({ "reason": err }),
3006 source: ConsoleFrameSource {
3007 kind: ConsoleFrameSourceKind::Synthetic,
3008 source_cursor: None,
3009 },
3010 source_event_id: None,
3011 interaction_id: Some(interaction_id),
3012 turn_id: None,
3013 run_id: None,
3014 parent_frame_id: Some(user_frame.id.clone()),
3015 caused_by_frame_id: Some(user_frame.id),
3016 };
3017 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
3018 tracing::warn!(
3019 error = %append_err,
3020 "failed to append console send failure frame"
3021 );
3022 }
3023 }
3024 }
3025 });
3026}
3027
3028async fn append_steer_delivery_terminal(
3029 inner: &AggregatorInner,
3030 user_frame: &ConsoleFrame,
3031 interaction_id: &str,
3032) -> ConsoleLogResult<AppendOutcome> {
3033 append_and_emit(
3034 inner,
3035 NewConsoleFrame {
3036 id: None,
3037 dedupe_key: format!("steer-delivered:{}", user_frame.id),
3038 timestamp_ms: current_time_ms(),
3039 runtime_key: user_frame.runtime_key.clone(),
3040 identity: user_frame.identity.clone(),
3041 conversation_id: user_frame.conversation_id.clone(),
3042 session_id: user_frame.session_id.clone(),
3043 kind: "interaction_complete".to_string(),
3044 status: ConsoleFrameStatus::Completed,
3045 payload: json!({
3046 "reason": "steer_delivered",
3047 "handling_mode": "steer",
3048 }),
3049 source: ConsoleFrameSource {
3050 kind: ConsoleFrameSourceKind::Synthetic,
3051 source_cursor: None,
3052 },
3053 source_event_id: None,
3054 interaction_id: Some(interaction_id.to_string()),
3055 turn_id: None,
3056 run_id: None,
3057 parent_frame_id: Some(user_frame.id.clone()),
3058 caused_by_frame_id: Some(user_frame.id.clone()),
3059 },
3060 )
3061 .await
3062}
3063
3064#[derive(Debug)]
3065pub enum ConsoleSendError {
3066 UnknownIdentity(String),
3067 AmbiguousIdentity {
3068 identity: String,
3069 candidates: String,
3070 },
3071 NotAddressable(String),
3072 Retired(String),
3073 InvalidContent(String),
3074 InvalidHandlingMode(String),
3075 InvalidRequest(String),
3076 IdempotencyConflict(String),
3077 State(&'static str),
3078 Dispatch(String),
3079 Log(ConsoleLogError),
3080}
3081
3082impl std::fmt::Display for ConsoleSendError {
3083 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3084 match self {
3085 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
3086 Self::AmbiguousIdentity {
3087 identity,
3088 candidates,
3089 } => write!(
3090 f,
3091 "ambiguous live identity alias {identity}: candidates [{candidates}]"
3092 ),
3093 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
3094 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
3095 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
3096 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
3097 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
3098 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
3099 Self::State(message) => write!(f, "console send state error: {message}"),
3100 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
3101 Self::Log(err) => write!(f, "console log error: {err}"),
3102 }
3103 }
3104}
3105
3106impl std::error::Error for ConsoleSendError {}
3107
3108async fn backfill_session_history(
3109 inner: Arc<AggregatorInner>,
3110 runtime_key: String,
3111 force_refresh: bool,
3112) -> ConsoleLogResult<()> {
3113 if !inner.options.session_history_backfill_enabled {
3114 return Ok(());
3115 }
3116 let Some(entry) = inner
3117 .runtimes
3118 .read()
3119 .ok()
3120 .and_then(|entries| entries.get(&runtime_key).cloned())
3121 else {
3122 return Ok(());
3123 };
3124 let members = Box::pin(member_sources_for_entry(&entry)).await;
3125 let mut targets = Vec::new();
3126 for resolved in members {
3127 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3128 continue;
3129 };
3130 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3131 continue;
3132 }
3133 let Some(session_id) = record.session_id.clone() else {
3134 continue;
3135 };
3136 targets.push(SessionBackfillTarget {
3137 entry: entry.clone(),
3138 record,
3139 session_id,
3140 });
3141 }
3142 backfill_session_history_targets(inner, targets, force_refresh).await
3143}
3144
3145#[derive(Clone)]
3146struct SessionBackfillTarget {
3147 entry: RuntimeEntry,
3148 record: ConsoleIdentityRecord,
3149 session_id: String,
3150}
3151
3152async fn backfill_session_history_targets(
3153 inner: Arc<AggregatorInner>,
3154 targets: Vec<SessionBackfillTarget>,
3155 force_refresh: bool,
3156) -> ConsoleLogResult<()> {
3157 let mut tasks = tokio::task::JoinSet::new();
3158 for target in targets {
3159 tasks.spawn(backfill_one_session_history(
3160 inner.clone(),
3161 target,
3162 force_refresh,
3163 ));
3164 }
3165 let mut first_error = None;
3166 while let Some(result) = tasks.join_next().await {
3167 match result {
3168 Ok(Ok(())) => {}
3169 Ok(Err(err)) => {
3170 if first_error.is_none() {
3171 first_error = Some(err);
3172 }
3173 }
3174 Err(err) => {
3175 if first_error.is_none() {
3176 first_error = Some(Box::new(std::io::Error::other(format!(
3177 "session backfill task failed: {err}"
3178 ))) as ConsoleLogError);
3179 }
3180 }
3181 }
3182 }
3183 if let Some(err) = first_error {
3184 Err(err)
3185 } else {
3186 Ok(())
3187 }
3188}
3189
3190async fn backfill_one_session_history(
3191 inner: Arc<AggregatorInner>,
3192 target: SessionBackfillTarget,
3193 force_refresh: bool,
3194) -> ConsoleLogResult<()> {
3195 let _permit = inner
3196 .session_backfill_permits
3197 .clone()
3198 .acquire_owned()
3199 .await
3200 .map_err(|err| -> ConsoleLogError {
3201 Box::new(std::io::Error::other(format!(
3202 "session backfill limiter closed: {err}"
3203 )))
3204 })?;
3205 let SessionBackfillTarget {
3206 entry,
3207 record,
3208 session_id,
3209 } = target;
3210 let watermark_runtime_key =
3211 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
3212 let write_epoch = entry.runtime.session_document_write_epoch(&session_id);
3220 if !force_refresh && let Some(epoch) = write_epoch {
3221 let epochs = inner
3222 .session_backfill_epochs
3223 .lock()
3224 .unwrap_or_else(std::sync::PoisonError::into_inner);
3225 if epochs.get(&watermark_runtime_key) == Some(&epoch) {
3226 return Ok(());
3227 }
3228 }
3229 let watermark = inner
3230 .store
3231 .source_watermark(
3232 &watermark_runtime_key,
3233 ConsoleFrameSourceKind::SessionHistory,
3234 )
3235 .await?;
3236 let now_ms = current_time_ms();
3237 let mut offset = watermark
3238 .as_deref()
3239 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
3240 .unwrap_or(0);
3241 if !force_refresh
3242 && watermark.as_deref().is_some_and(|watermark| {
3243 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
3244 })
3245 {
3246 return Ok(());
3247 }
3248 let mut completed_cleanly = true;
3251 loop {
3252 let page = match entry
3253 .runtime
3254 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
3255 .await
3256 {
3257 Ok(page) => page,
3258 Err(err) => {
3259 record_session_history_backfill_failure(
3260 &inner,
3261 &watermark_runtime_key,
3262 &entry.runtime_key,
3263 &record.identity,
3264 &session_id,
3265 offset,
3266 err.to_string(),
3267 )
3268 .await?;
3269 completed_cleanly = false;
3270 break;
3271 }
3272 };
3273 let page_value = match serde_json::to_value(page) {
3274 Ok(value) => value,
3275 Err(err) => {
3276 record_session_history_backfill_failure(
3277 &inner,
3278 &watermark_runtime_key,
3279 &entry.runtime_key,
3280 &record.identity,
3281 &session_id,
3282 offset,
3283 err.to_string(),
3284 )
3285 .await?;
3286 completed_cleanly = false;
3287 break;
3288 }
3289 };
3290 let base_offset = page_value
3291 .get("offset")
3292 .and_then(Value::as_u64)
3293 .unwrap_or(offset as u64) as usize;
3294 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3295 record_session_history_backfill_failure(
3296 &inner,
3297 &watermark_runtime_key,
3298 &entry.runtime_key,
3299 &record.identity,
3300 &session_id,
3301 offset,
3302 "session history page missing messages".to_string(),
3303 )
3304 .await?;
3305 completed_cleanly = false;
3306 break;
3307 };
3308 if messages.is_empty() {
3309 if offset > 0 {
3310 record_session_history_watermark(
3311 &inner,
3312 &watermark_runtime_key,
3313 &session_id,
3314 offset,
3315 )
3316 .await?;
3317 }
3318 break;
3319 }
3320 for (idx, message) in messages.iter().enumerate() {
3321 let absolute_offset = base_offset + idx;
3322 let frames = frames_from_session_history_message_with_namespace(
3323 &entry.runtime_key,
3324 &record.identity,
3325 &entry.identity_namespace,
3326 &session_id,
3327 absolute_offset,
3328 message.clone(),
3329 );
3330 for mut frame in frames {
3331 if history_frame_has_existing_counterpart(&inner, &frame).await? {
3332 continue;
3333 }
3334 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3335 frame.payload = redacted;
3336 frame.status = ConsoleFrameStatus::Redacted;
3337 }
3338 append_and_emit(&inner, frame).await?;
3339 }
3340 }
3341 let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3342 tracing::warn!(
3349 session_id = %session_id,
3350 requested_offset = offset,
3351 page_base_offset = base_offset,
3352 page_len = messages.len(),
3353 "session history backfill made no forward progress; stopping to avoid a livelock"
3354 );
3355 break;
3356 };
3357 offset = next_offset;
3358 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3359 .await?;
3360 let has_more = page_value
3361 .get("has_more")
3362 .and_then(Value::as_bool)
3363 .unwrap_or(false);
3364 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3365 break;
3366 }
3367 }
3368 if completed_cleanly && let Some(epoch) = write_epoch {
3369 inner
3370 .session_backfill_epochs
3371 .lock()
3372 .unwrap_or_else(std::sync::PoisonError::into_inner)
3373 .insert(watermark_runtime_key, epoch);
3374 }
3375 Ok(())
3376}
3377
3378fn advance_backfill_offset(
3383 requested_offset: usize,
3384 base_offset: usize,
3385 page_len: usize,
3386) -> Option<usize> {
3387 let next = base_offset.saturating_add(page_len);
3388 (next > requested_offset).then_some(next)
3389}
3390
3391async fn record_session_history_watermark(
3392 inner: &AggregatorInner,
3393 watermark_runtime_key: &str,
3394 session_id: &str,
3395 offset: usize,
3396) -> ConsoleLogResult<()> {
3397 inner
3398 .store
3399 .record_source_watermark(
3400 watermark_runtime_key,
3401 ConsoleFrameSourceKind::SessionHistory,
3402 &format_session_history_watermark(session_id, offset, current_time_ms()),
3403 )
3404 .await
3405}
3406
3407fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3408 if !inner.options.session_history_backfill_enabled {
3409 return;
3410 }
3411 tokio::spawn(async move {
3412 {
3413 let mut active = inner.active_session_backfills.lock().await;
3414 if !active.insert(runtime_key.clone()) {
3415 return;
3416 }
3417 }
3418 let result = Box::pin(backfill_session_history(
3419 inner.clone(),
3420 runtime_key.clone(),
3421 false,
3422 ))
3423 .await;
3424 let mut active = inner.active_session_backfills.lock().await;
3425 active.remove(&runtime_key);
3426 drop(active);
3427 if let Err(err) = result {
3428 tracing::warn!(
3429 runtime_key = %runtime_key,
3430 error = %err,
3431 "console session-history backfill failed"
3432 );
3433 }
3434 });
3435}
3436
3437fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3438 if !inner.options.session_history_backfill_enabled {
3439 return;
3440 }
3441 tokio::spawn(async move {
3442 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3443 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3444 loop {
3445 interval.tick().await;
3446 let runtime_still_registered = inner
3447 .runtimes
3448 .read()
3449 .ok()
3450 .is_some_and(|entries| entries.contains_key(&runtime_key));
3451 if !runtime_still_registered {
3452 break;
3453 }
3454 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3455 }
3456 });
3457}
3458
3459fn spawn_session_history_backfill_target(
3460 inner: Arc<AggregatorInner>,
3461 target: SessionBackfillTarget,
3462 force_refresh: bool,
3463) {
3464 if !inner.options.session_history_backfill_enabled {
3465 return;
3466 }
3467 tokio::spawn(async move {
3468 let active_key = targeted_session_history_active_key(&target, force_refresh);
3469 let result =
3470 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3471 if let Err(err) = result {
3472 tracing::warn!(
3473 active_key = %active_key,
3474 error = %err,
3475 "console targeted session-history backfill failed"
3476 );
3477 }
3478 });
3479}
3480
3481async fn run_targeted_session_history_backfill(
3482 inner: Arc<AggregatorInner>,
3483 target: SessionBackfillTarget,
3484 force_refresh: bool,
3485) -> ConsoleLogResult<()> {
3486 let active_key = targeted_session_history_active_key(&target, force_refresh);
3487 {
3488 let mut active = inner.active_session_backfills.lock().await;
3489 if !active.insert(active_key.clone()) {
3490 return Ok(());
3491 }
3492 }
3493 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3494 let mut active = inner.active_session_backfills.lock().await;
3495 active.remove(&active_key);
3496 result
3497}
3498
3499fn targeted_session_history_active_key(
3500 target: &SessionBackfillTarget,
3501 force_refresh: bool,
3502) -> String {
3503 let mode = if force_refresh { "force" } else { "refresh" };
3504 format!(
3505 "{}:session-history:{}:{mode}",
3506 target.entry.runtime_key, target.session_id
3507 )
3508}
3509
3510fn spawn_session_history_backfill_for_identity(
3511 inner: Arc<AggregatorInner>,
3512 identity: String,
3513 force_refresh: bool,
3514) {
3515 if !inner.options.session_history_backfill_enabled {
3516 return;
3517 }
3518 tokio::spawn(async move {
3519 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3520 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3521 }
3522 });
3523}
3524
3525fn spawn_opportunistic_session_history_backfill_for_identity(
3526 inner: Arc<AggregatorInner>,
3527 identity: String,
3528) {
3529 if !inner.options.session_history_backfill_enabled {
3530 return;
3531 }
3532 tokio::spawn(async move {
3533 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3534 let active_key = format!(
3535 "{}:session-history:{}",
3536 target.entry.runtime_key, target.session_id
3537 );
3538 {
3539 let mut seen = inner.opportunistic_session_backfills.lock().await;
3540 if !seen.insert(active_key) {
3541 continue;
3542 }
3543 }
3544 spawn_session_history_backfill_target(inner.clone(), target, false);
3545 }
3546 });
3547}
3548
3549#[cfg(test)]
3550async fn session_backfill_target_for_identity(
3551 inner: &AggregatorInner,
3552 identity: &str,
3553) -> Option<SessionBackfillTarget> {
3554 session_backfill_targets_for_identity(inner, identity)
3555 .await
3556 .into_iter()
3557 .next()
3558}
3559
3560async fn session_backfill_targets_for_identity(
3561 inner: &AggregatorInner,
3562 identity: &str,
3563) -> Vec<SessionBackfillTarget> {
3564 let entries = inner
3565 .runtimes
3566 .read()
3567 .ok()
3568 .map(|entries| entries.clone())
3569 .unwrap_or_default();
3570 let mut targets = Vec::new();
3571 for entry in entries.values() {
3572 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3573 if raw_identities.is_empty() {
3574 continue;
3575 }
3576 let mids = raw_identities
3577 .iter()
3578 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3579 .collect::<Vec<_>>();
3580 for resolved in Box::pin(member_sources_for_entry(entry))
3581 .await
3582 .into_iter()
3583 .filter(|candidate| {
3584 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3585 })
3586 {
3587 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3588 continue;
3589 };
3590 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3591 continue;
3592 }
3593 let Some(session_id) = record.session_id.clone() else {
3594 continue;
3595 };
3596 targets.push(SessionBackfillTarget {
3597 entry: entry.clone(),
3598 record,
3599 session_id,
3600 });
3601 }
3602 }
3603 targets
3604}
3605
3606fn resolved_member_matches_raw_identities(
3607 resolved: &ResolvedConsoleMember,
3608 raw_identities: &[String],
3609 mids: &[AgentIdentity],
3610) -> bool {
3611 mids.contains(&resolved.member.agent_identity)
3612 || crate::member_comms_id::durable_identity_label(&resolved.member.labels)
3613 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3614}
3615
3616async fn recover_lagged_source_events(
3617 inner: Arc<AggregatorInner>,
3618 runtime_key: &str,
3619 console_events: &ConsoleEventStore,
3620) -> ConsoleLogResult<()> {
3621 let watermark = inner
3622 .store
3623 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3624 .await?;
3625 match console_events.replay_all(watermark.as_deref()).await {
3626 Ok(events) => {
3627 for envelope in events {
3628 project_console_event(inner.clone(), runtime_key, envelope).await?;
3629 }
3630 }
3631 Err(err) => {
3632 append_source_gap(
3633 &inner,
3634 runtime_key,
3635 format!(
3636 "{}:{}:{}",
3637 err.error, err.stream, err.requested_last_event_id
3638 ),
3639 )
3640 .await?;
3641 }
3642 }
3643 Ok(())
3644}
3645
3646async fn append_source_gap(
3647 inner: &AggregatorInner,
3648 runtime_key: &str,
3649 reason: String,
3650) -> ConsoleLogResult<()> {
3651 append_and_emit(
3652 inner,
3653 NewConsoleFrame {
3654 id: None,
3655 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3656 timestamp_ms: current_time_ms(),
3657 runtime_key: runtime_key.to_string(),
3658 identity: "__console__".to_string(),
3659 conversation_id: None,
3660 session_id: None,
3661 kind: "replay_unavailable".to_string(),
3662 status: ConsoleFrameStatus::DeliveryFailed,
3663 payload: json!({
3664 "reason": reason,
3665 "source_kind": "console_event",
3666 }),
3667 source: ConsoleFrameSource {
3668 kind: ConsoleFrameSourceKind::Synthetic,
3669 source_cursor: None,
3670 },
3671 source_event_id: None,
3672 interaction_id: None,
3673 turn_id: None,
3674 run_id: None,
3675 parent_frame_id: None,
3676 caused_by_frame_id: None,
3677 },
3678 )
3679 .await?;
3680 let _ = inner
3681 .event_tx
3682 .send(ConsoleTimelineEvent::ReplayUnavailable {
3683 requested_cursor: format!("source-gap:{runtime_key}"),
3684 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3685 });
3686 Ok(())
3687}
3688
3689async fn record_session_history_backfill_failure(
3694 inner: &AggregatorInner,
3695 watermark_runtime_key: &str,
3696 runtime_key: &str,
3697 identity: &str,
3698 session_id: &str,
3699 offset: usize,
3700 reason: String,
3701) -> ConsoleLogResult<()> {
3702 record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3703 append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3704}
3705
3706fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3709 format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3710}
3711
3712async fn append_backfill_gap(
3713 inner: &AggregatorInner,
3714 runtime_key: &str,
3715 identity: &str,
3716 session_id: &str,
3717 reason: String,
3718) -> ConsoleLogResult<()> {
3719 append_and_emit(
3720 inner,
3721 NewConsoleFrame {
3722 id: None,
3723 dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3728 timestamp_ms: current_time_ms(),
3729 runtime_key: runtime_key.to_string(),
3730 identity: identity.to_string(),
3731 conversation_id: Some(identity.to_string()),
3732 session_id: Some(session_id.to_string()),
3733 kind: "replay_unavailable".to_string(),
3734 status: ConsoleFrameStatus::DeliveryFailed,
3735 payload: json!({
3736 "reason": reason,
3737 "source_kind": "session_history",
3738 }),
3739 source: ConsoleFrameSource {
3740 kind: ConsoleFrameSourceKind::Synthetic,
3741 source_cursor: None,
3742 },
3743 source_event_id: None,
3744 interaction_id: None,
3745 turn_id: None,
3746 run_id: None,
3747 parent_frame_id: None,
3748 caused_by_frame_id: None,
3749 },
3750 )
3751 .await?;
3752 Ok(())
3753}
3754
3755async fn project_console_event(
3756 inner: Arc<AggregatorInner>,
3757 runtime_key: &str,
3758 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3759) -> ConsoleLogResult<()> {
3760 let Some(entry) = inner
3761 .runtimes
3762 .read()
3763 .ok()
3764 .and_then(|entries| entries.get(runtime_key).cloned())
3765 else {
3766 return Ok(());
3767 };
3768 let mut frame = frame_from_console_event(&entry, envelope);
3769 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3770 frame.payload = redacted;
3771 frame.status = ConsoleFrameStatus::Redacted;
3772 }
3773 let source_cursor = frame
3774 .source_event_id
3775 .clone()
3776 .unwrap_or_else(|| frame.dedupe_key.clone());
3777 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3778 Some(frame.identity.clone())
3779 } else {
3780 None
3781 };
3782 let opportunistic_refresh_identity = if refresh_identity.is_none()
3783 && console_event_should_start_session_history_backfill(&frame)
3784 {
3785 Some(frame.identity.clone())
3786 } else {
3787 None
3788 };
3789 append_and_emit(&inner, frame).await?;
3790 inner
3791 .store
3792 .record_source_watermark(
3793 &entry.runtime_key,
3794 ConsoleFrameSourceKind::ConsoleEvent,
3795 &source_cursor,
3796 )
3797 .await?;
3798 if let Some(identity) = refresh_identity {
3799 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3800 } else if let Some(identity) = opportunistic_refresh_identity {
3801 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3802 }
3803 Ok(())
3804}
3805
3806fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3807 matches!(
3808 frame.kind.as_str(),
3809 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3810 ) || frame.session_id.is_some()
3811}
3812
3813fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3814 if frame.identity == SYSTEM_EVENT_IDENTITY {
3815 return false;
3816 }
3817 matches!(
3818 frame.kind.as_str(),
3819 "turn_started"
3820 | "run_started"
3821 | "reasoning_complete"
3822 | "tool_call_requested"
3823 | "tool_call"
3824 | "tool_execution_started"
3825 | "text_delta"
3826 | "system_notice"
3827 )
3828}
3829
3830async fn append_and_emit(
3831 inner: &AggregatorInner,
3832 frame: NewConsoleFrame,
3833) -> ConsoleLogResult<AppendOutcome> {
3834 let outcome = inner.store.append_if_absent(frame).await?;
3835 if outcome.disposition == AppendDisposition::Inserted {
3836 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3837 frame: outcome.frame.clone(),
3838 });
3839 }
3840 Ok(outcome)
3841}
3842
3843async fn update_frame_status_and_emit(
3844 inner: &AggregatorInner,
3845 frame_id: &str,
3846 status: ConsoleFrameStatus,
3847) -> ConsoleLogResult<Option<ConsoleFrame>> {
3848 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3849 return Ok(None);
3850 };
3851 let update_marker = NewConsoleFrame {
3852 id: None,
3853 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3854 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3855 runtime_key: updated.runtime_key.clone(),
3856 identity: updated.identity.clone(),
3857 conversation_id: updated.conversation_id.clone(),
3858 session_id: updated.session_id.clone(),
3859 kind: "frame_updated".to_string(),
3860 status: updated.status,
3861 payload: json!({ "frame": updated.clone() }),
3862 source: ConsoleFrameSource {
3863 kind: ConsoleFrameSourceKind::Synthetic,
3864 source_cursor: None,
3865 },
3866 source_event_id: None,
3867 interaction_id: updated.interaction_id.clone(),
3868 turn_id: updated.turn_id.clone(),
3869 run_id: updated.run_id.clone(),
3870 parent_frame_id: Some(updated.id.clone()),
3871 caused_by_frame_id: Some(updated.id.clone()),
3872 };
3873 let outcome = inner.store.append_if_absent(update_marker).await?;
3874 if outcome.disposition == AppendDisposition::Inserted {
3875 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3876 frame: outcome.frame,
3877 });
3878 }
3879 Ok(Some(updated))
3880}
3881
3882fn frame_from_console_event(
3883 entry: &RuntimeEntry,
3884 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3885) -> NewConsoleFrame {
3886 let event_id = envelope.event_id;
3887 let interaction_id = envelope.interaction_id;
3888 let event_type = envelope.event_type;
3889 let payload = envelope.data;
3890 let turn_id = payload
3891 .get("turn_id")
3892 .and_then(Value::as_str)
3893 .map(ToString::to_string);
3894 let run_id = payload
3895 .get("run_id")
3896 .and_then(Value::as_str)
3897 .map(ToString::to_string);
3898 let status = match event_type.as_str() {
3899 "interaction_started" => ConsoleFrameStatus::Accepted,
3900 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3901 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3902 _ => ConsoleFrameStatus::Delivered,
3903 };
3904 let identity = if envelope.identity == SYSTEM_EVENT_IDENTITY {
3909 envelope.identity
3910 } else {
3911 apply_namespace(&envelope.identity, &entry.identity_namespace)
3912 };
3913 let dedupe_key = reasoning_payload_text(&event_type, &payload)
3914 .map(|reasoning_text| {
3915 let turn_scope = interaction_id
3918 .as_deref()
3919 .or(turn_id.as_deref())
3920 .or(run_id.as_deref())
3921 .unwrap_or(event_id.as_str());
3922 format!(
3923 "console-reasoning:{}:{}:{}",
3924 entry.runtime_key,
3925 turn_scope,
3926 hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3927 )
3928 })
3929 .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3930 NewConsoleFrame {
3931 id: Some(event_id.clone()),
3932 dedupe_key,
3933 timestamp_ms: envelope.timestamp_ms,
3934 runtime_key: entry.runtime_key.clone(),
3935 identity: identity.clone(),
3936 conversation_id: Some(identity),
3937 session_id: payload
3938 .get("session_id")
3939 .and_then(Value::as_str)
3940 .map(ToString::to_string),
3941 kind: event_type,
3942 status,
3943 payload,
3944 source: ConsoleFrameSource {
3945 kind: ConsoleFrameSourceKind::ConsoleEvent,
3946 source_cursor: None,
3947 },
3948 source_event_id: Some(event_id),
3949 interaction_id,
3950 turn_id,
3951 run_id,
3952 parent_frame_id: None,
3953 caused_by_frame_id: None,
3954 }
3955}
3956
3957#[cfg(test)]
3958fn frame_from_session_history_message(
3959 runtime_key: &str,
3960 identity: &str,
3961 session_id: &str,
3962 offset: usize,
3963 message: Value,
3964) -> Option<NewConsoleFrame> {
3965 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3966 .into_iter()
3967 .next()
3968}
3969
3970#[cfg(test)]
3971fn frames_from_session_history_message(
3972 runtime_key: &str,
3973 identity: &str,
3974 session_id: &str,
3975 offset: usize,
3976 message: Value,
3977) -> Vec<NewConsoleFrame> {
3978 frames_from_session_history_message_with_namespace(
3979 runtime_key,
3980 identity,
3981 "",
3982 session_id,
3983 offset,
3984 message,
3985 )
3986}
3987
3988#[allow(clippy::too_many_arguments)]
3996fn session_history_assistant_image_frames(
3997 runtime_key: &str,
3998 identity: &str,
3999 session_id: &str,
4000 offset: usize,
4001 result_idx: usize,
4002 timestamp_ms: u64,
4003 tool_use_id: &str,
4004 result_text: &str,
4005) -> Vec<NewConsoleFrame> {
4006 let Ok(image_result) = serde_json::from_str::<
4007 meerkat_core::image_generation::ImageGenerationToolResult,
4008 >(result_text) else {
4009 return Vec::new();
4010 };
4011 image_result
4012 .images
4013 .iter()
4014 .enumerate()
4015 .map(|(image_idx, image)| NewConsoleFrame {
4016 id: None,
4017 dedupe_key: format!(
4020 "console-assistant-image:{runtime_key}:{}",
4021 image.blob_ref.blob_id
4022 ),
4023 timestamp_ms,
4024 runtime_key: runtime_key.to_string(),
4025 identity: identity.to_string(),
4026 conversation_id: Some(identity.to_string()),
4027 session_id: Some(session_id.to_string()),
4028 kind: "assistant_image".to_string(),
4029 status: ConsoleFrameStatus::Completed,
4030 payload: json!({
4031 "source_event_type": "session_history",
4032 "tool_call_id": tool_use_id,
4033 "image_id": image.image_id.0.to_string(),
4034 "blob_id": image.blob_ref.blob_id,
4035 "media_type": image.media_type.as_str(),
4036 "width": image.width,
4037 "height": image.height,
4038 "revised_prompt": image_result.revised_prompt.clone(),
4039 "type": "session_history",
4040 }),
4041 source: ConsoleFrameSource {
4042 kind: ConsoleFrameSourceKind::SessionHistory,
4043 source_cursor: Some(format!(
4044 "{session_id}:{offset}:{result_idx}:image:{image_idx}"
4045 )),
4046 },
4047 source_event_id: None,
4048 interaction_id: None,
4049 turn_id: None,
4050 run_id: None,
4051 parent_frame_id: None,
4052 caused_by_frame_id: None,
4053 })
4054 .collect()
4055}
4056
4057fn frames_from_session_history_message_with_namespace(
4058 runtime_key: &str,
4059 identity: &str,
4060 identity_namespace: &str,
4061 session_id: &str,
4062 offset: usize,
4063 message: Value,
4064) -> Vec<NewConsoleFrame> {
4065 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
4066 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
4067 return Vec::new();
4068 };
4069 if let Message::ToolResults {
4070 results,
4071 created_at,
4072 } = parsed
4073 {
4074 return results
4075 .into_iter()
4076 .enumerate()
4077 .flat_map(|(idx, result)| {
4078 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
4079 let result_text = result.text_content();
4080 let tool_use_id = result.tool_use_id.clone();
4081 let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
4082 let mut frames = vec![NewConsoleFrame {
4083 id: None,
4084 dedupe_key: format!(
4085 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
4086 ),
4087 timestamp_ms,
4088 runtime_key: runtime_key.to_string(),
4089 identity: identity.to_string(),
4090 conversation_id: Some(identity.to_string()),
4091 session_id: Some(session_id.to_string()),
4092 kind: "tool_execution_completed".to_string(),
4093 status: ConsoleFrameStatus::Completed,
4094 payload: json!({
4095 "id": tool_use_id,
4096 "tool_call_id": tool_use_id,
4097 "result": result_text,
4098 "content": content,
4099 "is_error": result.is_error,
4100 "source_event_type": "session_history",
4101 "type": "session_history",
4102 }),
4103 source: ConsoleFrameSource {
4104 kind: ConsoleFrameSourceKind::SessionHistory,
4105 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
4106 },
4107 source_event_id: None,
4108 interaction_id: None,
4109 turn_id: None,
4110 run_id: None,
4111 parent_frame_id: None,
4112 caused_by_frame_id: None,
4113 }];
4114 frames.extend(session_history_assistant_image_frames(
4122 runtime_key,
4123 identity,
4124 session_id,
4125 offset,
4126 idx,
4127 timestamp_ms,
4128 &tool_use_id,
4129 &result_text,
4130 ));
4131 frames
4132 })
4133 .collect();
4134 }
4135 let transcript_identity = match &parsed {
4142 Message::User(user) => Some(&user.identity),
4143 Message::BlockAssistant(assistant) => Some(&assistant.identity),
4144 _ => None,
4145 };
4146 let history_interaction_id = transcript_identity
4147 .and_then(|identity| identity.interaction_id.as_ref())
4148 .map(|id| id.0.to_string());
4149 let history_run_id = transcript_identity
4150 .and_then(|identity| identity.run_id.as_ref())
4151 .map(|id| id.0.to_string());
4152 let (kind, timestamp_ms, payload) = match &parsed {
4153 Message::User(user) => {
4154 if session_history_user_message_is_scaffold(&message) {
4155 return Vec::new();
4156 }
4157 (
4158 "user_input",
4159 user.created_at.timestamp_millis().max(0) as u64,
4160 json!({
4161 "content": user.content,
4162 "message": message,
4163 }),
4164 )
4165 }
4166 Message::BlockAssistant(assistant) => {
4169 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
4170 (
4171 "interaction_complete",
4172 assistant.created_at.timestamp_millis().max(0) as u64,
4173 json!({
4174 "result": text,
4175 "text": text,
4176 "message": message,
4177 "source_event_type": "session_history",
4178 "type": "session_history",
4179 }),
4180 )
4181 }
4182 Message::SystemNotice(notice) => (
4183 "system_notice",
4184 notice.created_at.timestamp_millis().max(0) as u64,
4185 json!({
4186 "message": message,
4187 "kind": notice.kind,
4188 "render_class": notice.kind.render_class(),
4189 "body": notice.body,
4190 "blocks": notice.blocks,
4191 "source_event_type": "session_history",
4192 "type": "session_history",
4193 }),
4194 ),
4195 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
4196 };
4197 let mut frames = vec![NewConsoleFrame {
4198 id: None,
4199 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
4200 timestamp_ms,
4201 runtime_key: runtime_key.to_string(),
4202 identity: identity.to_string(),
4203 conversation_id: Some(identity.to_string()),
4204 session_id: Some(session_id.to_string()),
4205 kind: kind.to_string(),
4206 status: ConsoleFrameStatus::Completed,
4207 payload,
4208 source: ConsoleFrameSource {
4209 kind: ConsoleFrameSourceKind::SessionHistory,
4210 source_cursor: Some(format!("{session_id}:{offset}")),
4211 },
4212 source_event_id: None,
4213 interaction_id: history_interaction_id,
4214 turn_id: None,
4215 run_id: history_run_id,
4216 parent_frame_id: None,
4217 caused_by_frame_id: None,
4218 }];
4219 if let Message::BlockAssistant(assistant) = &parsed {
4220 frames.extend(spawn_initial_message_frames_from_assistant(
4221 runtime_key,
4222 identity,
4223 identity_namespace,
4224 session_id,
4225 offset,
4226 assistant,
4227 &payload_hash,
4228 ));
4229 frames.extend(outgoing_comms_tool_call_frames_from_assistant(
4230 runtime_key,
4231 identity,
4232 session_id,
4233 offset,
4234 assistant,
4235 &payload_hash,
4236 ));
4237 }
4238 frames
4239}
4240
4241const OUTGOING_COMMS_SEND_TOOLS: [&str; 3] = ["send_message", "send_request", "send_response"];
4245
4246fn outgoing_comms_tool_call_frames_from_assistant(
4263 runtime_key: &str,
4264 identity: &str,
4265 session_id: &str,
4266 offset: usize,
4267 assistant: &meerkat_core::types::BlockAssistantMessage,
4268 payload_hash: &str,
4269) -> Vec<NewConsoleFrame> {
4270 let mut frames = Vec::new();
4271 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
4272 if !OUTGOING_COMMS_SEND_TOOLS.contains(&tool.name) {
4273 continue;
4274 }
4275 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
4276 continue;
4277 };
4278 frames.push(NewConsoleFrame {
4279 id: None,
4280 dedupe_key: format!(
4281 "session-history:{runtime_key}:{session_id}:{offset}:comms-out:{tool_idx}:{payload_hash}"
4282 ),
4283 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
4284 runtime_key: runtime_key.to_string(),
4285 identity: identity.to_string(),
4286 conversation_id: Some(identity.to_string()),
4287 session_id: Some(session_id.to_string()),
4288 kind: "tool_call_requested".to_string(),
4289 status: ConsoleFrameStatus::Completed,
4290 payload: json!({
4291 "id": tool.id,
4292 "tool_call_id": tool.id,
4293 "name": tool.name,
4294 "args": args,
4295 "source_event_type": "session_history",
4296 "type": "session_history",
4297 }),
4298 source: ConsoleFrameSource {
4299 kind: ConsoleFrameSourceKind::SessionHistory,
4300 source_cursor: Some(format!("{session_id}:{offset}:comms-out:{tool_idx}")),
4301 },
4302 source_event_id: None,
4303 interaction_id: None,
4304 turn_id: None,
4305 run_id: None,
4306 parent_frame_id: None,
4307 caused_by_frame_id: None,
4308 });
4309 }
4310 frames
4311}
4312
4313fn spawn_initial_message_frames_from_assistant(
4314 runtime_key: &str,
4315 parent_identity: &str,
4316 identity_namespace: &str,
4317 session_id: &str,
4318 offset: usize,
4319 assistant: &meerkat_core::types::BlockAssistantMessage,
4320 payload_hash: &str,
4321) -> Vec<NewConsoleFrame> {
4322 let mut frames = Vec::new();
4323 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
4324 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
4325 continue;
4326 }
4327 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
4328 continue;
4329 };
4330 for (spawn_idx, (target_identity, initial_message)) in
4331 spawn_initial_messages_from_tool_args(&args)
4332 .into_iter()
4333 .enumerate()
4334 {
4335 let target_identity = apply_namespace(&target_identity, identity_namespace);
4336 let message_content = match &initial_message {
4337 Value::String(text) => json!([
4338 {
4339 "type": "text",
4340 "text": text,
4341 }
4342 ]),
4343 other => other.clone(),
4344 };
4345 let message = match &initial_message {
4346 Value::String(text) => json!({
4347 "role": "user",
4348 "content": text,
4349 "created_at": assistant.created_at,
4350 }),
4351 other => json!({
4352 "role": "user",
4353 "content": other,
4354 "created_at": assistant.created_at,
4355 }),
4356 };
4357 frames.push(NewConsoleFrame {
4358 id: None,
4359 dedupe_key: format!(
4360 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
4361 ),
4362 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
4363 runtime_key: runtime_key.to_string(),
4364 identity: target_identity.clone(),
4365 conversation_id: Some(target_identity),
4366 session_id: None,
4367 kind: "user_input".to_string(),
4368 status: ConsoleFrameStatus::Completed,
4369 payload: json!({
4370 "content": message_content,
4371 "message": message,
4372 "source_event_type": "session_history_spawn_initial_message",
4373 "type": "session_history",
4374 "tool_call_id": tool.id,
4375 "parent_identity": parent_identity,
4376 "via_tool": tool.name,
4377 }),
4378 source: ConsoleFrameSource {
4379 kind: ConsoleFrameSourceKind::SessionHistory,
4380 source_cursor: Some(format!(
4381 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4382 )),
4383 },
4384 source_event_id: None,
4385 interaction_id: None,
4386 turn_id: None,
4387 run_id: None,
4388 parent_frame_id: None,
4389 caused_by_frame_id: None,
4390 });
4391 }
4392 }
4393 frames
4394}
4395
4396fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4397 let mut messages = Vec::new();
4398 if let Some(message) = spawn_initial_message_from_record(args) {
4399 messages.push(message);
4400 }
4401 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4402 for spec in specs {
4403 if let Some(message) = spawn_initial_message_from_record(spec) {
4404 messages.push(message);
4405 }
4406 }
4407 }
4408 messages
4409}
4410
4411fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4412 let target_identity = record
4413 .get("member_id")
4414 .or_else(|| record.get("agent_identity"))
4415 .and_then(Value::as_str)?
4416 .trim();
4417 if target_identity.is_empty() {
4418 return None;
4419 }
4420 let initial_message = record
4421 .get("initial_message")
4422 .or_else(|| record.get("task"))?
4423 .clone();
4424 Some((target_identity.to_string(), initial_message))
4425}
4426
4427fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4428 message
4429 .get("content")
4430 .is_some_and(scaffold_message_content_is_noise)
4431}
4432
4433fn scaffold_message_content_is_noise(value: &Value) -> bool {
4434 match value {
4435 Value::String(text) => scaffold_message_text_is_noise(text),
4436 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4437 Value::Object(map) => ["text", "content", "message"]
4438 .iter()
4439 .filter_map(|key| map.get(*key))
4440 .any(scaffold_message_content_is_noise),
4441 _ => false,
4442 }
4443}
4444
4445fn scaffold_message_text_is_noise(text: &str) -> bool {
4446 let trimmed = text.trim_start();
4447 trimmed.starts_with("[PEER UPDATE]")
4448 || trimmed
4449 .to_ascii_lowercase()
4450 .starts_with("you have been spawned")
4451}
4452
4453fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4454 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4455 rest.split(':').next()?.parse().ok()
4456}
4457
4458fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4459 format!("{session_id}:{offset}:{checked_at_ms}")
4460}
4461
4462fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4463 let Some(checked_at_ms) = watermark
4464 .rsplit_once(':')
4465 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4466 else {
4467 return false;
4468 };
4469 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4470 let ttl_ms = if offset > 0 {
4471 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4472 } else {
4473 SESSION_HISTORY_REFRESH_TTL_MS
4474 };
4475 now_ms.saturating_sub(checked_at_ms) < ttl_ms
4476}
4477
4478async fn history_frame_has_existing_counterpart(
4479 inner: &AggregatorInner,
4480 frame: &NewConsoleFrame,
4481) -> ConsoleLogResult<bool> {
4482 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4483 let Some(fingerprint) = fingerprint else {
4484 return Ok(false);
4485 };
4486 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4487 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4488 let mut after = None;
4489 loop {
4490 let page = inner
4491 .store
4492 .query_frames(ConsoleTimelineQuery {
4493 identity: Some(frame.identity.clone()),
4494 conversation_id: frame.conversation_id.clone(),
4495 after,
4496 limit: 1_000,
4497 })
4498 .await?;
4499 for existing in &page.frames {
4500 let same_session = existing.session_id == frame.session_id
4501 || existing.session_id.is_none()
4502 || frame.session_id.is_none();
4503 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4504 continue;
4505 }
4506 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4507 == Some(&fingerprint)
4508 {
4509 return Ok(true);
4510 }
4511 if assistant_terminal
4512 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4513 {
4514 let turn_key = existing
4515 .interaction_id
4516 .as_deref()
4517 .or(existing.turn_id.as_deref())
4518 .or(existing.run_id.as_deref())
4519 .unwrap_or("session");
4520 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4521 aggregated.push_str(delta);
4522 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4523 return Ok(true);
4524 }
4525 }
4526 }
4527 if page.frames.is_empty() || page.next_cursor.is_none() {
4528 return Ok(false);
4529 }
4530 after = page.next_cursor;
4531 }
4532}
4533
4534fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4535 format!("{runtime_key}:session-history:{session_id}")
4536}
4537
4538fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4539 match kind {
4540 "user_input" | "interaction_started" => payload
4541 .get("content")
4542 .map(stable_value_fingerprint)
4543 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4544 "tool_execution_completed" => tool_result_fingerprint(payload),
4545 "tool_call_requested" | "tool_call" | "tool_execution_started" => payload
4550 .get("tool_call_id")
4551 .or_else(|| payload.get("id"))
4552 .and_then(Value::as_str)
4553 .map(|tool_call_id| format!("tool-call:{tool_call_id}")),
4554 "text_delta" => {
4555 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4556 }
4557 kind if is_reasoning_event_kind(kind) => {
4558 reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4559 }
4560 "text_complete" | "interaction_complete" | "run_completed" => {
4561 assistant_terminal_fingerprint(kind, payload)
4562 }
4563 _ => None,
4564 }
4565}
4566
4567fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4568 let id = payload
4569 .get("tool_call_id")
4570 .or_else(|| payload.get("id"))
4571 .and_then(Value::as_str)
4572 .unwrap_or("");
4573 let result = payload
4574 .get("result")
4575 .or_else(|| payload.get("content"))
4576 .map(stable_value_fingerprint)?;
4577 Some(format!("{id}:{result}"))
4578}
4579
4580fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4581 match kind {
4582 "text_complete" | "interaction_complete" | "run_completed" => payload
4583 .get("text")
4584 .or_else(|| payload.get("result"))
4585 .or_else(|| payload.get("content"))
4586 .map(stable_value_fingerprint),
4587 _ => None,
4588 }
4589}
4590
4591fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4592 if kind != "text_delta" {
4593 return None;
4594 }
4595 payload
4596 .get("delta")
4597 .or_else(|| payload.get("text"))
4598 .or_else(|| payload.get("content"))
4599 .and_then(Value::as_str)
4600 .or_else(|| payload.as_str())
4601}
4602
4603fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4604 if !is_reasoning_event_kind(kind) {
4605 return None;
4606 }
4607 if kind == "reasoning_delta" {
4608 return payload
4609 .get("delta")
4610 .or_else(|| payload.get("text"))
4611 .or_else(|| payload.get("content"))
4612 .or_else(|| payload.get("summary"))
4613 .or_else(|| payload.get("reasoning"))
4614 .and_then(Value::as_str)
4615 .or_else(|| payload.as_str());
4616 }
4617 payload
4618 .get("summary")
4619 .or_else(|| payload.get("text"))
4620 .or_else(|| payload.get("content"))
4621 .or_else(|| payload.get("delta"))
4622 .or_else(|| payload.get("reasoning"))
4623 .and_then(Value::as_str)
4624 .or_else(|| payload.as_str())
4625}
4626
4627fn is_reasoning_event_kind(kind: &str) -> bool {
4628 matches!(
4629 kind,
4630 "reasoning_delta"
4631 | "reasoning_complete"
4632 | "reasoning_summary"
4633 | "reasoning_summary_delta"
4634 | "reasoning_summary_complete"
4635 )
4636}
4637
4638fn stable_value_fingerprint(value: &Value) -> String {
4639 if let Some(text) = content_value_text(value) {
4640 return normalize_transcript_fingerprint_text(&text);
4641 }
4642 match value {
4643 Value::String(text) => normalize_transcript_fingerprint_text(text),
4644 other => serde_json::to_string(other).unwrap_or_default(),
4645 }
4646}
4647
4648fn content_value_text(value: &Value) -> Option<String> {
4649 match value {
4650 Value::String(text) => Some(text.clone()),
4651 Value::Array(items) => {
4652 let text = items
4653 .iter()
4654 .filter_map(content_value_text)
4655 .collect::<String>();
4656 (!text.is_empty()).then_some(text)
4657 }
4658 Value::Object(map) => ["text", "content", "message", "blocks"]
4659 .iter()
4660 .filter_map(|key| map.get(*key))
4661 .find_map(content_value_text),
4662 _ => None,
4663 }
4664}
4665
4666fn normalize_transcript_fingerprint_text(text: &str) -> String {
4667 let trimmed = text.trim();
4668 trimmed
4669 .strip_prefix("[EVENT via rpc] ")
4670 .unwrap_or(trimmed)
4671 .trim()
4672 .to_string()
4673}
4674
4675#[derive(Debug, Clone, Copy)]
4676enum CachedIdentityVisibility {
4677 Visible,
4678 Hidden,
4679 Missing,
4680}
4681
4682async fn frame_is_visible(
4683 inner: &AggregatorInner,
4684 frame: &ConsoleFrame,
4685 allow_historical_identity: bool,
4686 identity_records: &[ConsoleIdentityRecord],
4687) -> ConsoleLogResult<bool> {
4688 let mut identity_visibility_cache = HashMap::new();
4689 Box::pin(frame_is_visible_cached(
4690 inner,
4691 frame,
4692 allow_historical_identity,
4693 &mut identity_visibility_cache,
4694 identity_records,
4695 ))
4696 .await
4697}
4698
4699async fn frame_is_visible_cached(
4700 inner: &AggregatorInner,
4701 frame: &ConsoleFrame,
4702 allow_historical_identity: bool,
4703 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4704 identity_records: &[ConsoleIdentityRecord],
4705) -> ConsoleLogResult<bool> {
4706 let entry = {
4707 let entries = inner
4708 .runtimes
4709 .read()
4710 .map_err(|_| runtime_registry_lock_error())?;
4711 if entries.is_empty() {
4712 return Ok(true);
4713 }
4714 let Some(entry) = entries.get(&frame.runtime_key) else {
4715 return Ok(false);
4716 };
4717 entry.clone()
4718 };
4719 if frame.identity != "__console__" && frame.identity != SYSTEM_EVENT_IDENTITY {
4727 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4728 let identity_visibility =
4729 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4730 cached
4731 } else {
4732 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4733 .unwrap_or_else(|| frame.identity.clone());
4734 let visibility = match identity_records.iter().find(|record| {
4735 record.runtime_key == frame.runtime_key
4736 && (record.identity == frame.identity
4737 || record.runtime_member_id == runtime_member_id)
4738 }) {
4739 Some(record) => {
4740 if Box::pin(console_identity_record_visible(&entry, record)).await {
4741 CachedIdentityVisibility::Visible
4742 } else {
4743 CachedIdentityVisibility::Hidden
4744 }
4745 }
4746 None => CachedIdentityVisibility::Missing,
4747 };
4748 identity_visibility_cache.insert(cache_key, visibility);
4749 visibility
4750 };
4751 match identity_visibility {
4752 CachedIdentityVisibility::Visible => {
4753 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4754 return Ok(false);
4755 }
4756 }
4757 CachedIdentityVisibility::Hidden => return Ok(false),
4758 CachedIdentityVisibility::Missing => {
4759 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4760 return Ok(false);
4761 }
4762 return Ok(
4763 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4764 );
4765 }
4766 }
4767 }
4768 Ok(entry.visibility_policy.frame_visible(frame))
4769}
4770
4771async fn identity_record_for_member(
4772 entry: &RuntimeEntry,
4773 handle: &MobHandle,
4774 member: &MobMemberListEntry,
4775) -> Option<ConsoleIdentityRecord> {
4776 let runtime_member_id =
4779 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4780 let durable_identity = crate::member_comms_id::durable_identity_label(&member.labels)
4781 .unwrap_or(runtime_member_id.as_str());
4782 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4783 let addressable = member_is_addressable(member);
4784 let visibility = match member.status {
4785 meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4786 ConsoleVisibility::RetiredReadable
4787 }
4788 meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4791 ConsoleVisibility::Unreachable
4792 }
4793 _ if addressable => ConsoleVisibility::Addressable,
4794 _ => ConsoleVisibility::Hidden,
4795 };
4796 let session_id = handle
4797 .resolve_bridge_session_id_observation(&member.agent_identity)
4798 .await
4799 .map(|sid| sid.to_string());
4800 let mut labels = member.labels.clone();
4801 labels
4802 .entry("role".to_string())
4803 .or_insert_with(|| member.role.to_string());
4804 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4810 if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4811 crate::console_spawn::merge_registered_labels(&mut labels, ®istered);
4812 }
4813 let display_name = labels
4814 .get("display_name")
4815 .cloned()
4816 .unwrap_or_else(|| runtime_member_id.clone());
4817 Some(ConsoleIdentityRecord {
4818 identity,
4819 display_name,
4820 runtime_key: entry.runtime_key.clone(),
4821 runtime_member_id,
4822 session_id,
4823 visibility,
4824 addressable,
4825 health: if addressable {
4826 "ready"
4827 } else {
4828 "hidden_by_policy"
4829 }
4830 .to_string(),
4831 topology_peers: member
4832 .wired_to
4833 .iter()
4834 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4835 .collect(),
4836 labels,
4837 })
4838}
4839
4840async fn identity_record_for_resolved_member(
4841 resolved: &ResolvedConsoleMember,
4842) -> Option<ConsoleIdentityRecord> {
4843 let mut record =
4844 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4845 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4846 record
4847 .labels
4848 .entry("source_mob_id".to_string())
4849 .or_insert_with(|| resolved.source_mob_id.clone());
4850 }
4851 Some(record)
4852}
4853
4854fn identity_record_for_status(
4855 entry: &RuntimeEntry,
4856 status: &crate::identity_first::IdentityStatus,
4857) -> ConsoleIdentityRecord {
4858 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4859 let runtime_member_id = status
4860 .agent_runtime_id
4861 .as_ref()
4862 .map(crate::identity_first::AgentRuntimeId::as_str)
4863 .unwrap_or_else(|| status.identity.as_str())
4864 .to_string();
4865 let addressable = status.addressability
4866 == crate::identity_first::AgentAddressability::Addressable
4867 && matches!(
4868 status.state,
4869 crate::identity_first::IdentityLifecycleState::Active
4870 | crate::identity_first::IdentityLifecycleState::Dormant
4871 | crate::identity_first::IdentityLifecycleState::Uninitialized
4872 );
4873 let visibility = match status.state {
4874 crate::identity_first::IdentityLifecycleState::Retiring => {
4875 ConsoleVisibility::RetiredReadable
4876 }
4877 crate::identity_first::IdentityLifecycleState::Broken
4878 | crate::identity_first::IdentityLifecycleState::Suspended => {
4879 ConsoleVisibility::Unreachable
4880 }
4881 _ if addressable => ConsoleVisibility::Addressable,
4882 _ => ConsoleVisibility::Hidden,
4883 };
4884 let health = match status.state {
4885 crate::identity_first::IdentityLifecycleState::Active => "ready",
4886 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4887 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4888 crate::identity_first::IdentityLifecycleState::Broken => "broken",
4889 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4890 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4891 }
4892 .to_string();
4893 let display_name = status
4894 .display_name
4895 .as_ref()
4896 .map(crate::identity_first::DisplayName::as_str)
4897 .unwrap_or_else(|| status.identity.as_str())
4898 .to_string();
4899 ConsoleIdentityRecord {
4900 identity,
4901 display_name,
4902 runtime_key: entry.runtime_key.clone(),
4903 runtime_member_id,
4904 session_id: status.session_id.as_ref().map(ToString::to_string),
4905 visibility,
4906 addressable,
4907 health,
4908 topology_peers: Vec::new(),
4909 labels: status.labels.clone(),
4910 }
4911}
4912
4913pub(crate) fn is_implicit_delegate_member(
4914 role: &str,
4915 labels: &std::collections::BTreeMap<String, String>,
4916) -> bool {
4917 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4918}
4919
4920fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4921 member
4922 .labels
4923 .get("addressable")
4924 .map(|value| !value.eq_ignore_ascii_case("false"))
4925 .unwrap_or(true)
4926}
4927
4928fn sendable_resolved_member_indices(matches: &[ResolvedConsoleMember]) -> Vec<usize> {
4929 matches
4930 .iter()
4931 .enumerate()
4932 .filter_map(|(index, resolved)| {
4933 (member_is_addressable(&resolved.member)
4934 && resolved.member.status == meerkat_mob::MobMemberStatus::Active)
4935 .then_some(index)
4936 })
4937 .collect()
4938}
4939
4940async fn authoritative_durable_sendable_member_index(
4941 requested_identity: &str,
4942 matches: &[ResolvedConsoleMember],
4943 sendable_indices: &[usize],
4944) -> Result<Option<usize>, ConsoleSendError> {
4945 if matches.iter().any(|resolved| {
4946 requested_identity_is_runtime_member_alias(
4947 requested_identity,
4948 &resolved.entry.identity_namespace,
4949 )
4950 }) {
4951 return Ok(None);
4952 }
4953
4954 let mut visited_runtime_keys = BTreeSet::new();
4955 let mut bindings = BTreeSet::new();
4956 for resolved in matches {
4957 let entry = &resolved.entry;
4958 if !visited_runtime_keys.insert(entry.runtime_key.clone()) {
4959 continue;
4960 }
4961 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
4962 continue;
4963 };
4964 for raw_identity in
4965 namespace_match_candidates(requested_identity, &entry.identity_namespace)
4966 {
4967 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&raw_identity) else {
4968 continue;
4969 };
4970 let Ok(status) = identity_runtime.status(&identity).await else {
4971 continue;
4972 };
4973 if status.state != crate::identity_first::IdentityLifecycleState::Active
4974 || status.addressability != crate::identity_first::AgentAddressability::Addressable
4975 {
4976 continue;
4977 }
4978 let Some(runtime_id) = status.agent_runtime_id.as_ref() else {
4979 continue;
4980 };
4981 let record = identity_record_for_status(entry, &status);
4982 if !Box::pin(console_identity_record_visible(entry, &record)).await {
4983 continue;
4984 }
4985 let mut projected_member_ids = BTreeSet::new();
4986 for index in sendable_indices {
4987 let resolved = &matches[*index];
4988 if resolved.entry.runtime_key != entry.runtime_key {
4989 continue;
4990 }
4991 let Some(live_record) = identity_record_for_resolved_member(resolved).await else {
4992 continue;
4993 };
4994 if live_record_realizes_durable_binding(&record, &live_record)
4995 && stale_durable_session_mismatch(&record, std::slice::from_ref(&live_record))
4996 .is_none()
4997 {
4998 projected_member_ids.insert(live_record.runtime_member_id);
4999 }
5000 }
5001 if projected_member_ids.is_empty() {
5002 bindings.insert((entry.runtime_key.clone(), runtime_id.as_str().to_string()));
5003 } else {
5004 bindings.extend(
5005 projected_member_ids
5006 .into_iter()
5007 .map(|member_id| (entry.runtime_key.clone(), member_id)),
5008 );
5009 }
5010 }
5011 }
5012
5013 if bindings.len() > 1 {
5014 let candidates = bindings
5015 .iter()
5016 .map(|(runtime_key, runtime_id)| format!("{runtime_id}@{runtime_key}"))
5017 .collect::<Vec<_>>()
5018 .join(", ");
5019 return Err(ConsoleSendError::AmbiguousIdentity {
5020 identity: requested_identity.to_string(),
5021 candidates,
5022 });
5023 }
5024 let Some((runtime_key, runtime_id)) = bindings.into_iter().next() else {
5025 return Ok(None);
5026 };
5027 let bound_indices = sendable_indices
5028 .iter()
5029 .copied()
5030 .filter(|index| {
5031 matches[*index].entry.runtime_key == runtime_key
5032 && matches[*index].runtime_identity == runtime_id
5033 })
5034 .collect::<Vec<_>>();
5035 match bound_indices.as_slice() {
5036 [index] => Ok(Some(*index)),
5037 [] => Ok(None),
5038 _ => {
5039 let candidates = bound_indices
5040 .iter()
5041 .map(|index| {
5042 format!(
5043 "{}@{}",
5044 matches[*index].runtime_identity, matches[*index].source_mob_id
5045 )
5046 })
5047 .collect::<Vec<_>>()
5048 .join(", ");
5049 Err(ConsoleSendError::AmbiguousIdentity {
5050 identity: requested_identity.to_string(),
5051 candidates,
5052 })
5053 }
5054 }
5055}
5056
5057fn apply_namespace(identity: &str, namespace: &str) -> String {
5058 let namespace = namespace.trim().trim_matches('/');
5059 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
5060 identity.to_string()
5061 } else {
5062 format!("{namespace}/{identity}")
5063 }
5064}
5065
5066fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
5067 let namespace = namespace.trim().trim_matches('/');
5068 if namespace.is_empty() {
5069 return Some(identity.to_string());
5070 }
5071 identity
5072 .strip_prefix(namespace)
5073 .and_then(|rest| rest.strip_prefix('/'))
5074 .map(ToString::to_string)
5075}
5076
5077fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
5078 let namespace = namespace.trim().trim_matches('/');
5079 if namespace.is_empty() {
5080 return vec![identity.to_string()];
5081 }
5082 let mut candidates = Vec::new();
5083 if let Some(stripped) = strip_namespace(identity, namespace) {
5084 candidates.push(stripped);
5085 candidates.push(identity.to_string());
5086 }
5087 candidates.sort();
5088 candidates.dedup();
5089 candidates
5090}
5091
5092fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
5093 namespace_match_candidates(identity, namespace)
5094 .iter()
5095 .any(|candidate| crate::member_comms_id::is_reserved_generated_alias(candidate))
5096}
5097
5098fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
5099 entries.iter().any(|entry| {
5100 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
5101 })
5102}
5103
5104fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
5105 if request.identity.trim().is_empty() {
5106 return Err(ConsoleSendError::InvalidRequest(
5107 "identity must be non-empty".to_string(),
5108 ));
5109 }
5110 if request.origin.trim().is_empty() {
5111 return Err(ConsoleSendError::InvalidRequest(
5112 "origin must be non-empty".to_string(),
5113 ));
5114 }
5115 if request.idempotency_key.trim().is_empty() {
5116 return Err(ConsoleSendError::InvalidRequest(
5117 "idempotency_key must be non-empty".to_string(),
5118 ));
5119 }
5120 Ok(())
5121}
5122
5123fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
5124 let content: ContentInput = serde_json::from_value(value.clone())
5125 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
5126 match &content {
5127 ContentInput::Text(text) if text.trim().is_empty() => Err(
5128 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
5129 ),
5130 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
5131 "content blocks must be non-empty".to_string(),
5132 )),
5133 _ => Ok(content),
5134 }
5135}
5136
5137fn parse_handling_mode(
5138 value: Option<&str>,
5139) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
5140 match value.unwrap_or("queue") {
5141 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
5142 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
5143 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
5144 }
5145}
5146
5147fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
5148 ConsoleInteractionAccepted {
5149 interaction_id: frame
5150 .interaction_id
5151 .clone()
5152 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
5153 identity: frame.identity.clone(),
5154 conversation_id: frame.conversation_id.clone(),
5155 session_id: frame.session_id.clone(),
5156 input_frame_id: frame.id.clone(),
5157 cursor: frame.cursor.clone(),
5158 status: frame.status,
5159 }
5160}
5161
5162fn send_dedupe_key(
5163 runtime_key: &str,
5164 identity: &str,
5165 origin: &str,
5166 idempotency_key: &str,
5167) -> String {
5168 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
5169}
5170
5171fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
5172 let content_json = serde_json::to_string(content).unwrap_or_default();
5173 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
5174}
5175
5176const CONSOLE_INTERACTION_UUID_NAMESPACE: uuid::Uuid = uuid::Uuid::from_bytes([
5186 0x6d, 0x6f, 0x62, 0x6b, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2d, 0x69,
5187]);
5188
5189fn identity_first_interaction_uuid(dedupe_key: &str) -> String {
5190 uuid::Uuid::new_v5(&CONSOLE_INTERACTION_UUID_NAMESPACE, dedupe_key.as_bytes()).to_string()
5191}
5192
5193fn hash_short(value: &str) -> String {
5194 let mut hasher = Sha256::new();
5195 hasher.update(value.as_bytes());
5196 let digest = hasher.finalize();
5197 to_hex(&digest[..8])
5198}
5199
5200fn to_hex(bytes: &[u8]) -> String {
5201 const HEX: &[u8; 16] = b"0123456789abcdef";
5202 let mut out = String::with_capacity(bytes.len() * 2);
5203 for byte in bytes {
5204 out.push(HEX[(byte >> 4) as usize] as char);
5205 out.push(HEX[(byte & 0x0f) as usize] as char);
5206 }
5207 out
5208}
5209
5210fn current_time_ms() -> u64 {
5211 match SystemTime::now().duration_since(UNIX_EPOCH) {
5212 Ok(duration) => duration.as_millis() as u64,
5213 Err(_) => 0,
5214 }
5215}
5216
5217fn runtime_registry_lock_error() -> ConsoleLogError {
5218 Box::new(std::io::Error::other(
5219 "console runtime registry lock poisoned",
5220 ))
5221}
5222
5223#[cfg(test)]
5224#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
5225mod tests {
5226 #[test]
5231 fn backfill_gap_dedupe_key_is_stable_per_session() {
5232 use super::backfill_gap_dedupe_key;
5233 let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
5234 let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
5235 assert_eq!(a, b, "same target must reuse one gap frame");
5236 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
5237 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
5238 assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
5241 }
5242
5243 use std::sync::Arc;
5244 use std::sync::atomic::{AtomicUsize, Ordering};
5245 use std::time::Duration;
5246 use std::time::Instant;
5247
5248 use futures::StreamExt;
5249 use meerkat::{AgentFactory, Config, build_ephemeral_service};
5250 use meerkat_client::types::LlmStream;
5251 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
5252 use meerkat_core::{
5253 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
5254 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
5255 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
5256 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
5257 StreamError,
5258 };
5259 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
5260 use serde_json::json;
5261
5262 use super::*;
5263 use crate::mob_handle_runtime::MobBootstrapSpec;
5264
5265 #[derive(Debug)]
5266 struct HideHiddenNoiseFrames;
5267
5268 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
5269 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
5270 frame.kind != "hidden_noise"
5271 }
5272 }
5273
5274 #[derive(Debug)]
5275 struct HideRuntimeMemberIdentity(&'static str);
5276
5277 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
5278 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5279 record.runtime_member_id != self.0
5280 }
5281 }
5282
5283 #[derive(Debug)]
5284 struct HideRuntimeMemberOnly(&'static str);
5285
5286 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
5287 fn member_visible(&self, member: &ConsoleMember) -> bool {
5288 member.agent_identity != self.0
5289 }
5290 }
5291
5292 #[derive(Debug)]
5293 struct HideConsoleIdentity(&'static str);
5294
5295 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
5296 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5297 record.identity != self.0
5298 }
5299 }
5300
5301 struct CountingConsoleLogStore {
5302 inner: InMemoryConsoleLogStore,
5303 source_watermark_calls: AtomicUsize,
5304 record_watermark_calls: AtomicUsize,
5305 }
5306
5307 impl CountingConsoleLogStore {
5308 fn new() -> Self {
5309 Self {
5310 inner: InMemoryConsoleLogStore::new(),
5311 source_watermark_calls: AtomicUsize::new(0),
5312 record_watermark_calls: AtomicUsize::new(0),
5313 }
5314 }
5315
5316 fn source_watermark_calls(&self) -> usize {
5317 self.source_watermark_calls.load(Ordering::SeqCst)
5318 }
5319 }
5320
5321 struct SlowTestClient {
5322 delay: Duration,
5323 }
5324
5325 #[async_trait::async_trait]
5326 impl LlmClient for SlowTestClient {
5327 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
5328 Ok(messages.to_vec())
5329 }
5330
5331 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
5332 let delay = self.delay;
5333 let delayed_text = futures::stream::once(async move {
5334 tokio::time::sleep(delay).await;
5335 Ok(LlmEvent::TextDelta {
5336 delta: "slow ok".to_string(),
5337 meta: None,
5338 })
5339 });
5340 let done = futures::stream::once(async {
5341 Ok(LlmEvent::Done {
5342 outcome: LlmDoneOutcome::Success {
5343 stop_reason: StopReason::EndTurn,
5344 },
5345 })
5346 });
5347 Box::pin(delayed_text.chain(done))
5348 }
5349
5350 fn provider(&self) -> meerkat_core::Provider {
5351 meerkat_core::Provider::Other
5352 }
5353
5354 async fn health_check(&self) -> Result<(), LlmError> {
5355 Ok(())
5356 }
5357 }
5358
5359 #[derive(Clone)]
5360 struct DelayedHistorySessionService {
5361 inner: Arc<dyn MobSessionService>,
5362 delay: Duration,
5363 read_calls: Arc<AtomicUsize>,
5364 active_reads: Arc<AtomicUsize>,
5365 max_active_reads: Arc<AtomicUsize>,
5366 }
5367
5368 impl DelayedHistorySessionService {
5369 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
5370 Self {
5371 inner,
5372 delay,
5373 read_calls: Arc::new(AtomicUsize::new(0)),
5374 active_reads: Arc::new(AtomicUsize::new(0)),
5375 max_active_reads: Arc::new(AtomicUsize::new(0)),
5376 }
5377 }
5378
5379 fn read_calls(&self) -> usize {
5380 self.read_calls.load(Ordering::SeqCst)
5381 }
5382
5383 fn max_active_reads(&self) -> usize {
5384 self.max_active_reads.load(Ordering::SeqCst)
5385 }
5386 }
5387
5388 #[async_trait::async_trait]
5389 impl SessionService for DelayedHistorySessionService {
5390 async fn create_session(
5391 &self,
5392 req: meerkat_core::CreateSessionRequest,
5393 ) -> Result<RunResult, SessionError> {
5394 self.inner.create_session(req).await
5395 }
5396
5397 async fn start_turn(
5398 &self,
5399 id: &SessionId,
5400 req: StartTurnRequest,
5401 ) -> Result<RunResult, SessionError> {
5402 self.inner.start_turn(id, req).await
5403 }
5404
5405 async fn reconcile_runtime_compaction_projections(
5406 &self,
5407 id: &SessionId,
5408 intents: Vec<meerkat_core::CompactionProjectionIntent>,
5409 ) -> Result<(), SessionError> {
5410 self.inner
5411 .reconcile_runtime_compaction_projections(id, intents)
5412 .await
5413 }
5414
5415 async fn abort_uncommitted_compaction_projections(
5416 &self,
5417 id: &SessionId,
5418 ) -> Result<(), SessionError> {
5419 self.inner
5420 .abort_uncommitted_compaction_projections(id)
5421 .await
5422 }
5423
5424 async fn abort_rejected_runtime_run_projections(
5425 &self,
5426 id: &SessionId,
5427 ) -> Result<(), SessionError> {
5428 self.inner.abort_rejected_runtime_run_projections(id).await
5429 }
5430
5431 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
5432 self.inner.interrupt(id).await
5433 }
5434
5435 async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
5436 self.inner.cancel_after_boundary(id).await
5437 }
5438
5439 async fn cancel_after_boundary_for_run(
5440 &self,
5441 id: &SessionId,
5442 expected_run_id: &meerkat_core::RunId,
5443 ) -> Result<(), SessionError> {
5444 self.inner
5445 .cancel_after_boundary_for_run(id, expected_run_id)
5446 .await
5447 }
5448
5449 async fn set_session_client(
5450 &self,
5451 id: &SessionId,
5452 client: Arc<dyn meerkat_core::AgentLlmClient>,
5453 ) -> Result<(), SessionError> {
5454 self.inner.set_session_client(id, client).await
5455 }
5456
5457 async fn hot_swap_session_llm_identity(
5458 &self,
5459 id: &SessionId,
5460 client: Arc<dyn meerkat_core::AgentLlmClient>,
5461 identity: meerkat_core::SessionLlmIdentity,
5462 request_policy: meerkat_core::SessionLlmRequestPolicy,
5463 ) -> Result<(), SessionError> {
5464 self.inner
5465 .hot_swap_session_llm_identity(id, client, identity, request_policy)
5466 .await
5467 }
5468
5469 async fn set_session_tool_visibility_state(
5470 &self,
5471 id: &SessionId,
5472 state: Option<meerkat_core::SessionToolVisibilityState>,
5473 ) -> Result<(), SessionError> {
5474 self.inner
5475 .set_session_tool_visibility_state(id, state)
5476 .await
5477 }
5478
5479 async fn update_session_mob_authority_context(
5480 &self,
5481 id: &SessionId,
5482 authority_context: Option<meerkat_core::service::MobToolAuthorityContext>,
5483 ) -> Result<(), SessionError> {
5484 self.inner
5485 .update_session_mob_authority_context(id, authority_context)
5486 .await
5487 }
5488
5489 async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
5490 self.inner.has_live_session(id).await
5491 }
5492
5493 async fn set_session_tool_filter(
5494 &self,
5495 id: &SessionId,
5496 filter: meerkat_core::ToolFilter,
5497 ) -> Result<(), SessionError> {
5498 self.inner.set_session_tool_filter(id, filter).await
5499 }
5500
5501 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
5502 self.inner.read(id).await
5503 }
5504
5505 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
5506 self.inner.list(query).await
5507 }
5508
5509 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
5510 self.inner.archive(id).await
5511 }
5512
5513 async fn subscribe_session_events(
5514 &self,
5515 id: &SessionId,
5516 ) -> Result<EventStream, StreamError> {
5517 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
5518 }
5519
5520 async fn record_live_terminal_error(
5521 &self,
5522 id: &SessionId,
5523 cause: meerkat_core::live_adapter::LiveAdapterErrorCode,
5524 ) -> Result<(), SessionError> {
5525 self.inner.record_live_terminal_error(id, cause).await
5526 }
5527
5528 async fn record_live_output_audio_degraded(
5529 &self,
5530 id: &SessionId,
5531 dropped: u64,
5532 ) -> Result<(), SessionError> {
5533 self.inner
5534 .record_live_output_audio_degraded(id, dropped)
5535 .await
5536 }
5537 }
5538
5539 #[async_trait::async_trait]
5540 impl SessionServiceCommsExt for DelayedHistorySessionService {
5541 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
5542 self.inner.comms_runtime(session_id).await
5543 }
5544
5545 async fn event_injector(
5546 &self,
5547 session_id: &SessionId,
5548 ) -> Option<Arc<dyn meerkat_core::EventInjector>> {
5549 self.inner.event_injector(session_id).await
5550 }
5551
5552 async fn interaction_event_injector(
5553 &self,
5554 session_id: &SessionId,
5555 ) -> Option<Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
5556 self.inner.interaction_event_injector(session_id).await
5557 }
5558 }
5559
5560 #[async_trait::async_trait]
5561 impl SessionServiceControlExt for DelayedHistorySessionService {
5562 async fn append_system_context(
5563 &self,
5564 id: &SessionId,
5565 req: AppendSystemContextRequest,
5566 ) -> Result<AppendSystemContextResult, SessionControlError> {
5567 self.inner.append_system_context(id, req).await
5568 }
5569
5570 async fn stage_tool_results(
5571 &self,
5572 id: &SessionId,
5573 req: meerkat_core::service::StageToolResultsRequest,
5574 ) -> Result<meerkat_core::service::StageToolResultsResult, SessionError> {
5575 self.inner.stage_tool_results(id, req).await
5576 }
5577 }
5578
5579 #[async_trait::async_trait]
5580 impl SessionServiceHistoryExt for DelayedHistorySessionService {
5581 async fn read_history(
5582 &self,
5583 id: &SessionId,
5584 query: SessionHistoryQuery,
5585 ) -> Result<SessionHistoryPage, SessionError> {
5586 self.read_calls.fetch_add(1, Ordering::SeqCst);
5587 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
5588 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
5589 tokio::time::sleep(self.delay).await;
5590 let result = self.inner.read_history(id, query).await;
5591 self.active_reads.fetch_sub(1, Ordering::SeqCst);
5592 result
5593 }
5594
5595 async fn read_transcript_revision(
5596 &self,
5597 id: &SessionId,
5598 query: meerkat_core::service::SessionTranscriptRevisionQuery,
5599 ) -> Result<meerkat_core::service::SessionTranscriptRevisionPage, SessionError> {
5600 self.inner.read_transcript_revision(id, query).await
5601 }
5602
5603 async fn list_transcript_revisions(
5604 &self,
5605 id: &SessionId,
5606 query: meerkat_core::service::SessionTranscriptRevisionListQuery,
5607 ) -> Result<meerkat_core::service::SessionTranscriptRevisionList, SessionError> {
5608 self.inner.list_transcript_revisions(id, query).await
5609 }
5610 }
5611
5612 #[async_trait::async_trait]
5613 impl MobSessionService for DelayedHistorySessionService {
5614 async fn prepare_session_for_resume(
5615 &self,
5616 session_id: &meerkat_core::types::SessionId,
5617 ) -> Result<(), meerkat_core::service::SessionError> {
5618 self.inner.prepare_session_for_resume(session_id).await
5619 }
5620 async fn materialize_session_for_resume(
5621 &self,
5622 session_id: &meerkat_core::types::SessionId,
5623 ) -> Result<meerkat_mob::ResumeSessionLoad, meerkat_core::service::SessionError> {
5624 self.inner.materialize_session_for_resume(session_id).await
5625 }
5626
5627 async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
5628 &self,
5629 session_id: &meerkat_core::types::SessionId,
5630 authority: &meerkat_core::CommittedSessionBoundaryAuthority,
5631 ) -> Result<(), meerkat_core::service::SessionError> {
5632 self.inner
5633 .acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
5634 session_id, authority,
5635 )
5636 .await
5637 }
5638 async fn load_session_for_resume(
5639 &self,
5640 session_id: &meerkat_core::types::SessionId,
5641 ) -> Result<meerkat_mob::ResumeSessionLoad, SessionError> {
5642 self.inner.load_session_for_resume(session_id).await
5643 }
5644
5645 async fn create_session_under_runtime_turn_boundary(
5646 &self,
5647 req: meerkat_core::service::CreateSessionRequest,
5648 ) -> Result<meerkat_core::RunResult, SessionError> {
5649 self.inner
5650 .create_session_under_runtime_turn_boundary(req)
5651 .await
5652 }
5653
5654 async fn create_session_with_actor_witness_under_runtime_turn_boundary(
5655 &self,
5656 req: meerkat_core::service::CreateSessionRequest,
5657 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
5658 ) -> Result<meerkat_core::RunResult, SessionError> {
5659 self.inner
5660 .create_session_with_actor_witness_under_runtime_turn_boundary(
5661 req,
5662 actor_witness_slot,
5663 )
5664 .await
5665 }
5666
5667 async fn create_session_with_machine_archived_resume_authority(
5668 &self,
5669 req: meerkat_core::service::CreateSessionRequest,
5670 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5671 ) -> Result<meerkat_core::RunResult, SessionError> {
5672 self.inner
5673 .create_session_with_machine_archived_resume_authority(req, authorization)
5674 .await
5675 }
5676
5677 async fn create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
5678 &self,
5679 req: meerkat_core::service::CreateSessionRequest,
5680 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5681 ) -> Result<meerkat_core::RunResult, SessionError> {
5682 self.inner
5683 .create_session_with_machine_archived_resume_authority_under_runtime_turn_boundary(
5684 req,
5685 authorization,
5686 )
5687 .await
5688 }
5689
5690 async fn create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
5691 &self,
5692 req: meerkat_core::service::CreateSessionRequest,
5693 authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
5694 actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
5695 ) -> Result<meerkat_core::RunResult, SessionError> {
5696 self.inner
5697 .create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
5698 req,
5699 authorization,
5700 actor_witness_slot,
5701 )
5702 .await
5703 }
5704
5705 async fn authorize_revivable_retired_session(
5706 &self,
5707 session_id: &SessionId,
5708 authority: meerkat_runtime::PreparedArchivedResumeCommitLease,
5709 ) -> Result<meerkat_runtime::AuthorizedArchivedResumeCommitLease, SessionError> {
5710 self.inner
5711 .authorize_revivable_retired_session(session_id, authority)
5712 .await
5713 }
5714
5715 async fn subscribe_session_events(
5716 &self,
5717 session_id: &SessionId,
5718 ) -> Result<EventStream, StreamError> {
5719 meerkat_mob::MobSessionService::subscribe_session_events(
5720 self.inner.as_ref(),
5721 session_id,
5722 )
5723 .await
5724 }
5725
5726 fn supports_persistent_sessions(&self) -> bool {
5727 self.inner.supports_persistent_sessions()
5728 }
5729
5730 async fn live_session_actor_registered(
5731 &self,
5732 session_id: &SessionId,
5733 ) -> Result<bool, SessionError> {
5734 self.inner.live_session_actor_registered(session_id).await
5735 }
5736
5737 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
5738 self.inner.runtime_adapter()
5739 }
5740
5741 fn supports_runtime_turn_apply(&self) -> bool {
5742 self.inner.supports_runtime_turn_apply()
5743 }
5744
5745 async fn interrupt_with_machine_authority(
5746 &self,
5747 session_id: &SessionId,
5748 authority: meerkat_runtime::MachineSessionControlAuthority,
5749 ) -> Result<(), SessionError> {
5750 self.inner
5751 .interrupt_with_machine_authority(session_id, authority)
5752 .await
5753 }
5754
5755 async fn cancel_after_boundary_with_machine_authority(
5756 &self,
5757 session_id: &SessionId,
5758 expected_run_id: &meerkat_core::RunId,
5759 authority: meerkat_runtime::MachineSessionControlAuthority,
5760 ) -> Result<(), SessionError> {
5761 self.inner
5762 .cancel_after_boundary_with_machine_authority(
5763 session_id,
5764 expected_run_id,
5765 authority,
5766 )
5767 .await
5768 }
5769
5770 async fn cancel_current_after_boundary_with_machine_authority(
5771 &self,
5772 session_id: &SessionId,
5773 authority: meerkat_runtime::MachineSessionControlAuthority,
5774 ) -> Result<(), SessionError> {
5775 self.inner
5776 .cancel_current_after_boundary_with_machine_authority(session_id, authority)
5777 .await
5778 }
5779
5780 async fn execution_snapshot(
5781 &self,
5782 session_id: &SessionId,
5783 ) -> Result<Option<meerkat_core::AgentExecutionSnapshot>, SessionError> {
5784 self.inner.execution_snapshot(session_id).await
5785 }
5786
5787 async fn tool_scope_snapshot(
5788 &self,
5789 session_id: &SessionId,
5790 ) -> Result<Option<meerkat_core::ToolScopeSnapshot>, SessionError> {
5791 self.inner.tool_scope_snapshot(session_id).await
5792 }
5793
5794 async fn external_tool_surface_snapshot(
5795 &self,
5796 session_id: &SessionId,
5797 ) -> Result<Option<meerkat_core::ExternalToolSurfaceSnapshot>, SessionError> {
5798 self.inner.external_tool_surface_snapshot(session_id).await
5799 }
5800
5801 async fn peer_ingress_runtime_snapshot(
5802 &self,
5803 session_id: &SessionId,
5804 ) -> Result<Option<meerkat_core::PeerIngressRuntimeSnapshot>, SessionError> {
5805 self.inner.peer_ingress_runtime_snapshot(session_id).await
5806 }
5807
5808 async fn session_known_to_archive_authority(
5809 &self,
5810 session_id: &SessionId,
5811 ) -> Result<bool, SessionError> {
5812 self.inner
5813 .session_known_to_archive_authority(session_id)
5814 .await
5815 }
5816
5817 async fn session_belongs_to_mob(
5818 &self,
5819 session_id: &SessionId,
5820 mob_id: &meerkat_mob::MobId,
5821 ) -> bool {
5822 self.inner.session_belongs_to_mob(session_id, mob_id).await
5823 }
5824
5825 async fn load_persisted_session(
5826 &self,
5827 session_id: &SessionId,
5828 ) -> Result<Option<meerkat_core::Session>, SessionError> {
5829 self.inner.load_persisted_session(session_id).await
5830 }
5831
5832 async fn load_revivable_retired_session(
5833 &self,
5834 session_id: &SessionId,
5835 ) -> Result<Option<meerkat_core::Session>, SessionError> {
5836 self.inner.load_revivable_retired_session(session_id).await
5837 }
5838
5839 async fn load_persisted_session_metadata(
5840 &self,
5841 session_id: &SessionId,
5842 ) -> Result<Option<meerkat_core::PersistedSessionMetadataView>, SessionError> {
5843 self.inner.load_persisted_session_metadata(session_id).await
5844 }
5845
5846 async fn archive_with_mob_lifecycle_authority(
5847 &self,
5848 session_id: &SessionId,
5849 ) -> Result<(), SessionError> {
5850 self.inner
5851 .archive_with_mob_lifecycle_authority(session_id)
5852 .await
5853 }
5854
5855 async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
5856 &self,
5857 session_id: &SessionId,
5858 ) -> Result<(), SessionError> {
5859 self.inner
5860 .archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(session_id)
5861 .await
5862 }
5863
5864 async fn apply_runtime_turn(
5865 &self,
5866 session_id: &SessionId,
5867 run_id: meerkat_core::RunId,
5868 req: StartTurnRequest,
5869 boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
5870 contributing_input_ids: Vec<meerkat_core::InputId>,
5871 ) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
5872 self.inner
5873 .apply_runtime_turn(session_id, run_id, req, boundary, contributing_input_ids)
5874 .await
5875 }
5876
5877 async fn prepare_transient_turn_context_for_active_turn(
5878 &self,
5879 session_id: &SessionId,
5880 expected_run_id: &meerkat_core::RunId,
5881 contexts: Vec<meerkat_core::lifecycle::run_primitive::TurnRequestContext>,
5882 ) -> Result<meerkat_core::CoreBoundaryStageOutput, meerkat_core::CoreBoundaryStageError>
5883 {
5884 self.inner
5885 .prepare_transient_turn_context_for_active_turn(
5886 session_id,
5887 expected_run_id,
5888 contexts,
5889 )
5890 .await
5891 }
5892
5893 async fn checkpoint_committed_runtime_session_snapshot(
5894 &self,
5895 session_id: &SessionId,
5896 session_snapshot: std::sync::Arc<Vec<u8>>,
5897 ) -> Result<(), SessionError> {
5898 self.inner
5899 .checkpoint_committed_runtime_session_snapshot(session_id, session_snapshot)
5900 .await
5901 }
5902
5903 async fn acquire_runtime_turn_finalization_guard(
5904 &self,
5905 session_id: &SessionId,
5906 ) -> Result<Box<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationGuard>, SessionError>
5907 {
5908 self.inner
5909 .acquire_runtime_turn_finalization_guard(session_id)
5910 .await
5911 }
5912
5913 async fn checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
5914 &self,
5915 session_id: &SessionId,
5916 session_snapshot: std::sync::Arc<Vec<u8>>,
5917 ) -> Result<(), SessionError> {
5918 self.inner
5919 .checkpoint_committed_runtime_session_snapshot_under_turn_finalization_boundary(
5920 session_id,
5921 session_snapshot,
5922 )
5923 .await
5924 }
5925
5926 async fn discard_live_session_after_runtime_stop_terminalized(
5927 &self,
5928 session_id: &SessionId,
5929 ) -> Result<(), SessionError> {
5930 self.inner
5931 .discard_live_session_after_runtime_stop_terminalized(session_id)
5932 .await
5933 }
5934
5935 async fn discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
5936 &self,
5937 session_id: &SessionId,
5938 ) -> Result<(), SessionError> {
5939 self.inner
5940 .discard_live_session_after_runtime_stop_terminalized_under_turn_finalization_boundary(
5941 session_id,
5942 )
5943 .await
5944 }
5945
5946 async fn publish_interaction_terminals(
5947 &self,
5948 session_id: &SessionId,
5949 events: &[meerkat_core::event::AgentEvent],
5950 ) -> Result<
5951 Vec<meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt>,
5952 SessionError,
5953 > {
5954 self.inner
5955 .publish_interaction_terminals(session_id, events)
5956 .await
5957 }
5958
5959 async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
5960 self.inner.discard_live_session(session_id).await
5961 }
5962
5963 async fn discard_live_session_under_runtime_turn_boundary(
5964 &self,
5965 session_id: &SessionId,
5966 ) -> Result<(), SessionError> {
5967 self.inner
5968 .discard_live_session_under_runtime_turn_boundary(session_id)
5969 .await
5970 }
5971
5972 async fn discard_live_session_actor_under_runtime_turn_boundary(
5973 &self,
5974 witness: &meerkat_session::LiveSessionActorWitness,
5975 ) -> Result<bool, SessionError> {
5976 self.inner
5977 .discard_live_session_actor_under_runtime_turn_boundary(witness)
5978 .await
5979 }
5980
5981 async fn await_event_projection_drain(
5982 &self,
5983 session_id: &SessionId,
5984 ) -> Result<bool, SessionError> {
5985 self.inner.await_event_projection_drain(session_id).await
5986 }
5987
5988 async fn cancel_all_checkpointers(&self) {
5989 self.inner.cancel_all_checkpointers().await;
5990 }
5991
5992 async fn rearm_all_checkpointers(&self) {
5993 self.inner.rearm_all_checkpointers().await;
5994 }
5995 }
5996
5997 #[async_trait::async_trait]
5998 impl ConsoleLogStore for CountingConsoleLogStore {
5999 async fn append_if_absent(
6000 &self,
6001 frame: NewConsoleFrame,
6002 ) -> ConsoleLogResult<AppendOutcome> {
6003 self.inner.append_if_absent(frame).await
6004 }
6005
6006 async fn update_frame_status(
6007 &self,
6008 frame_id: &str,
6009 status: ConsoleFrameStatus,
6010 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
6011 self.inner.update_frame_status(frame_id, status).await
6012 }
6013
6014 async fn query_frames(
6015 &self,
6016 query: ConsoleTimelineQuery,
6017 ) -> ConsoleLogResult<ConsoleTimelinePage> {
6018 self.inner.query_frames(query).await
6019 }
6020
6021 async fn query_windowed_frames(
6022 &self,
6023 query: ConsoleTimelineWindowQuery,
6024 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
6025 self.inner.query_windowed_frames(query).await
6026 }
6027
6028 async fn frame_by_dedupe_key(
6029 &self,
6030 dedupe_key: &str,
6031 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
6032 self.inner.frame_by_dedupe_key(dedupe_key).await
6033 }
6034
6035 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
6036 self.inner.latest_cursor().await
6037 }
6038
6039 async fn clear_frames(&self) -> ConsoleLogResult<()> {
6040 self.inner.clear_frames().await
6041 }
6042
6043 async fn record_source_watermark(
6044 &self,
6045 runtime_key: &str,
6046 source_kind: ConsoleFrameSourceKind,
6047 source_cursor: &str,
6048 ) -> ConsoleLogResult<()> {
6049 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
6050 self.inner
6051 .record_source_watermark(runtime_key, source_kind, source_cursor)
6052 .await
6053 }
6054
6055 async fn source_watermark(
6056 &self,
6057 runtime_key: &str,
6058 source_kind: ConsoleFrameSourceKind,
6059 ) -> ConsoleLogResult<Option<String>> {
6060 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
6061 self.inner.source_watermark(runtime_key, source_kind).await
6062 }
6063 }
6064
6065 async fn build_single_member_runtime() -> UnifiedRuntime {
6066 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
6067 }
6068
6069 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
6070 let definition = MobDefinition::from_toml(
6071 r#"
6072[mob]
6073id = "console-aggregator-perf-test"
6074
6075[profiles.worker]
6076model = "gpt-5.5"
6077external_addressable = true
6078
6079[profiles.worker.tools]
6080comms = true
6081"#,
6082 )
6083 .expect("definition parses");
6084 let runtime = UnifiedRuntime::builder()
6085 .definition(definition)
6086 .default_llm_client(client)
6087 .build()
6088 .await
6089 .expect("runtime builds");
6090 runtime
6091 .spawn(SpawnMemberSpec::from_wire(
6092 "worker".to_string(),
6093 "agent-a".to_string(),
6094 Some("You are agent-a.".into()),
6095 None,
6096 None,
6097 ))
6098 .await
6099 .expect("member spawns");
6100 runtime
6101 }
6102
6103 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
6104 let definition = MobDefinition::from_toml(&format!(
6105 r#"
6106[mob]
6107id = "{mob_id}"
6108
6109[profiles.worker]
6110model = "gpt-5.5"
6111external_addressable = true
6112
6113[profiles.worker.tools]
6114comms = true
6115"#
6116 ))
6117 .expect("definition parses");
6118 UnifiedRuntime::builder()
6119 .definition(definition)
6120 .default_llm_client(Arc::new(TestClient::default()))
6121 .build()
6122 .await
6123 .expect("runtime builds")
6124 }
6125
6126 async fn spawn_trusted_identity_projected_member(
6131 runtime: &UnifiedRuntime,
6132 mut spec: SpawnMemberSpec,
6133 ) {
6134 let alias = spec.identity.to_string();
6135 spec.identity = crate::member_comms_id::mob_member_id(&alias);
6136 runtime
6137 .mob_handle()
6138 .spawn_spec(spec)
6139 .await
6140 .expect("trusted identity-projected member spawns");
6141 }
6142
6143 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6144 async fn send_selection_ignores_a_retiring_generation_when_one_active_generation_remains() {
6145 let runtime = build_empty_runtime("console-send-retiring-generation-test").await;
6146 let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
6147 for generation in 0..=1 {
6148 spawn_trusted_identity_projected_member(
6149 &runtime,
6150 SpawnMemberSpec::from_wire(
6151 "worker".to_string(),
6152 format!("rt:builder:{generation}"),
6153 Some("You are Builder.".into()),
6154 None,
6155 None,
6156 )
6157 .with_labels(labels.clone()),
6158 )
6159 .await;
6160 }
6161
6162 let aggregator = MobKitConsoleAggregator::in_memory();
6163 aggregator.register_runtime_handles_with_policy(
6164 "runtime-a",
6165 "",
6166 runtime.mob_runtime().clone(),
6167 None,
6168 runtime.console_events(),
6169 Arc::new(AllowAllConsoleVisibilityPolicy),
6170 );
6171 let mut matches = aggregator.resolve_visible_members("builder").await;
6172 assert_eq!(matches.len(), 2);
6173 assert_eq!(sendable_resolved_member_indices(&matches).len(), 2);
6174
6175 matches[0].member.status = meerkat_mob::MobMemberStatus::Retiring;
6176 let sendable = sendable_resolved_member_indices(&matches);
6177 assert_eq!(sendable.len(), 1);
6178 assert_ne!(
6179 matches[sendable[0]].member.status,
6180 meerkat_mob::MobMemberStatus::Retiring
6181 );
6182
6183 let _ = runtime.mob_handle().stop().await;
6184 }
6185
6186 async fn build_stress_runtime(
6187 member_count: usize,
6188 history_delay: Duration,
6189 ) -> (
6190 tempfile::TempDir,
6191 Arc<UnifiedRuntime>,
6192 DelayedHistorySessionService,
6193 ) {
6194 let temp_dir = tempfile::tempdir().expect("temp dir");
6195 let session_path = temp_dir.path().join("sessions");
6196 std::fs::create_dir_all(&session_path).expect("session path");
6197 let factory = AgentFactory::new(&session_path).comms(true);
6198 let base_service = Arc::new(build_ephemeral_service(
6199 factory,
6200 Config::default(),
6201 member_count + 8,
6202 ));
6203 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
6204 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
6205 let definition = MobDefinition::from_toml(
6206 r#"
6207[mob]
6208id = "console-aggregator-stress-test"
6209
6210[profiles.worker]
6211model = "gpt-5.5"
6212external_addressable = true
6213
6214[profiles.worker.tools]
6215comms = true
6216"#,
6217 )
6218 .expect("definition parses");
6219 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
6220 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
6221 allow_ephemeral_sessions: true,
6222 notify_orchestrator_on_resume: true,
6223 default_llm_client: Some(Arc::new(TestClient::default())),
6224 });
6225 let runtime = Arc::new(
6226 UnifiedRuntime::bootstrap(
6227 spec,
6228 crate::types::MobKitConfig {
6229 modules: Vec::new(),
6230 discovery: crate::types::DiscoverySpec {
6231 namespace: "stress".to_string(),
6232 modules: Vec::new(),
6233 },
6234 pre_spawn: Vec::new(),
6235 },
6236 Duration::from_secs(5),
6237 )
6238 .await
6239 .expect("runtime boots"),
6240 );
6241 for idx in 0..member_count {
6242 runtime
6243 .spawn(SpawnMemberSpec::from_wire(
6244 "worker".to_string(),
6245 format!("agent-{idx}"),
6246 Some(format!("You are agent-{idx}.").into()),
6247 None,
6248 None,
6249 ))
6250 .await
6251 .expect("member spawns");
6252 }
6253 (temp_dir, runtime, delayed_service)
6254 }
6255
6256 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
6257 RuntimeEntry {
6258 runtime_key: runtime_key.to_string(),
6259 identity_namespace: "test".to_string(),
6260 runtime: runtime.mob_runtime().clone(),
6261 identity_runtime: runtime.identity_runtime().cloned(),
6262 console_events: runtime.console_events(),
6263 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6264 }
6265 }
6266
6267 fn console_envelope_for_test(
6268 event_id: &str,
6269 interaction_id: Option<&str>,
6270 event_type: &str,
6271 data: Value,
6272 ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
6273 crate::console_contracts::ConsoleIdentityEventEnvelope {
6274 event_id: event_id.to_string(),
6275 interaction_id: interaction_id.map(ToString::to_string),
6276 identity: "agent-a".to_string(),
6277 event_type: event_type.to_string(),
6278 timestamp_ms: 1_000,
6279 data,
6280 }
6281 }
6282
6283 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
6284 ConsoleIdentityRecord {
6285 identity: identity.to_string(),
6286 display_name: identity.to_string(),
6287 runtime_key: "runtime-cache".to_string(),
6288 runtime_member_id: identity.to_string(),
6289 session_id: Some(format!("session-{identity}")),
6290 visibility: ConsoleVisibility::Addressable,
6291 addressable: true,
6292 health: "ready".to_string(),
6293 topology_peers: Vec::new(),
6294 labels: BTreeMap::new(),
6295 }
6296 }
6297
6298 #[test]
6299 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
6300 let mut durable = identity_record_for_test("test/review:singleton");
6301 durable.runtime_member_id = "rt:review:singleton:1".to_string();
6302 durable.session_id = Some("session-stale".to_string());
6303 durable
6304 .labels
6305 .insert("agent_identity".to_string(), "review:singleton".to_string());
6306
6307 let mut live = identity_record_for_test("test/review:singleton");
6308 live.runtime_member_id = "rt:review:singleton:0".to_string();
6309 live.session_id = Some("session-live".to_string());
6310 live.labels
6311 .insert("agent_identity".to_string(), "review:singleton".to_string());
6312
6313 let records = dedupe_identity_records(vec![durable, live]);
6314
6315 assert_eq!(
6316 records.len(),
6317 2,
6318 "list projection must expose stale durable/live alias split-brain instead of picking one record"
6319 );
6320 assert!(
6321 records
6322 .iter()
6323 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
6324 );
6325 assert!(
6326 records
6327 .iter()
6328 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
6329 );
6330 }
6331
6332 #[test]
6333 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
6334 let mut first = identity_record_for_test("test/review:singleton");
6335 first.runtime_member_id = "rt:review:singleton:0".to_string();
6336 first
6337 .labels
6338 .insert("agent_identity".to_string(), "review:singleton".to_string());
6339
6340 let mut second = identity_record_for_test("test/review:singleton");
6341 second.runtime_member_id = "rt:review:singleton:1".to_string();
6342 second
6343 .labels
6344 .insert("agent_identity".to_string(), "review:singleton".to_string());
6345
6346 let records = dedupe_identity_records(vec![first, second]);
6347
6348 assert_eq!(
6349 records.len(),
6350 2,
6351 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
6352 );
6353 }
6354
6355 #[test]
6356 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
6357 let mut first = identity_record_for_test("test/review:singleton");
6358 first.runtime_member_id = "rt:review:singleton:0".to_string();
6359 first.session_id = None;
6360 first
6361 .labels
6362 .insert("agent_identity".to_string(), "review:singleton".to_string());
6363 first
6364 .labels
6365 .insert("source_mob_id".to_string(), "mob-a".to_string());
6366
6367 let mut second = first.clone();
6368 second
6369 .labels
6370 .insert("source_mob_id".to_string(), "mob-b".to_string());
6371
6372 let records = dedupe_identity_records(vec![first, second]);
6373
6374 assert_eq!(
6375 records.len(),
6376 2,
6377 "same member/session aliases from different source mobs must remain visible"
6378 );
6379 }
6380
6381 #[test]
6382 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
6383 let durable = identity_record_for_test("test/review:singleton");
6384 let live = durable.clone();
6385
6386 let records = dedupe_identity_records(vec![durable, live]);
6387
6388 assert_eq!(
6389 records.len(),
6390 1,
6391 "healthy durable and primary live projections should not duplicate the roster"
6392 );
6393 }
6394
6395 #[test]
6396 fn stale_durable_record_error_detects_session_split_brain() {
6397 let mut durable = identity_record_for_test("test/review:singleton");
6398 durable.runtime_member_id = "rt:review:singleton:0".to_string();
6399 durable.session_id = Some("session-durable".to_string());
6400
6401 let mut live = durable.clone();
6402 live.session_id = Some("session-live".to_string());
6403
6404 let err = stale_durable_record_error("review:singleton", &durable, &[live])
6405 .expect("session mismatch must be stale split-brain");
6406
6407 assert!(
6408 err.contains("stale durable identity alias"),
6409 "unexpected error: {err}"
6410 );
6411 assert!(
6412 err.contains("session-durable") && err.contains("session-live"),
6413 "error should name both sessions: {err}"
6414 );
6415 }
6416
6417 #[test]
6418 fn stable_external_member_alias_realizes_generated_durable_binding() {
6419 let mut durable = identity_record_for_test("target-smoke");
6420 durable.runtime_member_id = "rt:target-smoke:0".to_string();
6421
6422 let mut live = durable.clone();
6423 live.runtime_member_id = "target-smoke".to_string();
6424 live.labels
6425 .insert("agent_identity".to_string(), "target-smoke".to_string());
6426
6427 assert!(live_record_realizes_durable_binding(&durable, &live));
6428 assert_eq!(
6429 stale_durable_record_error("target-smoke", &durable, &[live]),
6430 None,
6431 "the bridge-owned stable alias must not be classified as a stale generated binding"
6432 );
6433 }
6434
6435 #[test]
6436 fn stable_external_member_alias_still_fences_session_split_brain() {
6437 let mut durable = identity_record_for_test("target-smoke");
6438 durable.runtime_member_id = "rt:target-smoke:0".to_string();
6439 durable.session_id = Some("session-durable".to_string());
6440
6441 let mut live = durable.clone();
6442 live.runtime_member_id = "target-smoke".to_string();
6443 live.session_id = Some("session-live".to_string());
6444 live.labels
6445 .insert("agent_identity".to_string(), "target-smoke".to_string());
6446
6447 let error = stale_durable_record_error("target-smoke", &durable, &[live])
6448 .expect("stable aliases must not hide a session split-brain");
6449 assert!(error.contains("session-durable"));
6450 assert!(error.contains("session-live"));
6451 }
6452
6453 async fn identity_runtime_for_test(
6454 identities: &[&str],
6455 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
6456 {
6457 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6458 crate::identity_first::IdentityRuntimeConfig {
6459 continuity_store: Arc::new(
6460 crate::identity_first::LocalContinuityStore::in_memory()?
6461 ),
6462 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6463 runtime_instance_id: "console-aggregator-identity-test".to_string(),
6464 has_runtime_store: true,
6465 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6466 bridge: None,
6467 default_timeout: None,
6468 },
6469 ));
6470 for identity in identities {
6471 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
6472 let record = crate::identity_first::ContinuityRecord {
6473 identity: identity.clone(),
6474 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
6475 "rt:{}:0",
6476 identity.as_str()
6477 ))?,
6478 session_id: SessionId::new(),
6479 generation: crate::identity_first::ContinuityGeneration::new(0),
6480 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6481 };
6482 runtime
6483 .register(
6484 crate::identity_first::DurableAgentSpec {
6485 identity: identity.clone(),
6486 profile: meerkat_mob::ProfileName::from("worker"),
6487 addressability: crate::identity_first::AgentAddressability::Addressable,
6488 display_name: None,
6489 labels: BTreeMap::new(),
6490 context: None,
6491 additional_instructions: Vec::new(),
6492 initial_message: None,
6493 runtime_mode_override: None,
6494 backend: None,
6495 binding: None,
6496 },
6497 crate::identity_first::IdentityLifecycleState::Active,
6498 Some(record),
6499 Some(crate::identity_first::LeaseGrant {
6500 identity,
6501 fencing_token: crate::identity_first::FencingToken::new(1),
6502 ttl: Duration::from_mins(5),
6503 }),
6504 )
6505 .await;
6506 }
6507 Ok(runtime)
6508 }
6509
6510 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6511 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
6512 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6513 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
6514 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
6515 let aggregator = MobKitConsoleAggregator::in_memory();
6516 aggregator.register_runtime_handles_with_policy(
6517 "runtime-a",
6518 "",
6519 runtime.mob_runtime().clone(),
6520 Some(identity_runtime),
6521 runtime.console_events(),
6522 Arc::new(AllowAllConsoleVisibilityPolicy),
6523 );
6524
6525 assert!(
6526 aggregator
6527 .inspect_identity("rt:review:singleton:0")
6528 .await?
6529 .is_none(),
6530 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
6531 );
6532 assert!(
6533 !aggregator.retire_identity("rt:review:singleton:0").await?,
6534 "runtime-member-id shaped requests must not retire a durable rt:* identity"
6535 );
6536 let reserve_err = aggregator
6537 .reserve_identity_first_interaction(
6538 ConsoleSendRequest {
6539 identity: "rt:review:singleton:0".to_string(),
6540 content: json!("hello"),
6541 origin: "console:test".to_string(),
6542 idempotency_key: "rt-shadow-reserve".to_string(),
6543 handling_mode: Some("queue".to_string()),
6544 },
6545 None,
6546 )
6547 .await
6548 .expect_err(
6549 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
6550 );
6551 assert!(
6552 reserve_err.to_string().contains("unknown identity"),
6553 "unexpected reserve error: {reserve_err}"
6554 );
6555 let send_err = aggregator
6556 .send(ConsoleSendRequest {
6557 identity: "rt:review:singleton:0".to_string(),
6558 content: json!("hello"),
6559 origin: "console:test".to_string(),
6560 idempotency_key: "rt-shadow-send".to_string(),
6561 handling_mode: Some("queue".to_string()),
6562 })
6563 .await
6564 .expect_err(
6565 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
6566 );
6567 assert!(
6568 send_err.to_string().contains("unknown identity"),
6569 "unexpected send error: {send_err}"
6570 );
6571 let blob_err = match aggregator
6572 .binary_blob_store_for_identity("rt:review:singleton:0")
6573 .await
6574 {
6575 Ok(_) => panic!(
6576 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
6577 ),
6578 Err(err) => err,
6579 };
6580 assert!(
6581 blob_err.to_string().contains("unknown identity"),
6582 "unexpected blob error: {blob_err}"
6583 );
6584
6585 let _ = runtime.mob_handle().stop().await;
6586 Ok(())
6587 }
6588
6589 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6590 async fn namespaced_current_runtime_alias_retires_through_durable_authority()
6591 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6592 let runtime = build_empty_runtime("namespaced-runtime-alias-retire-test").await;
6593 let runtime_alias = "rt:review:owned:0";
6594 spawn_trusted_identity_projected_member(
6595 &runtime,
6596 SpawnMemberSpec::from_wire(
6597 "worker".to_string(),
6598 runtime_alias.to_string(),
6599 Some("You are the owned review agent.".into()),
6600 None,
6601 None,
6602 )
6603 .with_labels(BTreeMap::from([(
6604 "agent_identity".to_string(),
6605 "review:owned".to_string(),
6606 )])),
6607 )
6608 .await;
6609 let live_session_id = runtime
6610 .mob_handle()
6611 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
6612 runtime_alias,
6613 ))
6614 .await
6615 .ok_or("live member has no bridge session")?;
6616
6617 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6618 crate::identity_first::IdentityRuntimeConfig {
6619 continuity_store: Arc::new(
6620 crate::identity_first::LocalContinuityStore::in_memory()?
6621 ),
6622 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6623 runtime_instance_id: "namespaced-runtime-alias-retire-test".to_string(),
6624 has_runtime_store: true,
6625 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6626 bridge: None,
6627 default_timeout: None,
6628 },
6629 ));
6630 let durable_identity = crate::identity_first::AgentIdentity::parse("review:owned")?;
6631 identity_runtime
6632 .register(
6633 crate::identity_first::DurableAgentSpec {
6634 identity: durable_identity.clone(),
6635 profile: meerkat_mob::ProfileName::from("worker"),
6636 addressability: crate::identity_first::AgentAddressability::Addressable,
6637 display_name: None,
6638 labels: BTreeMap::new(),
6639 context: None,
6640 additional_instructions: Vec::new(),
6641 initial_message: None,
6642 runtime_mode_override: None,
6643 backend: None,
6644 binding: None,
6645 },
6646 crate::identity_first::IdentityLifecycleState::Active,
6647 Some(crate::identity_first::ContinuityRecord {
6648 identity: durable_identity.clone(),
6649 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(runtime_alias)?,
6650 session_id: live_session_id,
6651 generation: crate::identity_first::ContinuityGeneration::new(0),
6652 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6653 }),
6654 Some(crate::identity_first::LeaseGrant {
6655 identity: durable_identity.clone(),
6656 fencing_token: crate::identity_first::FencingToken::new(1),
6657 ttl: Duration::from_mins(5),
6658 }),
6659 )
6660 .await;
6661
6662 let aggregator = MobKitConsoleAggregator::in_memory();
6663 aggregator.register_runtime_handles_with_policy(
6664 "runtime-a",
6665 "tenant",
6666 runtime.mob_runtime().clone(),
6667 Some(identity_runtime.clone()),
6668 runtime.console_events(),
6669 Arc::new(AllowAllConsoleVisibilityPolicy),
6670 );
6671
6672 assert!(
6673 aggregator
6674 .retire_identity("tenant/rt:review:owned:0")
6675 .await?,
6676 "the namespaced current runtime alias must resolve"
6677 );
6678 assert_eq!(
6679 identity_runtime.status(&durable_identity).await?.state,
6680 crate::identity_first::IdentityLifecycleState::Retiring,
6681 "retirement must advance the durable lifecycle authority"
6682 );
6683 let raw_member = runtime
6684 .mob_handle()
6685 .list_members_including_retiring()
6686 .await
6687 .into_iter()
6688 .find(|entry| {
6689 crate::member_comms_id::runtime_alias_str(entry.agent_identity.as_str())
6690 == runtime_alias
6691 })
6692 .ok_or("raw member disappeared")?;
6693 assert_eq!(
6694 raw_member.status,
6695 meerkat_mob::MobMemberStatus::Active,
6696 "a bridge-less durable retire proves the aggregator did not fall through to raw Mob"
6697 );
6698
6699 let _ = runtime.mob_handle().stop().await;
6700 Ok(())
6701 }
6702
6703 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6704 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
6705 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6706 let runtime = build_empty_runtime("identity-label-test").await;
6707 let mut labels = BTreeMap::new();
6708 labels.insert(
6709 "agent_identity".to_string(),
6710 "channel:C0SMOKEOB3".to_string(),
6711 );
6712 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
6713 spawn_trusted_identity_projected_member(
6714 &runtime,
6715 SpawnMemberSpec::from_wire(
6716 "worker".to_string(),
6717 "rt:channel:C0SMOKEOB3:0".to_string(),
6718 Some("You are C0SMOKEOB3.".into()),
6719 None,
6720 None,
6721 )
6722 .with_labels(labels),
6723 )
6724 .await;
6725
6726 let entry = RuntimeEntry {
6727 runtime_key: "runtime-a".to_string(),
6728 identity_namespace: String::new(),
6729 runtime: runtime.mob_runtime().clone(),
6730 identity_runtime: runtime.identity_runtime().cloned(),
6731 console_events: runtime.console_events(),
6732 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6733 };
6734 let aggregator = MobKitConsoleAggregator::in_memory();
6735 aggregator
6736 .inner
6737 .runtimes
6738 .write()
6739 .expect("runtime registry")
6740 .insert("runtime-a".to_string(), entry);
6741
6742 let records = aggregator.list_identities_fresh().await?;
6743 let record = records
6744 .iter()
6745 .find(|record| record.identity == "channel:C0SMOKEOB3")
6746 .expect("durable identity is exposed");
6747 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
6748
6749 let inspection = aggregator
6750 .inspect_identity("channel:C0SMOKEOB3")
6751 .await?
6752 .expect("durable identity resolves back to runtime member");
6753 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
6754 assert_eq!(
6755 inspection.identity.runtime_member_id,
6756 "rt:channel:C0SMOKEOB3:0"
6757 );
6758
6759 let _ = runtime.mob_handle().stop().await;
6760 Ok(())
6761 }
6762
6763 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6772 async fn identity_scoped_live_stream_sees_members_unknown_to_the_read_model()
6773 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6774 let runtime = build_empty_runtime("live-tail-unknown-identity-test").await;
6775 let aggregator = MobKitConsoleAggregator::in_memory();
6776 aggregator.register_runtime_handles_with_policy(
6777 "runtime-a",
6778 "",
6779 runtime.mob_runtime().clone(),
6780 runtime.identity_runtime().cloned(),
6781 runtime.console_events(),
6782 Arc::new(AllowAllConsoleVisibilityPolicy),
6783 );
6784 let _ = aggregator.list_identities_fresh().await?;
6787
6788 runtime
6789 .spawn(SpawnMemberSpec::from_wire(
6790 "worker".to_string(),
6791 "late-member".to_string(),
6792 Some("You are the late member.".into()),
6793 None,
6794 None,
6795 ))
6796 .await?;
6797 let outcome = aggregator
6798 .store()
6799 .append_if_absent(NewConsoleFrame {
6800 id: None,
6801 dedupe_key: "late-member-live-frame".to_string(),
6802 timestamp_ms: 1,
6803 runtime_key: "runtime-a".to_string(),
6804 identity: "late-member".to_string(),
6805 conversation_id: Some("late-member".to_string()),
6806 session_id: None,
6807 kind: "text_delta".to_string(),
6808 status: ConsoleFrameStatus::Completed,
6809 payload: json!({"delta": "hi"}),
6810 source: ConsoleFrameSource {
6811 kind: ConsoleFrameSourceKind::ConsoleEvent,
6812 source_cursor: None,
6813 },
6814 source_event_id: None,
6815 interaction_id: None,
6816 turn_id: None,
6817 run_id: None,
6818 parent_frame_id: None,
6819 caused_by_frame_id: None,
6820 })
6821 .await?;
6822 let event = ConsoleTimelineEvent::ConsoleFrame {
6823 frame: outcome.frame,
6824 };
6825
6826 assert!(
6829 aggregator
6830 .timeline_event_visible_for_subscriber(&event, Some("late-member"))
6831 .await,
6832 "identity-scoped live stream must not starve on a read-model miss"
6833 );
6834 let visible_to_other = aggregator
6841 .timeline_event_visible_for_subscriber(&event, Some("someone-else"))
6842 .await;
6843 if visible_to_other {
6844 assert!(
6845 aggregator.timeline_event_visible(&event).await,
6846 "the allowance is strictly own-identity: someone-else saw the \
6847 frame while the read model still treats the identity as \
6848 unknown (unscoped gate hides it)"
6849 );
6850 }
6851
6852 let mut converged = false;
6855 for _ in 0..40 {
6856 if aggregator.timeline_event_visible(&event).await {
6857 converged = true;
6858 break;
6859 }
6860 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
6861 }
6862 assert!(
6863 converged,
6864 "the unknown-identity miss must self-heal the read model"
6865 );
6866
6867 let _ = runtime.mob_handle().stop().await;
6868 Ok(())
6869 }
6870
6871 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6876 async fn unregister_runtime_stops_live_projection()
6877 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6878 let runtime = build_empty_runtime("unregister-live-projection-test").await;
6879 let aggregator = MobKitConsoleAggregator::in_memory();
6880 aggregator.register_runtime_handles_with_policy(
6881 "runtime-a",
6882 "",
6883 runtime.mob_runtime().clone(),
6884 runtime.identity_runtime().cloned(),
6885 runtime.console_events(),
6886 Arc::new(AllowAllConsoleVisibilityPolicy),
6887 );
6888
6889 aggregator.unregister_runtime("runtime-a");
6890 assert!(
6891 aggregator
6892 .inner
6893 .runtimes
6894 .read()
6895 .expect("runtime registry")
6896 .is_empty(),
6897 "unregister must remove the registry entry"
6898 );
6899 assert!(
6900 aggregator
6901 .inner
6902 .live_projection_shutdowns
6903 .lock()
6904 .expect("shutdown registry")
6905 .is_empty(),
6906 "unregister must consume the live-projection shutdown handle"
6907 );
6908 aggregator.unregister_runtime("runtime-a");
6910
6911 aggregator.register_runtime_handles_with_policy(
6913 "runtime-a",
6914 "",
6915 runtime.mob_runtime().clone(),
6916 runtime.identity_runtime().cloned(),
6917 runtime.console_events(),
6918 Arc::new(AllowAllConsoleVisibilityPolicy),
6919 );
6920 assert_eq!(
6921 aggregator
6922 .inner
6923 .live_projection_shutdowns
6924 .lock()
6925 .expect("shutdown registry")
6926 .len(),
6927 1
6928 );
6929
6930 let _ = runtime.mob_handle().stop().await;
6931 Ok(())
6932 }
6933
6934 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6935 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
6936 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6937 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
6938 let mut labels = BTreeMap::new();
6939 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6940 spawn_trusted_identity_projected_member(
6941 &runtime,
6942 SpawnMemberSpec::from_wire(
6943 "worker".to_string(),
6944 "rt:review:singleton:0".to_string(),
6945 Some("You are the Review Agent.".into()),
6946 None,
6947 None,
6948 )
6949 .with_labels(labels),
6950 )
6951 .await;
6952 let hidden_session_id = runtime
6953 .mob_handle()
6954 .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
6955 "rt:review:singleton:0",
6956 ))
6957 .await
6958 .map(|sid| sid.to_string());
6959 let mut duplicate_labels = BTreeMap::new();
6960 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6961 spawn_trusted_identity_projected_member(
6962 &runtime,
6963 SpawnMemberSpec::from_wire(
6964 "worker".to_string(),
6965 "rt:review:singleton:1".to_string(),
6966 Some("You are a visible duplicate Review Agent.".into()),
6967 None,
6968 None,
6969 )
6970 .with_labels(duplicate_labels),
6971 )
6972 .await;
6973
6974 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6975 let aggregator = MobKitConsoleAggregator::in_memory();
6976 aggregator.register_runtime_handles_with_policy(
6977 "runtime-a",
6978 "",
6979 runtime.mob_runtime().clone(),
6980 Some(identity_runtime),
6981 runtime.console_events(),
6982 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
6983 );
6984
6985 assert!(
6986 aggregator
6987 .inspect_identity("review:singleton")
6988 .await?
6989 .is_none(),
6990 "member-hidden live alias must not inspect through aggregator"
6991 );
6992 let records = aggregator.list_identities_fresh().await?;
6993 assert!(
6994 records
6995 .iter()
6996 .all(|record| record.identity != "review:singleton"),
6997 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
6998 );
6999 let appended = aggregator
7000 .store()
7001 .append_if_absent(NewConsoleFrame {
7002 id: None,
7003 dedupe_key: "member-hidden-frame".to_string(),
7004 timestamp_ms: 1,
7005 runtime_key: "runtime-a".to_string(),
7006 identity: "review:singleton".to_string(),
7007 conversation_id: Some("review:singleton".to_string()),
7008 session_id: hidden_session_id,
7009 kind: "text_delta".to_string(),
7010 status: ConsoleFrameStatus::Delivered,
7011 payload: json!({ "delta": "hidden" }),
7012 source: ConsoleFrameSource {
7013 kind: ConsoleFrameSourceKind::ConsoleEvent,
7014 source_cursor: None,
7015 },
7016 source_event_id: Some("member-hidden-frame".to_string()),
7017 interaction_id: None,
7018 turn_id: None,
7019 run_id: None,
7020 parent_frame_id: None,
7021 caused_by_frame_id: None,
7022 })
7023 .await?;
7024 assert!(
7025 !aggregator
7026 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
7027 frame: appended.frame.clone()
7028 })
7029 .await,
7030 "member-hidden live alias must not pass timeline event visibility"
7031 );
7032 let identity_only = aggregator
7033 .store()
7034 .append_if_absent(NewConsoleFrame {
7035 id: None,
7036 dedupe_key: "member-hidden-identity-only-frame".to_string(),
7037 timestamp_ms: 2,
7038 runtime_key: "runtime-a".to_string(),
7039 identity: "review:singleton".to_string(),
7040 conversation_id: Some("review:singleton".to_string()),
7041 session_id: None,
7042 kind: "text_delta".to_string(),
7043 status: ConsoleFrameStatus::Delivered,
7044 payload: json!({ "delta": "identity-only-hidden" }),
7045 source: ConsoleFrameSource {
7046 kind: ConsoleFrameSourceKind::ConsoleEvent,
7047 source_cursor: None,
7048 },
7049 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
7050 interaction_id: None,
7051 turn_id: None,
7052 run_id: None,
7053 parent_frame_id: None,
7054 caused_by_frame_id: None,
7055 })
7056 .await?;
7057 assert!(
7058 !aggregator
7059 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
7060 frame: identity_only.frame.clone()
7061 })
7062 .await,
7063 "member-hidden durable alias must not leak identity-only timeline frames"
7064 );
7065 let page = aggregator
7066 .query_timeline(ConsoleTimelineQuery {
7067 identity: Some("review:singleton".to_string()),
7068 limit: 10,
7069 ..ConsoleTimelineQuery::default()
7070 })
7071 .await?;
7072 assert!(
7073 page.frames.is_empty(),
7074 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
7075 );
7076 assert!(
7077 !aggregator.retire_identity("review:singleton").await?,
7078 "member-hidden live alias must not retire through aggregator"
7079 );
7080 let reserve_err = aggregator
7081 .reserve_identity_first_interaction(
7082 ConsoleSendRequest {
7083 identity: "review:singleton".to_string(),
7084 content: json!("hello"),
7085 origin: "console:test".to_string(),
7086 idempotency_key: "member-hidden-reserve".to_string(),
7087 handling_mode: Some("queue".to_string()),
7088 },
7089 None,
7090 )
7091 .await
7092 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
7093 assert!(
7094 reserve_err.to_string().contains("unknown identity"),
7095 "unexpected reserve error: {reserve_err}"
7096 );
7097 let send_err = aggregator
7098 .send(ConsoleSendRequest {
7099 identity: "review:singleton".to_string(),
7100 content: json!("hello"),
7101 origin: "console:test".to_string(),
7102 idempotency_key: "member-hidden-send".to_string(),
7103 handling_mode: Some("queue".to_string()),
7104 })
7105 .await
7106 .expect_err("member-hidden live alias must not receive sends");
7107 assert!(
7108 send_err.to_string().contains("unknown identity"),
7109 "unexpected send error: {send_err}"
7110 );
7111 let blob_result = aggregator
7112 .binary_blob_store_for_identity("review:singleton")
7113 .await;
7114 let Err(blob_err) = blob_result else {
7115 panic!("member-hidden live alias must not expose blob store");
7116 };
7117 assert!(
7118 blob_err.to_string().contains("unknown identity"),
7119 "unexpected blob error: {blob_err}"
7120 );
7121
7122 let _ = runtime.mob_handle().stop().await;
7123 Ok(())
7124 }
7125
7126 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7127 async fn explicit_session_backfill_target_resolves_durable_identity_label()
7128 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7129 let runtime = build_empty_runtime("identity-label-backfill-test").await;
7130 let mut labels = BTreeMap::new();
7131 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7132 spawn_trusted_identity_projected_member(
7133 &runtime,
7134 SpawnMemberSpec::from_wire(
7135 "worker".to_string(),
7136 "rt:review:singleton:0".to_string(),
7137 Some("You are the Review Agent.".into()),
7138 None,
7139 None,
7140 )
7141 .with_labels(labels),
7142 )
7143 .await;
7144
7145 let entry = RuntimeEntry {
7146 runtime_key: "runtime-a".to_string(),
7147 identity_namespace: String::new(),
7148 runtime: runtime.mob_runtime().clone(),
7149 identity_runtime: runtime.identity_runtime().cloned(),
7150 console_events: runtime.console_events(),
7151 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7152 };
7153 let aggregator = MobKitConsoleAggregator::in_memory();
7154 aggregator
7155 .inner
7156 .runtimes
7157 .write()
7158 .expect("runtime registry")
7159 .insert("runtime-a".to_string(), entry);
7160
7161 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
7162 .await
7163 .expect("durable label should resolve targeted session backfill");
7164 assert_eq!(target.record.identity, "review:singleton");
7165 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
7166
7167 let _ = runtime.mob_handle().stop().await;
7168 Ok(())
7169 }
7170
7171 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7172 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
7173 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7174 let runtime = build_empty_runtime("identity-stale-durable-test").await;
7175 let mut labels = BTreeMap::new();
7176 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7177 spawn_trusted_identity_projected_member(
7178 &runtime,
7179 SpawnMemberSpec::from_wire(
7180 "worker".to_string(),
7181 "rt:review:singleton:0".to_string(),
7182 Some("You are the live Review Agent.".into()),
7183 None,
7184 None,
7185 )
7186 .with_labels(labels),
7187 )
7188 .await;
7189
7190 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7191 crate::identity_first::IdentityRuntimeConfig {
7192 continuity_store: Arc::new(
7193 crate::identity_first::LocalContinuityStore::in_memory()?
7194 ),
7195 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7196 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
7197 has_runtime_store: true,
7198 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7199 bridge: None,
7200 default_timeout: None,
7201 },
7202 ));
7203 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7204 let record = crate::identity_first::ContinuityRecord {
7205 identity: identity.clone(),
7206 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7207 "rt:review:singleton:1",
7208 )?,
7209 session_id: SessionId::new(),
7210 generation: crate::identity_first::ContinuityGeneration::new(1),
7211 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7212 };
7213 identity_runtime
7214 .register(
7215 crate::identity_first::DurableAgentSpec {
7216 identity: identity.clone(),
7217 profile: meerkat_mob::ProfileName::from("worker"),
7218 addressability: crate::identity_first::AgentAddressability::Addressable,
7219 display_name: None,
7220 labels: BTreeMap::new(),
7221 context: None,
7222 additional_instructions: Vec::new(),
7223 initial_message: None,
7224 runtime_mode_override: None,
7225 backend: None,
7226 binding: None,
7227 },
7228 crate::identity_first::IdentityLifecycleState::Active,
7229 Some(record),
7230 Some(crate::identity_first::LeaseGrant {
7231 identity,
7232 fencing_token: crate::identity_first::FencingToken::new(7),
7233 ttl: Duration::from_mins(5),
7234 }),
7235 )
7236 .await;
7237
7238 let aggregator = MobKitConsoleAggregator::in_memory();
7239 aggregator
7240 .inner
7241 .runtimes
7242 .write()
7243 .expect("runtime registry")
7244 .insert(
7245 "runtime-a".to_string(),
7246 RuntimeEntry {
7247 runtime_key: "runtime-a".to_string(),
7248 identity_namespace: String::new(),
7249 runtime: runtime.mob_runtime().clone(),
7250 identity_runtime: Some(identity_runtime),
7251 console_events: runtime.console_events(),
7252 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7253 },
7254 );
7255
7256 let err = aggregator
7257 .inspect_identity("review:singleton")
7258 .await
7259 .expect_err("stale durable binding must not inspect as healthy");
7260 assert!(
7261 err.to_string().contains("stale durable identity alias"),
7262 "unexpected error: {err}"
7263 );
7264 let retire_err = aggregator
7265 .retire_identity("review:singleton")
7266 .await
7267 .expect_err("stale durable binding must not retire as healthy");
7268 assert!(
7269 retire_err
7270 .to_string()
7271 .contains("stale durable identity alias"),
7272 "unexpected retire error: {retire_err}"
7273 );
7274 let send_err = aggregator
7275 .send(ConsoleSendRequest {
7276 identity: "review:singleton".to_string(),
7277 content: json!("hello"),
7278 origin: "console:test".to_string(),
7279 idempotency_key: "stale-durable-send".to_string(),
7280 handling_mode: Some("queue".to_string()),
7281 })
7282 .await
7283 .expect_err("stale durable binding must not send to the wrong live member");
7284 assert!(
7285 send_err
7286 .to_string()
7287 .contains("stale durable identity alias"),
7288 "unexpected send error: {send_err}"
7289 );
7290 let reserve_err = aggregator
7291 .reserve_identity_first_interaction(
7292 ConsoleSendRequest {
7293 identity: "review:singleton".to_string(),
7294 content: json!("hello"),
7295 origin: "console:test".to_string(),
7296 idempotency_key: "stale-durable-reserve".to_string(),
7297 handling_mode: Some("queue".to_string()),
7298 },
7299 None,
7300 )
7301 .await
7302 .expect_err("identity-first reserve must not bypass stale durable binding");
7303 assert!(
7304 reserve_err
7305 .to_string()
7306 .contains("stale durable identity alias"),
7307 "unexpected reserve error: {reserve_err}"
7308 );
7309 let blob_err = match aggregator
7310 .binary_blob_store_for_identity("review:singleton")
7311 .await
7312 {
7313 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
7314 Err(err) => err,
7315 };
7316 assert!(
7317 blob_err
7318 .to_string()
7319 .contains("stale durable identity alias"),
7320 "unexpected blob error: {blob_err}"
7321 );
7322 let records = aggregator.list_identities_fresh().await?;
7323 assert!(
7324 records
7325 .iter()
7326 .any(|record| record.identity == "review:singleton"
7327 && record.runtime_member_id == "rt:review:singleton:0"),
7328 "live alias should still be listed; records: {records:#?}"
7329 );
7330 assert!(
7331 records
7332 .iter()
7333 .any(|record| record.identity == "review:singleton"
7334 && record.runtime_member_id == "rt:review:singleton:1"),
7335 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
7336 );
7337
7338 let _ = runtime.mob_handle().stop().await;
7339 Ok(())
7340 }
7341
7342 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7343 async fn identity_first_alias_self_heals_session_only_split_brain()
7344 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7345 let runtime = build_empty_runtime("identity-session-rebind-test").await;
7346 let mut labels = BTreeMap::new();
7347 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7348 spawn_trusted_identity_projected_member(
7349 &runtime,
7350 SpawnMemberSpec::from_wire(
7351 "worker".to_string(),
7352 "rt:review:singleton:0".to_string(),
7353 Some("You are the live Review Agent.".into()),
7354 None,
7355 None,
7356 )
7357 .with_labels(labels),
7358 )
7359 .await;
7360 let live_session_id = runtime
7361 .mob_handle()
7362 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
7363 "rt:review:singleton:0",
7364 ))
7365 .await
7366 .expect("live member has bridge session");
7367
7368 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7369 crate::identity_first::IdentityRuntimeConfig {
7370 continuity_store: Arc::new(
7371 crate::identity_first::LocalContinuityStore::in_memory()?
7372 ),
7373 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7374 runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
7375 has_runtime_store: true,
7376 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7377 bridge: None,
7378 default_timeout: None,
7379 },
7380 ));
7381 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7382 let stale_session_id = SessionId::new();
7383 assert_ne!(stale_session_id, live_session_id);
7384 let record = crate::identity_first::ContinuityRecord {
7385 identity: identity.clone(),
7386 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7387 "rt:review:singleton:0",
7388 )?,
7389 session_id: stale_session_id,
7390 generation: crate::identity_first::ContinuityGeneration::new(1),
7391 checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
7392 };
7393 identity_runtime
7394 .register(
7395 crate::identity_first::DurableAgentSpec {
7396 identity: identity.clone(),
7397 profile: meerkat_mob::ProfileName::from("worker"),
7398 addressability: crate::identity_first::AgentAddressability::Addressable,
7399 display_name: None,
7400 labels: BTreeMap::new(),
7401 context: None,
7402 additional_instructions: Vec::new(),
7403 initial_message: None,
7404 runtime_mode_override: None,
7405 backend: None,
7406 binding: None,
7407 },
7408 crate::identity_first::IdentityLifecycleState::Active,
7409 Some(record),
7410 Some(crate::identity_first::LeaseGrant {
7411 identity: identity.clone(),
7412 fencing_token: crate::identity_first::FencingToken::new(7),
7413 ttl: Duration::from_mins(5),
7414 }),
7415 )
7416 .await;
7417
7418 let aggregator = MobKitConsoleAggregator::in_memory();
7419 aggregator
7420 .inner
7421 .runtimes
7422 .write()
7423 .expect("runtime registry")
7424 .insert(
7425 "runtime-a".to_string(),
7426 RuntimeEntry {
7427 runtime_key: "runtime-a".to_string(),
7428 identity_namespace: String::new(),
7429 runtime: runtime.mob_runtime().clone(),
7430 identity_runtime: Some(identity_runtime.clone()),
7431 console_events: runtime.console_events(),
7432 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7433 },
7434 );
7435
7436 aggregator
7437 .reserve_identity_first_interaction(
7438 ConsoleSendRequest {
7439 identity: "review:singleton".to_string(),
7440 content: json!("hello"),
7441 origin: "console:test".to_string(),
7442 idempotency_key: "session-rebind-reserve".to_string(),
7443 handling_mode: Some("queue".to_string()),
7444 },
7445 None,
7446 )
7447 .await
7448 .expect("session-only split-brain should self-heal before reserve");
7449
7450 let rebound_status = identity_runtime.status(&identity).await?;
7451 assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
7452
7453 let inspection = aggregator
7454 .inspect_identity("review:singleton")
7455 .await?
7456 .expect("rebound identity should inspect");
7457 let live_session_string = live_session_id.to_string();
7458 assert_eq!(
7459 inspection.identity.session_id.as_deref(),
7460 Some(live_session_string.as_str())
7461 );
7462
7463 let _ = runtime.mob_handle().stop().await;
7464 Ok(())
7465 }
7466
7467 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7468 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
7469 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7470 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
7471 let mut labels = BTreeMap::new();
7472 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
7473 spawn_trusted_identity_projected_member(
7474 &runtime,
7475 SpawnMemberSpec::from_wire(
7476 "worker".to_string(),
7477 "rt:review:singleton:0".to_string(),
7478 Some("You are the mislabeled Review Agent.".into()),
7479 None,
7480 None,
7481 )
7482 .with_labels(labels),
7483 )
7484 .await;
7485
7486 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7487 crate::identity_first::IdentityRuntimeConfig {
7488 continuity_store: Arc::new(
7489 crate::identity_first::LocalContinuityStore::in_memory()?
7490 ),
7491 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7492 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
7493 has_runtime_store: true,
7494 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7495 bridge: None,
7496 default_timeout: None,
7497 },
7498 ));
7499 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7500 let record = crate::identity_first::ContinuityRecord {
7501 identity: identity.clone(),
7502 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7503 "rt:review:singleton:0",
7504 )?,
7505 session_id: SessionId::new(),
7506 generation: crate::identity_first::ContinuityGeneration::new(1),
7507 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7508 };
7509 identity_runtime
7510 .register(
7511 crate::identity_first::DurableAgentSpec {
7512 identity: identity.clone(),
7513 profile: meerkat_mob::ProfileName::from("worker"),
7514 addressability: crate::identity_first::AgentAddressability::Addressable,
7515 display_name: None,
7516 labels: BTreeMap::new(),
7517 context: None,
7518 additional_instructions: Vec::new(),
7519 initial_message: None,
7520 runtime_mode_override: None,
7521 backend: None,
7522 binding: None,
7523 },
7524 crate::identity_first::IdentityLifecycleState::Active,
7525 Some(record),
7526 Some(crate::identity_first::LeaseGrant {
7527 identity,
7528 fencing_token: crate::identity_first::FencingToken::new(7),
7529 ttl: Duration::from_mins(5),
7530 }),
7531 )
7532 .await;
7533
7534 let aggregator = MobKitConsoleAggregator::in_memory();
7535 aggregator
7536 .inner
7537 .runtimes
7538 .write()
7539 .expect("runtime registry")
7540 .insert(
7541 "runtime-a".to_string(),
7542 RuntimeEntry {
7543 runtime_key: "runtime-a".to_string(),
7544 identity_namespace: String::new(),
7545 runtime: runtime.mob_runtime().clone(),
7546 identity_runtime: Some(identity_runtime),
7547 console_events: runtime.console_events(),
7548 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7549 },
7550 );
7551
7552 let err = aggregator
7553 .inspect_identity("review:singleton")
7554 .await
7555 .expect_err("wrong projected live identity must not inspect as healthy");
7556 assert!(
7557 err.to_string()
7558 .contains("projects identity other:singleton"),
7559 "unexpected error: {err}"
7560 );
7561 let retire_err = aggregator
7562 .retire_identity("review:singleton")
7563 .await
7564 .expect_err("wrong projected live identity must not retire as healthy");
7565 assert!(
7566 retire_err
7567 .to_string()
7568 .contains("projects identity other:singleton"),
7569 "unexpected retire error: {retire_err}"
7570 );
7571 let send_err = aggregator
7572 .send(ConsoleSendRequest {
7573 identity: "review:singleton".to_string(),
7574 content: json!("hello"),
7575 origin: "console:test".to_string(),
7576 idempotency_key: "wrong-projection-send".to_string(),
7577 handling_mode: Some("queue".to_string()),
7578 })
7579 .await
7580 .expect_err("wrong projected live identity must not send as unknown");
7581 assert!(
7582 send_err
7583 .to_string()
7584 .contains("projects identity other:singleton"),
7585 "unexpected send error: {send_err}"
7586 );
7587 let reserve_err = aggregator
7588 .reserve_identity_first_interaction(
7589 ConsoleSendRequest {
7590 identity: "review:singleton".to_string(),
7591 content: json!("hello"),
7592 origin: "console:test".to_string(),
7593 idempotency_key: "wrong-projection-reserve".to_string(),
7594 handling_mode: Some("queue".to_string()),
7595 },
7596 None,
7597 )
7598 .await
7599 .expect_err("identity-first reserve must not bypass wrong live projection");
7600 assert!(
7601 reserve_err
7602 .to_string()
7603 .contains("projects identity other:singleton"),
7604 "unexpected reserve error: {reserve_err}"
7605 );
7606 let blob_err = match aggregator
7607 .binary_blob_store_for_identity("review:singleton")
7608 .await
7609 {
7610 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
7611 Err(err) => err,
7612 };
7613 assert!(
7614 blob_err
7615 .to_string()
7616 .contains("projects identity other:singleton"),
7617 "unexpected blob error: {blob_err}"
7618 );
7619 let projected_err = aggregator
7620 .inspect_identity("other:singleton")
7621 .await
7622 .expect_err("wrong projected alias must not inspect through live-only fallback");
7623 assert!(
7624 projected_err
7625 .to_string()
7626 .contains("identity runtime binding for review:singleton"),
7627 "unexpected projected-alias error: {projected_err}"
7628 );
7629 let projected_retire_err = aggregator
7630 .retire_identity("other:singleton")
7631 .await
7632 .expect_err("wrong projected alias must not retire through live-only fallback");
7633 assert!(
7634 projected_retire_err
7635 .to_string()
7636 .contains("identity runtime binding for review:singleton"),
7637 "unexpected projected-alias retire error: {projected_retire_err}"
7638 );
7639
7640 let _ = runtime.mob_handle().stop().await;
7641 Ok(())
7642 }
7643
7644 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7645 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
7646 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7647 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
7648 let mut labels = BTreeMap::new();
7649 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
7650 spawn_trusted_identity_projected_member(
7651 &runtime,
7652 SpawnMemberSpec::from_wire(
7653 "worker".to_string(),
7654 "rt:review:singleton:0".to_string(),
7655 Some("You are a hidden wrong projection.".into()),
7656 None,
7657 None,
7658 )
7659 .with_labels(labels),
7660 )
7661 .await;
7662
7663 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7664 crate::identity_first::IdentityRuntimeConfig {
7665 continuity_store: Arc::new(
7666 crate::identity_first::LocalContinuityStore::in_memory()?
7667 ),
7668 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7669 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
7670 has_runtime_store: true,
7671 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7672 bridge: None,
7673 default_timeout: None,
7674 },
7675 ));
7676 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7677 identity_runtime
7678 .register(
7679 crate::identity_first::DurableAgentSpec {
7680 identity: identity.clone(),
7681 profile: meerkat_mob::ProfileName::from("worker"),
7682 addressability: crate::identity_first::AgentAddressability::Addressable,
7683 display_name: None,
7684 labels: BTreeMap::new(),
7685 context: None,
7686 additional_instructions: Vec::new(),
7687 initial_message: None,
7688 runtime_mode_override: None,
7689 backend: None,
7690 binding: None,
7691 },
7692 crate::identity_first::IdentityLifecycleState::Active,
7693 Some(crate::identity_first::ContinuityRecord {
7694 identity: identity.clone(),
7695 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7696 "rt:review:singleton:0",
7697 )?,
7698 session_id: SessionId::new(),
7699 generation: crate::identity_first::ContinuityGeneration::new(1),
7700 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7701 }),
7702 Some(crate::identity_first::LeaseGrant {
7703 identity,
7704 fencing_token: crate::identity_first::FencingToken::new(7),
7705 ttl: Duration::from_mins(5),
7706 }),
7707 )
7708 .await;
7709
7710 let aggregator = MobKitConsoleAggregator::in_memory();
7711 aggregator
7712 .inner
7713 .runtimes
7714 .write()
7715 .expect("runtime registry")
7716 .insert(
7717 "runtime-a".to_string(),
7718 RuntimeEntry {
7719 runtime_key: "runtime-a".to_string(),
7720 identity_namespace: String::new(),
7721 runtime: runtime.mob_runtime().clone(),
7722 identity_runtime: Some(identity_runtime),
7723 console_events: runtime.console_events(),
7724 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
7725 },
7726 );
7727
7728 let err = aggregator
7729 .inspect_identity("review:singleton")
7730 .await
7731 .expect_err("hidden wrong projection must still fail durable binding validation");
7732 assert!(
7733 err.to_string()
7734 .contains("projects identity other:singleton"),
7735 "unexpected hidden wrong-projection error: {err}"
7736 );
7737 let reserve_err = aggregator
7738 .reserve_identity_first_interaction(
7739 ConsoleSendRequest {
7740 identity: "review:singleton".to_string(),
7741 content: json!("hello"),
7742 origin: "console:test".to_string(),
7743 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
7744 handling_mode: Some("queue".to_string()),
7745 },
7746 None,
7747 )
7748 .await
7749 .expect_err("reserve must not hide wrong projection from durable validation");
7750 assert!(
7751 reserve_err
7752 .to_string()
7753 .contains("projects identity other:singleton"),
7754 "unexpected reserve error: {reserve_err}"
7755 );
7756
7757 let _ = runtime.mob_handle().stop().await;
7758 Ok(())
7759 }
7760
7761 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7762 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
7763 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7764 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
7765 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
7766 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7767 crate::identity_first::IdentityRuntimeConfig {
7768 continuity_store: Arc::new(
7769 crate::identity_first::LocalContinuityStore::in_memory()?
7770 ),
7771 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7772 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
7773 has_runtime_store: true,
7774 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7775 bridge: None,
7776 default_timeout: None,
7777 },
7778 ));
7779 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7780 crate::identity_first::IdentityRuntimeConfig {
7781 continuity_store: Arc::new(
7782 crate::identity_first::LocalContinuityStore::in_memory()?
7783 ),
7784 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7785 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
7786 has_runtime_store: true,
7787 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7788 bridge: None,
7789 default_timeout: None,
7790 },
7791 ));
7792 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
7793 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7794 identity_runtime
7795 .register(
7796 crate::identity_first::DurableAgentSpec {
7797 identity: identity.clone(),
7798 profile: meerkat_mob::ProfileName::from("worker"),
7799 addressability: crate::identity_first::AgentAddressability::Addressable,
7800 display_name: None,
7801 labels: BTreeMap::new(),
7802 context: None,
7803 additional_instructions: Vec::new(),
7804 initial_message: None,
7805 runtime_mode_override: None,
7806 backend: None,
7807 binding: None,
7808 },
7809 crate::identity_first::IdentityLifecycleState::Active,
7810 Some(crate::identity_first::ContinuityRecord {
7811 identity: identity.clone(),
7812 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7813 "rt:review:singleton:0",
7814 )?,
7815 session_id: SessionId::new(),
7816 generation: crate::identity_first::ContinuityGeneration::new(1),
7817 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7818 }),
7819 Some(crate::identity_first::LeaseGrant {
7820 identity,
7821 fencing_token: crate::identity_first::FencingToken::new(7),
7822 ttl: Duration::from_mins(5),
7823 }),
7824 )
7825 .await;
7826 }
7827
7828 let aggregator = MobKitConsoleAggregator::in_memory();
7829 {
7830 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
7831 runtimes.insert(
7832 "a-hidden".to_string(),
7833 RuntimeEntry {
7834 runtime_key: "a-hidden".to_string(),
7835 identity_namespace: String::new(),
7836 runtime: hidden_runtime.mob_runtime().clone(),
7837 identity_runtime: Some(hidden_identity_runtime),
7838 console_events: hidden_runtime.console_events(),
7839 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
7840 },
7841 );
7842 runtimes.insert(
7843 "b-visible".to_string(),
7844 RuntimeEntry {
7845 runtime_key: "b-visible".to_string(),
7846 identity_namespace: String::new(),
7847 runtime: visible_runtime.mob_runtime().clone(),
7848 identity_runtime: Some(visible_identity_runtime),
7849 console_events: visible_runtime.console_events(),
7850 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7851 },
7852 );
7853 }
7854
7855 let inspection = aggregator
7856 .inspect_identity("rt:review:singleton:0")
7857 .await?
7858 .expect("visible runtime-id match should be returned");
7859 assert_eq!(inspection.identity.runtime_key, "b-visible");
7860
7861 let _ = hidden_runtime.mob_handle().stop().await;
7862 let _ = visible_runtime.mob_handle().stop().await;
7863 Ok(())
7864 }
7865
7866 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7867 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
7868 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7869 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
7870 let mut primary_labels = BTreeMap::new();
7871 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7872 spawn_trusted_identity_projected_member(
7873 &runtime,
7874 SpawnMemberSpec::from_wire(
7875 "worker".to_string(),
7876 "rt:review:singleton:0".to_string(),
7877 Some("You are the primary Review Agent.".into()),
7878 None,
7879 None,
7880 )
7881 .with_labels(primary_labels),
7882 )
7883 .await;
7884
7885 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
7886 let mut delegate_labels = BTreeMap::new();
7887 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
7888 spawn_trusted_identity_projected_member(
7889 &delegate_runtime,
7890 SpawnMemberSpec::from_wire(
7891 "worker".to_string(),
7892 "rt:review:singleton:0".to_string(),
7893 Some("You are the wrong projected Review Agent.".into()),
7894 None,
7895 None,
7896 )
7897 .with_labels(delegate_labels),
7898 )
7899 .await;
7900 runtime
7901 .mob_runtime()
7902 .agent_mob_mcp_state()
7903 .expect("runtime should expose mob MCP state")
7904 .mob_insert_handle(
7905 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
7906 delegate_runtime.mob_handle().clone(),
7907 )
7908 .await;
7909
7910 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7911 crate::identity_first::IdentityRuntimeConfig {
7912 continuity_store: Arc::new(
7913 crate::identity_first::LocalContinuityStore::in_memory()?
7914 ),
7915 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7916 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
7917 has_runtime_store: true,
7918 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7919 bridge: None,
7920 default_timeout: None,
7921 },
7922 ));
7923 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7924 let record = crate::identity_first::ContinuityRecord {
7925 identity: identity.clone(),
7926 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7927 "rt:review:singleton:0",
7928 )?,
7929 session_id: SessionId::new(),
7930 generation: crate::identity_first::ContinuityGeneration::new(1),
7931 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7932 };
7933 identity_runtime
7934 .register(
7935 crate::identity_first::DurableAgentSpec {
7936 identity: identity.clone(),
7937 profile: meerkat_mob::ProfileName::from("worker"),
7938 addressability: crate::identity_first::AgentAddressability::Addressable,
7939 display_name: None,
7940 labels: BTreeMap::new(),
7941 context: None,
7942 additional_instructions: Vec::new(),
7943 initial_message: None,
7944 runtime_mode_override: None,
7945 backend: None,
7946 binding: None,
7947 },
7948 crate::identity_first::IdentityLifecycleState::Active,
7949 Some(record),
7950 Some(crate::identity_first::LeaseGrant {
7951 identity,
7952 fencing_token: crate::identity_first::FencingToken::new(7),
7953 ttl: Duration::from_mins(5),
7954 }),
7955 )
7956 .await;
7957
7958 let aggregator = MobKitConsoleAggregator::in_memory();
7959 aggregator
7960 .inner
7961 .runtimes
7962 .write()
7963 .expect("runtime registry")
7964 .insert(
7965 "runtime-a".to_string(),
7966 RuntimeEntry {
7967 runtime_key: "runtime-a".to_string(),
7968 identity_namespace: String::new(),
7969 runtime: runtime.mob_runtime().clone(),
7970 identity_runtime: Some(identity_runtime),
7971 console_events: runtime.console_events(),
7972 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7973 },
7974 );
7975
7976 let err = aggregator
7977 .inspect_identity("review:singleton")
7978 .await
7979 .expect_err("cross-source wrong projection must not be hidden by primary source");
7980 assert!(
7981 err.to_string()
7982 .contains("projects identity other:singleton")
7983 || err.to_string().contains("ambiguous live identity alias"),
7984 "unexpected error: {err}"
7985 );
7986
7987 let _ = delegate_runtime.mob_handle().stop().await;
7988 let _ = runtime.mob_handle().stop().await;
7989 Ok(())
7990 }
7991
7992 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7993 async fn durable_send_selects_bound_generation_when_stale_generation_is_still_active()
7994 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7995 let runtime = build_empty_runtime("identity-respawn-send-authority-test").await;
7996 let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
7997 for generation in 0..=1 {
7998 spawn_trusted_identity_projected_member(
7999 &runtime,
8000 SpawnMemberSpec::from_wire(
8001 "worker".to_string(),
8002 format!("rt:builder:{generation}"),
8003 Some(format!("You are Builder generation {generation}.").into()),
8004 None,
8005 None,
8006 )
8007 .with_labels(labels.clone()),
8008 )
8009 .await;
8010 }
8011 let bound_session_id = runtime
8012 .mob_handle()
8013 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
8014 "rt:builder:1",
8015 ))
8016 .await
8017 .expect("bound generation has a bridge session");
8018
8019 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
8020 crate::identity_first::IdentityRuntimeConfig {
8021 continuity_store: Arc::new(
8022 crate::identity_first::LocalContinuityStore::in_memory()?
8023 ),
8024 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
8025 runtime_instance_id: "console-aggregator-respawn-send-authority-test".to_string(),
8026 has_runtime_store: true,
8027 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
8028 bridge: None,
8029 default_timeout: None,
8030 },
8031 ));
8032 let identity = crate::identity_first::AgentIdentity::parse("builder")?;
8033 identity_runtime
8034 .register(
8035 crate::identity_first::DurableAgentSpec {
8036 identity: identity.clone(),
8037 profile: meerkat_mob::ProfileName::from("worker"),
8038 addressability: crate::identity_first::AgentAddressability::Addressable,
8039 display_name: None,
8040 labels: BTreeMap::new(),
8041 context: None,
8042 additional_instructions: Vec::new(),
8043 initial_message: None,
8044 runtime_mode_override: None,
8045 backend: None,
8046 binding: None,
8047 },
8048 crate::identity_first::IdentityLifecycleState::Active,
8049 Some(crate::identity_first::ContinuityRecord {
8050 identity: identity.clone(),
8051 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse("rt:builder:1")?,
8052 session_id: bound_session_id.clone(),
8053 generation: crate::identity_first::ContinuityGeneration::new(1),
8054 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
8055 }),
8056 Some(crate::identity_first::LeaseGrant {
8057 identity,
8058 fencing_token: crate::identity_first::FencingToken::new(9),
8059 ttl: Duration::from_mins(5),
8060 }),
8061 )
8062 .await;
8063
8064 let aggregator = MobKitConsoleAggregator::in_memory();
8065 aggregator
8066 .inner
8067 .runtimes
8068 .write()
8069 .expect("runtime registry")
8070 .insert(
8071 "default".to_string(),
8072 RuntimeEntry {
8073 runtime_key: "default".to_string(),
8074 identity_namespace: String::new(),
8075 runtime: runtime.mob_runtime().clone(),
8076 identity_runtime: Some(identity_runtime),
8077 console_events: runtime.console_events(),
8078 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8079 },
8080 );
8081
8082 let matches = aggregator.resolve_visible_members("builder").await;
8083 assert_eq!(
8084 matches.len(),
8085 2,
8086 "both Active generations remain observable"
8087 );
8088 assert_eq!(
8089 sendable_resolved_member_indices(&matches).len(),
8090 2,
8091 "the stale generation reproduces the pre-fix send ambiguity"
8092 );
8093 let resolved = aggregator
8094 .resolve_send_member("builder")
8095 .await?
8096 .expect("durable builder alias resolves");
8097 assert_eq!(
8098 resolved.runtime_identity, "rt:builder:1",
8099 "continuity binding must shadow the stale Active generation for sends"
8100 );
8101
8102 let inspect_error = aggregator
8103 .inspect_identity("builder")
8104 .await
8105 .expect_err("inspection must keep duplicate live aliases explicit");
8106 assert!(
8107 inspect_error
8108 .to_string()
8109 .contains("ambiguous live identity alias")
8110 );
8111 let retire_error = aggregator
8112 .retire_identity("builder")
8113 .await
8114 .expect_err("control must not silently retire through a durable alias");
8115 assert!(
8116 retire_error
8117 .to_string()
8118 .contains("ambiguous live identity alias")
8119 );
8120
8121 let accepted = aggregator
8122 .send(ConsoleSendRequest {
8123 identity: "builder".to_string(),
8124 content: json!("continue after respawn"),
8125 origin: "console:test".to_string(),
8126 idempotency_key: "respawn-bound-generation-send".to_string(),
8127 handling_mode: Some("queue".to_string()),
8128 })
8129 .await?;
8130 assert_eq!(accepted.session_id, Some(bound_session_id.to_string()));
8131
8132 let _ = runtime.mob_handle().stop().await;
8133 Ok(())
8134 }
8135
8136 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8137 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
8138 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8139 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
8140 let mut labels = BTreeMap::new();
8141 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
8142 spawn_trusted_identity_projected_member(
8143 &runtime,
8144 SpawnMemberSpec::from_wire(
8145 "worker".to_string(),
8146 "rt:review:singleton:0".to_string(),
8147 Some("You are the first Review Agent.".into()),
8148 None,
8149 None,
8150 )
8151 .with_labels(labels.clone()),
8152 )
8153 .await;
8154 spawn_trusted_identity_projected_member(
8155 &runtime,
8156 SpawnMemberSpec::from_wire(
8157 "worker".to_string(),
8158 "rt:review:singleton:1".to_string(),
8159 Some("You are the second Review Agent.".into()),
8160 None,
8161 None,
8162 )
8163 .with_labels(labels),
8164 )
8165 .await;
8166
8167 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
8168 crate::identity_first::IdentityRuntimeConfig {
8169 continuity_store: Arc::new(
8170 crate::identity_first::LocalContinuityStore::in_memory()?
8171 ),
8172 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
8173 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
8174 has_runtime_store: true,
8175 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
8176 bridge: None,
8177 default_timeout: None,
8178 },
8179 ));
8180 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
8181 identity_runtime
8182 .register(
8183 crate::identity_first::DurableAgentSpec {
8184 identity: identity.clone(),
8185 profile: meerkat_mob::ProfileName::from("worker"),
8186 addressability: crate::identity_first::AgentAddressability::Addressable,
8187 display_name: None,
8188 labels: BTreeMap::new(),
8189 context: None,
8190 additional_instructions: Vec::new(),
8191 initial_message: None,
8192 runtime_mode_override: None,
8193 backend: None,
8194 binding: None,
8195 },
8196 crate::identity_first::IdentityLifecycleState::Active,
8197 Some(crate::identity_first::ContinuityRecord {
8198 identity: identity.clone(),
8199 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
8200 "rt:review:singleton:0",
8201 )?,
8202 session_id: SessionId::new(),
8203 generation: crate::identity_first::ContinuityGeneration::new(1),
8204 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
8205 }),
8206 Some(crate::identity_first::LeaseGrant {
8207 identity,
8208 fencing_token: crate::identity_first::FencingToken::new(8),
8209 ttl: Duration::from_mins(5),
8210 }),
8211 )
8212 .await;
8213
8214 let aggregator = MobKitConsoleAggregator::in_memory();
8215 aggregator
8216 .inner
8217 .runtimes
8218 .write()
8219 .expect("runtime registry")
8220 .insert(
8221 "runtime-a".to_string(),
8222 RuntimeEntry {
8223 runtime_key: "runtime-a".to_string(),
8224 identity_namespace: String::new(),
8225 runtime: runtime.mob_runtime().clone(),
8226 identity_runtime: Some(identity_runtime),
8227 console_events: runtime.console_events(),
8228 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8229 },
8230 );
8231
8232 let err = aggregator
8233 .inspect_identity("review:singleton")
8234 .await
8235 .expect_err("duplicate live alias must not inspect via durable fast path");
8236 assert!(
8237 err.to_string().contains("ambiguous live identity alias"),
8238 "unexpected error: {err}"
8239 );
8240 let retire_err = aggregator
8241 .retire_identity("review:singleton")
8242 .await
8243 .expect_err("duplicate live alias must not retire via durable fast path");
8244 assert!(
8245 retire_err
8246 .to_string()
8247 .contains("ambiguous live identity alias"),
8248 "unexpected retire error: {retire_err}"
8249 );
8250 let runtime_id_err = aggregator
8251 .inspect_identity("rt:review:singleton:0")
8252 .await
8253 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
8254 assert!(
8255 runtime_id_err
8256 .to_string()
8257 .contains("ambiguous live identity alias"),
8258 "unexpected runtime-id inspect error: {runtime_id_err}"
8259 );
8260 let runtime_id_retire_err = aggregator
8261 .retire_identity("rt:review:singleton:0")
8262 .await
8263 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
8264 assert!(
8265 runtime_id_retire_err
8266 .to_string()
8267 .contains("ambiguous live identity alias")
8268 || runtime_id_retire_err
8269 .to_string()
8270 .contains("stale durable identity alias"),
8271 "unexpected runtime-id retire error: {runtime_id_retire_err}"
8272 );
8273
8274 aggregator
8275 .reserve_identity_first_interaction(
8276 ConsoleSendRequest {
8277 identity: "review:singleton".to_string(),
8278 content: json!("hello"),
8279 origin: "console:test".to_string(),
8280 idempotency_key: "duplicate-live-alias-durable-reserve".to_string(),
8281 handling_mode: Some("queue".to_string()),
8282 },
8283 None,
8284 )
8285 .await
8286 .expect("durable identity binding remains authoritative for interaction reserve");
8287
8288 let _ = runtime.mob_handle().stop().await;
8289 Ok(())
8290 }
8291
8292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8293 async fn inspect_identity_ignores_hidden_live_alias_candidates()
8294 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8295 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
8296 let mut labels = BTreeMap::new();
8297 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
8298 spawn_trusted_identity_projected_member(
8299 &runtime,
8300 SpawnMemberSpec::from_wire(
8301 "worker".to_string(),
8302 "rt:review:singleton:0".to_string(),
8303 Some("You are the visible Review Agent.".into()),
8304 None,
8305 None,
8306 )
8307 .with_labels(labels.clone()),
8308 )
8309 .await;
8310 spawn_trusted_identity_projected_member(
8311 &runtime,
8312 SpawnMemberSpec::from_wire(
8313 "worker".to_string(),
8314 "rt:review:singleton:1".to_string(),
8315 Some("You are hidden maintenance noise.".into()),
8316 None,
8317 None,
8318 )
8319 .with_labels(labels),
8320 )
8321 .await;
8322
8323 let aggregator = MobKitConsoleAggregator::in_memory();
8324 aggregator.register_runtime_handles_with_policy(
8325 "runtime-a",
8326 "",
8327 runtime.mob_runtime().clone(),
8328 None,
8329 runtime.console_events(),
8330 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
8331 );
8332
8333 let inspection = aggregator
8334 .inspect_identity("review:singleton")
8335 .await?
8336 .expect("visible live alias should inspect without hidden ambiguity");
8337 assert_eq!(inspection.identity.identity, "review:singleton");
8338 assert_eq!(
8339 inspection.identity.runtime_member_id,
8340 "rt:review:singleton:0"
8341 );
8342
8343 let retired = aggregator.retire_identity("review:singleton").await?;
8344 assert!(
8345 retired,
8346 "visible live alias should retire without hidden ambiguity"
8347 );
8348
8349 let _ = runtime.mob_handle().stop().await;
8350 Ok(())
8351 }
8352
8353 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8354 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
8355 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8356 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
8357 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
8358 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
8359 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
8360
8361 let aggregator = MobKitConsoleAggregator::in_memory();
8362 aggregator.register_runtime_handles_with_policy(
8363 "a-hidden",
8364 "",
8365 hidden_runtime.mob_runtime().clone(),
8366 Some(hidden_identity_runtime),
8367 hidden_runtime.console_events(),
8368 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
8369 );
8370 aggregator.register_runtime_handles_with_policy(
8371 "b-visible",
8372 "",
8373 visible_runtime.mob_runtime().clone(),
8374 Some(visible_identity_runtime),
8375 visible_runtime.console_events(),
8376 Arc::new(AllowAllConsoleVisibilityPolicy),
8377 );
8378
8379 let accepted = aggregator
8380 .reserve_identity_first_interaction(
8381 ConsoleSendRequest {
8382 identity: "review:singleton".to_string(),
8383 content: json!("hello"),
8384 origin: "console:test".to_string(),
8385 idempotency_key: "visible-after-hidden-reserve".to_string(),
8386 handling_mode: Some("queue".to_string()),
8387 },
8388 None,
8389 )
8390 .await?;
8391 let page = aggregator
8392 .query_timeline(ConsoleTimelineQuery {
8393 identity: Some("review:singleton".to_string()),
8394 ..ConsoleTimelineQuery::default()
8395 })
8396 .await?;
8397 let frame = page
8398 .frames
8399 .iter()
8400 .find(|frame| frame.id == accepted.input_frame_id)
8401 .ok_or("accepted frame missing from timeline")?;
8402 assert_eq!(frame.runtime_key, "b-visible");
8403
8404 let _ = hidden_runtime.mob_handle().stop().await;
8405 let _ = visible_runtime.mob_handle().stop().await;
8406 Ok(())
8407 }
8408
8409 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8410 async fn identity_first_list_identities_projects_cached_topology_peers()
8411 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8412 let runtime = build_empty_runtime("identity-topology-cache-test").await;
8413 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
8414 identity_runtime
8415 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
8416 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
8417 crate::identity_first::AgentIdentity::parse("agent:beta")?,
8418 )?])
8419 .await;
8420
8421 let aggregator = MobKitConsoleAggregator::in_memory();
8422 aggregator.register_runtime_handles_with_policy(
8423 "identity-first",
8424 "",
8425 runtime.mob_runtime().clone(),
8426 Some(identity_runtime),
8427 runtime.console_events(),
8428 Arc::new(AllowAllConsoleVisibilityPolicy),
8429 );
8430
8431 let identities = aggregator.list_identities().await?;
8432 let alpha = identities
8433 .iter()
8434 .find(|record| record.identity == "agent:alpha")
8435 .ok_or("agent:alpha identity missing")?;
8436 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
8437 let inspection = aggregator
8438 .inspect_identity("agent:alpha")
8439 .await?
8440 .ok_or("agent:alpha inspection missing")?;
8441 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
8442
8443 let _ = runtime.mob_handle().stop().await;
8444 Ok(())
8445 }
8446
8447 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8448 async fn identity_first_list_identities_includes_member_only_spawned_workers()
8449 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
8450 let runtime = build_empty_runtime("identity-member-only-test").await;
8451 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
8452 let aggregator = MobKitConsoleAggregator::in_memory();
8453 aggregator.register_runtime_handles_with_policy(
8454 "identity-first",
8455 "",
8456 runtime.mob_runtime().clone(),
8457 Some(identity_runtime),
8458 runtime.console_events(),
8459 Arc::new(AllowAllConsoleVisibilityPolicy),
8460 );
8461
8462 runtime
8463 .spawn(SpawnMemberSpec::from_wire(
8464 "worker".to_string(),
8465 "agent:beta".to_string(),
8466 Some("You are beta.".into()),
8467 None,
8468 None,
8469 ))
8470 .await?;
8471
8472 let identities = aggregator.list_identities().await?;
8473 assert!(
8474 identities
8475 .iter()
8476 .any(|record| record.identity == "agent:beta"),
8477 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
8478 );
8479
8480 let _ = runtime.mob_handle().stop().await;
8481 Ok(())
8482 }
8483
8484 #[tokio::test]
8485 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
8486 let aggregator = MobKitConsoleAggregator::in_memory();
8487 let record = identity_record_for_test("agent-cached");
8488 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
8489 aggregator
8490 .inner
8491 .identity_read_model
8492 .primed
8493 .store(true, Ordering::Release);
8494 let _guard = aggregator
8495 .inner
8496 .identity_read_model
8497 .refresh_lock
8498 .clone()
8499 .lock_owned()
8500 .await;
8501
8502 let identities =
8503 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
8504 .await
8505 .expect("hot identity list should not wait for refresh lock")
8506 .expect("identity list succeeds");
8507
8508 assert_eq!(identities, vec![record]);
8509 }
8510
8511 #[tokio::test]
8512 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
8513 let aggregator = MobKitConsoleAggregator::in_memory();
8514 let guard = aggregator
8515 .inner
8516 .identity_read_model
8517 .refresh_lock
8518 .clone()
8519 .lock_owned()
8520 .await;
8521 let waiter = tokio::spawn({
8522 let aggregator = aggregator.clone();
8523 async move { aggregator.list_identities().await }
8524 });
8525
8526 tokio::time::sleep(Duration::from_millis(20)).await;
8527 assert!(
8528 !waiter.is_finished(),
8529 "cold identity list should wait for the in-flight refresh to finish"
8530 );
8531
8532 let record = identity_record_for_test("agent-primed");
8533 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
8534 aggregator
8535 .inner
8536 .identity_read_model
8537 .primed
8538 .store(true, Ordering::Release);
8539 drop(guard);
8540
8541 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
8542 .await
8543 .expect("cold identity list waiter should resume")
8544 .expect("waiter joins")
8545 .expect("identity list succeeds");
8546 assert_eq!(identities, vec![record]);
8547 }
8548
8549 #[tokio::test]
8550 async fn query_timeline_reads_from_aggregate_store() {
8551 let aggregator = MobKitConsoleAggregator::in_memory();
8552 let frame = NewConsoleFrame {
8553 id: None,
8554 dedupe_key: "event-1".to_string(),
8555 timestamp_ms: 1,
8556 runtime_key: "runtime-a".to_string(),
8557 identity: "agent-a".to_string(),
8558 conversation_id: Some("agent-a".to_string()),
8559 session_id: None,
8560 kind: "text_delta".to_string(),
8561 status: ConsoleFrameStatus::Delivered,
8562 payload: json!({ "delta": "hello" }),
8563 source: ConsoleFrameSource {
8564 kind: ConsoleFrameSourceKind::ConsoleEvent,
8565 source_cursor: None,
8566 },
8567 source_event_id: Some("event-1".to_string()),
8568 interaction_id: None,
8569 turn_id: None,
8570 run_id: None,
8571 parent_frame_id: None,
8572 caused_by_frame_id: None,
8573 };
8574 aggregator
8575 .store()
8576 .append_if_absent(frame)
8577 .await
8578 .expect("append frame");
8579
8580 let page = aggregator
8581 .query_timeline(ConsoleTimelineQuery {
8582 identity: Some("agent-a".to_string()),
8583 limit: 10,
8584 ..ConsoleTimelineQuery::default()
8585 })
8586 .await
8587 .expect("query timeline");
8588 assert_eq!(page.frames.len(), 1);
8589 assert_eq!(page.frames[0].kind, "text_delta");
8590 }
8591
8592 #[tokio::test]
8604 async fn system_identity_frames_are_visible_on_global_timeline_queries() {
8605 let aggregator = MobKitConsoleAggregator::in_memory();
8606 let runtime = build_single_member_runtime().await;
8607 aggregator
8611 .inner
8612 .runtimes
8613 .write()
8614 .expect("runtime registry")
8615 .insert(
8616 "runtime-a".to_string(),
8617 runtime_entry_for_test("runtime-a", &runtime),
8618 );
8619 let entry = runtime_entry_for_test("runtime-a", &runtime);
8620
8621 let envelope = |event_id: &str, identity: &str, event_type: &str| {
8622 crate::console_contracts::ConsoleIdentityEventEnvelope {
8623 event_id: event_id.to_string(),
8624 interaction_id: None,
8625 identity: identity.to_string(),
8626 event_type: event_type.to_string(),
8627 timestamp_ms: 1_000,
8628 data: json!({ "realm": "default" }),
8629 }
8630 };
8631
8632 let system_frame = frame_from_console_event(
8633 &entry,
8634 envelope("evt-mem-1", SYSTEM_EVENT_IDENTITY, "memory.dream.completed"),
8635 );
8636 assert_eq!(
8637 system_frame.identity, SYSTEM_EVENT_IDENTITY,
8638 "projection must not namespace the system identity"
8639 );
8640 let ghost_frame = frame_from_console_event(
8641 &entry,
8642 envelope("evt-ghost-1", "ghost-agent", "memory.dream.completed"),
8643 );
8644 assert_eq!(
8645 ghost_frame.identity, "test/ghost-agent",
8646 "ordinary identities keep the namespace"
8647 );
8648 for frame in [system_frame, ghost_frame] {
8649 aggregator
8650 .store()
8651 .append_if_absent(frame)
8652 .await
8653 .expect("append frame");
8654 }
8655
8656 let page = aggregator
8659 .query_timeline(ConsoleTimelineQuery {
8660 identity: None,
8661 limit: 10,
8662 ..ConsoleTimelineQuery::default()
8663 })
8664 .await
8665 .expect("query timeline");
8666 let identities: Vec<&str> = page
8667 .frames
8668 .iter()
8669 .map(|frame| frame.identity.as_str())
8670 .collect();
8671 assert!(
8672 identities.contains(&SYSTEM_EVENT_IDENTITY),
8673 "system-attributed memory event must be globally visible: {identities:?}"
8674 );
8675 assert!(
8676 !identities.contains(&"test/ghost-agent"),
8677 "unknown identities must stay hidden — the exemption is exact: {identities:?}"
8678 );
8679
8680 let _ = runtime.mob_handle().stop().await;
8681 }
8682
8683 #[tokio::test]
8684 async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
8685 let aggregator = MobKitConsoleAggregator::in_memory();
8686 let runtime = build_single_member_runtime().await;
8687 let entry = runtime_entry_for_test("runtime-a", &runtime);
8688 for event_type in ["reasoning_delta", "reasoning_complete"] {
8689 let frame = frame_from_console_event(
8690 &entry,
8691 console_envelope_for_test(
8692 event_type,
8693 Some("interaction-a"),
8694 event_type,
8695 json!({
8696 "delta": "I should inspect the file first.",
8697 "text": "I should inspect the file first.",
8698 "turn_id": "turn-a",
8699 }),
8700 ),
8701 );
8702 aggregator
8703 .store()
8704 .append_if_absent(frame)
8705 .await
8706 .expect("append reasoning frame");
8707 }
8708
8709 let page = aggregator
8710 .query_timeline(ConsoleTimelineQuery {
8711 identity: Some("test/agent-a".to_string()),
8712 limit: 10,
8713 ..ConsoleTimelineQuery::default()
8714 })
8715 .await
8716 .expect("query timeline");
8717
8718 assert_eq!(
8719 page.frames.len(),
8720 1,
8721 "identical reasoning re-emissions in one turn should collapse"
8722 );
8723 assert_eq!(page.frames[0].kind, "reasoning_delta");
8724 assert_eq!(
8725 page.frames[0].dedupe_key,
8726 format!(
8727 "console-reasoning:runtime-a:interaction-a:{}",
8728 hash_short("I should inspect the file first.")
8729 )
8730 );
8731 }
8732
8733 #[tokio::test]
8734 async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
8735 let aggregator = MobKitConsoleAggregator::in_memory();
8736 let runtime = build_single_member_runtime().await;
8737 let entry = runtime_entry_for_test("runtime-a", &runtime);
8738 for (event_id, text) in [
8739 ("reasoning-1", "I should inspect the file first."),
8740 ("reasoning-2", "Now I should run the focused test."),
8741 ] {
8742 let frame = frame_from_console_event(
8743 &entry,
8744 console_envelope_for_test(
8745 event_id,
8746 Some("interaction-a"),
8747 "reasoning_complete",
8748 json!({
8749 "text": text,
8750 "turn_id": "turn-a",
8751 }),
8752 ),
8753 );
8754 aggregator
8755 .store()
8756 .append_if_absent(frame)
8757 .await
8758 .expect("append reasoning frame");
8759 }
8760
8761 let page = aggregator
8762 .query_timeline(ConsoleTimelineQuery {
8763 identity: Some("test/agent-a".to_string()),
8764 limit: 10,
8765 ..ConsoleTimelineQuery::default()
8766 })
8767 .await
8768 .expect("query timeline");
8769
8770 assert_eq!(
8771 page.frames.len(),
8772 2,
8773 "distinct same-turn reasoning segments should not collapse"
8774 );
8775 assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
8776 }
8777
8778 #[tokio::test]
8779 async fn identity_recent_anchor_respects_query_limit() {
8780 let aggregator = MobKitConsoleAggregator::in_memory();
8781 aggregator
8782 .store()
8783 .append_if_absent(NewConsoleFrame {
8784 id: None,
8785 dedupe_key: "anchored-user-input".to_string(),
8786 timestamp_ms: 1,
8787 runtime_key: "runtime-a".to_string(),
8788 identity: "agent-a".to_string(),
8789 conversation_id: Some("agent-a".to_string()),
8790 session_id: None,
8791 kind: "user_input".to_string(),
8792 status: ConsoleFrameStatus::Delivered,
8793 payload: json!({ "content": "anchor me" }),
8794 source: ConsoleFrameSource {
8795 kind: ConsoleFrameSourceKind::Synthetic,
8796 source_cursor: None,
8797 },
8798 source_event_id: Some("anchored-user-input".to_string()),
8799 interaction_id: Some("turn-a".to_string()),
8800 turn_id: None,
8801 run_id: None,
8802 parent_frame_id: None,
8803 caused_by_frame_id: None,
8804 })
8805 .await
8806 .expect("append anchor");
8807 for idx in 2..=40 {
8808 aggregator
8809 .store()
8810 .append_if_absent(NewConsoleFrame {
8811 id: None,
8812 dedupe_key: format!("noisy-tail-{idx}"),
8813 timestamp_ms: idx,
8814 runtime_key: "runtime-a".to_string(),
8815 identity: "agent-a".to_string(),
8816 conversation_id: Some("agent-a".to_string()),
8817 session_id: None,
8818 kind: "reasoning_delta".to_string(),
8819 status: ConsoleFrameStatus::Delivered,
8820 payload: json!({ "delta": idx }),
8821 source: ConsoleFrameSource {
8822 kind: ConsoleFrameSourceKind::ConsoleEvent,
8823 source_cursor: None,
8824 },
8825 source_event_id: Some(format!("noisy-tail-{idx}")),
8826 interaction_id: Some("turn-a".to_string()),
8827 turn_id: None,
8828 run_id: None,
8829 parent_frame_id: None,
8830 caused_by_frame_id: None,
8831 })
8832 .await
8833 .expect("append noisy tail");
8834 }
8835
8836 let page = aggregator
8837 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8838 identity: Some("agent-a".to_string()),
8839 mode: ConsoleTimelineMode::Recent,
8840 limit: 5,
8841 ..ConsoleTimelineWindowQuery::default()
8842 })
8843 .await
8844 .expect("query recent identity timeline");
8845
8846 assert_eq!(
8847 page.frames.len(),
8848 5,
8849 "identity anchor merge must not exceed the requested limit"
8850 );
8851 assert!(
8852 page.frames
8853 .iter()
8854 .any(|frame| frame.dedupe_key == "anchored-user-input"),
8855 "the bounded result should still retain the useful turn anchor: {:#?}",
8856 page.frames
8857 );
8858 assert_eq!(
8859 page.frames.last().and_then(|frame| frame.cursor.seq()),
8860 Some(40)
8861 );
8862 }
8863
8864 #[tokio::test]
8865 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
8866 let aggregator = MobKitConsoleAggregator::in_memory();
8867 let runtime = build_single_member_runtime().await;
8868 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
8869 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
8870 aggregator
8871 .inner
8872 .runtimes
8873 .write()
8874 .expect("runtime registry")
8875 .insert("runtime-a".to_string(), entry);
8876
8877 for idx in 0..1_500 {
8878 aggregator
8879 .store()
8880 .append_if_absent(NewConsoleFrame {
8881 id: None,
8882 dedupe_key: format!("hidden-gap-{idx}"),
8883 timestamp_ms: idx,
8884 runtime_key: "runtime-a".to_string(),
8885 identity: "agent-a".to_string(),
8886 conversation_id: Some("agent-a".to_string()),
8887 session_id: None,
8888 kind: "hidden_noise".to_string(),
8889 status: ConsoleFrameStatus::Delivered,
8890 payload: json!({ "delta": idx }),
8891 source: ConsoleFrameSource {
8892 kind: ConsoleFrameSourceKind::ConsoleEvent,
8893 source_cursor: None,
8894 },
8895 source_event_id: Some(format!("hidden-gap-{idx}")),
8896 interaction_id: None,
8897 turn_id: None,
8898 run_id: None,
8899 parent_frame_id: None,
8900 caused_by_frame_id: None,
8901 })
8902 .await
8903 .expect("append hidden frame");
8904 }
8905 aggregator
8906 .store()
8907 .append_if_absent(NewConsoleFrame {
8908 id: None,
8909 dedupe_key: "visible-after-gap".to_string(),
8910 timestamp_ms: 1_501,
8911 runtime_key: "runtime-a".to_string(),
8912 identity: "agent-a".to_string(),
8913 conversation_id: Some("agent-a".to_string()),
8914 session_id: None,
8915 kind: "interaction_complete".to_string(),
8916 status: ConsoleFrameStatus::Delivered,
8917 payload: json!({ "text": "visible after hidden gap" }),
8918 source: ConsoleFrameSource {
8919 kind: ConsoleFrameSourceKind::ConsoleEvent,
8920 source_cursor: None,
8921 },
8922 source_event_id: Some("visible-after-gap".to_string()),
8923 interaction_id: Some("turn-visible".to_string()),
8924 turn_id: None,
8925 run_id: None,
8926 parent_frame_id: None,
8927 caused_by_frame_id: None,
8928 })
8929 .await
8930 .expect("append visible frame");
8931
8932 let page = aggregator
8933 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8934 identity: Some("agent-a".to_string()),
8935 mode: ConsoleTimelineMode::Since,
8936 limit: 1,
8937 ..ConsoleTimelineWindowQuery::default()
8938 })
8939 .await
8940 .expect("query timeline");
8941 assert_eq!(page.frames.len(), 1);
8942 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
8943 assert_eq!(
8944 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8945 Some(1_501)
8946 );
8947 let _ = runtime.mob_handle().stop().await;
8948 }
8949
8950 #[tokio::test]
8951 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
8952 let aggregator = MobKitConsoleAggregator::in_memory();
8953 for idx in 1..=300 {
8954 aggregator
8955 .store()
8956 .append_if_absent(NewConsoleFrame {
8957 id: None,
8958 dedupe_key: format!("visible-{idx}"),
8959 timestamp_ms: idx,
8960 runtime_key: "runtime-a".to_string(),
8961 identity: "agent-a".to_string(),
8962 conversation_id: Some("agent-a".to_string()),
8963 session_id: None,
8964 kind: "interaction_complete".to_string(),
8965 status: ConsoleFrameStatus::Delivered,
8966 payload: json!({ "text": format!("visible {idx}") }),
8967 source: ConsoleFrameSource {
8968 kind: ConsoleFrameSourceKind::ConsoleEvent,
8969 source_cursor: None,
8970 },
8971 source_event_id: Some(format!("visible-{idx}")),
8972 interaction_id: Some(format!("turn-{idx}")),
8973 turn_id: None,
8974 run_id: None,
8975 parent_frame_id: None,
8976 caused_by_frame_id: None,
8977 })
8978 .await
8979 .expect("append visible frame");
8980 }
8981
8982 let first = aggregator
8983 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8984 identity: Some("agent-a".to_string()),
8985 mode: ConsoleTimelineMode::Since,
8986 limit: 10,
8987 ..ConsoleTimelineWindowQuery::default()
8988 })
8989 .await
8990 .expect("query first page");
8991 assert_eq!(first.frames.len(), 10);
8992 assert_eq!(
8993 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8994 Some(10)
8995 );
8996
8997 let second = aggregator
8998 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8999 identity: Some("agent-a".to_string()),
9000 mode: ConsoleTimelineMode::Since,
9001 after: first.next_cursor,
9002 limit: 10,
9003 ..ConsoleTimelineWindowQuery::default()
9004 })
9005 .await
9006 .expect("query second page");
9007 assert_eq!(second.frames.len(), 10);
9008 assert_eq!(second.frames[0].dedupe_key, "visible-11");
9009 assert_eq!(
9010 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
9011 Some(20)
9012 );
9013 }
9014
9015 #[tokio::test]
9016 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
9017 let store = Arc::new(CountingConsoleLogStore::new());
9018 let aggregator = MobKitConsoleAggregator::new(store.clone());
9019 let runtime = build_single_member_runtime().await;
9020 aggregator
9021 .inner
9022 .runtimes
9023 .write()
9024 .expect("runtime registry")
9025 .insert(
9026 "runtime-a".to_string(),
9027 runtime_entry_for_test("runtime-a", &runtime),
9028 );
9029 let inserted = store
9030 .append_if_absent(NewConsoleFrame {
9031 id: None,
9032 dedupe_key: "event-1".to_string(),
9033 timestamp_ms: 1,
9034 runtime_key: "runtime-a".to_string(),
9035 identity: "agent-a".to_string(),
9036 conversation_id: Some("agent-a".to_string()),
9037 session_id: None,
9038 kind: "text_delta".to_string(),
9039 status: ConsoleFrameStatus::Delivered,
9040 payload: json!({ "delta": "hello" }),
9041 source: ConsoleFrameSource {
9042 kind: ConsoleFrameSourceKind::ConsoleEvent,
9043 source_cursor: None,
9044 },
9045 source_event_id: Some("event-1".to_string()),
9046 interaction_id: None,
9047 turn_id: None,
9048 run_id: None,
9049 parent_frame_id: None,
9050 caused_by_frame_id: None,
9051 })
9052 .await
9053 .expect("append frame");
9054
9055 let page = aggregator
9056 .query_timeline_windowed(ConsoleTimelineWindowQuery {
9057 identity: Some("agent-a".to_string()),
9058 mode: ConsoleTimelineMode::Since,
9059 after: Some(inserted.frame.cursor),
9060 limit: 10,
9061 ..ConsoleTimelineWindowQuery::default()
9062 })
9063 .await
9064 .expect("query continuation");
9065
9066 assert!(page.frames.is_empty());
9067 assert_eq!(
9068 store.source_watermark_calls(),
9069 0,
9070 "empty since continuation must not synchronously force session-history backfill"
9071 );
9072 let _ = runtime.mob_handle().stop().await;
9073 }
9074
9075 #[test]
9076 fn legacy_timeline_struct_literals_remain_source_compatible() {
9077 let query = ConsoleTimelineQuery {
9078 identity: Some("agent-a".to_string()),
9079 conversation_id: None,
9080 after: Some(ConsoleCursor::from("console:1")),
9081 limit: 10,
9082 };
9083 let page = ConsoleTimelinePage {
9084 frames: Vec::new(),
9085 next_cursor: query.after.clone(),
9086 };
9087
9088 assert_eq!(query.limit, 10);
9089 assert_eq!(
9090 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
9091 Some(1)
9092 );
9093 }
9094
9095 #[tokio::test]
9096 async fn query_timeline_is_store_local_for_registered_runtimes() {
9097 let store = Arc::new(CountingConsoleLogStore::new());
9098 let aggregator = MobKitConsoleAggregator::new(store.clone());
9099 let runtime = build_single_member_runtime().await;
9100 aggregator
9101 .inner
9102 .runtimes
9103 .write()
9104 .expect("runtime registry")
9105 .insert(
9106 "runtime-a".to_string(),
9107 runtime_entry_for_test("runtime-a", &runtime),
9108 );
9109 store
9110 .append_if_absent(NewConsoleFrame {
9111 id: None,
9112 dedupe_key: "event-1".to_string(),
9113 timestamp_ms: 1,
9114 runtime_key: "runtime-a".to_string(),
9115 identity: "agent-a".to_string(),
9116 conversation_id: Some("agent-a".to_string()),
9117 session_id: None,
9118 kind: "text_delta".to_string(),
9119 status: ConsoleFrameStatus::Delivered,
9120 payload: json!({ "delta": "hello" }),
9121 source: ConsoleFrameSource {
9122 kind: ConsoleFrameSourceKind::ConsoleEvent,
9123 source_cursor: None,
9124 },
9125 source_event_id: Some("event-1".to_string()),
9126 interaction_id: None,
9127 turn_id: None,
9128 run_id: None,
9129 parent_frame_id: None,
9130 caused_by_frame_id: None,
9131 })
9132 .await
9133 .expect("append frame");
9134
9135 let page = tokio::time::timeout(
9136 Duration::from_millis(250),
9137 aggregator.query_timeline(ConsoleTimelineQuery {
9138 identity: Some("agent-a".to_string()),
9139 limit: 10,
9140 ..ConsoleTimelineQuery::default()
9141 }),
9142 )
9143 .await
9144 .expect("timeline query should not wait for session history")
9145 .expect("timeline query succeeds");
9146
9147 assert_eq!(page.frames.len(), 1);
9148 assert_eq!(
9149 store.source_watermark_calls(),
9150 0,
9151 "query_timeline must not synchronously touch session-history watermarks"
9152 );
9153 let _ = runtime.mob_handle().stop().await;
9154 }
9155
9156 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9157 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
9158 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
9159 delay: Duration::from_secs(2),
9160 }))
9161 .await;
9162 let aggregator = MobKitConsoleAggregator::in_memory();
9163 aggregator
9164 .inner
9165 .runtimes
9166 .write()
9167 .expect("runtime registry")
9168 .insert(
9169 "runtime-a".to_string(),
9170 runtime_entry_for_test("runtime-a", &runtime),
9171 );
9172
9173 let start = Instant::now();
9174 let accepted = tokio::time::timeout(
9175 Duration::from_millis(300),
9176 aggregator.send(ConsoleSendRequest {
9177 identity: "test/agent-a".to_string(),
9178 content: json!("hello slow agent"),
9179 origin: "console:test".to_string(),
9180 idempotency_key: "nonblocking-send".to_string(),
9181 handling_mode: Some("queue".to_string()),
9182 }),
9183 )
9184 .await
9185 .expect("console send should return once the input is accepted")
9186 .expect("send succeeds");
9187
9188 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
9189 assert!(
9190 start.elapsed() < Duration::from_secs(1),
9191 "console send should not wait for the delayed LLM turn"
9192 );
9193
9194 wait_for_session_history_text(
9195 &aggregator,
9196 "test/agent-a",
9197 "slow ok",
9198 Duration::from_secs(5),
9199 )
9200 .await
9201 .expect("background dispatch should still complete and project history");
9202 let _ = runtime.mob_handle().stop().await;
9203 }
9204
9205 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9206 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
9207 {
9208 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
9209 let aggregator = MobKitConsoleAggregator::in_memory();
9210 aggregator.register_runtime(ConsoleRuntimeRegistration {
9211 runtime_key: "runtime-late".to_string(),
9212 runtime: runtime.clone(),
9213 identity_namespace: "late".to_string(),
9214 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
9215 });
9216
9217 runtime
9218 .spawn(SpawnMemberSpec::from_wire(
9219 "worker".to_string(),
9220 "agent-late".to_string(),
9221 Some("You are agent-late.".into()),
9222 None,
9223 None,
9224 ))
9225 .await
9226 .expect("late member spawns");
9227 let session_id = send_message_on_mob_with_mode(
9228 &runtime.mob_handle(),
9229 "agent-late",
9230 ContentInput::Text("hello after registration".to_string()),
9231 meerkat_core::types::HandlingMode::Queue,
9232 )
9233 .await
9234 .expect("direct member send succeeds");
9235 wait_for_identity_record(
9236 &aggregator,
9237 "late/agent-late",
9238 Some(session_id.as_str()),
9239 Duration::from_secs(5),
9240 )
9241 .await?;
9242
9243 wait_for_session_history_text(
9244 &aggregator,
9245 "late/agent-late",
9246 "You are agent-late.",
9247 Duration::from_secs(5),
9248 )
9249 .await?;
9250 let _ = runtime.mob_handle().stop().await;
9251 Ok(())
9252 }
9253
9254 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9255 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
9256 -> Result<(), String> {
9257 let runtime = build_single_member_runtime().await;
9258 let entry = runtime_entry_for_test("runtime-a", &runtime);
9259 let resolved = member_sources_for_entry(&entry)
9260 .await
9261 .into_iter()
9262 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
9263 .expect("agent-a member exists");
9264 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
9265 .await
9266 .expect("identity record exists");
9267 let session_id = record.session_id.expect("agent-a has a session");
9268 wait_for_runtime_session_history_text(
9269 &runtime,
9270 &session_id,
9271 "You are agent-a.",
9272 Duration::from_secs(5),
9273 )
9274 .await?;
9275
9276 let aggregator = MobKitConsoleAggregator::in_memory();
9277 aggregator
9278 .inner
9279 .runtimes
9280 .write()
9281 .expect("runtime registry")
9282 .insert("runtime-a".to_string(), entry);
9283 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
9284 aggregator
9285 .store()
9286 .record_source_watermark(
9287 &watermark_key,
9288 ConsoleFrameSourceKind::SessionHistory,
9289 &format_session_history_watermark(&session_id, 0, current_time_ms()),
9290 )
9291 .await
9292 .expect("record fresh empty watermark");
9293
9294 let page = tokio::time::timeout(
9298 Duration::from_mins(1),
9299 aggregator.query_timeline(ConsoleTimelineQuery {
9300 identity: Some("test/agent-a".to_string()),
9301 limit: 20,
9302 ..ConsoleTimelineQuery::default()
9303 }),
9304 )
9305 .await
9306 .expect("explicit identity query should not stall")
9307 .expect("query succeeds");
9308
9309 if !page.frames.is_empty() {
9310 assert!(
9311 page.frames.iter().any(|frame| {
9312 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
9313 && session_history_content_text(frame).as_deref()
9314 == Some("You are agent-a.")
9315 }),
9316 "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
9317 page.frames
9318 );
9319 }
9320 wait_for_session_history_text(
9321 &aggregator,
9322 "test/agent-a",
9323 "You are agent-a.",
9324 Duration::from_secs(5),
9325 )
9326 .await?;
9327 let _ = runtime.mob_handle().stop().await;
9328 Ok(())
9329 }
9330
9331 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9332 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
9333 {
9334 let store = Arc::new(CountingConsoleLogStore::new());
9335 let aggregator = MobKitConsoleAggregator::new(store.clone());
9336 let runtime = build_single_member_runtime().await;
9337 let entry = runtime_entry_for_test("runtime-a", &runtime);
9338 let resolved = member_sources_for_entry(&entry)
9339 .await
9340 .into_iter()
9341 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
9342 .expect("agent-a member exists");
9343 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
9344 .await
9345 .expect("identity record exists");
9346 let session_id = record.session_id.expect("agent-a has a session");
9347 wait_for_runtime_session_history_text(
9348 &runtime,
9349 &session_id,
9350 "You are agent-a.",
9351 Duration::from_secs(5),
9352 )
9353 .await?;
9354 aggregator
9355 .inner
9356 .runtimes
9357 .write()
9358 .expect("runtime registry")
9359 .insert("runtime-a".to_string(), entry);
9360
9361 store
9362 .append_if_absent(NewConsoleFrame {
9363 id: None,
9364 dedupe_key: "stale-session-history-agent-a".to_string(),
9365 timestamp_ms: 10,
9366 runtime_key: "runtime-a".to_string(),
9367 identity: "test/agent-a".to_string(),
9368 conversation_id: Some("test/agent-a".to_string()),
9369 session_id: Some(session_id.clone()),
9370 kind: "user_input".to_string(),
9371 status: ConsoleFrameStatus::Delivered,
9372 payload: json!({
9373 "text": "stale projected history",
9374 "type": "user_input",
9375 }),
9376 source: ConsoleFrameSource {
9377 kind: ConsoleFrameSourceKind::SessionHistory,
9378 source_cursor: Some("stale-session-history-agent-a".to_string()),
9379 },
9380 source_event_id: Some("stale-session-history-agent-a".to_string()),
9381 interaction_id: None,
9382 turn_id: None,
9383 run_id: None,
9384 parent_frame_id: None,
9385 caused_by_frame_id: None,
9386 })
9387 .await
9388 .expect("append stale history frame");
9389
9390 let fresh_prompt = "fresh prompt after stale history";
9391 let sent_session_id = send_message_on_mob_with_mode(
9392 &runtime.mob_handle(),
9393 "agent-a",
9394 ContentInput::Text(fresh_prompt.to_string()),
9395 meerkat_core::types::HandlingMode::Queue,
9396 )
9397 .await
9398 .expect("direct member send succeeds");
9399 assert_eq!(sent_session_id, session_id);
9400 wait_for_runtime_session_history_text(
9401 &runtime,
9402 &session_id,
9403 fresh_prompt,
9404 Duration::from_secs(5),
9405 )
9406 .await?;
9407
9408 let page = aggregator
9409 .query_timeline(ConsoleTimelineQuery {
9410 identity: Some("test/agent-a".to_string()),
9411 limit: 50,
9412 ..ConsoleTimelineQuery::default()
9413 })
9414 .await
9415 .expect("query child timeline");
9416
9417 assert!(
9418 page.frames.iter().any(|frame| {
9419 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
9420 && session_history_content_text(frame).as_deref()
9421 == Some("stale projected history")
9422 }),
9423 "explicit identity query should return existing history before async refresh; frames: {:#?}",
9424 page.frames
9425 );
9426 wait_for_session_history_text(
9427 &aggregator,
9428 "test/agent-a",
9429 fresh_prompt,
9430 Duration::from_secs(5),
9431 )
9432 .await?;
9433 assert!(
9434 store.source_watermark_calls() > 0,
9435 "stale existing session history should force a targeted source refresh"
9436 );
9437 let _ = runtime.mob_handle().stop().await;
9438 Ok(())
9439 }
9440
9441 async fn wait_for_session_history_text(
9442 aggregator: &MobKitConsoleAggregator,
9443 identity: &str,
9444 expected: &str,
9445 timeout: Duration,
9446 ) -> Result<(), String> {
9447 let deadline = Instant::now() + timeout;
9448 let mut observed = Vec::new();
9449 while Instant::now() < deadline {
9450 let _ = aggregator.list_identities().await;
9451 let page = aggregator
9452 .query_timeline(ConsoleTimelineQuery {
9453 identity: Some(identity.to_string()),
9454 limit: 20,
9455 ..ConsoleTimelineQuery::default()
9456 })
9457 .await
9458 .expect("query timeline");
9459 observed = page.frames;
9460 if observed.iter().any(|frame| {
9461 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
9462 && session_history_content_text(frame).as_deref() == Some(expected)
9463 }) {
9464 return Ok(());
9465 }
9466 tokio::time::sleep(Duration::from_millis(25)).await;
9467 }
9468
9469 Err(format!(
9470 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
9471 ))
9472 }
9473
9474 async fn wait_for_session_backfill_target(
9475 aggregator: &MobKitConsoleAggregator,
9476 identity: &str,
9477 timeout: Duration,
9478 ) -> Result<String, String> {
9479 let deadline = Instant::now() + timeout;
9480 while Instant::now() < deadline {
9481 if let Some(target) =
9482 session_backfill_target_for_identity(&aggregator.inner, identity).await
9483 {
9484 return Ok(target.session_id);
9485 }
9486 tokio::time::sleep(Duration::from_millis(25)).await;
9487 }
9488 Err(format!(
9489 "session backfill target for {identity:?} was not resolvable"
9490 ))
9491 }
9492
9493 async fn wait_for_runtime_session_history_text(
9494 runtime: &UnifiedRuntime,
9495 session_id: &str,
9496 expected: &str,
9497 timeout: Duration,
9498 ) -> Result<(), String> {
9499 let deadline = Instant::now() + timeout;
9500 let mut observed = Vec::new();
9501 while Instant::now() < deadline {
9502 let page = runtime
9503 .mob_runtime()
9504 .read_session_history(session_id, 0, Some(20))
9505 .await
9506 .map_err(|err| err.to_string())?;
9507 observed = page.messages;
9508 if observed.iter().enumerate().any(|(idx, message)| {
9509 let Some(message) = serde_json::to_value(message).ok() else {
9510 return false;
9511 };
9512 frames_from_session_history_message(
9513 "runtime-a",
9514 "test/agent-a",
9515 session_id,
9516 idx,
9517 message,
9518 )
9519 .iter()
9520 .any(|frame| {
9521 matches!(
9522 frame.kind.as_str(),
9523 "user_input" | "system_notice" | "interaction_complete"
9524 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
9525 })
9526 }) {
9527 return Ok(());
9528 }
9529 tokio::time::sleep(Duration::from_millis(25)).await;
9530 }
9531
9532 Err(format!(
9533 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
9534 ))
9535 }
9536
9537 async fn wait_for_identity_record(
9538 aggregator: &MobKitConsoleAggregator,
9539 identity: &str,
9540 session_id: Option<&str>,
9541 timeout: Duration,
9542 ) -> Result<(), String> {
9543 let deadline = Instant::now() + timeout;
9544 let mut observed = Vec::new();
9545 while Instant::now() < deadline {
9546 observed = aggregator
9547 .list_identities()
9548 .await
9549 .map_err(|err| err.to_string())?;
9550 if observed.iter().any(|record| {
9551 record.identity == identity && record.session_id.as_deref() == session_id
9552 }) {
9553 return Ok(());
9554 }
9555 tokio::time::sleep(Duration::from_millis(25)).await;
9556 }
9557
9558 Err(format!(
9559 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
9560 ))
9561 }
9562
9563 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
9564 session_history_payload_text(&frame.payload)
9565 }
9566
9567 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
9568 session_history_payload_text(&frame.payload)
9569 }
9570
9571 fn session_history_payload_text(payload: &Value) -> Option<String> {
9572 if let Some(text) = payload.get("text").and_then(Value::as_str) {
9573 return Some(text.to_string());
9574 }
9575 if let Some(text) = payload.get("result").and_then(Value::as_str) {
9576 return Some(text.to_string());
9577 }
9578 if let Some(text) = payload.get("body").and_then(Value::as_str) {
9579 return Some(text.to_string());
9580 }
9581 match payload.get("content")? {
9582 Value::String(text) => Some(text.clone()),
9583 Value::Array(blocks) => Some(
9584 blocks
9585 .iter()
9586 .filter_map(|block| block.get("text").and_then(Value::as_str))
9587 .collect::<Vec<_>>()
9588 .join(""),
9589 ),
9590 _ => None,
9591 }
9592 }
9593
9594 #[tokio::test]
9595 async fn query_timeline_handles_large_store_without_backfill_calls() {
9596 let store = Arc::new(CountingConsoleLogStore::new());
9597 let aggregator = MobKitConsoleAggregator::new(store.clone());
9598 for idx in 0..5_000 {
9599 store
9600 .append_if_absent(NewConsoleFrame {
9601 id: None,
9602 dedupe_key: format!("event-{idx}"),
9603 timestamp_ms: idx,
9604 runtime_key: "runtime-a".to_string(),
9605 identity: "agent-a".to_string(),
9606 conversation_id: Some("agent-a".to_string()),
9607 session_id: Some("session-a".to_string()),
9608 kind: "text_delta".to_string(),
9609 status: ConsoleFrameStatus::Delivered,
9610 payload: json!({ "delta": idx }),
9611 source: ConsoleFrameSource {
9612 kind: ConsoleFrameSourceKind::ConsoleEvent,
9613 source_cursor: None,
9614 },
9615 source_event_id: Some(format!("event-{idx}")),
9616 interaction_id: None,
9617 turn_id: None,
9618 run_id: None,
9619 parent_frame_id: None,
9620 caused_by_frame_id: None,
9621 })
9622 .await
9623 .expect("append frame");
9624 }
9625
9626 let page = aggregator
9627 .query_timeline(ConsoleTimelineQuery {
9628 identity: Some("agent-a".to_string()),
9629 limit: 1_000,
9630 ..ConsoleTimelineQuery::default()
9631 })
9632 .await
9633 .expect("large query");
9634
9635 assert_eq!(page.frames.len(), 1_000);
9636 assert_eq!(store.source_watermark_calls(), 0);
9637 }
9638
9639 #[tokio::test]
9640 async fn query_timeline_rejects_future_cursor_after_store_reset() {
9641 let aggregator = MobKitConsoleAggregator::in_memory();
9642 let err = aggregator
9643 .query_timeline_windowed(ConsoleTimelineWindowQuery {
9644 after: Some(ConsoleCursor::from("console:99")),
9645 limit: 10,
9646 ..ConsoleTimelineWindowQuery::default()
9647 })
9648 .await
9649 .expect_err("future cursor on empty/reset store must be replay-unavailable");
9650
9651 assert!(
9652 err.to_string()
9653 .contains("beyond the current store frontier"),
9654 "unexpected error: {err}"
9655 );
9656 }
9657
9658 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9659 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
9660 let store = Arc::new(CountingConsoleLogStore::new());
9661 let aggregator = MobKitConsoleAggregator::new(store.clone());
9662 let runtime = build_single_member_runtime().await;
9663 aggregator
9664 .inner
9665 .runtimes
9666 .write()
9667 .expect("runtime registry")
9668 .insert(
9669 "runtime-a".to_string(),
9670 runtime_entry_for_test("runtime-a", &runtime),
9671 );
9672 let session_id =
9673 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
9674 .await
9675 .expect("spawned member is resolvable for targeted backfill");
9676 wait_for_runtime_session_history_text(
9677 &runtime,
9678 &session_id,
9679 "You are agent-a.",
9680 Duration::from_secs(5),
9681 )
9682 .await
9683 .expect("spawned member kickoff is readable before live-frame race");
9684 store
9685 .append_if_absent(NewConsoleFrame {
9686 id: None,
9687 dedupe_key: "live-tool-before-history".to_string(),
9688 timestamp_ms: 10,
9689 runtime_key: "runtime-a".to_string(),
9690 identity: "test/agent-a".to_string(),
9691 conversation_id: Some("test/agent-a".to_string()),
9692 session_id: None,
9693 kind: "tool_execution_started".to_string(),
9694 status: ConsoleFrameStatus::Delivered,
9695 payload: json!({
9696 "id": "call-live",
9697 "name": "king_search",
9698 "source_event_type": "tool_execution_started",
9699 "type": "tool_execution_started",
9700 }),
9701 source: ConsoleFrameSource {
9702 kind: ConsoleFrameSourceKind::ConsoleEvent,
9703 source_cursor: Some("live-tool-before-history".to_string()),
9704 },
9705 source_event_id: Some("live-tool-before-history".to_string()),
9706 interaction_id: None,
9707 turn_id: None,
9708 run_id: None,
9709 parent_frame_id: None,
9710 caused_by_frame_id: None,
9711 })
9712 .await
9713 .expect("append live tool frame");
9714
9715 let page = aggregator
9716 .query_timeline(ConsoleTimelineQuery {
9717 identity: Some("test/agent-a".to_string()),
9718 limit: 20,
9719 ..ConsoleTimelineQuery::default()
9720 })
9721 .await
9722 .expect("query child timeline");
9723
9724 assert!(
9725 page.frames
9726 .iter()
9727 .any(|frame| frame.kind == "tool_execution_started"),
9728 "initial explicit child timeline should return existing store frames without waiting for session history"
9729 );
9730
9731 let mut observed_backfill = false;
9732 for _ in 0..80 {
9733 let page = aggregator
9734 .query_timeline(ConsoleTimelineQuery {
9735 identity: Some("test/agent-a".to_string()),
9736 limit: 20,
9737 ..ConsoleTimelineQuery::default()
9738 })
9739 .await
9740 .expect("query child timeline after scheduled backfill");
9741 if page.frames.iter().any(|frame| {
9742 frame.kind == "user_input"
9743 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
9744 && frame.payload.to_string().contains("You are agent-a.")
9745 }) {
9746 observed_backfill = true;
9747 break;
9748 }
9749 tokio::time::sleep(Duration::from_millis(25)).await;
9750 }
9751 assert!(
9752 observed_backfill,
9753 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
9754 );
9755 let _ = runtime.mob_handle().stop().await;
9756 }
9757
9758 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9759 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
9760 let store = Arc::new(CountingConsoleLogStore::new());
9761 let aggregator = MobKitConsoleAggregator::new(store.clone());
9762 let (_temp, runtime, _delayed_service) =
9763 build_stress_runtime(64, Duration::from_millis(0)).await;
9764 aggregator
9765 .inner
9766 .runtimes
9767 .write()
9768 .expect("runtime registry")
9769 .insert(
9770 "runtime-a".to_string(),
9771 runtime_entry_for_test("runtime-a", runtime.as_ref()),
9772 );
9773 for idx in 0..1_000 {
9774 store
9775 .append_if_absent(NewConsoleFrame {
9776 id: None,
9777 dedupe_key: format!("event-{idx}"),
9778 timestamp_ms: idx,
9779 runtime_key: "runtime-a".to_string(),
9780 identity: "test/agent-0".to_string(),
9781 conversation_id: Some("test/agent-0".to_string()),
9782 session_id: Some("session-a".to_string()),
9783 kind: "text_delta".to_string(),
9784 status: ConsoleFrameStatus::Delivered,
9785 payload: json!({ "delta": idx }),
9786 source: ConsoleFrameSource {
9787 kind: ConsoleFrameSourceKind::ConsoleEvent,
9788 source_cursor: None,
9789 },
9790 source_event_id: Some(format!("event-{idx}")),
9791 interaction_id: None,
9792 turn_id: None,
9793 run_id: None,
9794 parent_frame_id: None,
9795 caused_by_frame_id: None,
9796 })
9797 .await
9798 .expect("append frame");
9799 }
9800
9801 let page = tokio::time::timeout(
9808 Duration::from_mins(1),
9809 aggregator.query_timeline(ConsoleTimelineQuery {
9810 identity: Some("test/agent-0".to_string()),
9811 limit: 1_000,
9812 ..ConsoleTimelineQuery::default()
9813 }),
9814 )
9815 .await
9816 .expect("identity timeline query should not hang or rediscover the roster per frame")
9817 .expect("timeline query succeeds");
9818
9819 assert_eq!(page.frames.len(), 1_000);
9820 assert_eq!(store.source_watermark_calls(), 0);
9821 let _ = runtime.mob_handle().stop().await;
9822 }
9823
9824 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9825 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
9826 const MEMBER_COUNT: usize = 32;
9827 let (_temp, runtime, delayed_service) =
9828 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
9829 let aggregator = MobKitConsoleAggregator::in_memory();
9830 aggregator
9831 .inner
9832 .runtimes
9833 .write()
9834 .expect("runtime registry")
9835 .insert(
9836 "runtime-stress".to_string(),
9837 runtime_entry_for_test("runtime-stress", &runtime),
9838 );
9839
9840 let started = Instant::now();
9841 aggregator
9842 .refresh_session_history()
9843 .await
9844 .expect("stress refresh");
9845 let elapsed = started.elapsed();
9846
9847 assert!(
9848 delayed_service.read_calls() >= MEMBER_COUNT,
9849 "expected at least one history read per member, saw {}",
9850 delayed_service.read_calls()
9851 );
9852 assert!(
9853 delayed_service.max_active_reads() > 1,
9854 "session history backfill should fan out instead of reading members serially"
9855 );
9856 assert!(
9857 delayed_service.max_active_reads()
9858 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
9859 "session history backfill should respect the default concurrency limit"
9860 );
9861 assert!(
9862 elapsed < Duration::from_millis(600),
9863 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
9864 MEMBER_COUNT * 40
9865 );
9866 let _ = runtime.mob_handle().stop().await;
9867 }
9868
9869 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9870 async fn session_history_backfill_respects_configured_concurrency_limit() {
9871 const MEMBER_COUNT: usize = 16;
9872 let (_temp, runtime, delayed_service) =
9873 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
9874 let aggregator =
9875 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
9876 max_concurrent_session_backfills: 4,
9877 ..ConsoleAggregatorOptions::default()
9878 });
9879 aggregator
9880 .inner
9881 .runtimes
9882 .write()
9883 .expect("runtime registry")
9884 .insert(
9885 "runtime-stress".to_string(),
9886 runtime_entry_for_test("runtime-stress", &runtime),
9887 );
9888
9889 aggregator
9890 .refresh_session_history()
9891 .await
9892 .expect("stress refresh");
9893
9894 assert!(
9895 delayed_service.max_active_reads() <= 4,
9896 "configured concurrency limit should bound session history reads, saw {}",
9897 delayed_service.max_active_reads()
9898 );
9899 let _ = runtime.mob_handle().stop().await;
9900 }
9901
9902 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
9903 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
9904 const MEMBER_COUNT: usize = 24;
9905 const LIVE_EVENT_COUNT: usize = 2_048;
9906 let (_temp, runtime, _delayed_service) =
9907 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
9908 let aggregator = MobKitConsoleAggregator::in_memory();
9909 aggregator.register_runtime(ConsoleRuntimeRegistration {
9910 runtime_key: "runtime-burst".to_string(),
9911 runtime: runtime.clone(),
9912 identity_namespace: "stress".to_string(),
9913 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
9914 });
9915 let console_events = runtime.console_events();
9916 for idx in 0..LIVE_EVENT_COUNT {
9917 console_events
9918 .append(
9919 "agent-0",
9920 Some("burst-turn".to_string()),
9921 "text_delta",
9922 json!({ "delta": format!("frame-{idx}") }),
9923 )
9924 .await;
9925 }
9926
9927 let deadline = Instant::now() + Duration::from_secs(30);
9932 let mut observed = 0;
9933 while Instant::now() < deadline {
9934 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
9935 if observed >= LIVE_EVENT_COUNT {
9936 break;
9937 }
9938 tokio::time::sleep(Duration::from_millis(25)).await;
9939 }
9940
9941 assert_eq!(
9942 observed, LIVE_EVENT_COUNT,
9943 "live pump should not drop frames while slow background backfill is running"
9944 );
9945 let _ = runtime.mob_handle().stop().await;
9946 }
9947
9948 async fn count_console_event_frames(
9949 aggregator: &MobKitConsoleAggregator,
9950 identity: &str,
9951 ) -> usize {
9952 let mut after = None;
9953 let mut count = 0;
9954 loop {
9955 let page = aggregator
9956 .query_timeline(ConsoleTimelineQuery {
9957 identity: Some(identity.to_string()),
9958 after,
9959 limit: 1_000,
9960 ..ConsoleTimelineQuery::default()
9961 })
9962 .await
9963 .expect("query burst timeline");
9964 if page.frames.is_empty() {
9965 break;
9966 }
9967 count += page
9968 .frames
9969 .iter()
9970 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
9971 .count();
9972 after = page.next_cursor;
9973 if after.is_none() {
9974 break;
9975 }
9976 }
9977 count
9978 }
9979
9980 #[tokio::test]
9981 async fn status_updates_get_replayable_aggregate_cursors() {
9982 let aggregator = MobKitConsoleAggregator::in_memory();
9983 let frame = NewConsoleFrame {
9984 id: None,
9985 dedupe_key: "send-1".to_string(),
9986 timestamp_ms: 1,
9987 runtime_key: "runtime-a".to_string(),
9988 identity: "agent-a".to_string(),
9989 conversation_id: Some("agent-a".to_string()),
9990 session_id: Some("session-1".to_string()),
9991 kind: "user_input".to_string(),
9992 status: ConsoleFrameStatus::Accepted,
9993 payload: json!({ "content": "hello" }),
9994 source: ConsoleFrameSource {
9995 kind: ConsoleFrameSourceKind::Send,
9996 source_cursor: None,
9997 },
9998 source_event_id: None,
9999 interaction_id: Some("interaction-1".to_string()),
10000 turn_id: None,
10001 run_id: None,
10002 parent_frame_id: None,
10003 caused_by_frame_id: None,
10004 };
10005 let inserted = aggregator
10006 .store()
10007 .append_if_absent(frame)
10008 .await
10009 .expect("append frame");
10010
10011 update_frame_status_and_emit(
10012 &aggregator.inner,
10013 &inserted.frame.id,
10014 ConsoleFrameStatus::Delivered,
10015 )
10016 .await
10017 .expect("update status");
10018
10019 let page = aggregator
10020 .query_timeline(ConsoleTimelineQuery {
10021 identity: Some("agent-a".to_string()),
10022 after: Some(inserted.frame.cursor.clone()),
10023 limit: 10,
10024 ..ConsoleTimelineQuery::default()
10025 })
10026 .await
10027 .expect("query timeline");
10028 assert_eq!(page.frames.len(), 1);
10029 assert_eq!(page.frames[0].kind, "frame_updated");
10030 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
10031 assert_eq!(
10032 page.frames[0]
10033 .payload
10034 .get("frame")
10035 .and_then(|frame| frame.get("status"))
10036 .and_then(Value::as_str),
10037 Some("delivered")
10038 );
10039 }
10040
10041 #[tokio::test]
10042 async fn steer_delivery_appends_terminal_control_frame() {
10043 let aggregator = MobKitConsoleAggregator::in_memory();
10044 let frame = NewConsoleFrame {
10045 id: None,
10046 dedupe_key: "send-steer-1".to_string(),
10047 timestamp_ms: 1,
10048 runtime_key: "runtime-a".to_string(),
10049 identity: "agent-a".to_string(),
10050 conversation_id: Some("agent-a".to_string()),
10051 session_id: Some("session-1".to_string()),
10052 kind: "user_input".to_string(),
10053 status: ConsoleFrameStatus::Delivered,
10054 payload: json!({
10055 "content": "operator steer",
10056 "handling_mode": "steer",
10057 }),
10058 source: ConsoleFrameSource {
10059 kind: ConsoleFrameSourceKind::Send,
10060 source_cursor: None,
10061 },
10062 source_event_id: None,
10063 interaction_id: Some("interaction-steer-1".to_string()),
10064 turn_id: None,
10065 run_id: None,
10066 parent_frame_id: None,
10067 caused_by_frame_id: None,
10068 };
10069 let inserted = aggregator
10070 .store()
10071 .append_if_absent(frame)
10072 .await
10073 .expect("append steer input");
10074
10075 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
10076 .await
10077 .expect("append steer terminal");
10078
10079 let page = aggregator
10080 .query_timeline(ConsoleTimelineQuery {
10081 identity: Some("agent-a".to_string()),
10082 after: Some(inserted.frame.cursor.clone()),
10083 limit: 10,
10084 ..ConsoleTimelineQuery::default()
10085 })
10086 .await
10087 .expect("query timeline");
10088
10089 assert_eq!(page.frames.len(), 1);
10090 let terminal = &page.frames[0];
10091 assert_eq!(terminal.kind, "interaction_complete");
10092 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
10093 assert_eq!(
10094 terminal.interaction_id.as_deref(),
10095 Some("interaction-steer-1")
10096 );
10097 assert_eq!(
10098 terminal.parent_frame_id.as_deref(),
10099 Some(inserted.frame.id.as_str())
10100 );
10101 assert_eq!(
10102 terminal.payload.get("reason").and_then(Value::as_str),
10103 Some("steer_delivered")
10104 );
10105 }
10106
10107 #[test]
10108 fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
10109 let live = transcript_fingerprint(
10110 "reasoning_complete",
10111 &json!({ "text": "I should inspect the file first." }),
10112 );
10113 let replay = transcript_fingerprint(
10114 "reasoning_complete",
10115 &json!({ "summary": "I should inspect the file first." }),
10116 );
10117 let delta = transcript_fingerprint(
10118 "reasoning_delta",
10119 &json!({ "delta": "I should inspect the file first." }),
10120 );
10121
10122 assert_eq!(live.as_deref(), Some("I should inspect the file first."));
10123 assert_eq!(live, replay);
10124 assert_eq!(live, delta);
10125 }
10126
10127 #[tokio::test]
10128 async fn history_counterpart_scan_is_not_capped_to_one_page() {
10129 let aggregator = MobKitConsoleAggregator::in_memory();
10130 for idx in 0..1_005 {
10131 aggregator
10132 .store()
10133 .append_if_absent(NewConsoleFrame {
10134 id: None,
10135 dedupe_key: format!("filler-{idx}"),
10136 timestamp_ms: idx,
10137 runtime_key: "runtime-a".to_string(),
10138 identity: "agent-a".to_string(),
10139 conversation_id: Some("agent-a".to_string()),
10140 session_id: Some("session-a".to_string()),
10141 kind: "text_delta".to_string(),
10142 status: ConsoleFrameStatus::Completed,
10143 payload: json!({ "delta": idx }),
10144 source: ConsoleFrameSource {
10145 kind: ConsoleFrameSourceKind::ConsoleEvent,
10146 source_cursor: None,
10147 },
10148 source_event_id: Some(format!("filler-{idx}")),
10149 interaction_id: None,
10150 turn_id: None,
10151 run_id: None,
10152 parent_frame_id: None,
10153 caused_by_frame_id: None,
10154 })
10155 .await
10156 .expect("append filler");
10157 }
10158 aggregator
10159 .store()
10160 .append_if_absent(NewConsoleFrame {
10161 id: None,
10162 dedupe_key: "live-user-input".to_string(),
10163 timestamp_ms: 2_000,
10164 runtime_key: "runtime-a".to_string(),
10165 identity: "agent-a".to_string(),
10166 conversation_id: Some("agent-a".to_string()),
10167 session_id: Some("session-a".to_string()),
10168 kind: "user_input".to_string(),
10169 status: ConsoleFrameStatus::Delivered,
10170 payload: json!({ "content": "already here" }),
10171 source: ConsoleFrameSource {
10172 kind: ConsoleFrameSourceKind::ConsoleEvent,
10173 source_cursor: None,
10174 },
10175 source_event_id: Some("live-user-input".to_string()),
10176 interaction_id: None,
10177 turn_id: None,
10178 run_id: None,
10179 parent_frame_id: None,
10180 caused_by_frame_id: None,
10181 })
10182 .await
10183 .expect("append live input");
10184
10185 let history = NewConsoleFrame {
10186 id: None,
10187 dedupe_key: "history-user-input".to_string(),
10188 timestamp_ms: 3_000,
10189 runtime_key: "runtime-a".to_string(),
10190 identity: "agent-a".to_string(),
10191 conversation_id: Some("agent-a".to_string()),
10192 session_id: Some("session-a".to_string()),
10193 kind: "user_input".to_string(),
10194 status: ConsoleFrameStatus::Completed,
10195 payload: json!({ "content": "already here" }),
10196 source: ConsoleFrameSource {
10197 kind: ConsoleFrameSourceKind::SessionHistory,
10198 source_cursor: Some("session-a:1006".to_string()),
10199 },
10200 source_event_id: None,
10201 interaction_id: None,
10202 turn_id: None,
10203 run_id: None,
10204 parent_frame_id: None,
10205 caused_by_frame_id: None,
10206 };
10207
10208 assert!(
10209 history_frame_has_existing_counterpart(&aggregator.inner, &history)
10210 .await
10211 .expect("counterpart scan")
10212 );
10213 }
10214
10215 #[tokio::test]
10216 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
10217 let aggregator = MobKitConsoleAggregator::in_memory();
10218 aggregator
10219 .store()
10220 .append_if_absent(NewConsoleFrame {
10221 id: None,
10222 dedupe_key: "live-user-input".to_string(),
10223 timestamp_ms: 2_000,
10224 runtime_key: "runtime-a".to_string(),
10225 identity: "agent-a".to_string(),
10226 conversation_id: Some("agent-a".to_string()),
10227 session_id: Some("session-a".to_string()),
10228 kind: "user_input".to_string(),
10229 status: ConsoleFrameStatus::Delivered,
10230 payload: json!({ "content": "hello from operator" }),
10231 source: ConsoleFrameSource {
10232 kind: ConsoleFrameSourceKind::ConsoleEvent,
10233 source_cursor: None,
10234 },
10235 source_event_id: Some("live-user-input".to_string()),
10236 interaction_id: None,
10237 turn_id: None,
10238 run_id: None,
10239 parent_frame_id: None,
10240 caused_by_frame_id: None,
10241 })
10242 .await
10243 .expect("append live input");
10244
10245 let history = NewConsoleFrame {
10246 id: None,
10247 dedupe_key: "history-user-input".to_string(),
10248 timestamp_ms: 3_000,
10249 runtime_key: "runtime-a".to_string(),
10250 identity: "agent-a".to_string(),
10251 conversation_id: Some("agent-a".to_string()),
10252 session_id: Some("session-a".to_string()),
10253 kind: "user_input".to_string(),
10254 status: ConsoleFrameStatus::Completed,
10255 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
10256 source: ConsoleFrameSource {
10257 kind: ConsoleFrameSourceKind::SessionHistory,
10258 source_cursor: Some("session-a:2".to_string()),
10259 },
10260 source_event_id: None,
10261 interaction_id: None,
10262 turn_id: None,
10263 run_id: None,
10264 parent_frame_id: None,
10265 caused_by_frame_id: None,
10266 };
10267
10268 assert!(
10269 history_frame_has_existing_counterpart(&aggregator.inner, &history)
10270 .await
10271 .expect("counterpart scan")
10272 );
10273 }
10274
10275 #[tokio::test]
10276 async fn history_counterpart_scan_matches_content_block_user_prompts() {
10277 let aggregator = MobKitConsoleAggregator::in_memory();
10278 aggregator
10279 .store()
10280 .append_if_absent(NewConsoleFrame {
10281 id: None,
10282 dedupe_key: "live-user-input".to_string(),
10283 timestamp_ms: 2_000,
10284 runtime_key: "runtime-a".to_string(),
10285 identity: "agent-a".to_string(),
10286 conversation_id: Some("agent-a".to_string()),
10287 session_id: Some("session-a".to_string()),
10288 kind: "user_input".to_string(),
10289 status: ConsoleFrameStatus::Delivered,
10290 payload: json!({ "content": "hello from operator" }),
10291 source: ConsoleFrameSource {
10292 kind: ConsoleFrameSourceKind::ConsoleEvent,
10293 source_cursor: None,
10294 },
10295 source_event_id: Some("live-user-input".to_string()),
10296 interaction_id: None,
10297 turn_id: None,
10298 run_id: None,
10299 parent_frame_id: None,
10300 caused_by_frame_id: None,
10301 })
10302 .await
10303 .expect("append live input");
10304
10305 let history = NewConsoleFrame {
10306 id: None,
10307 dedupe_key: "history-user-input".to_string(),
10308 timestamp_ms: 3_000,
10309 runtime_key: "runtime-a".to_string(),
10310 identity: "agent-a".to_string(),
10311 conversation_id: Some("agent-a".to_string()),
10312 session_id: Some("session-a".to_string()),
10313 kind: "user_input".to_string(),
10314 status: ConsoleFrameStatus::Completed,
10315 payload: json!({
10316 "content": [{ "type": "text", "text": "hello from operator" }]
10317 }),
10318 source: ConsoleFrameSource {
10319 kind: ConsoleFrameSourceKind::SessionHistory,
10320 source_cursor: Some("session-a:2".to_string()),
10321 },
10322 source_event_id: None,
10323 interaction_id: None,
10324 turn_id: None,
10325 run_id: None,
10326 parent_frame_id: None,
10327 caused_by_frame_id: None,
10328 };
10329
10330 assert!(
10331 history_frame_has_existing_counterpart(&aggregator.inner, &history)
10332 .await
10333 .expect("counterpart scan")
10334 );
10335 }
10336
10337 #[tokio::test]
10338 async fn history_counterpart_scan_matches_live_tool_results() {
10339 let aggregator = MobKitConsoleAggregator::in_memory();
10340 aggregator
10341 .store()
10342 .append_if_absent(NewConsoleFrame {
10343 id: None,
10344 dedupe_key: "live-tool-result".to_string(),
10345 timestamp_ms: 2_000,
10346 runtime_key: "runtime-a".to_string(),
10347 identity: "agent-a".to_string(),
10348 conversation_id: Some("agent-a".to_string()),
10349 session_id: Some("session-a".to_string()),
10350 kind: "tool_execution_completed".to_string(),
10351 status: ConsoleFrameStatus::Delivered,
10352 payload: json!({
10353 "id": "call-1",
10354 "tool_call_id": "call-1",
10355 "result": "{ \"count\": 70 }"
10356 }),
10357 source: ConsoleFrameSource {
10358 kind: ConsoleFrameSourceKind::ConsoleEvent,
10359 source_cursor: None,
10360 },
10361 source_event_id: Some("live-tool-result".to_string()),
10362 interaction_id: None,
10363 turn_id: None,
10364 run_id: None,
10365 parent_frame_id: None,
10366 caused_by_frame_id: None,
10367 })
10368 .await
10369 .expect("append live tool result");
10370
10371 let history = NewConsoleFrame {
10372 id: None,
10373 dedupe_key: "history-tool-result".to_string(),
10374 timestamp_ms: 3_000,
10375 runtime_key: "runtime-a".to_string(),
10376 identity: "agent-a".to_string(),
10377 conversation_id: Some("agent-a".to_string()),
10378 session_id: Some("session-a".to_string()),
10379 kind: "tool_execution_completed".to_string(),
10380 status: ConsoleFrameStatus::Completed,
10381 payload: json!({
10382 "id": "call-1",
10383 "tool_call_id": "call-1",
10384 "result": "{ \"count\": 70 }",
10385 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
10386 }),
10387 source: ConsoleFrameSource {
10388 kind: ConsoleFrameSourceKind::SessionHistory,
10389 source_cursor: Some("session-a:3:0".to_string()),
10390 },
10391 source_event_id: None,
10392 interaction_id: None,
10393 turn_id: None,
10394 run_id: None,
10395 parent_frame_id: None,
10396 caused_by_frame_id: None,
10397 };
10398
10399 assert!(
10400 history_frame_has_existing_counterpart(&aggregator.inner, &history)
10401 .await
10402 .expect("counterpart scan")
10403 );
10404 }
10405
10406 #[tokio::test]
10407 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
10408 let aggregator = MobKitConsoleAggregator::in_memory();
10409 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
10410 aggregator
10411 .store()
10412 .append_if_absent(NewConsoleFrame {
10413 id: None,
10414 dedupe_key: format!("live-delta-{idx}"),
10415 timestamp_ms: 2_000 + idx as u64,
10416 runtime_key: "runtime-a".to_string(),
10417 identity: "agent-a".to_string(),
10418 conversation_id: Some("agent-a".to_string()),
10419 session_id: Some("session-a".to_string()),
10420 kind: "text_delta".to_string(),
10421 status: ConsoleFrameStatus::Delivered,
10422 payload: json!({ "delta": delta }),
10423 source: ConsoleFrameSource {
10424 kind: ConsoleFrameSourceKind::ConsoleEvent,
10425 source_cursor: None,
10426 },
10427 source_event_id: Some(format!("live-delta-{idx}")),
10428 interaction_id: Some("turn-a".to_string()),
10429 turn_id: None,
10430 run_id: None,
10431 parent_frame_id: None,
10432 caused_by_frame_id: None,
10433 })
10434 .await
10435 .expect("append live delta");
10436 }
10437
10438 let history = NewConsoleFrame {
10439 id: None,
10440 dedupe_key: "history-assistant-complete".to_string(),
10441 timestamp_ms: 3_000,
10442 runtime_key: "runtime-a".to_string(),
10443 identity: "agent-a".to_string(),
10444 conversation_id: Some("agent-a".to_string()),
10445 session_id: Some("session-a".to_string()),
10446 kind: "interaction_complete".to_string(),
10447 status: ConsoleFrameStatus::Completed,
10448 payload: json!({ "result": "Ready and standing by." }),
10449 source: ConsoleFrameSource {
10450 kind: ConsoleFrameSourceKind::SessionHistory,
10451 source_cursor: Some("session-a:3".to_string()),
10452 },
10453 source_event_id: None,
10454 interaction_id: None,
10455 turn_id: None,
10456 run_id: None,
10457 parent_frame_id: None,
10458 caused_by_frame_id: None,
10459 };
10460
10461 assert!(
10462 history_frame_has_existing_counterpart(&aggregator.inner, &history)
10463 .await
10464 .expect("counterpart scan")
10465 );
10466 }
10467
10468 #[test]
10469 fn session_history_watermark_key_is_session_scoped() {
10470 assert_ne!(
10471 session_history_watermark_runtime_key("runtime-a", "session-1"),
10472 session_history_watermark_runtime_key("runtime-a", "session-2")
10473 );
10474 }
10475
10476 #[test]
10477 fn backfill_offset_advances_only_on_strict_forward_progress() {
10478 assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
10480 assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
10482 assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
10484 assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
10486
10487 assert_eq!(advance_backfill_offset(500, 0, 500), None); assert_eq!(advance_backfill_offset(500, 0, 400), None); assert_eq!(advance_backfill_offset(1000, 500, 500), None); assert_eq!(advance_backfill_offset(10, 10, 0), None); assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
10496 }
10497
10498 #[test]
10499 fn session_history_watermarks_are_cursor_and_ttl_aware() {
10500 let legacy = "session:with:colon:42";
10501 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
10502 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
10503
10504 assert_eq!(
10505 parse_session_history_watermark(legacy, "session:with:colon"),
10506 Some(42)
10507 );
10508 assert_eq!(
10509 parse_session_history_watermark(&checked, "session:with:colon"),
10510 Some(43)
10511 );
10512 assert!(session_history_watermark_is_fresh(
10513 &checked,
10514 "session:with:colon",
10515 1_500
10516 ));
10517 assert!(!session_history_watermark_is_fresh(
10518 &checked,
10519 "session:with:colon",
10520 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
10521 ));
10522 assert!(session_history_watermark_is_fresh(
10523 &empty_checked,
10524 "session:with:colon",
10525 1_500
10526 ));
10527 assert!(!session_history_watermark_is_fresh(
10528 &empty_checked,
10529 "session:with:colon",
10530 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
10531 ));
10532 }
10533
10534 #[test]
10535 fn session_history_messages_project_to_renderable_frames() {
10536 let user = frame_from_session_history_message(
10537 "runtime-a",
10538 "agent-a",
10539 "session-a",
10540 0,
10541 json!({
10542 "role": "user",
10543 "content": "hello",
10544 "timestamp_ms": 10
10545 }),
10546 )
10547 .expect("user history frame");
10548 let assistant = frame_from_session_history_message(
10553 "runtime-a",
10554 "agent-a",
10555 "session-a",
10556 1,
10557 json!({
10558 "role": "block_assistant",
10559 "blocks": [
10560 { "block_type": "text", "data": { "text": "hi there" } }
10561 ],
10562 "stop_reason": "end_turn",
10563 "timestamp_ms": 11
10564 }),
10565 )
10566 .expect("assistant history frame");
10567
10568 assert_eq!(user.kind, "user_input");
10569 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
10570 assert_eq!(
10571 user.payload["content"],
10572 json!([{ "type": "text", "text": "hello" }])
10573 );
10574 assert_eq!(assistant.kind, "interaction_complete");
10575 assert_eq!(assistant.payload["text"], json!("hi there"));
10576 assert!(
10577 assistant
10578 .dedupe_key
10579 .starts_with("session-history:runtime-a:session-a:1:")
10580 );
10581 }
10582
10583 #[test]
10584 fn session_history_projection_filters_scaffold_user_messages() {
10585 let spawn_notice = frame_from_session_history_message(
10586 "runtime-a",
10587 "agent-a",
10588 "session-a",
10589 0,
10590 json!({
10591 "role": "user",
10592 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
10593 "timestamp_ms": 10
10594 }),
10595 );
10596 let peer_update = frame_from_session_history_message(
10597 "runtime-a",
10598 "agent-a",
10599 "session-a",
10600 1,
10601 json!({
10602 "role": "user",
10603 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
10604 "timestamp_ms": 11
10605 }),
10606 );
10607 let real_user = frame_from_session_history_message(
10608 "runtime-a",
10609 "agent-a",
10610 "session-a",
10611 2,
10612 json!({
10613 "role": "user",
10614 "content": "Please review the incident notes.",
10615 "timestamp_ms": 12
10616 }),
10617 );
10618
10619 assert!(spawn_notice.is_none());
10620 assert!(peer_update.is_none());
10621 assert!(real_user.is_some());
10622 }
10623
10624 #[test]
10625 fn session_history_projection_skips_non_transcript_messages() {
10626 let skipped = frame_from_session_history_message(
10627 "runtime-a",
10628 "agent-a",
10629 "session-a",
10630 0,
10631 json!({
10632 "content": "internal system prompt"
10633 }),
10634 );
10635 assert!(skipped.is_none());
10636 }
10637
10638 #[test]
10639 fn session_history_projection_extracts_assistant_blocks() {
10640 let frame = frame_from_session_history_message(
10641 "runtime-a",
10642 "agent-a",
10643 "session-a",
10644 0,
10645 json!({
10646 "role": "block_assistant",
10647 "blocks": [
10648 { "block_type": "text", "data": { "text": "hello " } },
10649 { "block_type": "text", "data": { "text": "there" } }
10650 ],
10651 "stop_reason": "end_turn"
10652 }),
10653 )
10654 .expect("assistant block history frame");
10655 assert_eq!(frame.payload["text"], json!("hello there"));
10656 }
10657
10658 #[test]
10659 fn session_history_projection_extracts_nested_text_block_data() {
10660 let frame = frame_from_session_history_message(
10661 "runtime-a",
10662 "agent-a",
10663 "session-a",
10664 0,
10665 json!({
10666 "role": "block_assistant",
10667 "blocks": [
10668 {
10669 "block_type": "text",
10670 "data": { "text": "Ready and standing by." }
10671 }
10672 ],
10673 "stop_reason": "end_turn",
10674 "created_at": "1970-01-01T00:00:00.010Z"
10675 }),
10676 )
10677 .expect("assistant block history frame");
10678
10679 assert_eq!(frame.kind, "interaction_complete");
10680 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
10681 }
10682
10683 #[test]
10684 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
10685 let frame = frame_from_session_history_message(
10686 "runtime-a",
10687 "agent-a",
10688 "session-a",
10689 0,
10690 json!({
10691 "role": "block_assistant",
10692 "blocks": [
10693 {
10694 "block_type": "reasoning",
10695 "data": { "text": "**Planning**\nI should not be rendered." }
10696 },
10697 {
10698 "block_type": "text",
10699 "data": { "text": "Visible answer." }
10700 }
10701 ],
10702 "stop_reason": "end_turn",
10703 "created_at": "1970-01-01T00:00:00.010Z"
10704 }),
10705 )
10706 .expect("assistant block history frame");
10707
10708 assert_eq!(frame.kind, "interaction_complete");
10709 assert_eq!(frame.payload["result"], json!("Visible answer."));
10710 assert_eq!(frame.payload["text"], json!("Visible answer."));
10711 }
10712
10713 #[test]
10714 fn session_history_projection_leaves_reasoning_only_result_empty() {
10715 let frame = frame_from_session_history_message(
10716 "runtime-a",
10717 "agent-a",
10718 "session-a",
10719 0,
10720 json!({
10721 "role": "block_assistant",
10722 "blocks": [
10723 {
10724 "block_type": "reasoning",
10725 "data": { "text": "Private planning text." }
10726 },
10727 {
10728 "block_type": "tool_use",
10729 "data": { "id": "toolu-1", "name": "peers", "args": {} }
10730 }
10731 ],
10732 "stop_reason": "end_turn",
10733 "created_at": "1970-01-01T00:00:00.010Z"
10734 }),
10735 )
10736 .expect("assistant block history frame");
10737
10738 assert_eq!(frame.kind, "interaction_complete");
10739 assert_eq!(frame.payload["result"], json!(""));
10740 assert_eq!(frame.payload["text"], json!(""));
10741 }
10742
10743 #[test]
10748 fn session_history_projection_emits_outgoing_comms_tool_calls() {
10749 let frames = frames_from_session_history_message(
10750 "runtime-a",
10751 "agent-a",
10752 "session-a",
10753 3,
10754 json!({
10755 "role": "block_assistant",
10756 "blocks": [
10757 { "block_type": "text", "data": { "text": "Sending updates." } },
10758 {
10759 "block_type": "tool_use",
10760 "data": {
10761 "id": "toolu-send-1",
10762 "name": "send_message",
10763 "args": { "peer_id": "peer-1", "body": "lights updated" }
10764 }
10765 },
10766 {
10767 "block_type": "tool_use",
10768 "data": {
10769 "id": "toolu-req-1",
10770 "name": "send_request",
10771 "args": {
10772 "peer_id": "peer-1",
10773 "intent": "checksum_token",
10774 "params": { "subject": "status" }
10775 }
10776 }
10777 },
10778 {
10779 "block_type": "tool_use",
10780 "data": { "id": "toolu-other", "name": "workgraph_get", "args": {} }
10781 }
10782 ],
10783 "stop_reason": "tool_use",
10784 "created_at": "1970-01-01T00:00:00.010Z"
10785 }),
10786 );
10787
10788 let tool_frames: Vec<_> = frames
10789 .iter()
10790 .filter(|frame| frame.kind == "tool_call_requested")
10791 .collect();
10792 assert_eq!(
10793 tool_frames.len(),
10794 2,
10795 "exactly the two comms send calls project (not workgraph_get): {frames:?}"
10796 );
10797 let message_frame = tool_frames
10798 .iter()
10799 .find(|frame| frame.payload["name"] == json!("send_message"))
10800 .expect("send_message history frame");
10801 assert_eq!(message_frame.payload["tool_call_id"], json!("toolu-send-1"));
10802 assert_eq!(
10803 message_frame.payload["args"]["body"],
10804 json!("lights updated")
10805 );
10806 assert_eq!(message_frame.conversation_id.as_deref(), Some("agent-a"));
10807 let request_frame = tool_frames
10808 .iter()
10809 .find(|frame| frame.payload["name"] == json!("send_request"))
10810 .expect("send_request history frame");
10811 assert_eq!(
10812 request_frame.payload["args"]["params"]["subject"],
10813 json!("status")
10814 );
10815
10816 let history_fingerprint =
10819 transcript_fingerprint(&message_frame.kind, &message_frame.payload);
10820 let live_fingerprint = transcript_fingerprint(
10821 "tool_call_requested",
10822 &json!({
10823 "id": "toolu-send-1",
10824 "tool_call_id": "toolu-send-1",
10825 "name": "send_message",
10826 "args": { "peer_id": "peer-1", "body": "lights updated" },
10827 "source_event_type": "tool_call_requested",
10828 }),
10829 );
10830 assert_eq!(history_fingerprint, live_fingerprint);
10831 assert!(history_fingerprint.is_some());
10832 }
10833
10834 #[test]
10835 fn session_history_projection_preserves_tool_results() {
10836 let frames = frames_from_session_history_message(
10837 "runtime-a",
10838 "agent-a",
10839 "session-a",
10840 5,
10841 json!({
10842 "role": "tool_results",
10843 "results": [
10844 {
10845 "tool_use_id": "call-peers",
10846 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
10847 "is_error": false
10848 }
10849 ],
10850 "created_at": "1970-01-01T00:00:00.050Z"
10851 }),
10852 );
10853
10854 assert_eq!(frames.len(), 1);
10855 let frame = &frames[0];
10856 assert_eq!(frame.kind, "tool_execution_completed");
10857 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
10858 assert_eq!(
10859 frame.payload["result"],
10860 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
10861 );
10862 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
10863 assert_eq!(frame.timestamp_ms, 50);
10864 }
10865
10866 #[test]
10867 fn session_history_projection_derives_assistant_image_from_image_tool_result() {
10868 let image_result = json!({
10875 "operation_id": "00000000-0000-0000-0000-0000000000aa",
10876 "terminal": { "terminal": "generated" },
10877 "images": [
10878 {
10879 "image_id": "00000000-0000-0000-0000-000000000051",
10880 "blob_ref": {
10881 "blob_id": "sha256:test-generated-image",
10882 "media_type": "image/png"
10883 },
10884 "media_type": "image/png",
10885 "width": 1024,
10886 "height": 768
10887 }
10888 ],
10889 "provider_text": { "disposition": "not_emitted" },
10890 "revised_prompt": { "disposition": "not_requested" },
10891 "native_metadata": { "provider": "not_emitted" }
10892 });
10893 serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
10896 image_result.clone(),
10897 )
10898 .expect("fixture must be a valid ImageGenerationToolResult");
10899 let result_text = serde_json::to_string(&image_result).expect("serialize image result");
10900
10901 let frames = frames_from_session_history_message(
10902 "runtime-a",
10903 "artist:singleton",
10904 "session-a",
10905 6,
10906 json!({
10907 "role": "tool_results",
10908 "results": [
10909 {
10910 "tool_use_id": "call-image",
10911 "content": result_text,
10912 "is_error": false
10913 }
10914 ],
10915 "created_at": "1970-01-01T00:00:00.060Z"
10916 }),
10917 );
10918
10919 assert!(
10921 frames
10922 .iter()
10923 .any(|frame| frame.kind == "tool_execution_completed"),
10924 "tool_execution_completed frame must still be present: {frames:#?}"
10925 );
10926 let image_frame = frames
10928 .iter()
10929 .find(|frame| frame.kind == "assistant_image")
10930 .expect("backfill must derive an assistant_image frame");
10931 assert_eq!(
10932 image_frame.payload["blob_id"],
10933 json!("sha256:test-generated-image")
10934 );
10935 assert_eq!(
10936 image_frame.payload["image_id"],
10937 json!("00000000-0000-0000-0000-000000000051")
10938 );
10939 assert_eq!(image_frame.payload["media_type"], json!("image/png"));
10940 assert_eq!(image_frame.payload["width"], json!(1024));
10941 assert_eq!(image_frame.payload["height"], json!(768));
10942 assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
10943 assert_eq!(
10944 image_frame.source.kind,
10945 ConsoleFrameSourceKind::SessionHistory
10946 );
10947 assert_eq!(image_frame.timestamp_ms, 60);
10948 }
10949
10950 #[test]
10951 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
10952 let frames = frames_from_session_history_message(
10953 "runtime-a",
10954 "review:singleton",
10955 "session-a",
10956 7,
10957 json!({
10958 "role": "block_assistant",
10959 "blocks": [
10960 {
10961 "block_type": "tool_use",
10962 "data": {
10963 "id": "call-spawn",
10964 "name": "mob_spawn_member",
10965 "args": {
10966 "member_id": "review-worker-alpha",
10967 "profile": "review-worker",
10968 "initial_message": "Review Initiative Alpha and save the result."
10969 }
10970 }
10971 }
10972 ],
10973 "stop_reason": "tool_use",
10974 "created_at": "1970-01-01T00:00:00.070Z"
10975 }),
10976 );
10977
10978 assert_eq!(frames.len(), 2);
10979 let child = frames
10980 .iter()
10981 .find(|frame| frame.identity == "review-worker-alpha")
10982 .expect("spawned member initial message frame");
10983 assert_eq!(child.kind, "user_input");
10984 assert_eq!(child.timestamp_ms, 70);
10985 assert_eq!(
10986 child.payload["message"]["content"],
10987 "Review Initiative Alpha and save the result."
10988 );
10989 assert_eq!(child.payload["parent_identity"], "review:singleton");
10990 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
10991 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
10992 }
10993
10994 #[test]
10995 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
10996 let frames = frames_from_session_history_message(
10997 "runtime-a",
10998 "review:singleton",
10999 "session-a",
11000 8,
11001 json!({
11002 "role": "block_assistant",
11003 "blocks": [
11004 {
11005 "block_type": "tool_use",
11006 "data": {
11007 "id": "call-spawn-many",
11008 "name": "mob_spawn_member",
11009 "args": {
11010 "specs": [
11011 {
11012 "agent_identity": "review-worker-alpha",
11013 "profile": "review-worker",
11014 "initial_message": "Review Initiative Alpha."
11015 },
11016 {
11017 "agent_identity": "review-worker-beta",
11018 "profile": "review-worker",
11019 "initial_message": "Review Initiative Beta."
11020 }
11021 ]
11022 }
11023 }
11024 }
11025 ],
11026 "stop_reason": "tool_use",
11027 "created_at": "1970-01-01T00:00:00.080Z"
11028 }),
11029 );
11030
11031 assert_eq!(frames.len(), 3);
11032 let child_messages = frames
11033 .iter()
11034 .filter(|frame| frame.kind == "user_input")
11035 .map(|frame| {
11036 (
11037 frame.identity.as_str(),
11038 frame.payload["message"]["content"]
11039 .as_str()
11040 .unwrap_or_default(),
11041 )
11042 })
11043 .collect::<Vec<_>>();
11044 assert_eq!(
11045 child_messages,
11046 vec![
11047 ("review-worker-alpha", "Review Initiative Alpha."),
11048 ("review-worker-beta", "Review Initiative Beta."),
11049 ]
11050 );
11051 }
11052
11053 #[tokio::test]
11054 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
11055 let aggregator = MobKitConsoleAggregator::in_memory();
11056 let frames = frames_from_session_history_message(
11057 "runtime-a",
11058 "review:singleton",
11059 "session-a",
11060 9,
11061 json!({
11062 "role": "block_assistant",
11063 "blocks": [
11064 {
11065 "block_type": "tool_use",
11066 "data": {
11067 "id": "call-spawn-many",
11068 "name": "mob_spawn_member",
11069 "args": {
11070 "specs": [
11071 {
11072 "agent_identity": "review-worker-alpha",
11073 "profile": "review-worker",
11074 "initial_message": "Review Initiative Alpha."
11075 },
11076 {
11077 "agent_identity": "review-worker-beta",
11078 "profile": "review-worker",
11079 "initial_message": "Review Initiative Beta."
11080 }
11081 ]
11082 }
11083 }
11084 }
11085 ],
11086 "stop_reason": "tool_use",
11087 "created_at": "1970-01-01T00:00:00.090Z"
11088 }),
11089 );
11090 for frame in frames {
11091 aggregator
11092 .store()
11093 .append_if_absent(frame)
11094 .await
11095 .expect("append projected session-history frame");
11096 }
11097
11098 let child_page = aggregator
11099 .query_timeline(ConsoleTimelineQuery {
11100 identity: Some("review-worker-alpha".to_string()),
11101 limit: 20,
11102 ..ConsoleTimelineQuery::default()
11103 })
11104 .await
11105 .expect("query child timeline");
11106 assert_eq!(child_page.frames.len(), 1);
11107 assert_eq!(child_page.frames[0].kind, "user_input");
11108 assert_eq!(
11109 child_page.frames[0].payload["message"]["content"],
11110 "Review Initiative Alpha."
11111 );
11112 assert_eq!(
11113 child_page.frames[0].payload["parent_identity"],
11114 "review:singleton"
11115 );
11116
11117 let parent_page = aggregator
11118 .query_timeline(ConsoleTimelineQuery {
11119 identity: Some("review:singleton".to_string()),
11120 limit: 20,
11121 ..ConsoleTimelineQuery::default()
11122 })
11123 .await
11124 .expect("query parent timeline");
11125 assert!(
11126 parent_page
11127 .frames
11128 .iter()
11129 .all(|frame| frame.identity == "review:singleton"),
11130 "child initial messages must not leak into parent timeline: {:#?}",
11131 parent_page.frames
11132 );
11133 }
11134
11135 #[test]
11136 fn session_history_frames_carry_transcript_interaction_identity() {
11137 let interaction = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
11141 let run = "886313e1-3b8a-5372-9b90-0c9aee199e5d";
11142 let frames = frames_from_session_history_message_with_namespace(
11143 "runtime-a",
11144 "helper",
11145 "",
11146 "session-a",
11147 3,
11148 json!({
11149 "role": "user",
11150 "content": "please ack",
11151 "identity": { "interaction_id": interaction, "run_id": run },
11152 "created_at": "1970-01-01T00:00:00.100Z"
11153 }),
11154 );
11155 assert_eq!(frames.len(), 1, "{frames:#?}");
11156 assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
11157 assert_eq!(frames[0].run_id.as_deref(), Some(run));
11158
11159 let frames = frames_from_session_history_message_with_namespace(
11160 "runtime-a",
11161 "helper",
11162 "",
11163 "session-a",
11164 4,
11165 json!({
11166 "role": "block_assistant",
11167 "blocks": [ { "block_type": "text", "data": { "text": "ACK" } } ],
11168 "identity": { "interaction_id": interaction },
11169 "stop_reason": "end_turn",
11170 "created_at": "1970-01-01T00:00:00.200Z"
11171 }),
11172 );
11173 assert!(!frames.is_empty(), "assistant history frame expected");
11174 assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
11175 assert_eq!(frames[0].run_id, None);
11176
11177 let frames = frames_from_session_history_message_with_namespace(
11180 "runtime-a",
11181 "helper",
11182 "",
11183 "session-a",
11184 5,
11185 json!({
11186 "role": "user",
11187 "content": "legacy row",
11188 "created_at": "1970-01-01T00:00:00.300Z"
11189 }),
11190 );
11191 assert_eq!(frames.len(), 1);
11192 assert_eq!(frames[0].interaction_id, None);
11193 }
11194
11195 #[test]
11196 fn identity_first_interaction_ids_are_deterministic_uuids() {
11197 let a = identity_first_interaction_uuid("send:key:1");
11198 let b = identity_first_interaction_uuid("send:key:1");
11199 let c = identity_first_interaction_uuid("send:key:2");
11200 assert_eq!(a, b, "idempotent retries must mint the same id");
11201 assert_ne!(a, c);
11202 assert!(
11203 a.parse::<uuid::Uuid>().is_ok(),
11204 "identity-first interaction ids must be UUID-form (the console's \
11205 authoritative twin-identity scheme): {a}"
11206 );
11207 }
11208
11209 #[test]
11210 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
11211 let frames = frames_from_session_history_message_with_namespace(
11212 "runtime-a",
11213 "test/review:singleton",
11214 "test",
11215 "session-a",
11216 10,
11217 json!({
11218 "role": "block_assistant",
11219 "blocks": [
11220 {
11221 "block_type": "tool_use",
11222 "data": {
11223 "id": "call-spawn-many",
11224 "name": "mob_spawn_member",
11225 "args": {
11226 "specs": [
11227 {
11228 "agent_identity": "review-worker-alpha",
11229 "profile": "review-worker",
11230 "initial_message": "Review Initiative Alpha."
11231 },
11232 {
11233 "agent_identity": "test/review-worker-beta",
11234 "profile": "review-worker",
11235 "initial_message": "Review Initiative Beta."
11236 }
11237 ]
11238 }
11239 }
11240 }
11241 ],
11242 "stop_reason": "tool_use",
11243 "created_at": "1970-01-01T00:00:00.100Z"
11244 }),
11245 );
11246
11247 let child_identities = frames
11248 .iter()
11249 .filter(|frame| frame.kind == "user_input")
11250 .map(|frame| frame.identity.as_str())
11251 .collect::<Vec<_>>();
11252 assert_eq!(
11253 child_identities,
11254 vec!["test/review-worker-alpha", "test/review-worker-beta"]
11255 );
11256 }
11257
11258 #[tokio::test]
11259 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
11260 let aggregator = MobKitConsoleAggregator::in_memory();
11261 aggregator
11262 .inner
11263 .identity_read_model
11264 .replace(vec![
11265 identity_record_for_test("test/review:singleton"),
11266 identity_record_for_test("test/review-worker-alpha"),
11267 ])
11268 .await;
11269 let frames = frames_from_session_history_message_with_namespace(
11270 "runtime-a",
11271 "test/review:singleton",
11272 "test",
11273 "session-a",
11274 11,
11275 json!({
11276 "role": "block_assistant",
11277 "blocks": [
11278 {
11279 "block_type": "tool_use",
11280 "data": {
11281 "id": "call-spawn",
11282 "name": "mob_spawn_member",
11283 "args": {
11284 "agent_identity": "review-worker-alpha",
11285 "profile": "review-worker",
11286 "initial_message": "Review Initiative Alpha."
11287 }
11288 }
11289 }
11290 ],
11291 "stop_reason": "tool_use",
11292 "created_at": "1970-01-01T00:00:00.110Z"
11293 }),
11294 );
11295 for frame in frames {
11296 aggregator
11297 .store()
11298 .append_if_absent(frame)
11299 .await
11300 .expect("append projected namespaced session-history frame");
11301 }
11302
11303 let namespaced_child_page = aggregator
11304 .query_timeline(ConsoleTimelineQuery {
11305 identity: Some("test/review-worker-alpha".to_string()),
11306 limit: 20,
11307 ..ConsoleTimelineQuery::default()
11308 })
11309 .await
11310 .expect("query namespaced child timeline");
11311 assert_eq!(namespaced_child_page.frames.len(), 1);
11312 assert_eq!(
11313 namespaced_child_page.frames[0].identity,
11314 "test/review-worker-alpha"
11315 );
11316 assert_eq!(
11317 namespaced_child_page.frames[0].payload["message"]["content"],
11318 "Review Initiative Alpha."
11319 );
11320
11321 let raw_child_page = aggregator
11322 .query_timeline(ConsoleTimelineQuery {
11323 identity: Some("review-worker-alpha".to_string()),
11324 limit: 20,
11325 ..ConsoleTimelineQuery::default()
11326 })
11327 .await
11328 .expect("query raw child timeline");
11329 assert!(
11330 raw_child_page.frames.is_empty(),
11331 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
11332 raw_child_page.frames
11333 );
11334 }
11335
11336 #[test]
11337 fn namespace_helpers_preserve_namespace_named_member_identity() {
11338 assert_eq!(apply_namespace("test", "test"), "test/test");
11339 assert_eq!(
11340 strip_namespace("test/test", "test").as_deref(),
11341 Some("test")
11342 );
11343 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
11344 assert_eq!(
11345 strip_namespace("test/worker", "test").as_deref(),
11346 Some("worker")
11347 );
11348 assert_eq!(strip_namespace("test", "test"), None);
11349 }
11350
11351 #[test]
11352 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
11353 assert!(!member_id_matches_durable_identity(
11354 "rt:review:singleton:0",
11355 "review:singleton"
11356 ));
11357 assert!(!member_id_matches_durable_identity(
11358 "review:singleton:gen1",
11359 "review:singleton"
11360 ));
11361 assert!(!member_id_matches_durable_identity(
11362 "review:singleton:1",
11363 "review:singleton"
11364 ));
11365 assert!(!member_id_matches_durable_identity(
11366 "rt:review:singleton:qa:0",
11367 "review:singleton"
11368 ));
11369 assert!(!member_id_matches_durable_identity(
11370 "review:singleton:qa",
11371 "review:singleton"
11372 ));
11373 }
11374
11375 #[test]
11376 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
11377 let frame = frame_from_session_history_message(
11378 "runtime-a",
11379 "agent-a",
11380 "session-a",
11381 0,
11382 json!({
11383 "role": "user",
11384 "content": "hello",
11385 "created_at": "2026-05-12T05:00:06.564227Z"
11386 }),
11387 )
11388 .expect("user history frame");
11389
11390 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
11391 }
11392
11393 async fn wait_for_kickoff_frames(
11394 aggregator: &MobKitConsoleAggregator,
11395 identity: &str,
11396 timeout: Duration,
11397 ) -> Vec<ConsoleFrame> {
11398 let deadline = Instant::now() + timeout;
11399 let mut observed = Vec::new();
11400 while Instant::now() < deadline {
11401 let page = aggregator
11402 .query_timeline(ConsoleTimelineQuery {
11403 identity: Some(identity.to_string()),
11404 limit: 50,
11405 ..ConsoleTimelineQuery::default()
11406 })
11407 .await
11408 .expect("query timeline");
11409 observed = page.frames;
11410 if observed.iter().any(|frame| frame.kind == "user_input") {
11411 break;
11412 }
11413 tokio::time::sleep(Duration::from_millis(25)).await;
11414 }
11415 observed
11416 }
11417
11418 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11419 async fn spawn_registry_primes_access_lineage() {
11420 let runtime = build_empty_runtime("spawn-access-lineage-test").await;
11421 let sink = runtime
11422 .mob_runtime()
11423 .console_spawn_sink()
11424 .expect("unified runtime installs the console spawn sink at bootstrap");
11425
11426 runtime
11427 .spawn(SpawnMemberSpec::from_wire(
11428 "worker".to_string(),
11429 "worker-3".to_string(),
11430 None,
11431 None,
11432 None,
11433 ))
11434 .await
11435 .expect("member spawns");
11436 sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
11437 mob_id: Some("spawn-access-lineage-test".to_string()),
11438 member_id: "worker-3".to_string(),
11439 identity: "worker-3".to_string(),
11440 initial_message: None,
11441 labels: BTreeMap::new(),
11442 spawned_by: Some("ops-lead".to_string()),
11443 via_tool: "mob_spawn_member".to_string(),
11444 })
11445 .await;
11446
11447 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
11448 enabled: true,
11449 admins: vec!["root@example.test".to_string()],
11450 groups: BTreeMap::new(),
11451 rules: vec![crate::access::AccessRule {
11452 id: "bob-ops-lead".to_string(),
11453 subjects: vec!["bob@example.test".to_string()],
11454 actions: vec!["agent.view".to_string()],
11455 agents: vec!["ops-lead".to_string()],
11456 ..crate::access::AccessRule::default()
11457 }],
11458 })
11459 .expect("controller");
11460
11461 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
11462 .await;
11463
11464 let bob = controller.view_for_subject(Some("bob@example.test"));
11465 assert!(
11466 bob.can_view_agent("worker-3"),
11467 "spawn registry lineage must let bob see the member ops-lead spawned"
11468 );
11469 assert!(
11470 !bob.can_view_agent("unrelated-agent"),
11471 "agents outside the granted lineage stay denied"
11472 );
11473
11474 let _ = runtime.mob_handle().stop().await;
11475 }
11476
11477 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11478 async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
11479 let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
11480
11481 let mut imposter = SpawnMemberSpec::from_wire(
11484 "worker".to_string(),
11485 "imposter".to_string(),
11486 None,
11487 None,
11488 None,
11489 );
11490 imposter.labels = Some(BTreeMap::from([(
11491 "spawned_by".to_string(),
11492 "ops-lead".to_string(),
11493 )]));
11494 runtime.spawn(imposter).await.expect("imposter spawns");
11495
11496 let mut evader = SpawnMemberSpec::from_wire(
11497 "worker".to_string(),
11498 "evader".to_string(),
11499 None,
11500 None,
11501 None,
11502 );
11503 evader.labels = Some(BTreeMap::from([(
11504 "spawned_by".to_string(),
11505 "secret-lead".to_string(),
11506 )]));
11507 runtime.spawn(evader).await.expect("evader spawns");
11508
11509 let aggregator = MobKitConsoleAggregator::in_memory();
11511 aggregator.register_runtime_handles_with_policy(
11512 "runtime-a",
11513 "",
11514 runtime.mob_runtime().clone(),
11515 None,
11516 runtime.console_events(),
11517 Arc::new(AllowAllConsoleVisibilityPolicy),
11518 );
11519 let identities = aggregator
11520 .list_identities_fresh()
11521 .await
11522 .expect("list identities");
11523 let imposter_record = identities
11524 .iter()
11525 .find(|record| record.identity == "imposter")
11526 .expect("imposter listed");
11527 assert!(
11528 !imposter_record.labels.contains_key("spawned_by"),
11529 "identity records must drop roster-claimed lineage when the spawn registry has none"
11530 );
11531
11532 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
11533 enabled: true,
11534 admins: vec!["root@example.test".to_string()],
11535 groups: BTreeMap::new(),
11536 rules: vec![
11537 crate::access::AccessRule {
11538 id: "bob-ops-lead".to_string(),
11539 subjects: vec!["bob@example.test".to_string()],
11540 actions: vec!["agent.view".to_string()],
11541 agents: vec!["ops-lead".to_string()],
11542 ..crate::access::AccessRule::default()
11543 },
11544 crate::access::AccessRule {
11545 id: "carol-view-all".to_string(),
11546 subjects: vec!["carol@example.test".to_string()],
11547 actions: vec!["agent.view".to_string()],
11548 agents: vec!["*".to_string()],
11549 ..crate::access::AccessRule::default()
11550 },
11551 crate::access::AccessRule {
11552 id: "hide-secret-lead".to_string(),
11553 effect: crate::access::AccessEffect::Deny,
11554 actions: vec!["agent.*".to_string()],
11555 agents: vec!["secret-lead".to_string()],
11556 ..crate::access::AccessRule::default()
11557 },
11558 ],
11559 })
11560 .expect("controller");
11561
11562 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
11563 .await;
11564
11565 let bob = controller.view_for_subject(Some("bob@example.test"));
11566 assert!(
11567 !bob.can_view_agent("imposter"),
11568 "a roster-claimed spawned_by must not mint inherited visibility"
11569 );
11570 let carol = controller.view_for_subject(Some("carol@example.test"));
11571 assert!(
11572 carol.can_view_agent("evader"),
11573 "a roster-claimed spawned_by must not let a member hide behind a denied parent"
11574 );
11575
11576 let _ = runtime.mob_handle().stop().await;
11577 }
11578
11579 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11580 async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
11581 let runtime = build_empty_runtime("agent-spawn-console-test").await;
11582
11583 let sink = runtime
11587 .mob_runtime()
11588 .console_spawn_sink()
11589 .expect("unified runtime installs the console spawn sink at bootstrap");
11590
11591 runtime
11595 .spawn(SpawnMemberSpec::from_wire(
11596 "worker".to_string(),
11597 "worker-3".to_string(),
11598 None,
11599 None,
11600 None,
11601 ))
11602 .await
11603 .expect("member spawns");
11604
11605 let seed = crate::console_spawn::ConsoleSpawnSeed {
11606 mob_id: Some("agent-spawn-console-test".to_string()),
11607 member_id: "worker-3".to_string(),
11608 identity: "worker-3".to_string(),
11609 initial_message: Some(json!("Kick off the person hunt")),
11610 labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
11611 spawned_by: Some("ops-lead".to_string()),
11612 via_tool: "mob_spawn_member".to_string(),
11613 };
11614 sink.project_spawned_member(&seed).await;
11615 sink.project_spawned_member(&seed).await;
11617
11618 let aggregator = MobKitConsoleAggregator::in_memory();
11619 aggregator.register_runtime_handles_with_policy(
11620 "runtime-a",
11621 "test",
11622 runtime.mob_runtime().clone(),
11623 None,
11624 runtime.console_events(),
11625 Arc::new(AllowAllConsoleVisibilityPolicy),
11626 );
11627
11628 let frames =
11629 wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
11630 let kickoffs = frames
11631 .iter()
11632 .filter(|frame| frame.kind == "user_input")
11633 .collect::<Vec<_>>();
11634 assert_eq!(
11635 kickoffs.len(),
11636 1,
11637 "member timeline must contain exactly one kickoff frame; got {frames:#?}"
11638 );
11639 assert_eq!(
11640 kickoffs[0].payload["content"][0]["text"],
11641 "Kick off the person hunt"
11642 );
11643
11644 runtime
11647 .project_console_event_from_unified(&crate::types::EventEnvelope {
11648 event_id: "evt-live-worker-3".to_string(),
11649 source: "test".to_string(),
11650 timestamp_ms: current_time_ms(),
11651 event: crate::types::UnifiedEvent::Agent {
11652 agent_id: "worker-3:0".to_string(),
11653 event_type: "text_delta".to_string(),
11654 payload: Some(json!({ "delta": "on it" })),
11655 },
11656 })
11657 .await;
11658
11659 let deadline = Instant::now() + Duration::from_secs(5);
11660 let mut live_frame = None;
11661 while Instant::now() < deadline && live_frame.is_none() {
11662 let page = aggregator
11663 .query_timeline(ConsoleTimelineQuery {
11664 identity: Some("test/worker-3".to_string()),
11665 limit: 50,
11666 ..ConsoleTimelineQuery::default()
11667 })
11668 .await
11669 .expect("query timeline");
11670 live_frame = page
11671 .frames
11672 .into_iter()
11673 .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
11674 if live_frame.is_none() {
11675 tokio::time::sleep(Duration::from_millis(25)).await;
11676 }
11677 }
11678 let live_frame = live_frame.expect("live runtime event lands in the member chat");
11679 assert_eq!(live_frame.identity, "test/worker-3");
11680
11681 let identities = aggregator
11684 .list_identities_fresh()
11685 .await
11686 .expect("list identities");
11687 let record = identities
11688 .iter()
11689 .find(|record| record.identity == "test/worker-3")
11690 .unwrap_or_else(|| {
11691 panic!("spawned member missing from identity list: {identities:#?}")
11692 });
11693 assert_eq!(
11694 record.labels.get("group").map(String::as_str),
11695 Some("workers")
11696 );
11697 assert_eq!(
11698 record.labels.get("spawned_by").map(String::as_str),
11699 Some("ops-lead")
11700 );
11701
11702 let _ = runtime.mob_handle().stop().await;
11703 }
11704}