1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images,
24 is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32 SourceIngestionTransition,
33};
34pub use store::{
35 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36 SqliteConsoleLogStore,
37};
38pub use types::{
39 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59 CachedOnly,
60 IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65 inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70 pub session_history_backfill_enabled: bool,
71 pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75 fn default() -> Self {
76 Self {
77 session_history_backfill_enabled: true,
78 max_concurrent_session_backfills: 16,
79 }
80 }
81}
82
83struct AggregatorInner {
84 store: Arc<dyn ConsoleLogStore>,
85 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89 session_backfill_permits: Arc<Semaphore>,
90 identity_read_model: ConsoleIdentityReadModel,
91 options: ConsoleAggregatorOptions,
92 live_projection_shutdowns: std::sync::Mutex<BTreeMap<String, tokio::sync::watch::Sender<bool>>>,
98}
99
100#[derive(Clone)]
101struct ConsoleIdentityReadModel {
102 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
103 refresh_lock: Arc<tokio::sync::Mutex<()>>,
104 primed: Arc<AtomicBool>,
105}
106
107impl Default for ConsoleIdentityReadModel {
108 fn default() -> Self {
109 Self {
110 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
111 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
112 primed: Arc::new(AtomicBool::new(false)),
113 }
114 }
115}
116
117impl ConsoleIdentityReadModel {
118 async fn snapshot(
119 &self,
120 inner: Arc<AggregatorInner>,
121 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
122 if !self.primed.load(Ordering::Acquire) {
123 Box::pin(self.prime_now(inner)).await?;
124 }
125 Ok(self.inner.read().await.clone())
126 }
127
128 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
129 self.inner.read().await.clone()
130 }
131
132 async fn refresh_now_if_idle(
133 &self,
134 inner: Arc<AggregatorInner>,
135 mode: IdentityCollectionMode,
136 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
137 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
138 return None;
139 };
140 let result = Box::pin(collect_identity_records(&inner, mode)).await;
141 drop(guard);
142 match result {
143 Ok(identities) => {
144 self.replace(identities.clone()).await;
145 Some(Ok(identities))
146 }
147 Err(err) => Some(Err(err)),
148 }
149 }
150
151 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
152 if self.primed.load(Ordering::Acquire) {
153 return Ok(());
154 }
155 let _guard = self.refresh_lock.clone().lock_owned().await;
156 if self.primed.load(Ordering::Acquire) {
157 return Ok(());
158 }
159 let identities = Box::pin(collect_identity_records(
160 &inner,
161 IdentityCollectionMode::CachedOnly,
162 ))
163 .await?;
164 *self.inner.write().await = identities;
165 self.primed.store(true, Ordering::Release);
166 Ok(())
167 }
168
169 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
170 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
171 return;
172 };
173 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
174 return;
175 };
176 let read_model = self.clone();
177 runtime_handle.spawn(async move {
178 let _guard = guard;
179 match Box::pin(collect_identity_records(
180 &inner,
181 IdentityCollectionMode::IncludeLiveMembers,
182 ))
183 .await
184 {
185 Ok(identities) => {
186 *read_model.inner.write().await = identities;
187 read_model.primed.store(true, Ordering::Release);
188 }
189 Err(err) => {
190 tracing::warn!(error = %err, "console identity read-model refresh failed");
191 }
192 }
193 });
194 }
195
196 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
197 *self.inner.write().await = identities;
198 self.primed.store(true, Ordering::Release);
199 }
200}
201
202#[derive(Clone)]
203struct RuntimeEntry {
204 runtime_key: String,
205 identity_namespace: String,
206 runtime: MobRuntime,
207 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
208 console_events: ConsoleEventStore,
209 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
210}
211
212#[derive(Clone)]
213struct ResolvedConsoleMember {
214 entry: RuntimeEntry,
215 handle: MobHandle,
216 member: MobMemberListEntry,
217 source_mob_id: String,
218 runtime_identity: String,
219}
220
221fn console_member_for_resolved_member(
222 resolved: &ResolvedConsoleMember,
223 record: &ConsoleIdentityRecord,
224) -> ConsoleMember {
225 ConsoleMember {
226 agent_identity: crate::member_comms_id::runtime_alias_str(
227 resolved.member.agent_identity.as_str(),
228 )
229 .into_owned(),
230 role: resolved.member.role.to_string(),
231 state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
232 model_capabilities: Default::default(),
233 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
234 session_id: record.session_id.clone(),
235 wired_to: resolved
236 .member
237 .wired_to
238 .iter()
239 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
240 .collect(),
241 labels: record.labels.clone(),
242 progress: None,
243 }
244}
245
246async fn resolved_member_visible(
247 resolved: &ResolvedConsoleMember,
248 record: &ConsoleIdentityRecord,
249) -> bool {
250 if !raw_resolved_member_visible(resolved, record) {
251 return false;
252 }
253 !Box::pin(live_record_shadowed_by_hidden_durable_binding(
254 resolved, record,
255 ))
256 .await
257}
258
259fn raw_resolved_member_visible(
260 resolved: &ResolvedConsoleMember,
261 record: &ConsoleIdentityRecord,
262) -> bool {
263 resolved.entry.visibility_policy.identity_visible(record)
264 && resolved
265 .entry
266 .visibility_policy
267 .member_visible(&console_member_for_resolved_member(resolved, record))
268}
269
270pub trait ConsoleVisibilityPolicy: Send + Sync {
271 fn include_implicit_delegate_members(&self) -> bool {
272 true
273 }
274
275 fn member_visible(&self, _member: &ConsoleMember) -> bool {
276 true
277 }
278
279 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
280 true
281 }
282
283 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
284 true
285 }
286
287 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
288 None
289 }
290}
291
292#[derive(Debug, Default)]
293pub struct AllowAllConsoleVisibilityPolicy;
294
295impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
296
297#[derive(Debug, Default)]
298pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
299
300impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
301 fn include_implicit_delegate_members(&self) -> bool {
302 false
303 }
304
305 fn member_visible(&self, member: &ConsoleMember) -> bool {
306 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
307 }
308
309 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
310 !is_implicit_delegate_member(
311 record
312 .labels
313 .get("role")
314 .map(String::as_str)
315 .unwrap_or_default(),
316 &record.labels,
317 )
318 }
319}
320
321#[derive(Clone)]
322pub struct ConsoleRuntimeRegistration {
323 pub runtime_key: String,
324 pub runtime: Arc<UnifiedRuntime>,
325 pub identity_namespace: String,
326 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
327}
328
329impl MobKitConsoleAggregator {
330 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
331 Self::new_with_options(store, ConsoleAggregatorOptions::default())
332 }
333
334 pub fn new_with_options(
335 store: Arc<dyn ConsoleLogStore>,
336 mut options: ConsoleAggregatorOptions,
337 ) -> Self {
338 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
339 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
340 Self {
341 inner: Arc::new(AggregatorInner {
342 store,
343 runtimes: RwLock::new(BTreeMap::new()),
344 event_tx,
345 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
346 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
347 session_backfill_permits: Arc::new(Semaphore::new(
348 options.max_concurrent_session_backfills,
349 )),
350 identity_read_model: ConsoleIdentityReadModel::default(),
351 options,
352 live_projection_shutdowns: std::sync::Mutex::new(BTreeMap::new()),
353 }),
354 }
355 }
356
357 pub fn in_memory() -> Self {
358 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
359 }
360
361 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
362 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
363 }
364
365 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
366 self.inner.event_tx.subscribe()
367 }
368
369 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
370 self.inner.store.clone()
371 }
372
373 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
374 let identity_runtime = registration.runtime.identity_runtime().cloned();
375 self.register_runtime_handles_with_policy(
376 registration.runtime_key,
377 registration.identity_namespace,
378 registration.runtime.mob_runtime().clone(),
379 identity_runtime,
380 registration.runtime.console_events(),
381 registration.visibility_policy,
382 );
383 }
384
385 pub(crate) fn register_runtime_handles_with_policy(
386 &self,
387 runtime_key: impl Into<String>,
388 identity_namespace: impl Into<String>,
389 runtime: MobRuntime,
390 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
391 console_events: ConsoleEventStore,
392 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
393 ) {
394 let runtime_key = runtime_key.into();
395 let identity_namespace = identity_namespace.into();
396 let entry = RuntimeEntry {
397 runtime_key: runtime_key.clone(),
398 identity_namespace,
399 runtime,
400 identity_runtime,
401 console_events: console_events.clone(),
402 visibility_policy,
403 };
404 if let Ok(mut runtimes) = self.inner.runtimes.write() {
405 runtimes.insert(runtime_key.clone(), entry);
406 }
407 let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false);
410 if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
411 && let Some(previous) = shutdowns.insert(runtime_key.clone(), shutdown_tx)
412 {
413 let _ = previous.send(true);
414 }
415 self.inner
416 .identity_read_model
417 .refresh_soon(self.inner.clone());
418 let inner = self.inner.clone();
419 let events_for_live = console_events.clone();
420 let events_for_live_recovery = console_events.clone();
421 let runtime_key_for_live = runtime_key.clone();
422 tokio::spawn(async move {
423 let mut rx = events_for_live.subscribe();
424 loop {
425 tokio::select! {
426 changed = shutdown_rx.changed() => {
427 if changed.is_err() || *shutdown_rx.borrow() {
429 break;
430 }
431 }
432 received = rx.recv() => match received {
433 Ok(envelope) => {
434 let _ = project_console_event(
435 inner.clone(),
436 &runtime_key_for_live,
437 envelope,
438 )
439 .await;
440 }
441 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
442 let _ = recover_lagged_source_events(
443 inner.clone(),
444 &runtime_key_for_live,
445 &events_for_live_recovery,
446 )
447 .await;
448 }
449 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
450 },
451 }
452 }
453 });
454
455 let inner = self.inner.clone();
456 let events_for_replay = console_events;
457 let runtime_key_for_replay = runtime_key;
458 tokio::spawn(async move {
459 let mut ingestion_state = SourceIngestionState::Registered;
460 if let Ok((next, _effects)) =
461 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
462 {
463 ingestion_state = next;
464 }
465 if let Ok(events) = events_for_replay.replay_all(None).await {
466 for envelope in events {
467 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
468 .await;
469 }
470 }
471 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
472 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
473 if let Ok((next, _effects)) =
474 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
475 {
476 ingestion_state = next;
477 }
478 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
479 });
480 }
481
482 pub fn unregister_runtime(&self, runtime_key: &str) {
492 let removed = self
493 .inner
494 .runtimes
495 .write()
496 .ok()
497 .and_then(|mut runtimes| runtimes.remove(runtime_key))
498 .is_some();
499 if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
500 && let Some(shutdown) = shutdowns.remove(runtime_key)
501 {
502 let _ = shutdown.send(true);
503 }
504 if removed {
505 self.inner
506 .identity_read_model
507 .refresh_soon(self.inner.clone());
508 }
509 }
510
511 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
512 if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
513 self.inner.clone(),
514 IdentityCollectionMode::IncludeLiveMembers,
515 ))
516 .await
517 {
518 let identities = identities?;
519 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
520 return Ok(identities);
521 }
522 self.inner
523 .identity_read_model
524 .refresh_soon(self.inner.clone());
525 let identities =
526 Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
527 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
528 Ok(identities)
529 }
530
531 #[cfg(test)]
532 pub(crate) async fn list_identities_fresh(
533 &self,
534 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
535 let _guard = self
536 .inner
537 .identity_read_model
538 .refresh_lock
539 .clone()
540 .lock_owned()
541 .await;
542 let identities =
543 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
544 .await?;
545 self.inner
546 .identity_read_model
547 .replace(identities.clone())
548 .await;
549 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
550 Ok(identities)
551 }
552
553 pub async fn inspect_identity(
554 &self,
555 identity: &str,
556 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
557 let entries = self
558 .inner
559 .runtimes
560 .read()
561 .map_err(|_| runtime_registry_lock_error())?
562 .values()
563 .cloned()
564 .collect::<Vec<_>>();
565 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
566 let mut durable_matches = Vec::new();
567 if !requested_identity_has_runtime_member_alias(identity, &entries) {
568 for entry in &entries {
569 let Some(identity_runtime) = entry.identity_runtime.clone() else {
570 continue;
571 };
572 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
573 {
574 let Ok(parsed_identity) =
575 crate::identity_first::AgentIdentity::parse(&raw_identity)
576 else {
577 continue;
578 };
579 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
580 continue;
581 };
582 let mut record = identity_record_for_status(entry, &status);
583 let topology_peers =
584 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
585 record.topology_peers = topology_peers
586 .get(&record.identity)
587 .cloned()
588 .unwrap_or_default();
589 if Box::pin(console_identity_record_visible(entry, &record)).await {
590 durable_matches.push((
591 entry.clone(),
592 identity_runtime.clone(),
593 parsed_identity,
594 record,
595 ));
596 }
597 }
598 }
599 }
600 if durable_matches.len() > 1 {
601 let candidates = durable_matches
602 .iter()
603 .map(|(_, _, _, record)| record.runtime_member_id.clone())
604 .collect::<Vec<_>>()
605 .join(", ");
606 return Err(format!(
607 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
608 )
609 .into());
610 }
611 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
612 durable_matches.into_iter().next()
613 {
614 let durable_live_records = Box::pin(self.live_records_for_durable_record(
615 &entry,
616 identity,
617 &record,
618 &live_records,
619 ))
620 .await;
621 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
622 &entry,
623 &durable_live_records,
624 ))
625 .await;
626 if let Some(ambiguous_error) =
627 ambiguous_live_alias_error(identity, &visible_durable_live_records)
628 {
629 return Err(ambiguous_error.into());
630 }
631 if let Some(rebound_record) = self_heal_durable_session_mismatch(
632 &entry,
633 identity_runtime.as_ref(),
634 &parsed_identity,
635 identity,
636 &record,
637 &durable_live_records,
638 )
639 .await
640 .map_err(ConsoleLogError::from)?
641 {
642 record = rebound_record;
643 }
644 if let Some(stale_error) =
645 stale_durable_record_error(identity, &record, &durable_live_records)
646 {
647 return Err(stale_error.into());
648 }
649 if let Some(session_id) = record.session_id.clone() {
650 spawn_session_history_backfill_target(
651 self.inner.clone(),
652 SessionBackfillTarget {
653 entry,
654 record: record.clone(),
655 session_id,
656 },
657 false,
658 );
659 }
660 return Ok(Some(ConsoleIdentityInspection {
661 peers: record.topology_peers.clone(),
662 identity: record,
663 }));
664 }
665 let mut runtime_id_matches = Vec::new();
666 for entry in &entries {
667 let Some(identity_runtime) = entry.identity_runtime.clone() else {
668 continue;
669 };
670 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
671 continue;
672 };
673 let Some(status) = identity_runtime
674 .statuses()
675 .await
676 .into_iter()
677 .find(|status| {
678 status
679 .agent_runtime_id
680 .as_ref()
681 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
682 })
683 else {
684 continue;
685 };
686 let mut record = identity_record_for_status(entry, &status);
687 let topology_peers =
688 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
689 record.topology_peers = topology_peers
690 .get(&record.identity)
691 .cloned()
692 .unwrap_or_default();
693 if Box::pin(console_identity_record_visible(entry, &record)).await {
694 runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
695 }
696 }
697 if runtime_id_matches.len() > 1 {
698 let candidates = runtime_id_matches
699 .iter()
700 .map(|(_, _, _, record)| record.identity.clone())
701 .collect::<Vec<_>>()
702 .join(", ");
703 return Err(format!(
704 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
705 )
706 .into());
707 }
708 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
709 runtime_id_matches.into_iter().next()
710 {
711 let durable_live_records = Box::pin(self.live_records_for_durable_record(
712 &entry,
713 identity,
714 &record,
715 &live_records,
716 ))
717 .await;
718 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
719 &entry,
720 &durable_live_records,
721 ))
722 .await;
723 if let Some(ambiguous_error) =
724 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
725 {
726 return Err(ambiguous_error.into());
727 }
728 if let Some(rebound_record) = self_heal_durable_session_mismatch(
729 &entry,
730 identity_runtime.as_ref(),
731 &parsed_identity,
732 identity,
733 &record,
734 &durable_live_records,
735 )
736 .await
737 .map_err(ConsoleLogError::from)?
738 {
739 record = rebound_record;
740 }
741 if let Some(stale_error) =
742 stale_durable_record_error(identity, &record, &durable_live_records)
743 {
744 return Err(stale_error.into());
745 }
746 if let Some(session_id) = record.session_id.clone() {
747 spawn_session_history_backfill_target(
748 self.inner.clone(),
749 SessionBackfillTarget {
750 entry: entry.clone(),
751 record: record.clone(),
752 session_id,
753 },
754 false,
755 );
756 }
757 return Ok(Some(ConsoleIdentityInspection {
758 peers: record.topology_peers.clone(),
759 identity: record,
760 }));
761 }
762
763 let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
764 if live_matches.len() > 1 {
765 let candidates = live_matches
766 .iter()
767 .map(|resolved| resolved.runtime_identity.clone())
768 .collect::<Vec<_>>()
769 .join(", ");
770 return Err(format!(
771 "ambiguous live identity alias {identity}: candidates [{candidates}]"
772 )
773 .into());
774 }
775 if let Some(resolved) = live_matches.into_iter().next() {
776 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
777 return Ok(None);
778 };
779 if let Some(stale_error) =
780 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
781 {
782 return Err(stale_error.into());
783 }
784 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
785 && let Ok(parsed_identity) =
786 crate::identity_first::AgentIdentity::parse(&record.identity)
787 && let Ok(status) = identity_runtime.status(&parsed_identity).await
788 {
789 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
790 let durable_live_records = Box::pin(self.live_records_for_durable_record(
791 &resolved.entry,
792 identity,
793 &durable_record,
794 &live_records,
795 ))
796 .await;
797 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
798 &resolved.entry,
799 &durable_live_records,
800 ))
801 .await;
802 if let Some(ambiguous_error) =
803 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
804 {
805 return Err(ambiguous_error.into());
806 }
807 if let Some(rebound_record) = self_heal_durable_session_mismatch(
808 &resolved.entry,
809 identity_runtime.as_ref(),
810 &parsed_identity,
811 identity,
812 &durable_record,
813 &durable_live_records,
814 )
815 .await
816 .map_err(ConsoleLogError::from)?
817 {
818 durable_record = rebound_record;
819 }
820 if let Some(stale_error) =
821 stale_durable_record_error(identity, &durable_record, &durable_live_records)
822 {
823 return Err(stale_error.into());
824 }
825 if status
826 .agent_runtime_id
827 .as_ref()
828 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
829 {
830 return Ok(None);
831 }
832 }
833 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
834 return Ok(None);
835 }
836 if let Some(session_id) = record.session_id.clone() {
837 spawn_session_history_backfill_target(
838 self.inner.clone(),
839 SessionBackfillTarget {
840 entry: resolved.entry.clone(),
841 record: record.clone(),
842 session_id,
843 },
844 false,
845 );
846 }
847 let peers = resolved
848 .member
849 .wired_to
850 .iter()
851 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
852 .collect();
853 return Ok(Some(ConsoleIdentityInspection {
854 identity: record,
855 peers,
856 }));
857 }
858
859 Ok(None)
860 }
861
862 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
863 let entries = self
864 .inner
865 .runtimes
866 .read()
867 .map_err(|_| runtime_registry_lock_error())?
868 .clone();
869 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
870 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
871 let mut durable_matches = Vec::new();
872 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
873 for entry in entries.values() {
874 let Some(identity_runtime) = entry.identity_runtime.clone() else {
875 continue;
876 };
877 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
878 {
879 let Ok(parsed_identity) =
880 crate::identity_first::AgentIdentity::parse(&raw_identity)
881 else {
882 continue;
883 };
884 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
885 continue;
886 };
887 let record = identity_record_for_status(entry, &status);
888 if !Box::pin(console_identity_record_visible(entry, &record)).await {
889 continue;
890 }
891 durable_matches.push((
892 entry.clone(),
893 identity_runtime.clone(),
894 parsed_identity,
895 record,
896 ));
897 }
898 }
899 }
900 if durable_matches.len() > 1 {
901 let candidates = durable_matches
902 .iter()
903 .map(|(_, _, identity, _)| identity.as_str().to_string())
904 .collect::<Vec<_>>()
905 .join(", ");
906 return Err(format!(
907 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
908 )
909 .into());
910 }
911 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
912 durable_matches.into_iter().next()
913 {
914 let durable_live_records = Box::pin(self.live_records_for_durable_record(
915 &entry,
916 identity,
917 &record,
918 &live_records,
919 ))
920 .await;
921 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
922 &entry,
923 &durable_live_records,
924 ))
925 .await;
926 if let Some(ambiguous_error) =
927 ambiguous_live_alias_error(identity, &visible_durable_live_records)
928 {
929 return Err(ambiguous_error.into());
930 }
931 if let Some(rebound_record) = self_heal_durable_session_mismatch(
932 &entry,
933 identity_runtime.as_ref(),
934 &parsed_identity,
935 identity,
936 &record,
937 &durable_live_records,
938 )
939 .await
940 .map_err(ConsoleLogError::from)?
941 {
942 record = rebound_record;
943 }
944 if let Some(stale_error) =
945 stale_durable_record_error(identity, &record, &durable_live_records)
946 {
947 return Err(stale_error.into());
948 }
949 identity_runtime
950 .retire(&parsed_identity)
951 .await
952 .map_err(|err| -> ConsoleLogError {
953 format!("retire failed for {identity}: {err}").into()
954 })?;
955 {
956 let mut durable_identities =
957 namespace_match_candidates(identity, &entry.identity_namespace);
958 durable_identities.push(parsed_identity.as_str().to_string());
959 durable_identities.sort();
960 durable_identities.dedup();
961 if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
962 &entry,
963 &durable_identities,
964 ))
965 .await
966 {
967 tracing::warn!(
968 identity,
969 error = %err,
970 "stale console member cleanup failed after durable identity retire"
971 );
972 }
973 }
974 return Ok(true);
975 }
976 let matches = Box::pin(self.resolve_visible_members(identity)).await;
977 if matches.len() > 1 {
978 let candidates = matches
979 .iter()
980 .map(|resolved| resolved.runtime_identity.clone())
981 .collect::<Vec<_>>()
982 .join(", ");
983 return Err(format!(
984 "ambiguous live identity alias {identity}: candidates [{candidates}]"
985 )
986 .into());
987 }
988 let mut live_retired_any = false;
989 for resolved in matches {
990 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
991 continue;
992 };
993 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
994 continue;
995 }
996 if let Some(stale_error) =
997 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
998 {
999 return Err(stale_error.into());
1000 }
1001 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
1002 && let Ok(parsed_identity) =
1003 crate::identity_first::AgentIdentity::parse(&record.identity)
1004 && let Ok(status) = identity_runtime.status(&parsed_identity).await
1005 {
1006 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
1007 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1008 &resolved.entry,
1009 identity,
1010 &durable_record,
1011 &live_records,
1012 ))
1013 .await;
1014 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1015 &resolved.entry,
1016 &durable_live_records,
1017 ))
1018 .await;
1019 if let Some(ambiguous_error) =
1020 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
1021 {
1022 return Err(ambiguous_error.into());
1023 }
1024 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1025 &resolved.entry,
1026 identity_runtime.as_ref(),
1027 &parsed_identity,
1028 identity,
1029 &durable_record,
1030 &durable_live_records,
1031 )
1032 .await
1033 .map_err(ConsoleLogError::from)?
1034 {
1035 durable_record = rebound_record;
1036 }
1037 if let Some(stale_error) =
1038 stale_durable_record_error(identity, &durable_record, &durable_live_records)
1039 {
1040 return Err(stale_error.into());
1041 }
1042 if status
1043 .agent_runtime_id
1044 .as_ref()
1045 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
1046 {
1047 continue;
1048 }
1049 }
1050 resolved
1051 .handle
1052 .retire(crate::member_comms_id::mob_member_id(
1053 resolved.runtime_identity.as_str(),
1054 ))
1055 .await
1056 .map_err(|err| -> ConsoleLogError {
1057 format!("retire failed for {identity}: {err}").into()
1058 })
1059 .or_else(|err| {
1060 if lifecycle_archive_cleanup_completed(&err.to_string()) {
1061 Ok(())
1062 } else {
1063 Err(err)
1064 }
1065 })?;
1066 live_retired_any = true;
1067 }
1068 Ok(live_retired_any)
1069 }
1070
1071 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1072 self.inner.store.clear_frames().await
1073 }
1074
1075 pub async fn query_timeline(
1076 &self,
1077 query: ConsoleTimelineQuery,
1078 ) -> ConsoleLogResult<ConsoleTimelinePage> {
1079 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1080 Ok(ConsoleTimelinePage {
1081 frames: page.frames,
1082 next_cursor: page.next_cursor,
1083 })
1084 }
1085
1086 pub async fn query_timeline_windowed(
1087 &self,
1088 query: ConsoleTimelineWindowQuery,
1089 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1090 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1091 .await?;
1092 if query.after.is_none()
1093 && query.before.is_none()
1094 && let Some(identity) = query.identity.clone()
1095 {
1096 let mut probe_query = query.clone();
1097 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1098 let page = self.inner.store.query_windowed_frames(probe_query).await?;
1099 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1100 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1101 }
1102 }
1103 Box::pin(self.query_timeline_visible(query)).await
1104 }
1105
1106 async fn reject_after_cursor_beyond_store_frontier(
1107 &self,
1108 after: Option<&ConsoleCursor>,
1109 ) -> ConsoleLogResult<()> {
1110 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1111 return Ok(());
1112 };
1113 let latest_seq = self
1114 .inner
1115 .store
1116 .latest_cursor()
1117 .await?
1118 .and_then(|cursor| cursor.seq())
1119 .unwrap_or(0);
1120 if after_seq > latest_seq {
1121 return Err(std::io::Error::other(
1122 "timeline replay cursor is beyond the current store frontier",
1123 )
1124 .into());
1125 }
1126 Ok(())
1127 }
1128
1129 async fn query_timeline_visible(
1130 &self,
1131 query: ConsoleTimelineWindowQuery,
1132 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1133 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1134 let mut scan_query = query.clone();
1135 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1136 let mut visible_frames = Vec::with_capacity(requested_limit);
1137 let mut anchor_frames = Vec::new();
1138 let mut identity_visibility_cache = HashMap::new();
1139 let identity_records = self.inner.identity_read_model.current().await;
1140 let mut next_cursor = query.after.clone();
1141 let mut since_last_scanned_cursor = query.after.clone();
1142 let mut since_last_delivered_cursor = None;
1143 let mut since_stopped_at_visible_limit = false;
1144 let mut latest_cursor = None;
1145 let mut exhausted = false;
1146 let mut scanned = 0usize;
1147
1148 loop {
1149 let page = self
1150 .inner
1151 .store
1152 .query_windowed_frames(scan_query.clone())
1153 .await?;
1154 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1155 if page.frames.is_empty() {
1156 exhausted = true;
1157 break;
1158 }
1159 let raw_len = page.frames.len();
1160 scanned = scanned.saturating_add(raw_len);
1161 match query.mode {
1162 ConsoleTimelineMode::Since => {
1163 if let Some(cursor) = page.next_cursor.clone() {
1164 since_last_scanned_cursor = Some(cursor.clone());
1165 scan_query.after = Some(cursor);
1166 }
1167 }
1168 ConsoleTimelineMode::Recent => {
1169 if let Some(first) = page.frames.first() {
1170 scan_query.before = Some(first.cursor.clone());
1171 }
1172 next_cursor = page.next_cursor.clone().or(next_cursor);
1173 }
1174 }
1175
1176 for frame in page.frames {
1177 let allow_historical_identity =
1178 query.identity.as_deref() == Some(frame.identity.as_str());
1179 if Box::pin(frame_is_visible_cached(
1180 &self.inner,
1181 &frame,
1182 allow_historical_identity,
1183 &mut identity_visibility_cache,
1184 &identity_records,
1185 ))
1186 .await
1187 .unwrap_or(false)
1188 {
1189 if query.mode == ConsoleTimelineMode::Recent
1190 && query.identity.is_some()
1191 && is_identity_timeline_anchor_frame(&frame)
1192 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1193 {
1194 anchor_frames.push(frame.clone());
1195 }
1196 match query.mode {
1197 ConsoleTimelineMode::Since => {
1198 if visible_frames.len() >= requested_limit {
1199 since_stopped_at_visible_limit = true;
1200 break;
1201 }
1202 since_last_delivered_cursor = Some(frame.cursor.clone());
1203 visible_frames.push(frame);
1204 if visible_frames.len() >= requested_limit {
1205 since_stopped_at_visible_limit = true;
1206 exhausted = false;
1207 break;
1208 }
1209 }
1210 ConsoleTimelineMode::Recent => {
1211 visible_frames.push(frame);
1212 }
1213 }
1214 }
1215 }
1216 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1217 && query.identity.is_some()
1218 && anchor_frames.is_empty()
1219 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1220 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1221 break;
1222 }
1223 if since_stopped_at_visible_limit {
1224 break;
1225 }
1226 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1227 exhausted = true;
1228 break;
1229 }
1230 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1231 break;
1232 }
1233 }
1234
1235 if query.mode == ConsoleTimelineMode::Recent {
1236 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1237 if visible_frames.len() > requested_limit {
1238 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1239 }
1240 if !anchor_frames.is_empty() {
1241 let anchor_cursors = anchor_frames
1242 .iter()
1243 .map(|frame| frame.cursor.clone())
1244 .collect::<Vec<_>>();
1245 let mut merged = anchor_frames;
1246 for frame in visible_frames {
1247 if !merged
1248 .iter()
1249 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1250 {
1251 merged.push(frame);
1252 }
1253 }
1254 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1255 visible_frames = merged;
1256 if visible_frames.len() > requested_limit {
1257 let mut anchors = Vec::new();
1258 let mut tail = Vec::new();
1259 for frame in visible_frames {
1260 if anchor_cursors.contains(&frame.cursor) {
1261 anchors.push(frame);
1262 } else {
1263 tail.push(frame);
1264 }
1265 }
1266 visible_frames = if anchors.len() >= requested_limit {
1267 anchors.split_off(anchors.len() - requested_limit)
1268 } else {
1269 let keep_tail = requested_limit.saturating_sub(anchors.len());
1270 if tail.len() > keep_tail {
1271 tail = tail.split_off(tail.len() - keep_tail);
1272 }
1273 anchors.extend(tail);
1274 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1275 anchors
1276 };
1277 }
1278 }
1279 }
1280
1281 Ok(ConsoleTimelineWindowPage {
1282 frames: visible_frames,
1283 next_cursor: match query.mode {
1284 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1285 since_last_delivered_cursor.or(since_last_scanned_cursor)
1286 }
1287 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1288 ConsoleTimelineMode::Recent => next_cursor,
1289 },
1290 latest_cursor,
1291 exhausted,
1292 })
1293 }
1294
1295 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1296 let runtime_keys = self
1297 .inner
1298 .runtimes
1299 .read()
1300 .map_err(|_| runtime_registry_lock_error())?
1301 .keys()
1302 .cloned()
1303 .collect::<Vec<_>>();
1304 let results =
1305 join_all(runtime_keys.into_iter().map(|runtime_key| {
1306 backfill_session_history(self.inner.clone(), runtime_key, true)
1307 }))
1308 .await;
1309 for result in results {
1310 result?;
1311 }
1312 Ok(())
1313 }
1314
1315 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1316 self.inner.store.latest_cursor().await
1317 }
1318
1319 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1320 self.timeline_event_visible_for_subscriber(event, None)
1321 .await
1322 }
1323
1324 pub async fn timeline_event_visible_for_subscriber(
1342 &self,
1343 event: &ConsoleTimelineEvent,
1344 subscriber_identity: Option<&str>,
1345 ) -> bool {
1346 match event {
1347 ConsoleTimelineEvent::ConsoleFrame { frame }
1348 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1349 let identity_records = self.inner.identity_read_model.current().await;
1350 let identity_known = identity_records.iter().any(|record| {
1351 record.runtime_key == frame.runtime_key
1352 && (record.identity == frame.identity
1353 || record.runtime_member_id == frame.identity)
1354 });
1355 if !identity_known
1356 && frame.identity != "__console__"
1357 && frame.identity != SYSTEM_EVENT_IDENTITY
1358 {
1359 self.inner
1364 .identity_read_model
1365 .refresh_soon(self.inner.clone());
1366 }
1367 let allow_historical_identity =
1368 subscriber_identity == Some(frame.identity.as_str());
1369 Box::pin(frame_is_visible(
1370 &self.inner,
1371 frame,
1372 allow_historical_identity,
1373 &identity_records,
1374 ))
1375 .await
1376 .unwrap_or(false)
1377 }
1378 ConsoleTimelineEvent::SnapshotStarted { .. }
1379 | ConsoleTimelineEvent::SnapshotComplete { .. }
1380 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1381 }
1382 }
1383
1384 pub async fn timeline_frame_visible_for_query(
1385 &self,
1386 frame: &ConsoleFrame,
1387 identity: Option<&str>,
1388 ) -> bool {
1389 let allow_historical_identity = identity == Some(frame.identity.as_str());
1390 let identity_records = self.inner.identity_read_model.current().await;
1391 Box::pin(frame_is_visible(
1392 &self.inner,
1393 frame,
1394 allow_historical_identity,
1395 &identity_records,
1396 ))
1397 .await
1398 .unwrap_or(false)
1399 }
1400
1401 pub async fn send(
1402 &self,
1403 request: ConsoleSendRequest,
1404 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1405 validate_send_request(&request)?;
1406 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1407 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1408 };
1409 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1410 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1411 };
1412 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1413 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1414 }
1415 if !member_is_addressable(&resolved.member) {
1416 return Err(ConsoleSendError::NotAddressable(request.identity));
1417 }
1418 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1419 return Err(ConsoleSendError::Retired(request.identity));
1420 }
1421
1422 let content = content_input_from_value(&request.content)?;
1423 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1424 assert_member_accepts_images(
1425 &resolved.handle,
1426 resolved.entry.runtime.session_service(),
1427 &resolved.runtime_identity,
1428 &content,
1429 )
1430 .await
1431 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1432
1433 let dedupe_key = send_dedupe_key(
1434 &resolved.entry.runtime_key,
1435 &request.identity,
1436 &request.origin,
1437 &request.idempotency_key,
1438 );
1439 let handling_mode_value = request
1440 .handling_mode
1441 .as_deref()
1442 .unwrap_or("queue")
1443 .to_string();
1444 let request_fingerprint =
1445 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1446 if let Some(existing) = self
1447 .inner
1448 .store
1449 .frame_by_dedupe_key(&dedupe_key)
1450 .await
1451 .map_err(ConsoleSendError::Log)?
1452 {
1453 let same_request = existing.source.source_cursor.as_deref()
1454 == Some(request_fingerprint.as_str())
1455 || existing.source.source_cursor.is_none()
1456 && existing.payload.get("origin").and_then(Value::as_str)
1457 == Some(request.origin.as_str())
1458 && existing.payload.get("content") == Some(&request.content)
1459 && existing
1460 .payload
1461 .get("handling_mode")
1462 .and_then(Value::as_str)
1463 == Some(handling_mode_value.as_str());
1464 if !same_request {
1465 return Err(ConsoleSendError::IdempotencyConflict(
1466 request.idempotency_key,
1467 ));
1468 }
1469 return Ok(accepted_from_frame(&existing));
1470 }
1471
1472 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1473 resolved
1474 .entry
1475 .console_events
1476 .reserve_interaction_value(
1477 &resolved.runtime_identity,
1478 Some(resolved.runtime_identity.as_str()),
1479 &interaction_id,
1480 &request.origin,
1481 request.content.clone(),
1482 )
1483 .await
1484 .map_err(ConsoleSendError::State)?;
1485 let session_id = resolved
1486 .handle
1487 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1488 resolved.runtime_identity.as_str(),
1489 ))
1490 .await
1491 .map(|sid| sid.to_string());
1492 let mut new_frame = NewConsoleFrame {
1493 id: None,
1494 dedupe_key,
1495 timestamp_ms: current_time_ms(),
1496 runtime_key: resolved.entry.runtime_key.clone(),
1497 identity: request.identity.clone(),
1498 conversation_id: Some(request.identity.clone()),
1499 session_id: session_id.clone(),
1500 kind: "user_input".to_string(),
1501 status: ConsoleFrameStatus::Accepted,
1502 payload: json!({
1503 "content": request.content,
1504 "origin": request.origin,
1505 "idempotency_key": request.idempotency_key,
1506 "handling_mode": handling_mode_value,
1507 }),
1508 source: ConsoleFrameSource {
1509 kind: ConsoleFrameSourceKind::Send,
1510 source_cursor: Some(request_fingerprint),
1511 },
1512 source_event_id: None,
1513 interaction_id: Some(interaction_id.clone()),
1514 turn_id: None,
1515 run_id: None,
1516 parent_frame_id: None,
1517 caused_by_frame_id: None,
1518 };
1519 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1520 new_frame.payload = redacted;
1521 new_frame.status = ConsoleFrameStatus::Redacted;
1522 }
1523
1524 let _ = SendState::Requested
1525 .apply(SendTransition::PersistAccepted)
1526 .map_err(ConsoleSendError::State)?;
1527 let outcome = self
1528 .inner
1529 .store
1530 .append_if_absent(new_frame)
1531 .await
1532 .map_err(ConsoleSendError::Log)?;
1533 if outcome.disposition == AppendDisposition::Existing {
1534 return Ok(accepted_from_frame(&outcome.frame));
1535 }
1536 let _ = self
1537 .inner
1538 .event_tx
1539 .send(ConsoleTimelineEvent::ConsoleFrame {
1540 frame: outcome.frame.clone(),
1541 });
1542 let accepted = accepted_from_frame(&outcome.frame);
1543
1544 let (dispatching, _effects) = SendState::AcceptedPersisted
1545 .apply(SendTransition::StartDispatch)
1546 .map_err(ConsoleSendError::State)?;
1547 update_frame_status_and_emit(
1548 &self.inner,
1549 &outcome.frame.id,
1550 ConsoleFrameStatus::Dispatching,
1551 )
1552 .await
1553 .map_err(ConsoleSendError::Log)?;
1554
1555 spawn_console_send_dispatch(
1556 self.inner.clone(),
1557 resolved,
1558 content,
1559 handling_mode,
1560 dispatching,
1561 outcome.frame,
1562 interaction_id,
1563 );
1564 Ok(accepted)
1565 }
1566
1567 pub async fn reserve_identity_first_interaction(
1568 &self,
1569 request: ConsoleSendRequest,
1570 session_id: Option<&str>,
1571 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1572 validate_send_request(&request)?;
1573 let _content = content_input_from_value(&request.content)?;
1574 let runtime_key =
1575 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1576 let handling_mode_value = request
1577 .handling_mode
1578 .as_deref()
1579 .unwrap_or("queue")
1580 .to_string();
1581 let dedupe_key = send_dedupe_key(
1582 &runtime_key,
1583 &request.identity,
1584 &request.origin,
1585 &request.idempotency_key,
1586 );
1587 let request_fingerprint =
1588 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1589 if let Some(existing) = self
1590 .inner
1591 .store
1592 .frame_by_dedupe_key(&dedupe_key)
1593 .await
1594 .map_err(ConsoleSendError::Log)?
1595 {
1596 let same_request = existing.source.source_cursor.as_deref()
1597 == Some(request_fingerprint.as_str())
1598 || existing.source.source_cursor.is_none()
1599 && existing.payload.get("origin").and_then(Value::as_str)
1600 == Some(request.origin.as_str())
1601 && existing.payload.get("content") == Some(&request.content)
1602 && existing
1603 .payload
1604 .get("handling_mode")
1605 .and_then(Value::as_str)
1606 == Some(handling_mode_value.as_str());
1607 if !same_request {
1608 return Err(ConsoleSendError::IdempotencyConflict(
1609 request.idempotency_key,
1610 ));
1611 }
1612 return Ok(accepted_from_frame(&existing));
1613 }
1614
1615 let interaction_id = identity_first_interaction_uuid(&dedupe_key);
1616 let new_frame = NewConsoleFrame {
1617 id: None,
1618 dedupe_key,
1619 timestamp_ms: current_time_ms(),
1620 runtime_key,
1621 identity: request.identity.clone(),
1622 conversation_id: Some(request.identity.clone()),
1623 session_id: session_id.map(ToString::to_string),
1624 kind: "user_input".to_string(),
1625 status: ConsoleFrameStatus::Accepted,
1626 payload: json!({
1627 "content": request.content,
1628 "origin": request.origin,
1629 "idempotency_key": request.idempotency_key,
1630 "handling_mode": handling_mode_value,
1631 }),
1632 source: ConsoleFrameSource {
1633 kind: ConsoleFrameSourceKind::Send,
1634 source_cursor: Some(request_fingerprint),
1635 },
1636 source_event_id: None,
1637 interaction_id: Some(interaction_id),
1638 turn_id: None,
1639 run_id: None,
1640 parent_frame_id: None,
1641 caused_by_frame_id: None,
1642 };
1643 let outcome = self
1644 .inner
1645 .store
1646 .append_if_absent(new_frame)
1647 .await
1648 .map_err(ConsoleSendError::Log)?;
1649 let _ = self
1650 .inner
1651 .event_tx
1652 .send(ConsoleTimelineEvent::ConsoleFrame {
1653 frame: outcome.frame.clone(),
1654 });
1655 Ok(accepted_from_frame(&outcome.frame))
1656 }
1657
1658 async fn runtime_key_for_identity_first_send(
1659 &self,
1660 identity: &str,
1661 ) -> Result<String, ConsoleSendError> {
1662 let entries = self
1663 .inner
1664 .runtimes
1665 .read()
1666 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1667 .clone();
1668 if entries.is_empty() {
1669 return Ok("identity-first".to_string());
1670 }
1671
1672 let mut identity_runtime_keys = Vec::new();
1673 let mut hidden_durable_match = false;
1674 let mut durable_matches = Vec::new();
1675 for entry in entries.values() {
1676 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1677 continue;
1678 };
1679 identity_runtime_keys.push(entry.runtime_key.clone());
1680 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1681 continue;
1682 }
1683 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1684 {
1685 let Ok(parsed_identity) =
1686 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1687 else {
1688 continue;
1689 };
1690 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1691 let record = identity_record_for_status(entry, &status);
1692 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1693 hidden_durable_match = true;
1694 continue;
1695 }
1696 durable_matches.push((
1697 entry.clone(),
1698 identity_runtime.clone(),
1699 parsed_identity,
1700 record,
1701 ));
1702 }
1703 }
1704 }
1705 if durable_matches.len() > 1 {
1706 let candidates = durable_matches
1707 .iter()
1708 .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1709 .collect::<Vec<_>>()
1710 .join(", ");
1711 return Err(ConsoleSendError::InvalidRequest(format!(
1712 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1713 )));
1714 }
1715 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1716 durable_matches.into_iter().next()
1717 {
1718 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1719 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1720 &entry,
1721 identity,
1722 &record,
1723 &live_records,
1724 ))
1725 .await;
1726 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1727 &entry,
1728 identity_runtime.as_ref(),
1729 &parsed_identity,
1730 identity,
1731 &record,
1732 &durable_live_records,
1733 )
1734 .await
1735 .map_err(ConsoleSendError::InvalidRequest)?
1736 {
1737 record = rebound_record;
1738 }
1739 if let Some(stale_error) =
1740 stale_durable_record_error(identity, &record, &durable_live_records)
1741 {
1742 return Err(ConsoleSendError::InvalidRequest(stale_error));
1743 }
1744 return Ok(entry.runtime_key);
1745 }
1746 if entries.values().any(|entry| {
1747 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1748 }) {
1749 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1750 }
1751 if hidden_durable_match {
1752 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1753 }
1754
1755 match identity_runtime_keys.as_slice() {
1756 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1757 [runtime_key] => Ok(runtime_key.clone()),
1758 _ => Err(ConsoleSendError::InvalidRequest(format!(
1759 "identity-first send for '{identity}' did not match exactly one registered runtime"
1760 ))),
1761 }
1762 }
1763
1764 pub async fn mark_interaction_delivery_failed(
1765 &self,
1766 input_frame_id: &str,
1767 ) -> Result<(), ConsoleSendError> {
1768 update_frame_status_and_emit(
1769 &self.inner,
1770 input_frame_id,
1771 ConsoleFrameStatus::DeliveryFailed,
1772 )
1773 .await
1774 .map_err(ConsoleSendError::Log)?;
1775 Ok(())
1776 }
1777
1778 pub async fn mark_interaction_delivered(
1779 &self,
1780 input_frame_id: &str,
1781 ) -> Result<(), ConsoleSendError> {
1782 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1783 .await
1784 .map_err(ConsoleSendError::Log)?;
1785 Ok(())
1786 }
1787
1788 pub async fn mark_steer_interaction_delivered(
1789 &self,
1790 input_frame_id: &str,
1791 interaction_id: &str,
1792 ) -> Result<(), ConsoleSendError> {
1793 let Some(updated) = update_frame_status_and_emit(
1794 &self.inner,
1795 input_frame_id,
1796 ConsoleFrameStatus::Delivered,
1797 )
1798 .await
1799 .map_err(ConsoleSendError::Log)?
1800 else {
1801 return Ok(());
1802 };
1803 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1804 .await
1805 .map_err(ConsoleSendError::Log)?;
1806 Ok(())
1807 }
1808
1809 pub async fn binary_blob_store_for_identity(
1810 &self,
1811 identity: &str,
1812 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1813 if identity.trim().is_empty() {
1814 return Err(ConsoleSendError::InvalidRequest(
1815 "identity must be non-empty".to_string(),
1816 ));
1817 }
1818 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1819 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1820 };
1821 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1822 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1823 };
1824 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1825 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1826 }
1827 if !member_is_addressable(&resolved.member) {
1828 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1829 }
1830 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1831 return Err(ConsoleSendError::Retired(identity.to_string()));
1832 }
1833 Ok(resolved.entry.runtime.binary_blob_store())
1834 }
1835
1836 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1837 self.inner
1838 .runtimes
1839 .read()
1840 .map(|entries| {
1841 entries
1842 .values()
1843 .filter_map(|entry| entry.runtime.binary_blob_store())
1844 .collect()
1845 })
1846 .unwrap_or_default()
1847 }
1848
1849 async fn resolve_send_member(
1850 &self,
1851 identity: &str,
1852 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1853 let mut matches = Box::pin(self.resolve_visible_members(identity)).await;
1854 let sendable_indices = sendable_resolved_member_indices(&matches);
1863 if sendable_indices.len() > 1 {
1864 if let Some(index) = Box::pin(authoritative_durable_sendable_member_index(
1865 identity,
1866 &matches,
1867 &sendable_indices,
1868 ))
1869 .await?
1870 {
1871 let resolved = matches.swap_remove(index);
1872 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1873 && let Some(stale_error) =
1874 reconcile_stale_live_record_binding(&resolved.entry, identity, &record)
1875 .await
1876 {
1877 return Err(ConsoleSendError::InvalidRequest(stale_error));
1878 }
1879 if let Some(stale_error) =
1880 Box::pin(self.durable_send_binding_error(identity)).await?
1881 {
1882 return Err(ConsoleSendError::InvalidRequest(stale_error));
1883 }
1884 return Ok(Some(resolved));
1885 }
1886 let candidates = sendable_indices
1887 .iter()
1888 .map(|index| matches[*index].runtime_identity.clone())
1889 .collect::<Vec<_>>()
1890 .join(", ");
1891 return Err(ConsoleSendError::AmbiguousIdentity {
1892 identity: identity.to_string(),
1893 candidates,
1894 });
1895 }
1896 if let Some(index) = sendable_indices.into_iter().next() {
1897 let resolved = matches.swap_remove(index);
1898 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1899 && let Some(stale_error) =
1900 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1901 {
1902 return Err(ConsoleSendError::InvalidRequest(stale_error));
1903 }
1904 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1905 return Err(ConsoleSendError::InvalidRequest(stale_error));
1906 }
1907 return Ok(Some(resolved));
1908 }
1909 if matches.len() > 1 {
1910 let candidates = matches
1911 .iter()
1912 .map(|resolved| resolved.runtime_identity.clone())
1913 .collect::<Vec<_>>()
1914 .join(", ");
1915 return Err(ConsoleSendError::AmbiguousIdentity {
1916 identity: identity.to_string(),
1917 candidates,
1918 });
1919 }
1920 let Some(resolved) = matches.into_iter().next() else {
1921 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1922 return Err(ConsoleSendError::InvalidRequest(stale_error));
1923 }
1924 return Ok(None);
1925 };
1926 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1927 && let Some(stale_error) =
1928 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1929 {
1930 return Err(ConsoleSendError::InvalidRequest(stale_error));
1931 }
1932 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1933 return Err(ConsoleSendError::InvalidRequest(stale_error));
1934 }
1935 Ok(Some(resolved))
1936 }
1937
1938 async fn durable_send_binding_error(
1939 &self,
1940 identity: &str,
1941 ) -> Result<Option<String>, ConsoleSendError> {
1942 let entries = self
1943 .inner
1944 .runtimes
1945 .read()
1946 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1947 .clone();
1948 if entries.values().any(|entry| {
1949 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1950 }) {
1951 return Ok(None);
1952 }
1953 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1954 let mut durable_matches = Vec::new();
1955 for entry in entries.values() {
1956 let Some(identity_runtime) = entry.identity_runtime.clone() else {
1957 continue;
1958 };
1959 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1960 let Ok(parsed_identity) =
1961 crate::identity_first::AgentIdentity::parse(&raw_identity)
1962 else {
1963 continue;
1964 };
1965 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1966 continue;
1967 };
1968 let record = identity_record_for_status(entry, &status);
1969 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1970 continue;
1971 }
1972 durable_matches.push((
1973 entry.clone(),
1974 identity_runtime.clone(),
1975 parsed_identity,
1976 record,
1977 ));
1978 }
1979 }
1980 if durable_matches.len() > 1 {
1981 let candidates = durable_matches
1982 .iter()
1983 .map(|(_, _, _, record)| record.runtime_member_id.clone())
1984 .collect::<Vec<_>>()
1985 .join(", ");
1986 return Err(ConsoleSendError::AmbiguousIdentity {
1987 identity: identity.to_string(),
1988 candidates,
1989 });
1990 }
1991 let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1992 durable_matches.into_iter().next()
1993 else {
1994 return Ok(None);
1995 };
1996 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1997 &entry,
1998 identity,
1999 &durable_record,
2000 &live_records,
2001 ))
2002 .await;
2003 if let Some(rebound_record) = self_heal_durable_session_mismatch(
2004 &entry,
2005 identity_runtime.as_ref(),
2006 &parsed_identity,
2007 identity,
2008 &durable_record,
2009 &durable_live_records,
2010 )
2011 .await
2012 .map_err(ConsoleSendError::InvalidRequest)?
2013 {
2014 durable_record = rebound_record;
2015 }
2016 Ok(stale_durable_record_error(
2017 identity,
2018 &durable_record,
2019 &durable_live_records,
2020 ))
2021 }
2022
2023 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2024 let entries = self
2025 .inner
2026 .runtimes
2027 .read()
2028 .ok()
2029 .map(|entries| entries.clone())
2030 .unwrap_or_default();
2031 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2032 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2033 for entry in entries.values() {
2034 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2035 if raw_identities.is_empty() {
2036 continue;
2037 }
2038 let mids = raw_identities
2039 .iter()
2040 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
2041 .collect::<Vec<_>>();
2042 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2043 let session_id = resolved
2044 .handle
2045 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
2046 .await
2047 .map(|sid| sid.to_string())
2048 .unwrap_or_default();
2049 if mids.contains(&resolved.member.agent_identity) {
2050 exact_matches.push((session_id, resolved));
2051 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
2052 {
2053 label_matches.push((session_id, resolved));
2054 }
2055 }
2056 }
2057 let mut matches = exact_matches;
2058 matches.extend(label_matches);
2059 let mut seen_members = BTreeSet::new();
2060 matches.retain(|(session_id, resolved)| {
2061 seen_members.insert((
2062 resolved.entry.runtime_key.clone(),
2063 resolved.source_mob_id.clone(),
2064 session_id.clone(),
2065 resolved.runtime_identity.clone(),
2066 ))
2067 });
2068 matches.sort_by(|left, right| right.0.cmp(&left.0));
2069 matches.into_iter().map(|(_, resolved)| resolved).collect()
2070 }
2071
2072 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2073 let mut visible = Vec::new();
2074 for resolved in Box::pin(self.resolve_members(identity)).await {
2075 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2076 continue;
2077 };
2078 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2079 visible.push(resolved);
2080 }
2081 }
2082 visible
2083 }
2084
2085 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
2086 let mut records = Vec::new();
2087 for resolved in Box::pin(self.resolve_members(identity)).await {
2088 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2089 continue;
2090 };
2091 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2092 records.push(record);
2093 }
2094 }
2095 records
2096 }
2097
2098 async fn live_records_for_durable_record(
2099 &self,
2100 entry: &RuntimeEntry,
2101 requested_identity: &str,
2102 durable: &ConsoleIdentityRecord,
2103 requested_live_records: &[ConsoleIdentityRecord],
2104 ) -> Vec<ConsoleIdentityRecord> {
2105 let mut records = if durable.identity == requested_identity {
2106 requested_live_records.to_vec()
2107 } else {
2108 Box::pin(self.live_records_for_identity(&durable.identity)).await
2109 };
2110 for runtime_record in Box::pin(live_records_for_runtime_member(
2111 entry,
2112 &durable.runtime_member_id,
2113 ))
2114 .await
2115 {
2116 if records.iter().any(|record| {
2117 record.runtime_key == runtime_record.runtime_key
2118 && record.runtime_member_id == runtime_record.runtime_member_id
2119 && record.session_id == runtime_record.session_id
2120 && record.identity == runtime_record.identity
2121 }) {
2122 continue;
2123 }
2124 records.push(runtime_record);
2125 }
2126 records
2127 }
2128}
2129
2130async fn console_identity_record_visible(
2131 entry: &RuntimeEntry,
2132 record: &ConsoleIdentityRecord,
2133) -> bool {
2134 if !entry.visibility_policy.identity_visible(record) {
2135 return false;
2136 }
2137 let mut bound_live_member_seen = false;
2138 let mut wrong_live_projection_seen = false;
2139 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2140 if resolved.runtime_identity != record.runtime_member_id {
2141 continue;
2142 }
2143 bound_live_member_seen = true;
2144 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2145 continue;
2146 };
2147 if live_record.identity != record.identity {
2148 wrong_live_projection_seen = true;
2149 continue;
2150 }
2151 if raw_resolved_member_visible(&resolved, &live_record) {
2152 return true;
2153 }
2154 }
2155 if wrong_live_projection_seen {
2156 return true;
2157 }
2158 !bound_live_member_seen
2159}
2160
2161async fn live_record_shadowed_by_hidden_durable_binding(
2162 resolved: &ResolvedConsoleMember,
2163 record: &ConsoleIdentityRecord,
2164) -> bool {
2165 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2166 return false;
2167 };
2168 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2169 return false;
2170 };
2171 let Ok(status) = identity_runtime.status(&identity).await else {
2172 return false;
2173 };
2174 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2175 return false;
2176 };
2177 if bound_runtime_id.as_str() == resolved.runtime_identity {
2178 return false;
2179 }
2180 for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2181 if candidate.runtime_identity != bound_runtime_id.as_str() {
2182 continue;
2183 }
2184 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2185 continue;
2186 };
2187 if bound_record.identity == record.identity
2188 && !raw_resolved_member_visible(&candidate, &bound_record)
2189 {
2190 return true;
2191 }
2192 }
2193 false
2194}
2195
2196async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2197 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2198 .unwrap_or_else(|| frame.identity.clone());
2199 let frame_session_id = frame.session_id.as_deref();
2200 let mut saw_hidden = false;
2201 let mut saw_visible = false;
2202 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2203 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2204 continue;
2205 };
2206 let session_matches =
2207 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2208 let runtime_matches = record.runtime_member_id == runtime_member_id;
2209 let identity_matches = record.identity == frame.identity;
2210 if !session_matches && !runtime_matches && !identity_matches {
2211 continue;
2212 }
2213 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2214 return true;
2215 }
2216 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2217 saw_visible = true;
2218 } else {
2219 saw_hidden = true;
2220 }
2221 }
2222 saw_hidden && !saw_visible
2223}
2224
2225async fn live_records_for_runtime_member(
2226 entry: &RuntimeEntry,
2227 runtime_member_id: &str,
2228) -> Vec<ConsoleIdentityRecord> {
2229 let mut records = Vec::new();
2230 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2231 if resolved.runtime_identity != runtime_member_id {
2232 continue;
2233 }
2234 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2235 continue;
2236 };
2237 records.push(record);
2238 }
2239 records
2240}
2241
2242async fn visible_live_records_for_entry(
2243 entry: &RuntimeEntry,
2244 records: &[ConsoleIdentityRecord],
2245) -> Vec<ConsoleIdentityRecord> {
2246 let mut visible = Vec::new();
2247 for record in records {
2248 if record.runtime_key != entry.runtime_key
2249 || Box::pin(console_identity_record_visible(entry, record)).await
2250 {
2251 visible.push(record.clone());
2252 }
2253 }
2254 visible
2255}
2256
2257async fn reconcile_stale_live_record_binding(
2258 entry: &RuntimeEntry,
2259 requested_identity: &str,
2260 live_record: &ConsoleIdentityRecord,
2261) -> Option<String> {
2262 let identity_runtime = entry.identity_runtime.as_ref()?;
2263 for status in identity_runtime.statuses().await {
2264 let matches_runtime = status
2265 .agent_runtime_id
2266 .as_ref()
2267 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2268 if !matches_runtime {
2269 continue;
2270 }
2271 let durable_record = identity_record_for_status(entry, &status);
2272 if let Some(session_mismatch) =
2273 stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2274 {
2275 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2276 continue;
2277 };
2278 let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2279 return Some(format!(
2280 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2281 ));
2282 };
2283 return match identity_runtime
2284 .rebind_session_after_live_respawn(&status.identity, session_id)
2285 .await
2286 {
2287 Ok(_) => {
2288 tracing::warn!(
2289 requested_identity,
2290 identity = %durable_record.identity,
2291 runtime_member_id = %durable_record.runtime_member_id,
2292 old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2293 new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2294 "self-healed stale durable identity alias session mismatch"
2295 );
2296 None
2297 }
2298 Err(err) => Some(format!(
2299 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2300 durable_record.identity,
2301 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2302 )),
2303 };
2304 }
2305 if let Some(stale_error) = stale_durable_record_error(
2306 requested_identity,
2307 &durable_record,
2308 std::slice::from_ref(live_record),
2309 ) {
2310 return Some(stale_error);
2311 }
2312 }
2313 None
2314}
2315
2316async fn self_heal_durable_session_mismatch(
2317 entry: &RuntimeEntry,
2318 identity_runtime: &crate::identity_first::IdentityRuntime,
2319 parsed_identity: &crate::identity_first::AgentIdentity,
2320 requested_identity: &str,
2321 durable: &ConsoleIdentityRecord,
2322 live_records: &[ConsoleIdentityRecord],
2323) -> Result<Option<ConsoleIdentityRecord>, String> {
2324 let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2325 return Ok(None);
2326 };
2327 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2328 return Ok(None);
2329 };
2330 let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2331 format!(
2332 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2333 )
2334 })?;
2335 identity_runtime
2336 .rebind_session_after_live_respawn(parsed_identity, session_id)
2337 .await
2338 .map_err(|err| {
2339 format!(
2340 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2341 durable.identity, live_session_id
2342 )
2343 })?;
2344 tracing::warn!(
2345 requested_identity,
2346 identity = %durable.identity,
2347 runtime_member_id = %durable.runtime_member_id,
2348 old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2349 new_session_id = live_session_id,
2350 "self-healed stale durable identity alias session mismatch"
2351 );
2352 let status = identity_runtime
2353 .status(parsed_identity)
2354 .await
2355 .map_err(|err| {
2356 format!(
2357 "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2358 durable.identity
2359 )
2360 })?;
2361 Ok(Some(identity_record_for_status(entry, &status)))
2362}
2363
2364fn stale_durable_session_mismatch<'a>(
2365 durable: &ConsoleIdentityRecord,
2366 live_records: &'a [ConsoleIdentityRecord],
2367) -> Option<&'a ConsoleIdentityRecord> {
2368 live_records.iter().find(|record| {
2369 record.identity == durable.identity
2370 && record.runtime_member_id == durable.runtime_member_id
2371 && durable.session_id.is_some()
2372 && record.session_id.is_some()
2373 && durable.session_id != record.session_id
2374 })
2375}
2376
2377fn stale_durable_record_error(
2378 requested_identity: &str,
2379 durable: &ConsoleIdentityRecord,
2380 live_records: &[ConsoleIdentityRecord],
2381) -> Option<String> {
2382 let matching_live = live_records
2383 .iter()
2384 .filter(|record| record.identity == durable.identity)
2385 .collect::<Vec<_>>();
2386 if let Some(wrong_projection) = live_records.iter().find(|record| {
2387 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2388 }) {
2389 return Some(format!(
2390 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2391 durable.identity, durable.runtime_member_id, wrong_projection.identity
2392 ));
2393 }
2394 if matching_live.is_empty() {
2395 return None;
2396 }
2397 if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2398 return Some(format!(
2399 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2400 durable.identity,
2401 durable.runtime_member_id,
2402 durable.session_id.as_deref().unwrap_or("<none>"),
2403 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2404 ));
2405 }
2406 if matching_live
2407 .iter()
2408 .any(|record| record.runtime_member_id == durable.runtime_member_id)
2409 {
2410 return None;
2411 }
2412 let live_candidates = matching_live
2413 .iter()
2414 .map(|record| record.runtime_member_id.as_str())
2415 .collect::<Vec<_>>()
2416 .join(", ");
2417 Some(format!(
2418 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2419 durable.identity, durable.runtime_member_id, live_candidates
2420 ))
2421}
2422
2423fn ambiguous_live_alias_error(
2424 requested_identity: &str,
2425 live_records: &[ConsoleIdentityRecord],
2426) -> Option<String> {
2427 if live_records.len() <= 1 {
2428 return None;
2429 }
2430 let candidates = live_records
2431 .iter()
2432 .map(|record| {
2433 format!(
2434 "{}@{}",
2435 record.runtime_member_id,
2436 record
2437 .labels
2438 .get("source_mob_id")
2439 .map(String::as_str)
2440 .unwrap_or(record.runtime_key.as_str())
2441 )
2442 })
2443 .collect::<Vec<_>>()
2444 .join(", ");
2445 Some(format!(
2446 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2447 ))
2448}
2449
2450fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2451 crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2454}
2455
2456async fn retire_stale_console_members_for_identity(
2457 handle: &MobHandle,
2458 durable_identities: &[String],
2459 keep_runtime_member_id: Option<&str>,
2460) -> Result<(), String> {
2461 let stale_members = handle
2462 .list_members_including_retiring()
2463 .await
2464 .into_iter()
2465 .filter(|member| {
2466 (durable_identities.iter().any(|durable_identity| {
2467 member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2468 }) || member.labels.get("agent_identity").is_some_and(|identity| {
2469 durable_identities
2470 .iter()
2471 .any(|durable_identity| identity == durable_identity)
2472 })) && keep_runtime_member_id
2473 .map(|keep| {
2474 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2476 != keep
2477 })
2478 .unwrap_or(true)
2479 })
2480 .map(|member| member.agent_identity)
2481 .collect::<Vec<_>>();
2482 for member_id in stale_members {
2483 match handle.retire(member_id).await {
2484 Ok(()) => {}
2485 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2486 Err(err) => return Err(err.to_string()),
2487 }
2488 }
2489 Ok(())
2490}
2491
2492async fn retire_stale_console_members_for_runtime_entry(
2493 entry: &RuntimeEntry,
2494 durable_identities: &[String],
2495) -> Result<(), String> {
2496 let primary_handle = entry.runtime.handle();
2497 let primary_mob_id = primary_handle.mob_id().to_string();
2498 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2499 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2500 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2501 .await
2502 .unwrap_or_default()
2503 {
2504 if mob_id.as_str() != primary_mob_id {
2505 handles.push((mob_id.to_string(), handle));
2506 }
2507 }
2508 }
2509 let mut seen_handles = BTreeSet::new();
2510 for (mob_id, handle) in handles {
2511 if !seen_handles.insert(mob_id) {
2512 continue;
2513 }
2514 retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2515 }
2516 Ok(())
2517}
2518
2519fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2520 is_recoverable_lifecycle_cleanup_error(error)
2521}
2522
2523fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2524 if frames.is_empty() {
2525 return true;
2526 }
2527 let latest_session_history_timestamp_ms = frames
2528 .iter()
2529 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2530 .map(|frame| frame.timestamp_ms)
2531 .max();
2532 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2533 return current_time_ms().saturating_sub(latest_timestamp_ms)
2534 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2535 }
2536 frames.iter().any(|frame| {
2537 matches!(
2538 frame.kind.as_str(),
2539 "turn_started"
2540 | "run_started"
2541 | "reasoning_delta"
2542 | "reasoning_complete"
2543 | "tool_call_requested"
2544 | "tool_call"
2545 | "tool_execution_started"
2546 | "tool_execution_completed"
2547 | "tool_result_received"
2548 )
2549 })
2550}
2551
2552fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2553 match frame.kind.as_str() {
2554 "user_input" | "run_started" => true,
2555 "interaction_started" => frame
2556 .payload
2557 .get("content")
2558 .or_else(|| frame.payload.get("prompt"))
2559 .is_some(),
2560 _ => false,
2561 }
2562}
2563
2564fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2565 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2566 for record in records {
2567 by_identity
2568 .entry(identity_record_dedupe_key(&record))
2569 .and_modify(|current| {
2570 if identity_record_prefer(&record, current) {
2571 *current = record.clone();
2572 }
2573 })
2574 .or_insert(record);
2575 }
2576 by_identity.into_values().collect()
2577}
2578
2579fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2580 format!(
2581 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2582 record.identity,
2583 record.runtime_key,
2584 record
2585 .labels
2586 .get("source_mob_id")
2587 .map(String::as_str)
2588 .unwrap_or(""),
2589 record.runtime_member_id,
2590 record.session_id.as_deref().unwrap_or("")
2591 )
2592}
2593
2594fn identity_record_prefer(
2595 candidate: &ConsoleIdentityRecord,
2596 current: &ConsoleIdentityRecord,
2597) -> bool {
2598 let candidate_is_live_label_projection =
2599 candidate
2600 .labels
2601 .get("agent_identity")
2602 .is_some_and(|identity| {
2603 identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2604 || candidate.runtime_member_id != candidate.identity
2605 });
2606 let current_is_live_label_projection =
2607 current
2608 .labels
2609 .get("agent_identity")
2610 .is_some_and(|identity| {
2611 identity == &strip_namespace(¤t.identity, "").unwrap_or_default()
2612 || current.runtime_member_id != current.identity
2613 });
2614 if candidate_is_live_label_projection != current_is_live_label_projection {
2615 return candidate_is_live_label_projection;
2616 }
2617 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2618 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2619 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2620 return candidate_has_distinct_runtime_binding;
2621 }
2622 let candidate_live = candidate.addressable && candidate.health != "retired";
2623 let current_live = current.addressable && current.health != "retired";
2624 if candidate_live != current_live {
2625 return candidate_live;
2626 }
2627 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2628}
2629
2630async fn collect_identity_records(
2631 inner: &Arc<AggregatorInner>,
2632 mode: IdentityCollectionMode,
2633) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2634 let entries = inner
2635 .runtimes
2636 .read()
2637 .map_err(|_| runtime_registry_lock_error())?
2638 .clone();
2639 let mut identities = Vec::new();
2640 for entry in entries.values() {
2641 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2642 let topology_peers =
2643 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2644 for status in identity_runtime.statuses().await {
2645 let mut record = identity_record_for_status(entry, &status);
2646 record.topology_peers = topology_peers
2647 .get(&record.identity)
2648 .cloned()
2649 .unwrap_or_default();
2650 if Box::pin(console_identity_record_visible(entry, &record)).await {
2651 identities.push(record);
2652 }
2653 }
2654 if mode == IdentityCollectionMode::CachedOnly {
2655 continue;
2656 }
2657 }
2658 let members = if entry.identity_runtime.is_some() {
2659 match Box::pin(tokio::time::timeout(
2660 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2661 member_sources_for_entry(entry),
2662 ))
2663 .await
2664 {
2665 Ok(members) => members,
2666 Err(_) => {
2667 tracing::warn!(
2668 runtime_key = %entry.runtime_key,
2669 "identity-first live member refresh timed out; keeping cached identity read model"
2670 );
2671 Vec::new()
2672 }
2673 }
2674 } else {
2675 Box::pin(member_sources_for_entry(entry)).await
2676 };
2677 for resolved in members {
2678 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2679 && Box::pin(resolved_member_visible(&resolved, &record)).await
2680 {
2681 identities.push(record);
2682 }
2683 }
2684 }
2685 Ok(dedupe_identity_records(identities))
2686}
2687
2688async fn identity_runtime_topology_peers(
2689 entry: &RuntimeEntry,
2690 identity_runtime: &crate::identity_first::IdentityRuntime,
2691) -> BTreeMap<String, Vec<String>> {
2692 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2693 for edge in identity_runtime.desired_peer_edges().await {
2694 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2695 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2696 peers.entry(a.clone()).or_default().insert(b.clone());
2697 peers.entry(b).or_default().insert(a);
2698 }
2699 peers
2700 .into_iter()
2701 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2702 .collect()
2703}
2704
2705fn spawn_identity_backfills_for_records(
2706 inner: Arc<AggregatorInner>,
2707 records: &[ConsoleIdentityRecord],
2708) {
2709 if !inner.options.session_history_backfill_enabled {
2710 return;
2711 }
2712 let entries = match inner.runtimes.read() {
2713 Ok(entries) => entries.clone(),
2714 Err(_) => return,
2715 };
2716 for record in records {
2717 if record.health == "dormant" || record.health == "uninitialized" {
2718 continue;
2719 }
2720 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2721 continue;
2722 };
2723 let Some(session_id) = record.session_id.clone() else {
2724 continue;
2725 };
2726 spawn_session_history_backfill_target(
2727 inner.clone(),
2728 SessionBackfillTarget {
2729 entry,
2730 record: record.clone(),
2731 session_id,
2732 },
2733 false,
2734 );
2735 }
2736}
2737
2738async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2739 let mut resolved = Vec::new();
2740 let primary_handle = entry.runtime.handle();
2741 let primary_mob_id = primary_handle.mob_id().to_string();
2742 for member in primary_handle.list_members_observation_snapshot().await {
2743 resolved.push(ResolvedConsoleMember {
2744 entry: entry.clone(),
2745 handle: primary_handle.clone(),
2746 runtime_identity: crate::member_comms_id::runtime_alias_str(
2749 member.agent_identity.as_str(),
2750 )
2751 .into_owned(),
2752 source_mob_id: primary_mob_id.clone(),
2753 member,
2754 });
2755 }
2756
2757 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2758 return resolved;
2759 };
2760 if !entry.visibility_policy.include_implicit_delegate_members() {
2761 return resolved;
2762 }
2763 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2764 .await
2765 .unwrap_or_default()
2766 {
2767 if mob_id.as_str() == primary_mob_id {
2768 continue;
2769 }
2770 for member in handle.list_members_observation_snapshot().await {
2771 resolved.push(ResolvedConsoleMember {
2772 entry: entry.clone(),
2773 handle: handle.clone(),
2774 runtime_identity: crate::member_comms_id::runtime_alias_str(
2777 member.agent_identity.as_str(),
2778 )
2779 .into_owned(),
2780 source_mob_id: mob_id.to_string(),
2781 member,
2782 });
2783 }
2784 }
2785 resolved
2786}
2787
2788async fn dispatch_message_to_resolved_member(
2789 resolved: &ResolvedConsoleMember,
2790 content: ContentInput,
2791 handling_mode: meerkat_core::types::HandlingMode,
2792) -> Result<String, String> {
2793 let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2794 match send_message_on_mob_with_mode(
2795 &resolved.handle,
2796 &resolved.runtime_identity,
2797 content.clone(),
2798 handling_mode,
2799 )
2800 .await
2801 {
2802 Ok(session_id) => Ok(session_id),
2803 Err(err) if is_not_externally_addressable(&err) => {
2804 let member = resolved
2805 .handle
2806 .member(&mid)
2807 .await
2808 .map_err(|err| err.to_string())?;
2809 let _receipt = member
2810 .internal_turn(content)
2811 .await
2812 .map_err(|err| err.to_string())?;
2813 resolved
2814 .handle
2815 .resolve_bridge_session_id_observation(&mid)
2816 .await
2817 .map(|sid| sid.to_string())
2818 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2819 }
2820 Err(err) => Err(err.to_string()),
2821 }
2822}
2823
2824fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2825 matches!(
2826 err,
2827 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2828 )
2829}
2830
2831fn spawn_console_send_dispatch(
2832 inner: Arc<AggregatorInner>,
2833 resolved: ResolvedConsoleMember,
2834 content: ContentInput,
2835 handling_mode: meerkat_core::types::HandlingMode,
2836 dispatching: SendState,
2837 user_frame: ConsoleFrame,
2838 interaction_id: String,
2839) {
2840 tokio::spawn(async move {
2841 match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2842 Ok(_) => {
2843 let _ = dispatching.apply(SendTransition::MarkDelivered);
2844 if let Err(err) = update_frame_status_and_emit(
2845 &inner,
2846 &user_frame.id,
2847 ConsoleFrameStatus::Delivered,
2848 )
2849 .await
2850 {
2851 tracing::warn!(
2852 frame_id = %user_frame.id,
2853 error = %err,
2854 "failed to update console send delivery status"
2855 );
2856 }
2857 if handling_mode == HandlingMode::Steer
2858 && let Err(err) =
2859 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2860 {
2861 tracing::warn!(
2862 frame_id = %user_frame.id,
2863 interaction_id = %interaction_id,
2864 error = %err,
2865 "failed to append console steer terminal frame"
2866 );
2867 }
2868 }
2869 Err(err) => {
2870 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2871 if let Err(update_err) = update_frame_status_and_emit(
2872 &inner,
2873 &user_frame.id,
2874 ConsoleFrameStatus::DeliveryFailed,
2875 )
2876 .await
2877 {
2878 tracing::warn!(
2879 frame_id = %user_frame.id,
2880 error = %update_err,
2881 "failed to update console send failure status"
2882 );
2883 }
2884 let failure_frame = NewConsoleFrame {
2885 id: None,
2886 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2887 timestamp_ms: current_time_ms(),
2888 runtime_key: user_frame.runtime_key,
2889 identity: user_frame.identity,
2890 conversation_id: user_frame.conversation_id,
2891 session_id: user_frame.session_id,
2892 kind: "message_delivery_failed".to_string(),
2893 status: ConsoleFrameStatus::DeliveryFailed,
2894 payload: json!({ "reason": err }),
2895 source: ConsoleFrameSource {
2896 kind: ConsoleFrameSourceKind::Synthetic,
2897 source_cursor: None,
2898 },
2899 source_event_id: None,
2900 interaction_id: Some(interaction_id),
2901 turn_id: None,
2902 run_id: None,
2903 parent_frame_id: Some(user_frame.id.clone()),
2904 caused_by_frame_id: Some(user_frame.id),
2905 };
2906 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2907 tracing::warn!(
2908 error = %append_err,
2909 "failed to append console send failure frame"
2910 );
2911 }
2912 }
2913 }
2914 });
2915}
2916
2917async fn append_steer_delivery_terminal(
2918 inner: &AggregatorInner,
2919 user_frame: &ConsoleFrame,
2920 interaction_id: &str,
2921) -> ConsoleLogResult<AppendOutcome> {
2922 append_and_emit(
2923 inner,
2924 NewConsoleFrame {
2925 id: None,
2926 dedupe_key: format!("steer-delivered:{}", user_frame.id),
2927 timestamp_ms: current_time_ms(),
2928 runtime_key: user_frame.runtime_key.clone(),
2929 identity: user_frame.identity.clone(),
2930 conversation_id: user_frame.conversation_id.clone(),
2931 session_id: user_frame.session_id.clone(),
2932 kind: "interaction_complete".to_string(),
2933 status: ConsoleFrameStatus::Completed,
2934 payload: json!({
2935 "reason": "steer_delivered",
2936 "handling_mode": "steer",
2937 }),
2938 source: ConsoleFrameSource {
2939 kind: ConsoleFrameSourceKind::Synthetic,
2940 source_cursor: None,
2941 },
2942 source_event_id: None,
2943 interaction_id: Some(interaction_id.to_string()),
2944 turn_id: None,
2945 run_id: None,
2946 parent_frame_id: Some(user_frame.id.clone()),
2947 caused_by_frame_id: Some(user_frame.id.clone()),
2948 },
2949 )
2950 .await
2951}
2952
2953#[derive(Debug)]
2954pub enum ConsoleSendError {
2955 UnknownIdentity(String),
2956 AmbiguousIdentity {
2957 identity: String,
2958 candidates: String,
2959 },
2960 NotAddressable(String),
2961 Retired(String),
2962 InvalidContent(String),
2963 InvalidHandlingMode(String),
2964 InvalidRequest(String),
2965 IdempotencyConflict(String),
2966 State(&'static str),
2967 Dispatch(String),
2968 Log(ConsoleLogError),
2969}
2970
2971impl std::fmt::Display for ConsoleSendError {
2972 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2973 match self {
2974 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2975 Self::AmbiguousIdentity {
2976 identity,
2977 candidates,
2978 } => write!(
2979 f,
2980 "ambiguous live identity alias {identity}: candidates [{candidates}]"
2981 ),
2982 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2983 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2984 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2985 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2986 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2987 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2988 Self::State(message) => write!(f, "console send state error: {message}"),
2989 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2990 Self::Log(err) => write!(f, "console log error: {err}"),
2991 }
2992 }
2993}
2994
2995impl std::error::Error for ConsoleSendError {}
2996
2997async fn backfill_session_history(
2998 inner: Arc<AggregatorInner>,
2999 runtime_key: String,
3000 force_refresh: bool,
3001) -> ConsoleLogResult<()> {
3002 if !inner.options.session_history_backfill_enabled {
3003 return Ok(());
3004 }
3005 let Some(entry) = inner
3006 .runtimes
3007 .read()
3008 .ok()
3009 .and_then(|entries| entries.get(&runtime_key).cloned())
3010 else {
3011 return Ok(());
3012 };
3013 let members = Box::pin(member_sources_for_entry(&entry)).await;
3014 let mut targets = Vec::new();
3015 for resolved in members {
3016 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3017 continue;
3018 };
3019 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3020 continue;
3021 }
3022 let Some(session_id) = record.session_id.clone() else {
3023 continue;
3024 };
3025 targets.push(SessionBackfillTarget {
3026 entry: entry.clone(),
3027 record,
3028 session_id,
3029 });
3030 }
3031 backfill_session_history_targets(inner, targets, force_refresh).await
3032}
3033
3034#[derive(Clone)]
3035struct SessionBackfillTarget {
3036 entry: RuntimeEntry,
3037 record: ConsoleIdentityRecord,
3038 session_id: String,
3039}
3040
3041async fn backfill_session_history_targets(
3042 inner: Arc<AggregatorInner>,
3043 targets: Vec<SessionBackfillTarget>,
3044 force_refresh: bool,
3045) -> ConsoleLogResult<()> {
3046 let mut tasks = tokio::task::JoinSet::new();
3047 for target in targets {
3048 tasks.spawn(backfill_one_session_history(
3049 inner.clone(),
3050 target,
3051 force_refresh,
3052 ));
3053 }
3054 let mut first_error = None;
3055 while let Some(result) = tasks.join_next().await {
3056 match result {
3057 Ok(Ok(())) => {}
3058 Ok(Err(err)) => {
3059 if first_error.is_none() {
3060 first_error = Some(err);
3061 }
3062 }
3063 Err(err) => {
3064 if first_error.is_none() {
3065 first_error = Some(Box::new(std::io::Error::other(format!(
3066 "session backfill task failed: {err}"
3067 ))) as ConsoleLogError);
3068 }
3069 }
3070 }
3071 }
3072 if let Some(err) = first_error {
3073 Err(err)
3074 } else {
3075 Ok(())
3076 }
3077}
3078
3079async fn backfill_one_session_history(
3080 inner: Arc<AggregatorInner>,
3081 target: SessionBackfillTarget,
3082 force_refresh: bool,
3083) -> ConsoleLogResult<()> {
3084 let _permit = inner
3085 .session_backfill_permits
3086 .clone()
3087 .acquire_owned()
3088 .await
3089 .map_err(|err| -> ConsoleLogError {
3090 Box::new(std::io::Error::other(format!(
3091 "session backfill limiter closed: {err}"
3092 )))
3093 })?;
3094 let SessionBackfillTarget {
3095 entry,
3096 record,
3097 session_id,
3098 } = target;
3099 let watermark_runtime_key =
3100 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
3101 let watermark = inner
3102 .store
3103 .source_watermark(
3104 &watermark_runtime_key,
3105 ConsoleFrameSourceKind::SessionHistory,
3106 )
3107 .await?;
3108 let now_ms = current_time_ms();
3109 let mut offset = watermark
3110 .as_deref()
3111 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
3112 .unwrap_or(0);
3113 if !force_refresh
3114 && watermark.as_deref().is_some_and(|watermark| {
3115 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
3116 })
3117 {
3118 return Ok(());
3119 }
3120 loop {
3121 let page = match entry
3122 .runtime
3123 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
3124 .await
3125 {
3126 Ok(page) => page,
3127 Err(err) => {
3128 record_session_history_backfill_failure(
3129 &inner,
3130 &watermark_runtime_key,
3131 &entry.runtime_key,
3132 &record.identity,
3133 &session_id,
3134 offset,
3135 err.to_string(),
3136 )
3137 .await?;
3138 break;
3139 }
3140 };
3141 let page_value = match serde_json::to_value(page) {
3142 Ok(value) => value,
3143 Err(err) => {
3144 record_session_history_backfill_failure(
3145 &inner,
3146 &watermark_runtime_key,
3147 &entry.runtime_key,
3148 &record.identity,
3149 &session_id,
3150 offset,
3151 err.to_string(),
3152 )
3153 .await?;
3154 break;
3155 }
3156 };
3157 let base_offset = page_value
3158 .get("offset")
3159 .and_then(Value::as_u64)
3160 .unwrap_or(offset as u64) as usize;
3161 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3162 record_session_history_backfill_failure(
3163 &inner,
3164 &watermark_runtime_key,
3165 &entry.runtime_key,
3166 &record.identity,
3167 &session_id,
3168 offset,
3169 "session history page missing messages".to_string(),
3170 )
3171 .await?;
3172 break;
3173 };
3174 if messages.is_empty() {
3175 if offset > 0 {
3176 record_session_history_watermark(
3177 &inner,
3178 &watermark_runtime_key,
3179 &session_id,
3180 offset,
3181 )
3182 .await?;
3183 }
3184 break;
3185 }
3186 for (idx, message) in messages.iter().enumerate() {
3187 let absolute_offset = base_offset + idx;
3188 let frames = frames_from_session_history_message_with_namespace(
3189 &entry.runtime_key,
3190 &record.identity,
3191 &entry.identity_namespace,
3192 &session_id,
3193 absolute_offset,
3194 message.clone(),
3195 );
3196 for mut frame in frames {
3197 if history_frame_has_existing_counterpart(&inner, &frame).await? {
3198 continue;
3199 }
3200 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3201 frame.payload = redacted;
3202 frame.status = ConsoleFrameStatus::Redacted;
3203 }
3204 append_and_emit(&inner, frame).await?;
3205 }
3206 }
3207 let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3208 tracing::warn!(
3215 session_id = %session_id,
3216 requested_offset = offset,
3217 page_base_offset = base_offset,
3218 page_len = messages.len(),
3219 "session history backfill made no forward progress; stopping to avoid a livelock"
3220 );
3221 break;
3222 };
3223 offset = next_offset;
3224 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3225 .await?;
3226 let has_more = page_value
3227 .get("has_more")
3228 .and_then(Value::as_bool)
3229 .unwrap_or(false);
3230 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3231 break;
3232 }
3233 }
3234 Ok(())
3235}
3236
3237fn advance_backfill_offset(
3242 requested_offset: usize,
3243 base_offset: usize,
3244 page_len: usize,
3245) -> Option<usize> {
3246 let next = base_offset.saturating_add(page_len);
3247 (next > requested_offset).then_some(next)
3248}
3249
3250async fn record_session_history_watermark(
3251 inner: &AggregatorInner,
3252 watermark_runtime_key: &str,
3253 session_id: &str,
3254 offset: usize,
3255) -> ConsoleLogResult<()> {
3256 inner
3257 .store
3258 .record_source_watermark(
3259 watermark_runtime_key,
3260 ConsoleFrameSourceKind::SessionHistory,
3261 &format_session_history_watermark(session_id, offset, current_time_ms()),
3262 )
3263 .await
3264}
3265
3266fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3267 if !inner.options.session_history_backfill_enabled {
3268 return;
3269 }
3270 tokio::spawn(async move {
3271 {
3272 let mut active = inner.active_session_backfills.lock().await;
3273 if !active.insert(runtime_key.clone()) {
3274 return;
3275 }
3276 }
3277 let result = Box::pin(backfill_session_history(
3278 inner.clone(),
3279 runtime_key.clone(),
3280 false,
3281 ))
3282 .await;
3283 let mut active = inner.active_session_backfills.lock().await;
3284 active.remove(&runtime_key);
3285 drop(active);
3286 if let Err(err) = result {
3287 tracing::warn!(
3288 runtime_key = %runtime_key,
3289 error = %err,
3290 "console session-history backfill failed"
3291 );
3292 }
3293 });
3294}
3295
3296fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3297 if !inner.options.session_history_backfill_enabled {
3298 return;
3299 }
3300 tokio::spawn(async move {
3301 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3302 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3303 loop {
3304 interval.tick().await;
3305 let runtime_still_registered = inner
3306 .runtimes
3307 .read()
3308 .ok()
3309 .is_some_and(|entries| entries.contains_key(&runtime_key));
3310 if !runtime_still_registered {
3311 break;
3312 }
3313 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3314 }
3315 });
3316}
3317
3318fn spawn_session_history_backfill_target(
3319 inner: Arc<AggregatorInner>,
3320 target: SessionBackfillTarget,
3321 force_refresh: bool,
3322) {
3323 if !inner.options.session_history_backfill_enabled {
3324 return;
3325 }
3326 tokio::spawn(async move {
3327 let active_key = targeted_session_history_active_key(&target, force_refresh);
3328 let result =
3329 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3330 if let Err(err) = result {
3331 tracing::warn!(
3332 active_key = %active_key,
3333 error = %err,
3334 "console targeted session-history backfill failed"
3335 );
3336 }
3337 });
3338}
3339
3340async fn run_targeted_session_history_backfill(
3341 inner: Arc<AggregatorInner>,
3342 target: SessionBackfillTarget,
3343 force_refresh: bool,
3344) -> ConsoleLogResult<()> {
3345 let active_key = targeted_session_history_active_key(&target, force_refresh);
3346 {
3347 let mut active = inner.active_session_backfills.lock().await;
3348 if !active.insert(active_key.clone()) {
3349 return Ok(());
3350 }
3351 }
3352 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3353 let mut active = inner.active_session_backfills.lock().await;
3354 active.remove(&active_key);
3355 result
3356}
3357
3358fn targeted_session_history_active_key(
3359 target: &SessionBackfillTarget,
3360 force_refresh: bool,
3361) -> String {
3362 let mode = if force_refresh { "force" } else { "refresh" };
3363 format!(
3364 "{}:session-history:{}:{mode}",
3365 target.entry.runtime_key, target.session_id
3366 )
3367}
3368
3369fn spawn_session_history_backfill_for_identity(
3370 inner: Arc<AggregatorInner>,
3371 identity: String,
3372 force_refresh: bool,
3373) {
3374 if !inner.options.session_history_backfill_enabled {
3375 return;
3376 }
3377 tokio::spawn(async move {
3378 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3379 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3380 }
3381 });
3382}
3383
3384fn spawn_opportunistic_session_history_backfill_for_identity(
3385 inner: Arc<AggregatorInner>,
3386 identity: String,
3387) {
3388 if !inner.options.session_history_backfill_enabled {
3389 return;
3390 }
3391 tokio::spawn(async move {
3392 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3393 let active_key = format!(
3394 "{}:session-history:{}",
3395 target.entry.runtime_key, target.session_id
3396 );
3397 {
3398 let mut seen = inner.opportunistic_session_backfills.lock().await;
3399 if !seen.insert(active_key) {
3400 continue;
3401 }
3402 }
3403 spawn_session_history_backfill_target(inner.clone(), target, false);
3404 }
3405 });
3406}
3407
3408#[cfg(test)]
3409async fn session_backfill_target_for_identity(
3410 inner: &AggregatorInner,
3411 identity: &str,
3412) -> Option<SessionBackfillTarget> {
3413 session_backfill_targets_for_identity(inner, identity)
3414 .await
3415 .into_iter()
3416 .next()
3417}
3418
3419async fn session_backfill_targets_for_identity(
3420 inner: &AggregatorInner,
3421 identity: &str,
3422) -> Vec<SessionBackfillTarget> {
3423 let entries = inner
3424 .runtimes
3425 .read()
3426 .ok()
3427 .map(|entries| entries.clone())
3428 .unwrap_or_default();
3429 let mut targets = Vec::new();
3430 for entry in entries.values() {
3431 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3432 if raw_identities.is_empty() {
3433 continue;
3434 }
3435 let mids = raw_identities
3436 .iter()
3437 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3438 .collect::<Vec<_>>();
3439 for resolved in Box::pin(member_sources_for_entry(entry))
3440 .await
3441 .into_iter()
3442 .filter(|candidate| {
3443 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3444 })
3445 {
3446 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3447 continue;
3448 };
3449 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3450 continue;
3451 }
3452 let Some(session_id) = record.session_id.clone() else {
3453 continue;
3454 };
3455 targets.push(SessionBackfillTarget {
3456 entry: entry.clone(),
3457 record,
3458 session_id,
3459 });
3460 }
3461 }
3462 targets
3463}
3464
3465fn resolved_member_matches_raw_identities(
3466 resolved: &ResolvedConsoleMember,
3467 raw_identities: &[String],
3468 mids: &[AgentIdentity],
3469) -> bool {
3470 mids.contains(&resolved.member.agent_identity)
3471 || resolved
3472 .member
3473 .labels
3474 .get("agent_identity")
3475 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3476}
3477
3478async fn recover_lagged_source_events(
3479 inner: Arc<AggregatorInner>,
3480 runtime_key: &str,
3481 console_events: &ConsoleEventStore,
3482) -> ConsoleLogResult<()> {
3483 let watermark = inner
3484 .store
3485 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3486 .await?;
3487 match console_events.replay_all(watermark.as_deref()).await {
3488 Ok(events) => {
3489 for envelope in events {
3490 project_console_event(inner.clone(), runtime_key, envelope).await?;
3491 }
3492 }
3493 Err(err) => {
3494 append_source_gap(
3495 &inner,
3496 runtime_key,
3497 format!(
3498 "{}:{}:{}",
3499 err.error, err.stream, err.requested_last_event_id
3500 ),
3501 )
3502 .await?;
3503 }
3504 }
3505 Ok(())
3506}
3507
3508async fn append_source_gap(
3509 inner: &AggregatorInner,
3510 runtime_key: &str,
3511 reason: String,
3512) -> ConsoleLogResult<()> {
3513 append_and_emit(
3514 inner,
3515 NewConsoleFrame {
3516 id: None,
3517 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3518 timestamp_ms: current_time_ms(),
3519 runtime_key: runtime_key.to_string(),
3520 identity: "__console__".to_string(),
3521 conversation_id: None,
3522 session_id: None,
3523 kind: "replay_unavailable".to_string(),
3524 status: ConsoleFrameStatus::DeliveryFailed,
3525 payload: json!({
3526 "reason": reason,
3527 "source_kind": "console_event",
3528 }),
3529 source: ConsoleFrameSource {
3530 kind: ConsoleFrameSourceKind::Synthetic,
3531 source_cursor: None,
3532 },
3533 source_event_id: None,
3534 interaction_id: None,
3535 turn_id: None,
3536 run_id: None,
3537 parent_frame_id: None,
3538 caused_by_frame_id: None,
3539 },
3540 )
3541 .await?;
3542 let _ = inner
3543 .event_tx
3544 .send(ConsoleTimelineEvent::ReplayUnavailable {
3545 requested_cursor: format!("source-gap:{runtime_key}"),
3546 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3547 });
3548 Ok(())
3549}
3550
3551async fn record_session_history_backfill_failure(
3556 inner: &AggregatorInner,
3557 watermark_runtime_key: &str,
3558 runtime_key: &str,
3559 identity: &str,
3560 session_id: &str,
3561 offset: usize,
3562 reason: String,
3563) -> ConsoleLogResult<()> {
3564 record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3565 append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3566}
3567
3568fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3571 format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3572}
3573
3574async fn append_backfill_gap(
3575 inner: &AggregatorInner,
3576 runtime_key: &str,
3577 identity: &str,
3578 session_id: &str,
3579 reason: String,
3580) -> ConsoleLogResult<()> {
3581 append_and_emit(
3582 inner,
3583 NewConsoleFrame {
3584 id: None,
3585 dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3590 timestamp_ms: current_time_ms(),
3591 runtime_key: runtime_key.to_string(),
3592 identity: identity.to_string(),
3593 conversation_id: Some(identity.to_string()),
3594 session_id: Some(session_id.to_string()),
3595 kind: "replay_unavailable".to_string(),
3596 status: ConsoleFrameStatus::DeliveryFailed,
3597 payload: json!({
3598 "reason": reason,
3599 "source_kind": "session_history",
3600 }),
3601 source: ConsoleFrameSource {
3602 kind: ConsoleFrameSourceKind::Synthetic,
3603 source_cursor: None,
3604 },
3605 source_event_id: None,
3606 interaction_id: None,
3607 turn_id: None,
3608 run_id: None,
3609 parent_frame_id: None,
3610 caused_by_frame_id: None,
3611 },
3612 )
3613 .await?;
3614 Ok(())
3615}
3616
3617async fn project_console_event(
3618 inner: Arc<AggregatorInner>,
3619 runtime_key: &str,
3620 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3621) -> ConsoleLogResult<()> {
3622 let Some(entry) = inner
3623 .runtimes
3624 .read()
3625 .ok()
3626 .and_then(|entries| entries.get(runtime_key).cloned())
3627 else {
3628 return Ok(());
3629 };
3630 let mut frame = frame_from_console_event(&entry, envelope);
3631 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3632 frame.payload = redacted;
3633 frame.status = ConsoleFrameStatus::Redacted;
3634 }
3635 let source_cursor = frame
3636 .source_event_id
3637 .clone()
3638 .unwrap_or_else(|| frame.dedupe_key.clone());
3639 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3640 Some(frame.identity.clone())
3641 } else {
3642 None
3643 };
3644 let opportunistic_refresh_identity = if refresh_identity.is_none()
3645 && console_event_should_start_session_history_backfill(&frame)
3646 {
3647 Some(frame.identity.clone())
3648 } else {
3649 None
3650 };
3651 append_and_emit(&inner, frame).await?;
3652 inner
3653 .store
3654 .record_source_watermark(
3655 &entry.runtime_key,
3656 ConsoleFrameSourceKind::ConsoleEvent,
3657 &source_cursor,
3658 )
3659 .await?;
3660 if let Some(identity) = refresh_identity {
3661 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3662 } else if let Some(identity) = opportunistic_refresh_identity {
3663 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3664 }
3665 Ok(())
3666}
3667
3668fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3669 matches!(
3670 frame.kind.as_str(),
3671 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3672 ) || frame.session_id.is_some()
3673}
3674
3675fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3676 if frame.identity == SYSTEM_EVENT_IDENTITY {
3677 return false;
3678 }
3679 matches!(
3680 frame.kind.as_str(),
3681 "turn_started"
3682 | "run_started"
3683 | "reasoning_complete"
3684 | "tool_call_requested"
3685 | "tool_call"
3686 | "tool_execution_started"
3687 | "text_delta"
3688 | "system_notice"
3689 )
3690}
3691
3692async fn append_and_emit(
3693 inner: &AggregatorInner,
3694 frame: NewConsoleFrame,
3695) -> ConsoleLogResult<AppendOutcome> {
3696 let outcome = inner.store.append_if_absent(frame).await?;
3697 if outcome.disposition == AppendDisposition::Inserted {
3698 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3699 frame: outcome.frame.clone(),
3700 });
3701 }
3702 Ok(outcome)
3703}
3704
3705async fn update_frame_status_and_emit(
3706 inner: &AggregatorInner,
3707 frame_id: &str,
3708 status: ConsoleFrameStatus,
3709) -> ConsoleLogResult<Option<ConsoleFrame>> {
3710 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3711 return Ok(None);
3712 };
3713 let update_marker = NewConsoleFrame {
3714 id: None,
3715 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3716 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3717 runtime_key: updated.runtime_key.clone(),
3718 identity: updated.identity.clone(),
3719 conversation_id: updated.conversation_id.clone(),
3720 session_id: updated.session_id.clone(),
3721 kind: "frame_updated".to_string(),
3722 status: updated.status,
3723 payload: json!({ "frame": updated.clone() }),
3724 source: ConsoleFrameSource {
3725 kind: ConsoleFrameSourceKind::Synthetic,
3726 source_cursor: None,
3727 },
3728 source_event_id: None,
3729 interaction_id: updated.interaction_id.clone(),
3730 turn_id: updated.turn_id.clone(),
3731 run_id: updated.run_id.clone(),
3732 parent_frame_id: Some(updated.id.clone()),
3733 caused_by_frame_id: Some(updated.id.clone()),
3734 };
3735 let outcome = inner.store.append_if_absent(update_marker).await?;
3736 if outcome.disposition == AppendDisposition::Inserted {
3737 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3738 frame: outcome.frame,
3739 });
3740 }
3741 Ok(Some(updated))
3742}
3743
3744fn frame_from_console_event(
3745 entry: &RuntimeEntry,
3746 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3747) -> NewConsoleFrame {
3748 let event_id = envelope.event_id;
3749 let interaction_id = envelope.interaction_id;
3750 let event_type = envelope.event_type;
3751 let payload = envelope.data;
3752 let turn_id = payload
3753 .get("turn_id")
3754 .and_then(Value::as_str)
3755 .map(ToString::to_string);
3756 let run_id = payload
3757 .get("run_id")
3758 .and_then(Value::as_str)
3759 .map(ToString::to_string);
3760 let status = match event_type.as_str() {
3761 "interaction_started" => ConsoleFrameStatus::Accepted,
3762 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3763 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3764 _ => ConsoleFrameStatus::Delivered,
3765 };
3766 let identity = if envelope.identity == SYSTEM_EVENT_IDENTITY {
3771 envelope.identity
3772 } else {
3773 apply_namespace(&envelope.identity, &entry.identity_namespace)
3774 };
3775 let dedupe_key = reasoning_payload_text(&event_type, &payload)
3776 .map(|reasoning_text| {
3777 let turn_scope = interaction_id
3780 .as_deref()
3781 .or(turn_id.as_deref())
3782 .or(run_id.as_deref())
3783 .unwrap_or(event_id.as_str());
3784 format!(
3785 "console-reasoning:{}:{}:{}",
3786 entry.runtime_key,
3787 turn_scope,
3788 hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3789 )
3790 })
3791 .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3792 NewConsoleFrame {
3793 id: Some(event_id.clone()),
3794 dedupe_key,
3795 timestamp_ms: envelope.timestamp_ms,
3796 runtime_key: entry.runtime_key.clone(),
3797 identity: identity.clone(),
3798 conversation_id: Some(identity),
3799 session_id: payload
3800 .get("session_id")
3801 .and_then(Value::as_str)
3802 .map(ToString::to_string),
3803 kind: event_type,
3804 status,
3805 payload,
3806 source: ConsoleFrameSource {
3807 kind: ConsoleFrameSourceKind::ConsoleEvent,
3808 source_cursor: None,
3809 },
3810 source_event_id: Some(event_id),
3811 interaction_id,
3812 turn_id,
3813 run_id,
3814 parent_frame_id: None,
3815 caused_by_frame_id: None,
3816 }
3817}
3818
3819#[cfg(test)]
3820fn frame_from_session_history_message(
3821 runtime_key: &str,
3822 identity: &str,
3823 session_id: &str,
3824 offset: usize,
3825 message: Value,
3826) -> Option<NewConsoleFrame> {
3827 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3828 .into_iter()
3829 .next()
3830}
3831
3832#[cfg(test)]
3833fn frames_from_session_history_message(
3834 runtime_key: &str,
3835 identity: &str,
3836 session_id: &str,
3837 offset: usize,
3838 message: Value,
3839) -> Vec<NewConsoleFrame> {
3840 frames_from_session_history_message_with_namespace(
3841 runtime_key,
3842 identity,
3843 "",
3844 session_id,
3845 offset,
3846 message,
3847 )
3848}
3849
3850#[allow(clippy::too_many_arguments)]
3858fn session_history_assistant_image_frames(
3859 runtime_key: &str,
3860 identity: &str,
3861 session_id: &str,
3862 offset: usize,
3863 result_idx: usize,
3864 timestamp_ms: u64,
3865 tool_use_id: &str,
3866 result_text: &str,
3867) -> Vec<NewConsoleFrame> {
3868 let Ok(image_result) = serde_json::from_str::<
3869 meerkat_core::image_generation::ImageGenerationToolResult,
3870 >(result_text) else {
3871 return Vec::new();
3872 };
3873 image_result
3874 .images
3875 .iter()
3876 .enumerate()
3877 .map(|(image_idx, image)| NewConsoleFrame {
3878 id: None,
3879 dedupe_key: format!(
3882 "console-assistant-image:{runtime_key}:{}",
3883 image.blob_ref.blob_id
3884 ),
3885 timestamp_ms,
3886 runtime_key: runtime_key.to_string(),
3887 identity: identity.to_string(),
3888 conversation_id: Some(identity.to_string()),
3889 session_id: Some(session_id.to_string()),
3890 kind: "assistant_image".to_string(),
3891 status: ConsoleFrameStatus::Completed,
3892 payload: json!({
3893 "source_event_type": "session_history",
3894 "tool_call_id": tool_use_id,
3895 "image_id": image.image_id.0.to_string(),
3896 "blob_id": image.blob_ref.blob_id,
3897 "media_type": image.media_type.as_str(),
3898 "width": image.width,
3899 "height": image.height,
3900 "revised_prompt": image_result.revised_prompt.clone(),
3901 "type": "session_history",
3902 }),
3903 source: ConsoleFrameSource {
3904 kind: ConsoleFrameSourceKind::SessionHistory,
3905 source_cursor: Some(format!(
3906 "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3907 )),
3908 },
3909 source_event_id: None,
3910 interaction_id: None,
3911 turn_id: None,
3912 run_id: None,
3913 parent_frame_id: None,
3914 caused_by_frame_id: None,
3915 })
3916 .collect()
3917}
3918
3919fn frames_from_session_history_message_with_namespace(
3920 runtime_key: &str,
3921 identity: &str,
3922 identity_namespace: &str,
3923 session_id: &str,
3924 offset: usize,
3925 message: Value,
3926) -> Vec<NewConsoleFrame> {
3927 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3928 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3929 return Vec::new();
3930 };
3931 if let Message::ToolResults {
3932 results,
3933 created_at,
3934 } = parsed
3935 {
3936 return results
3937 .into_iter()
3938 .enumerate()
3939 .flat_map(|(idx, result)| {
3940 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3941 let result_text = result.text_content();
3942 let tool_use_id = result.tool_use_id.clone();
3943 let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3944 let mut frames = vec![NewConsoleFrame {
3945 id: None,
3946 dedupe_key: format!(
3947 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3948 ),
3949 timestamp_ms,
3950 runtime_key: runtime_key.to_string(),
3951 identity: identity.to_string(),
3952 conversation_id: Some(identity.to_string()),
3953 session_id: Some(session_id.to_string()),
3954 kind: "tool_execution_completed".to_string(),
3955 status: ConsoleFrameStatus::Completed,
3956 payload: json!({
3957 "id": tool_use_id,
3958 "tool_call_id": tool_use_id,
3959 "result": result_text,
3960 "content": content,
3961 "is_error": result.is_error,
3962 "source_event_type": "session_history",
3963 "type": "session_history",
3964 }),
3965 source: ConsoleFrameSource {
3966 kind: ConsoleFrameSourceKind::SessionHistory,
3967 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3968 },
3969 source_event_id: None,
3970 interaction_id: None,
3971 turn_id: None,
3972 run_id: None,
3973 parent_frame_id: None,
3974 caused_by_frame_id: None,
3975 }];
3976 frames.extend(session_history_assistant_image_frames(
3984 runtime_key,
3985 identity,
3986 session_id,
3987 offset,
3988 idx,
3989 timestamp_ms,
3990 &tool_use_id,
3991 &result_text,
3992 ));
3993 frames
3994 })
3995 .collect();
3996 }
3997 let transcript_identity = match &parsed {
4004 Message::User(user) => Some(&user.identity),
4005 Message::BlockAssistant(assistant) => Some(&assistant.identity),
4006 _ => None,
4007 };
4008 let history_interaction_id = transcript_identity
4009 .and_then(|identity| identity.interaction_id.as_ref())
4010 .map(|id| id.0.to_string());
4011 let history_run_id = transcript_identity
4012 .and_then(|identity| identity.run_id.as_ref())
4013 .map(|id| id.0.to_string());
4014 let (kind, timestamp_ms, payload) = match &parsed {
4015 Message::User(user) => {
4016 if session_history_user_message_is_scaffold(&message) {
4017 return Vec::new();
4018 }
4019 (
4020 "user_input",
4021 user.created_at.timestamp_millis().max(0) as u64,
4022 json!({
4023 "content": user.content,
4024 "message": message,
4025 }),
4026 )
4027 }
4028 Message::BlockAssistant(assistant) => {
4031 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
4032 (
4033 "interaction_complete",
4034 assistant.created_at.timestamp_millis().max(0) as u64,
4035 json!({
4036 "result": text,
4037 "text": text,
4038 "message": message,
4039 "source_event_type": "session_history",
4040 "type": "session_history",
4041 }),
4042 )
4043 }
4044 Message::SystemNotice(notice) => (
4045 "system_notice",
4046 notice.created_at.timestamp_millis().max(0) as u64,
4047 json!({
4048 "message": message,
4049 "kind": notice.kind,
4050 "render_class": notice.kind.render_class(),
4051 "body": notice.body,
4052 "blocks": notice.blocks,
4053 "source_event_type": "session_history",
4054 "type": "session_history",
4055 }),
4056 ),
4057 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
4058 };
4059 let mut frames = vec![NewConsoleFrame {
4060 id: None,
4061 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
4062 timestamp_ms,
4063 runtime_key: runtime_key.to_string(),
4064 identity: identity.to_string(),
4065 conversation_id: Some(identity.to_string()),
4066 session_id: Some(session_id.to_string()),
4067 kind: kind.to_string(),
4068 status: ConsoleFrameStatus::Completed,
4069 payload,
4070 source: ConsoleFrameSource {
4071 kind: ConsoleFrameSourceKind::SessionHistory,
4072 source_cursor: Some(format!("{session_id}:{offset}")),
4073 },
4074 source_event_id: None,
4075 interaction_id: history_interaction_id,
4076 turn_id: None,
4077 run_id: history_run_id,
4078 parent_frame_id: None,
4079 caused_by_frame_id: None,
4080 }];
4081 if let Message::BlockAssistant(assistant) = &parsed {
4082 frames.extend(spawn_initial_message_frames_from_assistant(
4083 runtime_key,
4084 identity,
4085 identity_namespace,
4086 session_id,
4087 offset,
4088 assistant,
4089 &payload_hash,
4090 ));
4091 }
4092 frames
4093}
4094
4095fn spawn_initial_message_frames_from_assistant(
4096 runtime_key: &str,
4097 parent_identity: &str,
4098 identity_namespace: &str,
4099 session_id: &str,
4100 offset: usize,
4101 assistant: &meerkat_core::types::BlockAssistantMessage,
4102 payload_hash: &str,
4103) -> Vec<NewConsoleFrame> {
4104 let mut frames = Vec::new();
4105 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
4106 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
4107 continue;
4108 }
4109 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
4110 continue;
4111 };
4112 for (spawn_idx, (target_identity, initial_message)) in
4113 spawn_initial_messages_from_tool_args(&args)
4114 .into_iter()
4115 .enumerate()
4116 {
4117 let target_identity = apply_namespace(&target_identity, identity_namespace);
4118 let message_content = match &initial_message {
4119 Value::String(text) => json!([
4120 {
4121 "type": "text",
4122 "text": text,
4123 }
4124 ]),
4125 other => other.clone(),
4126 };
4127 let message = match &initial_message {
4128 Value::String(text) => json!({
4129 "role": "user",
4130 "content": text,
4131 "created_at": assistant.created_at,
4132 }),
4133 other => json!({
4134 "role": "user",
4135 "content": other,
4136 "created_at": assistant.created_at,
4137 }),
4138 };
4139 frames.push(NewConsoleFrame {
4140 id: None,
4141 dedupe_key: format!(
4142 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
4143 ),
4144 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
4145 runtime_key: runtime_key.to_string(),
4146 identity: target_identity.clone(),
4147 conversation_id: Some(target_identity),
4148 session_id: None,
4149 kind: "user_input".to_string(),
4150 status: ConsoleFrameStatus::Completed,
4151 payload: json!({
4152 "content": message_content,
4153 "message": message,
4154 "source_event_type": "session_history_spawn_initial_message",
4155 "type": "session_history",
4156 "tool_call_id": tool.id,
4157 "parent_identity": parent_identity,
4158 "via_tool": tool.name,
4159 }),
4160 source: ConsoleFrameSource {
4161 kind: ConsoleFrameSourceKind::SessionHistory,
4162 source_cursor: Some(format!(
4163 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4164 )),
4165 },
4166 source_event_id: None,
4167 interaction_id: None,
4168 turn_id: None,
4169 run_id: None,
4170 parent_frame_id: None,
4171 caused_by_frame_id: None,
4172 });
4173 }
4174 }
4175 frames
4176}
4177
4178fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4179 let mut messages = Vec::new();
4180 if let Some(message) = spawn_initial_message_from_record(args) {
4181 messages.push(message);
4182 }
4183 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4184 for spec in specs {
4185 if let Some(message) = spawn_initial_message_from_record(spec) {
4186 messages.push(message);
4187 }
4188 }
4189 }
4190 messages
4191}
4192
4193fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4194 let target_identity = record
4195 .get("member_id")
4196 .or_else(|| record.get("agent_identity"))
4197 .and_then(Value::as_str)?
4198 .trim();
4199 if target_identity.is_empty() {
4200 return None;
4201 }
4202 let initial_message = record
4203 .get("initial_message")
4204 .or_else(|| record.get("task"))?
4205 .clone();
4206 Some((target_identity.to_string(), initial_message))
4207}
4208
4209fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4210 message
4211 .get("content")
4212 .is_some_and(scaffold_message_content_is_noise)
4213}
4214
4215fn scaffold_message_content_is_noise(value: &Value) -> bool {
4216 match value {
4217 Value::String(text) => scaffold_message_text_is_noise(text),
4218 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4219 Value::Object(map) => ["text", "content", "message"]
4220 .iter()
4221 .filter_map(|key| map.get(*key))
4222 .any(scaffold_message_content_is_noise),
4223 _ => false,
4224 }
4225}
4226
4227fn scaffold_message_text_is_noise(text: &str) -> bool {
4228 let trimmed = text.trim_start();
4229 trimmed.starts_with("[PEER UPDATE]")
4230 || trimmed
4231 .to_ascii_lowercase()
4232 .starts_with("you have been spawned")
4233}
4234
4235fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4236 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4237 rest.split(':').next()?.parse().ok()
4238}
4239
4240fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4241 format!("{session_id}:{offset}:{checked_at_ms}")
4242}
4243
4244fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4245 let Some(checked_at_ms) = watermark
4246 .rsplit_once(':')
4247 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4248 else {
4249 return false;
4250 };
4251 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4252 let ttl_ms = if offset > 0 {
4253 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4254 } else {
4255 SESSION_HISTORY_REFRESH_TTL_MS
4256 };
4257 now_ms.saturating_sub(checked_at_ms) < ttl_ms
4258}
4259
4260async fn history_frame_has_existing_counterpart(
4261 inner: &AggregatorInner,
4262 frame: &NewConsoleFrame,
4263) -> ConsoleLogResult<bool> {
4264 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4265 let Some(fingerprint) = fingerprint else {
4266 return Ok(false);
4267 };
4268 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4269 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4270 let mut after = None;
4271 loop {
4272 let page = inner
4273 .store
4274 .query_frames(ConsoleTimelineQuery {
4275 identity: Some(frame.identity.clone()),
4276 conversation_id: frame.conversation_id.clone(),
4277 after,
4278 limit: 1_000,
4279 })
4280 .await?;
4281 for existing in &page.frames {
4282 let same_session = existing.session_id == frame.session_id
4283 || existing.session_id.is_none()
4284 || frame.session_id.is_none();
4285 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4286 continue;
4287 }
4288 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4289 == Some(&fingerprint)
4290 {
4291 return Ok(true);
4292 }
4293 if assistant_terminal
4294 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4295 {
4296 let turn_key = existing
4297 .interaction_id
4298 .as_deref()
4299 .or(existing.turn_id.as_deref())
4300 .or(existing.run_id.as_deref())
4301 .unwrap_or("session");
4302 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4303 aggregated.push_str(delta);
4304 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4305 return Ok(true);
4306 }
4307 }
4308 }
4309 if page.frames.is_empty() || page.next_cursor.is_none() {
4310 return Ok(false);
4311 }
4312 after = page.next_cursor;
4313 }
4314}
4315
4316fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4317 format!("{runtime_key}:session-history:{session_id}")
4318}
4319
4320fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4321 match kind {
4322 "user_input" | "interaction_started" => payload
4323 .get("content")
4324 .map(stable_value_fingerprint)
4325 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4326 "tool_execution_completed" => tool_result_fingerprint(payload),
4327 "text_delta" => {
4328 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4329 }
4330 kind if is_reasoning_event_kind(kind) => {
4331 reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4332 }
4333 "text_complete" | "interaction_complete" | "run_completed" => {
4334 assistant_terminal_fingerprint(kind, payload)
4335 }
4336 _ => None,
4337 }
4338}
4339
4340fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4341 let id = payload
4342 .get("tool_call_id")
4343 .or_else(|| payload.get("id"))
4344 .and_then(Value::as_str)
4345 .unwrap_or("");
4346 let result = payload
4347 .get("result")
4348 .or_else(|| payload.get("content"))
4349 .map(stable_value_fingerprint)?;
4350 Some(format!("{id}:{result}"))
4351}
4352
4353fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4354 match kind {
4355 "text_complete" | "interaction_complete" | "run_completed" => payload
4356 .get("text")
4357 .or_else(|| payload.get("result"))
4358 .or_else(|| payload.get("content"))
4359 .map(stable_value_fingerprint),
4360 _ => None,
4361 }
4362}
4363
4364fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4365 if kind != "text_delta" {
4366 return None;
4367 }
4368 payload
4369 .get("delta")
4370 .or_else(|| payload.get("text"))
4371 .or_else(|| payload.get("content"))
4372 .and_then(Value::as_str)
4373 .or_else(|| payload.as_str())
4374}
4375
4376fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4377 if !is_reasoning_event_kind(kind) {
4378 return None;
4379 }
4380 if kind == "reasoning_delta" {
4381 return payload
4382 .get("delta")
4383 .or_else(|| payload.get("text"))
4384 .or_else(|| payload.get("content"))
4385 .or_else(|| payload.get("summary"))
4386 .or_else(|| payload.get("reasoning"))
4387 .and_then(Value::as_str)
4388 .or_else(|| payload.as_str());
4389 }
4390 payload
4391 .get("summary")
4392 .or_else(|| payload.get("text"))
4393 .or_else(|| payload.get("content"))
4394 .or_else(|| payload.get("delta"))
4395 .or_else(|| payload.get("reasoning"))
4396 .and_then(Value::as_str)
4397 .or_else(|| payload.as_str())
4398}
4399
4400fn is_reasoning_event_kind(kind: &str) -> bool {
4401 matches!(
4402 kind,
4403 "reasoning_delta"
4404 | "reasoning_complete"
4405 | "reasoning_summary"
4406 | "reasoning_summary_delta"
4407 | "reasoning_summary_complete"
4408 )
4409}
4410
4411fn stable_value_fingerprint(value: &Value) -> String {
4412 if let Some(text) = content_value_text(value) {
4413 return normalize_transcript_fingerprint_text(&text);
4414 }
4415 match value {
4416 Value::String(text) => normalize_transcript_fingerprint_text(text),
4417 other => serde_json::to_string(other).unwrap_or_default(),
4418 }
4419}
4420
4421fn content_value_text(value: &Value) -> Option<String> {
4422 match value {
4423 Value::String(text) => Some(text.clone()),
4424 Value::Array(items) => {
4425 let text = items
4426 .iter()
4427 .filter_map(content_value_text)
4428 .collect::<String>();
4429 (!text.is_empty()).then_some(text)
4430 }
4431 Value::Object(map) => ["text", "content", "message", "blocks"]
4432 .iter()
4433 .filter_map(|key| map.get(*key))
4434 .find_map(content_value_text),
4435 _ => None,
4436 }
4437}
4438
4439fn normalize_transcript_fingerprint_text(text: &str) -> String {
4440 let trimmed = text.trim();
4441 trimmed
4442 .strip_prefix("[EVENT via rpc] ")
4443 .unwrap_or(trimmed)
4444 .trim()
4445 .to_string()
4446}
4447
4448#[derive(Debug, Clone, Copy)]
4449enum CachedIdentityVisibility {
4450 Visible,
4451 Hidden,
4452 Missing,
4453}
4454
4455async fn frame_is_visible(
4456 inner: &AggregatorInner,
4457 frame: &ConsoleFrame,
4458 allow_historical_identity: bool,
4459 identity_records: &[ConsoleIdentityRecord],
4460) -> ConsoleLogResult<bool> {
4461 let mut identity_visibility_cache = HashMap::new();
4462 Box::pin(frame_is_visible_cached(
4463 inner,
4464 frame,
4465 allow_historical_identity,
4466 &mut identity_visibility_cache,
4467 identity_records,
4468 ))
4469 .await
4470}
4471
4472async fn frame_is_visible_cached(
4473 inner: &AggregatorInner,
4474 frame: &ConsoleFrame,
4475 allow_historical_identity: bool,
4476 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4477 identity_records: &[ConsoleIdentityRecord],
4478) -> ConsoleLogResult<bool> {
4479 let entry = {
4480 let entries = inner
4481 .runtimes
4482 .read()
4483 .map_err(|_| runtime_registry_lock_error())?;
4484 if entries.is_empty() {
4485 return Ok(true);
4486 }
4487 let Some(entry) = entries.get(&frame.runtime_key) else {
4488 return Ok(false);
4489 };
4490 entry.clone()
4491 };
4492 if frame.identity != "__console__" && frame.identity != SYSTEM_EVENT_IDENTITY {
4500 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4501 let identity_visibility =
4502 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4503 cached
4504 } else {
4505 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4506 .unwrap_or_else(|| frame.identity.clone());
4507 let visibility = match identity_records.iter().find(|record| {
4508 record.runtime_key == frame.runtime_key
4509 && (record.identity == frame.identity
4510 || record.runtime_member_id == runtime_member_id)
4511 }) {
4512 Some(record) => {
4513 if Box::pin(console_identity_record_visible(&entry, record)).await {
4514 CachedIdentityVisibility::Visible
4515 } else {
4516 CachedIdentityVisibility::Hidden
4517 }
4518 }
4519 None => CachedIdentityVisibility::Missing,
4520 };
4521 identity_visibility_cache.insert(cache_key, visibility);
4522 visibility
4523 };
4524 match identity_visibility {
4525 CachedIdentityVisibility::Visible => {
4526 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4527 return Ok(false);
4528 }
4529 }
4530 CachedIdentityVisibility::Hidden => return Ok(false),
4531 CachedIdentityVisibility::Missing => {
4532 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4533 return Ok(false);
4534 }
4535 return Ok(
4536 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4537 );
4538 }
4539 }
4540 }
4541 Ok(entry.visibility_policy.frame_visible(frame))
4542}
4543
4544async fn identity_record_for_member(
4545 entry: &RuntimeEntry,
4546 handle: &MobHandle,
4547 member: &MobMemberListEntry,
4548) -> Option<ConsoleIdentityRecord> {
4549 let runtime_member_id =
4552 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4553 let durable_identity = member
4554 .labels
4555 .get("agent_identity")
4556 .filter(|value| !value.trim().is_empty())
4557 .map_or(runtime_member_id.as_str(), String::as_str);
4558 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4559 let addressable = member_is_addressable(member);
4560 let visibility = match member.status {
4561 meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4562 ConsoleVisibility::RetiredReadable
4563 }
4564 meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4567 ConsoleVisibility::Unreachable
4568 }
4569 _ if addressable => ConsoleVisibility::Addressable,
4570 _ => ConsoleVisibility::Hidden,
4571 };
4572 let session_id = handle
4573 .resolve_bridge_session_id_observation(&member.agent_identity)
4574 .await
4575 .map(|sid| sid.to_string());
4576 let mut labels = member.labels.clone();
4577 labels
4578 .entry("role".to_string())
4579 .or_insert_with(|| member.role.to_string());
4580 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4586 if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4587 crate::console_spawn::merge_registered_labels(&mut labels, ®istered);
4588 }
4589 let display_name = labels
4590 .get("display_name")
4591 .cloned()
4592 .unwrap_or_else(|| runtime_member_id.clone());
4593 Some(ConsoleIdentityRecord {
4594 identity,
4595 display_name,
4596 runtime_key: entry.runtime_key.clone(),
4597 runtime_member_id,
4598 session_id,
4599 visibility,
4600 addressable,
4601 health: if addressable {
4602 "ready"
4603 } else {
4604 "hidden_by_policy"
4605 }
4606 .to_string(),
4607 topology_peers: member
4608 .wired_to
4609 .iter()
4610 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4611 .collect(),
4612 labels,
4613 })
4614}
4615
4616async fn identity_record_for_resolved_member(
4617 resolved: &ResolvedConsoleMember,
4618) -> Option<ConsoleIdentityRecord> {
4619 let mut record =
4620 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4621 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4622 record
4623 .labels
4624 .entry("source_mob_id".to_string())
4625 .or_insert_with(|| resolved.source_mob_id.clone());
4626 }
4627 Some(record)
4628}
4629
4630fn identity_record_for_status(
4631 entry: &RuntimeEntry,
4632 status: &crate::identity_first::IdentityStatus,
4633) -> ConsoleIdentityRecord {
4634 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4635 let runtime_member_id = status
4636 .agent_runtime_id
4637 .as_ref()
4638 .map(crate::identity_first::AgentRuntimeId::as_str)
4639 .unwrap_or_else(|| status.identity.as_str())
4640 .to_string();
4641 let addressable = status.addressability
4642 == crate::identity_first::AgentAddressability::Addressable
4643 && matches!(
4644 status.state,
4645 crate::identity_first::IdentityLifecycleState::Active
4646 | crate::identity_first::IdentityLifecycleState::Dormant
4647 | crate::identity_first::IdentityLifecycleState::Uninitialized
4648 );
4649 let visibility = match status.state {
4650 crate::identity_first::IdentityLifecycleState::Retiring => {
4651 ConsoleVisibility::RetiredReadable
4652 }
4653 crate::identity_first::IdentityLifecycleState::Broken
4654 | crate::identity_first::IdentityLifecycleState::Suspended => {
4655 ConsoleVisibility::Unreachable
4656 }
4657 _ if addressable => ConsoleVisibility::Addressable,
4658 _ => ConsoleVisibility::Hidden,
4659 };
4660 let health = match status.state {
4661 crate::identity_first::IdentityLifecycleState::Active => "ready",
4662 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4663 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4664 crate::identity_first::IdentityLifecycleState::Broken => "broken",
4665 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4666 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4667 }
4668 .to_string();
4669 let display_name = status
4670 .display_name
4671 .as_ref()
4672 .map(crate::identity_first::DisplayName::as_str)
4673 .unwrap_or_else(|| status.identity.as_str())
4674 .to_string();
4675 ConsoleIdentityRecord {
4676 identity,
4677 display_name,
4678 runtime_key: entry.runtime_key.clone(),
4679 runtime_member_id,
4680 session_id: status.session_id.as_ref().map(ToString::to_string),
4681 visibility,
4682 addressable,
4683 health,
4684 topology_peers: Vec::new(),
4685 labels: status.labels.clone(),
4686 }
4687}
4688
4689pub(crate) fn is_implicit_delegate_member(
4690 role: &str,
4691 labels: &std::collections::BTreeMap<String, String>,
4692) -> bool {
4693 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4694}
4695
4696fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4697 member
4698 .labels
4699 .get("addressable")
4700 .map(|value| !value.eq_ignore_ascii_case("false"))
4701 .unwrap_or(true)
4702}
4703
4704fn sendable_resolved_member_indices(matches: &[ResolvedConsoleMember]) -> Vec<usize> {
4705 matches
4706 .iter()
4707 .enumerate()
4708 .filter_map(|(index, resolved)| {
4709 (member_is_addressable(&resolved.member)
4710 && resolved.member.status == meerkat_mob::MobMemberStatus::Active)
4711 .then_some(index)
4712 })
4713 .collect()
4714}
4715
4716async fn authoritative_durable_sendable_member_index(
4717 requested_identity: &str,
4718 matches: &[ResolvedConsoleMember],
4719 sendable_indices: &[usize],
4720) -> Result<Option<usize>, ConsoleSendError> {
4721 if matches.iter().any(|resolved| {
4722 requested_identity_is_runtime_member_alias(
4723 requested_identity,
4724 &resolved.entry.identity_namespace,
4725 )
4726 }) {
4727 return Ok(None);
4728 }
4729
4730 let mut visited_runtime_keys = BTreeSet::new();
4731 let mut bindings = BTreeSet::new();
4732 for resolved in matches {
4733 let entry = &resolved.entry;
4734 if !visited_runtime_keys.insert(entry.runtime_key.clone()) {
4735 continue;
4736 }
4737 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
4738 continue;
4739 };
4740 for raw_identity in
4741 namespace_match_candidates(requested_identity, &entry.identity_namespace)
4742 {
4743 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&raw_identity) else {
4744 continue;
4745 };
4746 let Ok(status) = identity_runtime.status(&identity).await else {
4747 continue;
4748 };
4749 if status.state != crate::identity_first::IdentityLifecycleState::Active
4750 || status.addressability != crate::identity_first::AgentAddressability::Addressable
4751 {
4752 continue;
4753 }
4754 let Some(runtime_id) = status.agent_runtime_id.as_ref() else {
4755 continue;
4756 };
4757 let record = identity_record_for_status(entry, &status);
4758 if !Box::pin(console_identity_record_visible(entry, &record)).await {
4759 continue;
4760 }
4761 bindings.insert((entry.runtime_key.clone(), runtime_id.as_str().to_string()));
4762 }
4763 }
4764
4765 if bindings.len() > 1 {
4766 let candidates = bindings
4767 .iter()
4768 .map(|(runtime_key, runtime_id)| format!("{runtime_id}@{runtime_key}"))
4769 .collect::<Vec<_>>()
4770 .join(", ");
4771 return Err(ConsoleSendError::AmbiguousIdentity {
4772 identity: requested_identity.to_string(),
4773 candidates,
4774 });
4775 }
4776 let Some((runtime_key, runtime_id)) = bindings.into_iter().next() else {
4777 return Ok(None);
4778 };
4779 let bound_indices = sendable_indices
4780 .iter()
4781 .copied()
4782 .filter(|index| {
4783 matches[*index].entry.runtime_key == runtime_key
4784 && matches[*index].runtime_identity == runtime_id
4785 })
4786 .collect::<Vec<_>>();
4787 match bound_indices.as_slice() {
4788 [index] => Ok(Some(*index)),
4789 [] => Ok(None),
4790 _ => {
4791 let candidates = bound_indices
4792 .iter()
4793 .map(|index| {
4794 format!(
4795 "{}@{}",
4796 matches[*index].runtime_identity, matches[*index].source_mob_id
4797 )
4798 })
4799 .collect::<Vec<_>>()
4800 .join(", ");
4801 Err(ConsoleSendError::AmbiguousIdentity {
4802 identity: requested_identity.to_string(),
4803 candidates,
4804 })
4805 }
4806 }
4807}
4808
4809fn apply_namespace(identity: &str, namespace: &str) -> String {
4810 let namespace = namespace.trim().trim_matches('/');
4811 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4812 identity.to_string()
4813 } else {
4814 format!("{namespace}/{identity}")
4815 }
4816}
4817
4818fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4819 let namespace = namespace.trim().trim_matches('/');
4820 if namespace.is_empty() {
4821 return Some(identity.to_string());
4822 }
4823 identity
4824 .strip_prefix(namespace)
4825 .and_then(|rest| rest.strip_prefix('/'))
4826 .map(ToString::to_string)
4827}
4828
4829fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4830 let namespace = namespace.trim().trim_matches('/');
4831 if namespace.is_empty() {
4832 return vec![identity.to_string()];
4833 }
4834 let mut candidates = Vec::new();
4835 if let Some(stripped) = strip_namespace(identity, namespace) {
4836 candidates.push(stripped);
4837 candidates.push(identity.to_string());
4838 }
4839 candidates.sort();
4840 candidates.dedup();
4841 candidates
4842}
4843
4844fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4845 namespace_match_candidates(identity, namespace)
4846 .iter()
4847 .any(|candidate| candidate.starts_with("rt:"))
4848}
4849
4850fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4851 entries.iter().any(|entry| {
4852 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4853 })
4854}
4855
4856fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4857 if request.identity.trim().is_empty() {
4858 return Err(ConsoleSendError::InvalidRequest(
4859 "identity must be non-empty".to_string(),
4860 ));
4861 }
4862 if request.origin.trim().is_empty() {
4863 return Err(ConsoleSendError::InvalidRequest(
4864 "origin must be non-empty".to_string(),
4865 ));
4866 }
4867 if request.idempotency_key.trim().is_empty() {
4868 return Err(ConsoleSendError::InvalidRequest(
4869 "idempotency_key must be non-empty".to_string(),
4870 ));
4871 }
4872 Ok(())
4873}
4874
4875fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4876 let content: ContentInput = serde_json::from_value(value.clone())
4877 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4878 match &content {
4879 ContentInput::Text(text) if text.trim().is_empty() => Err(
4880 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4881 ),
4882 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4883 "content blocks must be non-empty".to_string(),
4884 )),
4885 _ => Ok(content),
4886 }
4887}
4888
4889fn parse_handling_mode(
4890 value: Option<&str>,
4891) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4892 match value.unwrap_or("queue") {
4893 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4894 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4895 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4896 }
4897}
4898
4899fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4900 ConsoleInteractionAccepted {
4901 interaction_id: frame
4902 .interaction_id
4903 .clone()
4904 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4905 identity: frame.identity.clone(),
4906 conversation_id: frame.conversation_id.clone(),
4907 session_id: frame.session_id.clone(),
4908 input_frame_id: frame.id.clone(),
4909 cursor: frame.cursor.clone(),
4910 status: frame.status,
4911 }
4912}
4913
4914fn send_dedupe_key(
4915 runtime_key: &str,
4916 identity: &str,
4917 origin: &str,
4918 idempotency_key: &str,
4919) -> String {
4920 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4921}
4922
4923fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4924 let content_json = serde_json::to_string(content).unwrap_or_default();
4925 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4926}
4927
4928const CONSOLE_INTERACTION_UUID_NAMESPACE: uuid::Uuid = uuid::Uuid::from_bytes([
4938 0x6d, 0x6f, 0x62, 0x6b, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2d, 0x69,
4939]);
4940
4941fn identity_first_interaction_uuid(dedupe_key: &str) -> String {
4942 uuid::Uuid::new_v5(&CONSOLE_INTERACTION_UUID_NAMESPACE, dedupe_key.as_bytes()).to_string()
4943}
4944
4945fn hash_short(value: &str) -> String {
4946 let mut hasher = Sha256::new();
4947 hasher.update(value.as_bytes());
4948 let digest = hasher.finalize();
4949 to_hex(&digest[..8])
4950}
4951
4952fn to_hex(bytes: &[u8]) -> String {
4953 const HEX: &[u8; 16] = b"0123456789abcdef";
4954 let mut out = String::with_capacity(bytes.len() * 2);
4955 for byte in bytes {
4956 out.push(HEX[(byte >> 4) as usize] as char);
4957 out.push(HEX[(byte & 0x0f) as usize] as char);
4958 }
4959 out
4960}
4961
4962fn current_time_ms() -> u64 {
4963 match SystemTime::now().duration_since(UNIX_EPOCH) {
4964 Ok(duration) => duration.as_millis() as u64,
4965 Err(_) => 0,
4966 }
4967}
4968
4969fn runtime_registry_lock_error() -> ConsoleLogError {
4970 Box::new(std::io::Error::other(
4971 "console runtime registry lock poisoned",
4972 ))
4973}
4974
4975#[cfg(test)]
4976#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4977mod tests {
4978 #[test]
4983 fn backfill_gap_dedupe_key_is_stable_per_session() {
4984 use super::backfill_gap_dedupe_key;
4985 let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4986 let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4987 assert_eq!(a, b, "same target must reuse one gap frame");
4988 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4989 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4990 assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4993 }
4994
4995 use std::sync::Arc;
4996 use std::sync::atomic::{AtomicUsize, Ordering};
4997 use std::time::Duration;
4998 use std::time::Instant;
4999
5000 use futures::StreamExt;
5001 use meerkat::{AgentFactory, Config, build_ephemeral_service};
5002 use meerkat_client::types::LlmStream;
5003 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
5004 use meerkat_core::{
5005 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
5006 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
5007 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
5008 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
5009 StreamError,
5010 };
5011 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
5012 use serde_json::json;
5013
5014 use super::*;
5015 use crate::mob_handle_runtime::MobBootstrapSpec;
5016
5017 #[derive(Debug)]
5018 struct HideHiddenNoiseFrames;
5019
5020 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
5021 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
5022 frame.kind != "hidden_noise"
5023 }
5024 }
5025
5026 #[derive(Debug)]
5027 struct HideRuntimeMemberIdentity(&'static str);
5028
5029 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
5030 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5031 record.runtime_member_id != self.0
5032 }
5033 }
5034
5035 #[derive(Debug)]
5036 struct HideRuntimeMemberOnly(&'static str);
5037
5038 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
5039 fn member_visible(&self, member: &ConsoleMember) -> bool {
5040 member.agent_identity != self.0
5041 }
5042 }
5043
5044 #[derive(Debug)]
5045 struct HideConsoleIdentity(&'static str);
5046
5047 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
5048 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5049 record.identity != self.0
5050 }
5051 }
5052
5053 struct CountingConsoleLogStore {
5054 inner: InMemoryConsoleLogStore,
5055 source_watermark_calls: AtomicUsize,
5056 record_watermark_calls: AtomicUsize,
5057 }
5058
5059 impl CountingConsoleLogStore {
5060 fn new() -> Self {
5061 Self {
5062 inner: InMemoryConsoleLogStore::new(),
5063 source_watermark_calls: AtomicUsize::new(0),
5064 record_watermark_calls: AtomicUsize::new(0),
5065 }
5066 }
5067
5068 fn source_watermark_calls(&self) -> usize {
5069 self.source_watermark_calls.load(Ordering::SeqCst)
5070 }
5071 }
5072
5073 struct SlowTestClient {
5074 delay: Duration,
5075 }
5076
5077 #[async_trait::async_trait]
5078 impl LlmClient for SlowTestClient {
5079 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
5080 Ok(messages.to_vec())
5081 }
5082
5083 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
5084 let delay = self.delay;
5085 let delayed_text = futures::stream::once(async move {
5086 tokio::time::sleep(delay).await;
5087 Ok(LlmEvent::TextDelta {
5088 delta: "slow ok".to_string(),
5089 meta: None,
5090 })
5091 });
5092 let done = futures::stream::once(async {
5093 Ok(LlmEvent::Done {
5094 outcome: LlmDoneOutcome::Success {
5095 stop_reason: StopReason::EndTurn,
5096 },
5097 })
5098 });
5099 Box::pin(delayed_text.chain(done))
5100 }
5101
5102 fn provider(&self) -> meerkat_core::Provider {
5103 meerkat_core::Provider::Other
5104 }
5105
5106 async fn health_check(&self) -> Result<(), LlmError> {
5107 Ok(())
5108 }
5109 }
5110
5111 #[derive(Clone)]
5112 struct DelayedHistorySessionService {
5113 inner: Arc<dyn MobSessionService>,
5114 delay: Duration,
5115 read_calls: Arc<AtomicUsize>,
5116 active_reads: Arc<AtomicUsize>,
5117 max_active_reads: Arc<AtomicUsize>,
5118 }
5119
5120 impl DelayedHistorySessionService {
5121 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
5122 Self {
5123 inner,
5124 delay,
5125 read_calls: Arc::new(AtomicUsize::new(0)),
5126 active_reads: Arc::new(AtomicUsize::new(0)),
5127 max_active_reads: Arc::new(AtomicUsize::new(0)),
5128 }
5129 }
5130
5131 fn read_calls(&self) -> usize {
5132 self.read_calls.load(Ordering::SeqCst)
5133 }
5134
5135 fn max_active_reads(&self) -> usize {
5136 self.max_active_reads.load(Ordering::SeqCst)
5137 }
5138 }
5139
5140 #[async_trait::async_trait]
5141 impl SessionService for DelayedHistorySessionService {
5142 async fn create_session(
5143 &self,
5144 req: meerkat_core::CreateSessionRequest,
5145 ) -> Result<RunResult, SessionError> {
5146 self.inner.create_session(req).await
5147 }
5148
5149 async fn start_turn(
5150 &self,
5151 id: &SessionId,
5152 req: StartTurnRequest,
5153 ) -> Result<RunResult, SessionError> {
5154 self.inner.start_turn(id, req).await
5155 }
5156
5157 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
5158 self.inner.interrupt(id).await
5159 }
5160
5161 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
5162 self.inner.read(id).await
5163 }
5164
5165 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
5166 self.inner.list(query).await
5167 }
5168
5169 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
5170 self.inner.archive(id).await
5171 }
5172
5173 async fn subscribe_session_events(
5174 &self,
5175 id: &SessionId,
5176 ) -> Result<EventStream, StreamError> {
5177 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
5178 }
5179 }
5180
5181 #[async_trait::async_trait]
5182 impl SessionServiceCommsExt for DelayedHistorySessionService {
5183 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
5184 self.inner.comms_runtime(session_id).await
5185 }
5186 }
5187
5188 #[async_trait::async_trait]
5189 impl SessionServiceControlExt for DelayedHistorySessionService {
5190 async fn append_system_context(
5191 &self,
5192 id: &SessionId,
5193 req: AppendSystemContextRequest,
5194 ) -> Result<AppendSystemContextResult, SessionControlError> {
5195 self.inner.append_system_context(id, req).await
5196 }
5197 }
5198
5199 #[async_trait::async_trait]
5200 impl SessionServiceHistoryExt for DelayedHistorySessionService {
5201 async fn read_history(
5202 &self,
5203 id: &SessionId,
5204 query: SessionHistoryQuery,
5205 ) -> Result<SessionHistoryPage, SessionError> {
5206 self.read_calls.fetch_add(1, Ordering::SeqCst);
5207 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
5208 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
5209 tokio::time::sleep(self.delay).await;
5210 let result = self.inner.read_history(id, query).await;
5211 self.active_reads.fetch_sub(1, Ordering::SeqCst);
5212 result
5213 }
5214 }
5215
5216 #[async_trait::async_trait]
5217 impl MobSessionService for DelayedHistorySessionService {
5218 fn supports_persistent_sessions(&self) -> bool {
5219 self.inner.supports_persistent_sessions()
5220 }
5221
5222 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
5223 self.inner.runtime_adapter()
5224 }
5225
5226 async fn session_belongs_to_mob(
5227 &self,
5228 session_id: &SessionId,
5229 mob_id: &meerkat_mob::MobId,
5230 ) -> bool {
5231 self.inner.session_belongs_to_mob(session_id, mob_id).await
5232 }
5233
5234 async fn cancel_all_checkpointers(&self) {
5235 self.inner.cancel_all_checkpointers().await;
5236 }
5237
5238 async fn rearm_all_checkpointers(&self) {
5239 self.inner.rearm_all_checkpointers().await;
5240 }
5241 }
5242
5243 #[async_trait::async_trait]
5244 impl ConsoleLogStore for CountingConsoleLogStore {
5245 async fn append_if_absent(
5246 &self,
5247 frame: NewConsoleFrame,
5248 ) -> ConsoleLogResult<AppendOutcome> {
5249 self.inner.append_if_absent(frame).await
5250 }
5251
5252 async fn update_frame_status(
5253 &self,
5254 frame_id: &str,
5255 status: ConsoleFrameStatus,
5256 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5257 self.inner.update_frame_status(frame_id, status).await
5258 }
5259
5260 async fn query_frames(
5261 &self,
5262 query: ConsoleTimelineQuery,
5263 ) -> ConsoleLogResult<ConsoleTimelinePage> {
5264 self.inner.query_frames(query).await
5265 }
5266
5267 async fn query_windowed_frames(
5268 &self,
5269 query: ConsoleTimelineWindowQuery,
5270 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
5271 self.inner.query_windowed_frames(query).await
5272 }
5273
5274 async fn frame_by_dedupe_key(
5275 &self,
5276 dedupe_key: &str,
5277 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5278 self.inner.frame_by_dedupe_key(dedupe_key).await
5279 }
5280
5281 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
5282 self.inner.latest_cursor().await
5283 }
5284
5285 async fn clear_frames(&self) -> ConsoleLogResult<()> {
5286 self.inner.clear_frames().await
5287 }
5288
5289 async fn record_source_watermark(
5290 &self,
5291 runtime_key: &str,
5292 source_kind: ConsoleFrameSourceKind,
5293 source_cursor: &str,
5294 ) -> ConsoleLogResult<()> {
5295 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5296 self.inner
5297 .record_source_watermark(runtime_key, source_kind, source_cursor)
5298 .await
5299 }
5300
5301 async fn source_watermark(
5302 &self,
5303 runtime_key: &str,
5304 source_kind: ConsoleFrameSourceKind,
5305 ) -> ConsoleLogResult<Option<String>> {
5306 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5307 self.inner.source_watermark(runtime_key, source_kind).await
5308 }
5309 }
5310
5311 async fn build_single_member_runtime() -> UnifiedRuntime {
5312 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5313 }
5314
5315 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5316 let definition = MobDefinition::from_toml(
5317 r#"
5318[mob]
5319id = "console-aggregator-perf-test"
5320
5321[profiles.worker]
5322model = "gpt-5.5"
5323external_addressable = true
5324
5325[profiles.worker.tools]
5326comms = true
5327"#,
5328 )
5329 .expect("definition parses");
5330 let runtime = UnifiedRuntime::builder()
5331 .definition(definition)
5332 .default_llm_client(client)
5333 .build()
5334 .await
5335 .expect("runtime builds");
5336 runtime
5337 .spawn(SpawnMemberSpec::from_wire(
5338 "worker".to_string(),
5339 "agent-a".to_string(),
5340 Some("You are agent-a.".into()),
5341 None,
5342 None,
5343 ))
5344 .await
5345 .expect("member spawns");
5346 runtime
5347 }
5348
5349 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5350 let definition = MobDefinition::from_toml(&format!(
5351 r#"
5352[mob]
5353id = "{mob_id}"
5354
5355[profiles.worker]
5356model = "gpt-5.5"
5357external_addressable = true
5358
5359[profiles.worker.tools]
5360comms = true
5361"#
5362 ))
5363 .expect("definition parses");
5364 UnifiedRuntime::builder()
5365 .definition(definition)
5366 .default_llm_client(Arc::new(TestClient::default()))
5367 .build()
5368 .await
5369 .expect("runtime builds")
5370 }
5371
5372 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5373 async fn send_selection_ignores_a_retiring_generation_when_one_active_generation_remains() {
5374 let runtime = build_empty_runtime("console-send-retiring-generation-test").await;
5375 let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
5376 for generation in 0..=1 {
5377 runtime
5378 .spawn(
5379 SpawnMemberSpec::from_wire(
5380 "worker".to_string(),
5381 format!("rt:builder:{generation}"),
5382 Some("You are Builder.".into()),
5383 None,
5384 None,
5385 )
5386 .with_labels(labels.clone()),
5387 )
5388 .await
5389 .expect("member spawns");
5390 }
5391
5392 let aggregator = MobKitConsoleAggregator::in_memory();
5393 aggregator.register_runtime_handles_with_policy(
5394 "runtime-a",
5395 "",
5396 runtime.mob_runtime().clone(),
5397 None,
5398 runtime.console_events(),
5399 Arc::new(AllowAllConsoleVisibilityPolicy),
5400 );
5401 let mut matches = aggregator.resolve_visible_members("builder").await;
5402 assert_eq!(matches.len(), 2);
5403 assert_eq!(sendable_resolved_member_indices(&matches).len(), 2);
5404
5405 matches[0].member.status = meerkat_mob::MobMemberStatus::Retiring;
5406 let sendable = sendable_resolved_member_indices(&matches);
5407 assert_eq!(sendable.len(), 1);
5408 assert_ne!(
5409 matches[sendable[0]].member.status,
5410 meerkat_mob::MobMemberStatus::Retiring
5411 );
5412
5413 let _ = runtime.mob_handle().stop().await;
5414 }
5415
5416 async fn build_stress_runtime(
5417 member_count: usize,
5418 history_delay: Duration,
5419 ) -> (
5420 tempfile::TempDir,
5421 Arc<UnifiedRuntime>,
5422 DelayedHistorySessionService,
5423 ) {
5424 let temp_dir = tempfile::tempdir().expect("temp dir");
5425 let session_path = temp_dir.path().join("sessions");
5426 std::fs::create_dir_all(&session_path).expect("session path");
5427 let factory = AgentFactory::new(&session_path).comms(true);
5428 let base_service = Arc::new(build_ephemeral_service(
5429 factory,
5430 Config::default(),
5431 member_count + 8,
5432 ));
5433 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5434 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5435 let definition = MobDefinition::from_toml(
5436 r#"
5437[mob]
5438id = "console-aggregator-stress-test"
5439
5440[profiles.worker]
5441model = "gpt-5.5"
5442external_addressable = true
5443
5444[profiles.worker.tools]
5445comms = true
5446"#,
5447 )
5448 .expect("definition parses");
5449 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5450 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5451 allow_ephemeral_sessions: true,
5452 notify_orchestrator_on_resume: true,
5453 default_llm_client: Some(Arc::new(TestClient::default())),
5454 });
5455 let runtime = Arc::new(
5456 UnifiedRuntime::bootstrap(
5457 spec,
5458 crate::types::MobKitConfig {
5459 modules: Vec::new(),
5460 discovery: crate::types::DiscoverySpec {
5461 namespace: "stress".to_string(),
5462 modules: Vec::new(),
5463 },
5464 pre_spawn: Vec::new(),
5465 },
5466 Duration::from_secs(5),
5467 )
5468 .await
5469 .expect("runtime boots"),
5470 );
5471 for idx in 0..member_count {
5472 runtime
5473 .spawn(SpawnMemberSpec::from_wire(
5474 "worker".to_string(),
5475 format!("agent-{idx}"),
5476 Some(format!("You are agent-{idx}.").into()),
5477 None,
5478 None,
5479 ))
5480 .await
5481 .expect("member spawns");
5482 }
5483 (temp_dir, runtime, delayed_service)
5484 }
5485
5486 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5487 RuntimeEntry {
5488 runtime_key: runtime_key.to_string(),
5489 identity_namespace: "test".to_string(),
5490 runtime: runtime.mob_runtime().clone(),
5491 identity_runtime: runtime.identity_runtime().cloned(),
5492 console_events: runtime.console_events(),
5493 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5494 }
5495 }
5496
5497 fn console_envelope_for_test(
5498 event_id: &str,
5499 interaction_id: Option<&str>,
5500 event_type: &str,
5501 data: Value,
5502 ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5503 crate::console_contracts::ConsoleIdentityEventEnvelope {
5504 event_id: event_id.to_string(),
5505 interaction_id: interaction_id.map(ToString::to_string),
5506 identity: "agent-a".to_string(),
5507 event_type: event_type.to_string(),
5508 timestamp_ms: 1_000,
5509 data,
5510 }
5511 }
5512
5513 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5514 ConsoleIdentityRecord {
5515 identity: identity.to_string(),
5516 display_name: identity.to_string(),
5517 runtime_key: "runtime-cache".to_string(),
5518 runtime_member_id: identity.to_string(),
5519 session_id: Some(format!("session-{identity}")),
5520 visibility: ConsoleVisibility::Addressable,
5521 addressable: true,
5522 health: "ready".to_string(),
5523 topology_peers: Vec::new(),
5524 labels: BTreeMap::new(),
5525 }
5526 }
5527
5528 #[test]
5529 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5530 let mut durable = identity_record_for_test("test/review:singleton");
5531 durable.runtime_member_id = "rt:review:singleton:1".to_string();
5532 durable.session_id = Some("session-stale".to_string());
5533 durable
5534 .labels
5535 .insert("agent_identity".to_string(), "review:singleton".to_string());
5536
5537 let mut live = identity_record_for_test("test/review:singleton");
5538 live.runtime_member_id = "rt:review:singleton:0".to_string();
5539 live.session_id = Some("session-live".to_string());
5540 live.labels
5541 .insert("agent_identity".to_string(), "review:singleton".to_string());
5542
5543 let records = dedupe_identity_records(vec![durable, live]);
5544
5545 assert_eq!(
5546 records.len(),
5547 2,
5548 "list projection must expose stale durable/live alias split-brain instead of picking one record"
5549 );
5550 assert!(
5551 records
5552 .iter()
5553 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5554 );
5555 assert!(
5556 records
5557 .iter()
5558 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5559 );
5560 }
5561
5562 #[test]
5563 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5564 let mut first = identity_record_for_test("test/review:singleton");
5565 first.runtime_member_id = "rt:review:singleton:0".to_string();
5566 first
5567 .labels
5568 .insert("agent_identity".to_string(), "review:singleton".to_string());
5569
5570 let mut second = identity_record_for_test("test/review:singleton");
5571 second.runtime_member_id = "rt:review:singleton:1".to_string();
5572 second
5573 .labels
5574 .insert("agent_identity".to_string(), "review:singleton".to_string());
5575
5576 let records = dedupe_identity_records(vec![first, second]);
5577
5578 assert_eq!(
5579 records.len(),
5580 2,
5581 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5582 );
5583 }
5584
5585 #[test]
5586 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5587 let mut first = identity_record_for_test("test/review:singleton");
5588 first.runtime_member_id = "rt:review:singleton:0".to_string();
5589 first.session_id = None;
5590 first
5591 .labels
5592 .insert("agent_identity".to_string(), "review:singleton".to_string());
5593 first
5594 .labels
5595 .insert("source_mob_id".to_string(), "mob-a".to_string());
5596
5597 let mut second = first.clone();
5598 second
5599 .labels
5600 .insert("source_mob_id".to_string(), "mob-b".to_string());
5601
5602 let records = dedupe_identity_records(vec![first, second]);
5603
5604 assert_eq!(
5605 records.len(),
5606 2,
5607 "same member/session aliases from different source mobs must remain visible"
5608 );
5609 }
5610
5611 #[test]
5612 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5613 let durable = identity_record_for_test("test/review:singleton");
5614 let live = durable.clone();
5615
5616 let records = dedupe_identity_records(vec![durable, live]);
5617
5618 assert_eq!(
5619 records.len(),
5620 1,
5621 "healthy durable and primary live projections should not duplicate the roster"
5622 );
5623 }
5624
5625 #[test]
5626 fn stale_durable_record_error_detects_session_split_brain() {
5627 let mut durable = identity_record_for_test("test/review:singleton");
5628 durable.runtime_member_id = "rt:review:singleton:0".to_string();
5629 durable.session_id = Some("session-durable".to_string());
5630
5631 let mut live = durable.clone();
5632 live.session_id = Some("session-live".to_string());
5633
5634 let err = stale_durable_record_error("review:singleton", &durable, &[live])
5635 .expect("session mismatch must be stale split-brain");
5636
5637 assert!(
5638 err.contains("stale durable identity alias"),
5639 "unexpected error: {err}"
5640 );
5641 assert!(
5642 err.contains("session-durable") && err.contains("session-live"),
5643 "error should name both sessions: {err}"
5644 );
5645 }
5646
5647 async fn identity_runtime_for_test(
5648 identities: &[&str],
5649 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5650 {
5651 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5652 crate::identity_first::IdentityRuntimeConfig {
5653 continuity_store: Arc::new(
5654 crate::identity_first::LocalContinuityStore::in_memory()?
5655 ),
5656 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5657 runtime_instance_id: "console-aggregator-identity-test".to_string(),
5658 has_runtime_store: true,
5659 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5660 bridge: None,
5661 default_timeout: None,
5662 },
5663 ));
5664 for identity in identities {
5665 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5666 let record = crate::identity_first::ContinuityRecord {
5667 identity: identity.clone(),
5668 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5669 "rt:{}:0",
5670 identity.as_str()
5671 ))?,
5672 session_id: SessionId::new(),
5673 generation: crate::identity_first::ContinuityGeneration::new(0),
5674 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5675 };
5676 runtime
5677 .register(
5678 crate::identity_first::DurableAgentSpec {
5679 identity: identity.clone(),
5680 profile: meerkat_mob::ProfileName::from("worker"),
5681 addressability: crate::identity_first::AgentAddressability::Addressable,
5682 display_name: None,
5683 labels: BTreeMap::new(),
5684 context: None,
5685 additional_instructions: Vec::new(),
5686 initial_message: None,
5687 runtime_mode_override: None,
5688 backend: None,
5689 binding: None,
5690 },
5691 crate::identity_first::IdentityLifecycleState::Active,
5692 Some(record),
5693 Some(crate::identity_first::LeaseGrant {
5694 identity,
5695 fencing_token: crate::identity_first::FencingToken::new(1),
5696 ttl: Duration::from_mins(5),
5697 }),
5698 )
5699 .await;
5700 }
5701 Ok(runtime)
5702 }
5703
5704 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5705 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5706 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5707 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5708 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5709 let aggregator = MobKitConsoleAggregator::in_memory();
5710 aggregator.register_runtime_handles_with_policy(
5711 "runtime-a",
5712 "",
5713 runtime.mob_runtime().clone(),
5714 Some(identity_runtime),
5715 runtime.console_events(),
5716 Arc::new(AllowAllConsoleVisibilityPolicy),
5717 );
5718
5719 assert!(
5720 aggregator
5721 .inspect_identity("rt:review:singleton:0")
5722 .await?
5723 .is_none(),
5724 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5725 );
5726 assert!(
5727 !aggregator.retire_identity("rt:review:singleton:0").await?,
5728 "runtime-member-id shaped requests must not retire a durable rt:* identity"
5729 );
5730 let reserve_err = aggregator
5731 .reserve_identity_first_interaction(
5732 ConsoleSendRequest {
5733 identity: "rt:review:singleton:0".to_string(),
5734 content: json!("hello"),
5735 origin: "console:test".to_string(),
5736 idempotency_key: "rt-shadow-reserve".to_string(),
5737 handling_mode: Some("queue".to_string()),
5738 },
5739 None,
5740 )
5741 .await
5742 .expect_err(
5743 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5744 );
5745 assert!(
5746 reserve_err.to_string().contains("unknown identity"),
5747 "unexpected reserve error: {reserve_err}"
5748 );
5749 let send_err = aggregator
5750 .send(ConsoleSendRequest {
5751 identity: "rt:review:singleton:0".to_string(),
5752 content: json!("hello"),
5753 origin: "console:test".to_string(),
5754 idempotency_key: "rt-shadow-send".to_string(),
5755 handling_mode: Some("queue".to_string()),
5756 })
5757 .await
5758 .expect_err(
5759 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5760 );
5761 assert!(
5762 send_err.to_string().contains("unknown identity"),
5763 "unexpected send error: {send_err}"
5764 );
5765 let blob_err = match aggregator
5766 .binary_blob_store_for_identity("rt:review:singleton:0")
5767 .await
5768 {
5769 Ok(_) => panic!(
5770 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5771 ),
5772 Err(err) => err,
5773 };
5774 assert!(
5775 blob_err.to_string().contains("unknown identity"),
5776 "unexpected blob error: {blob_err}"
5777 );
5778
5779 let _ = runtime.mob_handle().stop().await;
5780 Ok(())
5781 }
5782
5783 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5784 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5785 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5786 let runtime = build_empty_runtime("identity-label-test").await;
5787 let mut labels = BTreeMap::new();
5788 labels.insert(
5789 "agent_identity".to_string(),
5790 "channel:C0SMOKEOB3".to_string(),
5791 );
5792 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5793 runtime
5794 .spawn(
5795 SpawnMemberSpec::from_wire(
5796 "worker".to_string(),
5797 "rt:channel:C0SMOKEOB3:0".to_string(),
5798 Some("You are C0SMOKEOB3.".into()),
5799 None,
5800 None,
5801 )
5802 .with_labels(labels),
5803 )
5804 .await
5805 .expect("member spawns");
5806
5807 let entry = RuntimeEntry {
5808 runtime_key: "runtime-a".to_string(),
5809 identity_namespace: String::new(),
5810 runtime: runtime.mob_runtime().clone(),
5811 identity_runtime: runtime.identity_runtime().cloned(),
5812 console_events: runtime.console_events(),
5813 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5814 };
5815 let aggregator = MobKitConsoleAggregator::in_memory();
5816 aggregator
5817 .inner
5818 .runtimes
5819 .write()
5820 .expect("runtime registry")
5821 .insert("runtime-a".to_string(), entry);
5822
5823 let records = aggregator.list_identities_fresh().await?;
5824 let record = records
5825 .iter()
5826 .find(|record| record.identity == "channel:C0SMOKEOB3")
5827 .expect("durable identity is exposed");
5828 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5829
5830 let inspection = aggregator
5831 .inspect_identity("channel:C0SMOKEOB3")
5832 .await?
5833 .expect("durable identity resolves back to runtime member");
5834 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5835 assert_eq!(
5836 inspection.identity.runtime_member_id,
5837 "rt:channel:C0SMOKEOB3:0"
5838 );
5839
5840 let _ = runtime.mob_handle().stop().await;
5841 Ok(())
5842 }
5843
5844 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5853 async fn identity_scoped_live_stream_sees_members_unknown_to_the_read_model()
5854 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5855 let runtime = build_empty_runtime("live-tail-unknown-identity-test").await;
5856 let aggregator = MobKitConsoleAggregator::in_memory();
5857 aggregator.register_runtime_handles_with_policy(
5858 "runtime-a",
5859 "",
5860 runtime.mob_runtime().clone(),
5861 runtime.identity_runtime().cloned(),
5862 runtime.console_events(),
5863 Arc::new(AllowAllConsoleVisibilityPolicy),
5864 );
5865 let _ = aggregator.list_identities_fresh().await?;
5868
5869 runtime
5870 .spawn(SpawnMemberSpec::from_wire(
5871 "worker".to_string(),
5872 "late-member".to_string(),
5873 Some("You are the late member.".into()),
5874 None,
5875 None,
5876 ))
5877 .await?;
5878 let outcome = aggregator
5879 .store()
5880 .append_if_absent(NewConsoleFrame {
5881 id: None,
5882 dedupe_key: "late-member-live-frame".to_string(),
5883 timestamp_ms: 1,
5884 runtime_key: "runtime-a".to_string(),
5885 identity: "late-member".to_string(),
5886 conversation_id: Some("late-member".to_string()),
5887 session_id: None,
5888 kind: "text_delta".to_string(),
5889 status: ConsoleFrameStatus::Completed,
5890 payload: json!({"delta": "hi"}),
5891 source: ConsoleFrameSource {
5892 kind: ConsoleFrameSourceKind::ConsoleEvent,
5893 source_cursor: None,
5894 },
5895 source_event_id: None,
5896 interaction_id: None,
5897 turn_id: None,
5898 run_id: None,
5899 parent_frame_id: None,
5900 caused_by_frame_id: None,
5901 })
5902 .await?;
5903 let event = ConsoleTimelineEvent::ConsoleFrame {
5904 frame: outcome.frame,
5905 };
5906
5907 assert!(
5910 aggregator
5911 .timeline_event_visible_for_subscriber(&event, Some("late-member"))
5912 .await,
5913 "identity-scoped live stream must not starve on a read-model miss"
5914 );
5915 let visible_to_other = aggregator
5922 .timeline_event_visible_for_subscriber(&event, Some("someone-else"))
5923 .await;
5924 if visible_to_other {
5925 assert!(
5926 aggregator.timeline_event_visible(&event).await,
5927 "the allowance is strictly own-identity: someone-else saw the \
5928 frame while the read model still treats the identity as \
5929 unknown (unscoped gate hides it)"
5930 );
5931 }
5932
5933 let mut converged = false;
5936 for _ in 0..40 {
5937 if aggregator.timeline_event_visible(&event).await {
5938 converged = true;
5939 break;
5940 }
5941 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
5942 }
5943 assert!(
5944 converged,
5945 "the unknown-identity miss must self-heal the read model"
5946 );
5947
5948 let _ = runtime.mob_handle().stop().await;
5949 Ok(())
5950 }
5951
5952 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5957 async fn unregister_runtime_stops_live_projection()
5958 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5959 let runtime = build_empty_runtime("unregister-live-projection-test").await;
5960 let aggregator = MobKitConsoleAggregator::in_memory();
5961 aggregator.register_runtime_handles_with_policy(
5962 "runtime-a",
5963 "",
5964 runtime.mob_runtime().clone(),
5965 runtime.identity_runtime().cloned(),
5966 runtime.console_events(),
5967 Arc::new(AllowAllConsoleVisibilityPolicy),
5968 );
5969
5970 aggregator.unregister_runtime("runtime-a");
5971 assert!(
5972 aggregator
5973 .inner
5974 .runtimes
5975 .read()
5976 .expect("runtime registry")
5977 .is_empty(),
5978 "unregister must remove the registry entry"
5979 );
5980 assert!(
5981 aggregator
5982 .inner
5983 .live_projection_shutdowns
5984 .lock()
5985 .expect("shutdown registry")
5986 .is_empty(),
5987 "unregister must consume the live-projection shutdown handle"
5988 );
5989 aggregator.unregister_runtime("runtime-a");
5991
5992 aggregator.register_runtime_handles_with_policy(
5994 "runtime-a",
5995 "",
5996 runtime.mob_runtime().clone(),
5997 runtime.identity_runtime().cloned(),
5998 runtime.console_events(),
5999 Arc::new(AllowAllConsoleVisibilityPolicy),
6000 );
6001 assert_eq!(
6002 aggregator
6003 .inner
6004 .live_projection_shutdowns
6005 .lock()
6006 .expect("shutdown registry")
6007 .len(),
6008 1
6009 );
6010
6011 let _ = runtime.mob_handle().stop().await;
6012 Ok(())
6013 }
6014
6015 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6016 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
6017 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6018 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
6019 let mut labels = BTreeMap::new();
6020 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6021 runtime
6022 .spawn(
6023 SpawnMemberSpec::from_wire(
6024 "worker".to_string(),
6025 "rt:review:singleton:0".to_string(),
6026 Some("You are the Review Agent.".into()),
6027 None,
6028 None,
6029 )
6030 .with_labels(labels),
6031 )
6032 .await?;
6033 let hidden_session_id = runtime
6034 .mob_handle()
6035 .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
6036 "rt:review:singleton:0",
6037 ))
6038 .await
6039 .map(|sid| sid.to_string());
6040 let mut duplicate_labels = BTreeMap::new();
6041 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6042 runtime
6043 .spawn(
6044 SpawnMemberSpec::from_wire(
6045 "worker".to_string(),
6046 "rt:review:singleton:1".to_string(),
6047 Some("You are a visible duplicate Review Agent.".into()),
6048 None,
6049 None,
6050 )
6051 .with_labels(duplicate_labels),
6052 )
6053 .await?;
6054
6055 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6056 let aggregator = MobKitConsoleAggregator::in_memory();
6057 aggregator.register_runtime_handles_with_policy(
6058 "runtime-a",
6059 "",
6060 runtime.mob_runtime().clone(),
6061 Some(identity_runtime),
6062 runtime.console_events(),
6063 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
6064 );
6065
6066 assert!(
6067 aggregator
6068 .inspect_identity("review:singleton")
6069 .await?
6070 .is_none(),
6071 "member-hidden live alias must not inspect through aggregator"
6072 );
6073 let records = aggregator.list_identities_fresh().await?;
6074 assert!(
6075 records
6076 .iter()
6077 .all(|record| record.identity != "review:singleton"),
6078 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
6079 );
6080 let appended = aggregator
6081 .store()
6082 .append_if_absent(NewConsoleFrame {
6083 id: None,
6084 dedupe_key: "member-hidden-frame".to_string(),
6085 timestamp_ms: 1,
6086 runtime_key: "runtime-a".to_string(),
6087 identity: "review:singleton".to_string(),
6088 conversation_id: Some("review:singleton".to_string()),
6089 session_id: hidden_session_id,
6090 kind: "text_delta".to_string(),
6091 status: ConsoleFrameStatus::Delivered,
6092 payload: json!({ "delta": "hidden" }),
6093 source: ConsoleFrameSource {
6094 kind: ConsoleFrameSourceKind::ConsoleEvent,
6095 source_cursor: None,
6096 },
6097 source_event_id: Some("member-hidden-frame".to_string()),
6098 interaction_id: None,
6099 turn_id: None,
6100 run_id: None,
6101 parent_frame_id: None,
6102 caused_by_frame_id: None,
6103 })
6104 .await?;
6105 assert!(
6106 !aggregator
6107 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
6108 frame: appended.frame.clone()
6109 })
6110 .await,
6111 "member-hidden live alias must not pass timeline event visibility"
6112 );
6113 let identity_only = aggregator
6114 .store()
6115 .append_if_absent(NewConsoleFrame {
6116 id: None,
6117 dedupe_key: "member-hidden-identity-only-frame".to_string(),
6118 timestamp_ms: 2,
6119 runtime_key: "runtime-a".to_string(),
6120 identity: "review:singleton".to_string(),
6121 conversation_id: Some("review:singleton".to_string()),
6122 session_id: None,
6123 kind: "text_delta".to_string(),
6124 status: ConsoleFrameStatus::Delivered,
6125 payload: json!({ "delta": "identity-only-hidden" }),
6126 source: ConsoleFrameSource {
6127 kind: ConsoleFrameSourceKind::ConsoleEvent,
6128 source_cursor: None,
6129 },
6130 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
6131 interaction_id: None,
6132 turn_id: None,
6133 run_id: None,
6134 parent_frame_id: None,
6135 caused_by_frame_id: None,
6136 })
6137 .await?;
6138 assert!(
6139 !aggregator
6140 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
6141 frame: identity_only.frame.clone()
6142 })
6143 .await,
6144 "member-hidden durable alias must not leak identity-only timeline frames"
6145 );
6146 let page = aggregator
6147 .query_timeline(ConsoleTimelineQuery {
6148 identity: Some("review:singleton".to_string()),
6149 limit: 10,
6150 ..ConsoleTimelineQuery::default()
6151 })
6152 .await?;
6153 assert!(
6154 page.frames.is_empty(),
6155 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
6156 );
6157 assert!(
6158 !aggregator.retire_identity("review:singleton").await?,
6159 "member-hidden live alias must not retire through aggregator"
6160 );
6161 let reserve_err = aggregator
6162 .reserve_identity_first_interaction(
6163 ConsoleSendRequest {
6164 identity: "review:singleton".to_string(),
6165 content: json!("hello"),
6166 origin: "console:test".to_string(),
6167 idempotency_key: "member-hidden-reserve".to_string(),
6168 handling_mode: Some("queue".to_string()),
6169 },
6170 None,
6171 )
6172 .await
6173 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
6174 assert!(
6175 reserve_err.to_string().contains("unknown identity"),
6176 "unexpected reserve error: {reserve_err}"
6177 );
6178 let send_err = aggregator
6179 .send(ConsoleSendRequest {
6180 identity: "review:singleton".to_string(),
6181 content: json!("hello"),
6182 origin: "console:test".to_string(),
6183 idempotency_key: "member-hidden-send".to_string(),
6184 handling_mode: Some("queue".to_string()),
6185 })
6186 .await
6187 .expect_err("member-hidden live alias must not receive sends");
6188 assert!(
6189 send_err.to_string().contains("unknown identity"),
6190 "unexpected send error: {send_err}"
6191 );
6192 let blob_result = aggregator
6193 .binary_blob_store_for_identity("review:singleton")
6194 .await;
6195 let Err(blob_err) = blob_result else {
6196 panic!("member-hidden live alias must not expose blob store");
6197 };
6198 assert!(
6199 blob_err.to_string().contains("unknown identity"),
6200 "unexpected blob error: {blob_err}"
6201 );
6202
6203 let _ = runtime.mob_handle().stop().await;
6204 Ok(())
6205 }
6206
6207 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6208 async fn explicit_session_backfill_target_resolves_durable_identity_label()
6209 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6210 let runtime = build_empty_runtime("identity-label-backfill-test").await;
6211 let mut labels = BTreeMap::new();
6212 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6213 runtime
6214 .spawn(
6215 SpawnMemberSpec::from_wire(
6216 "worker".to_string(),
6217 "rt:review:singleton:0".to_string(),
6218 Some("You are the Review Agent.".into()),
6219 None,
6220 None,
6221 )
6222 .with_labels(labels),
6223 )
6224 .await
6225 .expect("member spawns");
6226
6227 let entry = RuntimeEntry {
6228 runtime_key: "runtime-a".to_string(),
6229 identity_namespace: String::new(),
6230 runtime: runtime.mob_runtime().clone(),
6231 identity_runtime: runtime.identity_runtime().cloned(),
6232 console_events: runtime.console_events(),
6233 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6234 };
6235 let aggregator = MobKitConsoleAggregator::in_memory();
6236 aggregator
6237 .inner
6238 .runtimes
6239 .write()
6240 .expect("runtime registry")
6241 .insert("runtime-a".to_string(), entry);
6242
6243 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
6244 .await
6245 .expect("durable label should resolve targeted session backfill");
6246 assert_eq!(target.record.identity, "review:singleton");
6247 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
6248
6249 let _ = runtime.mob_handle().stop().await;
6250 Ok(())
6251 }
6252
6253 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6254 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
6255 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6256 let runtime = build_empty_runtime("identity-stale-durable-test").await;
6257 let mut labels = BTreeMap::new();
6258 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6259 runtime
6260 .spawn(
6261 SpawnMemberSpec::from_wire(
6262 "worker".to_string(),
6263 "rt:review:singleton:0".to_string(),
6264 Some("You are the live Review Agent.".into()),
6265 None,
6266 None,
6267 )
6268 .with_labels(labels),
6269 )
6270 .await
6271 .expect("member spawns");
6272
6273 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6274 crate::identity_first::IdentityRuntimeConfig {
6275 continuity_store: Arc::new(
6276 crate::identity_first::LocalContinuityStore::in_memory()?
6277 ),
6278 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6279 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
6280 has_runtime_store: true,
6281 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6282 bridge: None,
6283 default_timeout: None,
6284 },
6285 ));
6286 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6287 let record = crate::identity_first::ContinuityRecord {
6288 identity: identity.clone(),
6289 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6290 "rt:review:singleton:1",
6291 )?,
6292 session_id: SessionId::new(),
6293 generation: crate::identity_first::ContinuityGeneration::new(1),
6294 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6295 };
6296 identity_runtime
6297 .register(
6298 crate::identity_first::DurableAgentSpec {
6299 identity: identity.clone(),
6300 profile: meerkat_mob::ProfileName::from("worker"),
6301 addressability: crate::identity_first::AgentAddressability::Addressable,
6302 display_name: None,
6303 labels: BTreeMap::new(),
6304 context: None,
6305 additional_instructions: Vec::new(),
6306 initial_message: None,
6307 runtime_mode_override: None,
6308 backend: None,
6309 binding: None,
6310 },
6311 crate::identity_first::IdentityLifecycleState::Active,
6312 Some(record),
6313 Some(crate::identity_first::LeaseGrant {
6314 identity,
6315 fencing_token: crate::identity_first::FencingToken::new(7),
6316 ttl: Duration::from_mins(5),
6317 }),
6318 )
6319 .await;
6320
6321 let aggregator = MobKitConsoleAggregator::in_memory();
6322 aggregator
6323 .inner
6324 .runtimes
6325 .write()
6326 .expect("runtime registry")
6327 .insert(
6328 "runtime-a".to_string(),
6329 RuntimeEntry {
6330 runtime_key: "runtime-a".to_string(),
6331 identity_namespace: String::new(),
6332 runtime: runtime.mob_runtime().clone(),
6333 identity_runtime: Some(identity_runtime),
6334 console_events: runtime.console_events(),
6335 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6336 },
6337 );
6338
6339 let err = aggregator
6340 .inspect_identity("review:singleton")
6341 .await
6342 .expect_err("stale durable binding must not inspect as healthy");
6343 assert!(
6344 err.to_string().contains("stale durable identity alias"),
6345 "unexpected error: {err}"
6346 );
6347 let retire_err = aggregator
6348 .retire_identity("review:singleton")
6349 .await
6350 .expect_err("stale durable binding must not retire as healthy");
6351 assert!(
6352 retire_err
6353 .to_string()
6354 .contains("stale durable identity alias"),
6355 "unexpected retire error: {retire_err}"
6356 );
6357 let send_err = aggregator
6358 .send(ConsoleSendRequest {
6359 identity: "review:singleton".to_string(),
6360 content: json!("hello"),
6361 origin: "console:test".to_string(),
6362 idempotency_key: "stale-durable-send".to_string(),
6363 handling_mode: Some("queue".to_string()),
6364 })
6365 .await
6366 .expect_err("stale durable binding must not send to the wrong live member");
6367 assert!(
6368 send_err
6369 .to_string()
6370 .contains("stale durable identity alias"),
6371 "unexpected send error: {send_err}"
6372 );
6373 let reserve_err = aggregator
6374 .reserve_identity_first_interaction(
6375 ConsoleSendRequest {
6376 identity: "review:singleton".to_string(),
6377 content: json!("hello"),
6378 origin: "console:test".to_string(),
6379 idempotency_key: "stale-durable-reserve".to_string(),
6380 handling_mode: Some("queue".to_string()),
6381 },
6382 None,
6383 )
6384 .await
6385 .expect_err("identity-first reserve must not bypass stale durable binding");
6386 assert!(
6387 reserve_err
6388 .to_string()
6389 .contains("stale durable identity alias"),
6390 "unexpected reserve error: {reserve_err}"
6391 );
6392 let blob_err = match aggregator
6393 .binary_blob_store_for_identity("review:singleton")
6394 .await
6395 {
6396 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
6397 Err(err) => err,
6398 };
6399 assert!(
6400 blob_err
6401 .to_string()
6402 .contains("stale durable identity alias"),
6403 "unexpected blob error: {blob_err}"
6404 );
6405 let records = aggregator.list_identities_fresh().await?;
6406 assert!(
6407 records
6408 .iter()
6409 .any(|record| record.identity == "review:singleton"
6410 && record.runtime_member_id == "rt:review:singleton:0"),
6411 "live alias should still be listed; records: {records:#?}"
6412 );
6413 assert!(
6414 records
6415 .iter()
6416 .any(|record| record.identity == "review:singleton"
6417 && record.runtime_member_id == "rt:review:singleton:1"),
6418 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
6419 );
6420
6421 let _ = runtime.mob_handle().stop().await;
6422 Ok(())
6423 }
6424
6425 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6426 async fn identity_first_alias_self_heals_session_only_split_brain()
6427 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6428 let runtime = build_empty_runtime("identity-session-rebind-test").await;
6429 let mut labels = BTreeMap::new();
6430 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6431 runtime
6432 .spawn(
6433 SpawnMemberSpec::from_wire(
6434 "worker".to_string(),
6435 "rt:review:singleton:0".to_string(),
6436 Some("You are the live Review Agent.".into()),
6437 None,
6438 None,
6439 )
6440 .with_labels(labels),
6441 )
6442 .await
6443 .expect("member spawns");
6444 let live_session_id = runtime
6445 .mob_handle()
6446 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
6447 "rt:review:singleton:0",
6448 ))
6449 .await
6450 .expect("live member has bridge session");
6451
6452 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6453 crate::identity_first::IdentityRuntimeConfig {
6454 continuity_store: Arc::new(
6455 crate::identity_first::LocalContinuityStore::in_memory()?
6456 ),
6457 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6458 runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
6459 has_runtime_store: true,
6460 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6461 bridge: None,
6462 default_timeout: None,
6463 },
6464 ));
6465 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6466 let stale_session_id = SessionId::new();
6467 assert_ne!(stale_session_id, live_session_id);
6468 let record = crate::identity_first::ContinuityRecord {
6469 identity: identity.clone(),
6470 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6471 "rt:review:singleton:0",
6472 )?,
6473 session_id: stale_session_id,
6474 generation: crate::identity_first::ContinuityGeneration::new(1),
6475 checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
6476 };
6477 identity_runtime
6478 .register(
6479 crate::identity_first::DurableAgentSpec {
6480 identity: identity.clone(),
6481 profile: meerkat_mob::ProfileName::from("worker"),
6482 addressability: crate::identity_first::AgentAddressability::Addressable,
6483 display_name: None,
6484 labels: BTreeMap::new(),
6485 context: None,
6486 additional_instructions: Vec::new(),
6487 initial_message: None,
6488 runtime_mode_override: None,
6489 backend: None,
6490 binding: None,
6491 },
6492 crate::identity_first::IdentityLifecycleState::Active,
6493 Some(record),
6494 Some(crate::identity_first::LeaseGrant {
6495 identity: identity.clone(),
6496 fencing_token: crate::identity_first::FencingToken::new(7),
6497 ttl: Duration::from_mins(5),
6498 }),
6499 )
6500 .await;
6501
6502 let aggregator = MobKitConsoleAggregator::in_memory();
6503 aggregator
6504 .inner
6505 .runtimes
6506 .write()
6507 .expect("runtime registry")
6508 .insert(
6509 "runtime-a".to_string(),
6510 RuntimeEntry {
6511 runtime_key: "runtime-a".to_string(),
6512 identity_namespace: String::new(),
6513 runtime: runtime.mob_runtime().clone(),
6514 identity_runtime: Some(identity_runtime.clone()),
6515 console_events: runtime.console_events(),
6516 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6517 },
6518 );
6519
6520 aggregator
6521 .reserve_identity_first_interaction(
6522 ConsoleSendRequest {
6523 identity: "review:singleton".to_string(),
6524 content: json!("hello"),
6525 origin: "console:test".to_string(),
6526 idempotency_key: "session-rebind-reserve".to_string(),
6527 handling_mode: Some("queue".to_string()),
6528 },
6529 None,
6530 )
6531 .await
6532 .expect("session-only split-brain should self-heal before reserve");
6533
6534 let rebound_status = identity_runtime.status(&identity).await?;
6535 assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6536
6537 let inspection = aggregator
6538 .inspect_identity("review:singleton")
6539 .await?
6540 .expect("rebound identity should inspect");
6541 let live_session_string = live_session_id.to_string();
6542 assert_eq!(
6543 inspection.identity.session_id.as_deref(),
6544 Some(live_session_string.as_str())
6545 );
6546
6547 let _ = runtime.mob_handle().stop().await;
6548 Ok(())
6549 }
6550
6551 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6552 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6553 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6554 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6555 let mut labels = BTreeMap::new();
6556 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6557 runtime
6558 .spawn(
6559 SpawnMemberSpec::from_wire(
6560 "worker".to_string(),
6561 "rt:review:singleton:0".to_string(),
6562 Some("You are the mislabeled Review Agent.".into()),
6563 None,
6564 None,
6565 )
6566 .with_labels(labels),
6567 )
6568 .await
6569 .expect("member spawns");
6570
6571 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6572 crate::identity_first::IdentityRuntimeConfig {
6573 continuity_store: Arc::new(
6574 crate::identity_first::LocalContinuityStore::in_memory()?
6575 ),
6576 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6577 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6578 has_runtime_store: true,
6579 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6580 bridge: None,
6581 default_timeout: None,
6582 },
6583 ));
6584 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6585 let record = crate::identity_first::ContinuityRecord {
6586 identity: identity.clone(),
6587 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6588 "rt:review:singleton:0",
6589 )?,
6590 session_id: SessionId::new(),
6591 generation: crate::identity_first::ContinuityGeneration::new(1),
6592 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6593 };
6594 identity_runtime
6595 .register(
6596 crate::identity_first::DurableAgentSpec {
6597 identity: identity.clone(),
6598 profile: meerkat_mob::ProfileName::from("worker"),
6599 addressability: crate::identity_first::AgentAddressability::Addressable,
6600 display_name: None,
6601 labels: BTreeMap::new(),
6602 context: None,
6603 additional_instructions: Vec::new(),
6604 initial_message: None,
6605 runtime_mode_override: None,
6606 backend: None,
6607 binding: None,
6608 },
6609 crate::identity_first::IdentityLifecycleState::Active,
6610 Some(record),
6611 Some(crate::identity_first::LeaseGrant {
6612 identity,
6613 fencing_token: crate::identity_first::FencingToken::new(7),
6614 ttl: Duration::from_mins(5),
6615 }),
6616 )
6617 .await;
6618
6619 let aggregator = MobKitConsoleAggregator::in_memory();
6620 aggregator
6621 .inner
6622 .runtimes
6623 .write()
6624 .expect("runtime registry")
6625 .insert(
6626 "runtime-a".to_string(),
6627 RuntimeEntry {
6628 runtime_key: "runtime-a".to_string(),
6629 identity_namespace: String::new(),
6630 runtime: runtime.mob_runtime().clone(),
6631 identity_runtime: Some(identity_runtime),
6632 console_events: runtime.console_events(),
6633 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6634 },
6635 );
6636
6637 let err = aggregator
6638 .inspect_identity("review:singleton")
6639 .await
6640 .expect_err("wrong projected live identity must not inspect as healthy");
6641 assert!(
6642 err.to_string()
6643 .contains("projects identity other:singleton"),
6644 "unexpected error: {err}"
6645 );
6646 let retire_err = aggregator
6647 .retire_identity("review:singleton")
6648 .await
6649 .expect_err("wrong projected live identity must not retire as healthy");
6650 assert!(
6651 retire_err
6652 .to_string()
6653 .contains("projects identity other:singleton"),
6654 "unexpected retire error: {retire_err}"
6655 );
6656 let send_err = aggregator
6657 .send(ConsoleSendRequest {
6658 identity: "review:singleton".to_string(),
6659 content: json!("hello"),
6660 origin: "console:test".to_string(),
6661 idempotency_key: "wrong-projection-send".to_string(),
6662 handling_mode: Some("queue".to_string()),
6663 })
6664 .await
6665 .expect_err("wrong projected live identity must not send as unknown");
6666 assert!(
6667 send_err
6668 .to_string()
6669 .contains("projects identity other:singleton"),
6670 "unexpected send error: {send_err}"
6671 );
6672 let reserve_err = aggregator
6673 .reserve_identity_first_interaction(
6674 ConsoleSendRequest {
6675 identity: "review:singleton".to_string(),
6676 content: json!("hello"),
6677 origin: "console:test".to_string(),
6678 idempotency_key: "wrong-projection-reserve".to_string(),
6679 handling_mode: Some("queue".to_string()),
6680 },
6681 None,
6682 )
6683 .await
6684 .expect_err("identity-first reserve must not bypass wrong live projection");
6685 assert!(
6686 reserve_err
6687 .to_string()
6688 .contains("projects identity other:singleton"),
6689 "unexpected reserve error: {reserve_err}"
6690 );
6691 let blob_err = match aggregator
6692 .binary_blob_store_for_identity("review:singleton")
6693 .await
6694 {
6695 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6696 Err(err) => err,
6697 };
6698 assert!(
6699 blob_err
6700 .to_string()
6701 .contains("projects identity other:singleton"),
6702 "unexpected blob error: {blob_err}"
6703 );
6704 let projected_err = aggregator
6705 .inspect_identity("other:singleton")
6706 .await
6707 .expect_err("wrong projected alias must not inspect through live-only fallback");
6708 assert!(
6709 projected_err
6710 .to_string()
6711 .contains("identity runtime binding for review:singleton"),
6712 "unexpected projected-alias error: {projected_err}"
6713 );
6714 let projected_retire_err = aggregator
6715 .retire_identity("other:singleton")
6716 .await
6717 .expect_err("wrong projected alias must not retire through live-only fallback");
6718 assert!(
6719 projected_retire_err
6720 .to_string()
6721 .contains("identity runtime binding for review:singleton"),
6722 "unexpected projected-alias retire error: {projected_retire_err}"
6723 );
6724
6725 let _ = runtime.mob_handle().stop().await;
6726 Ok(())
6727 }
6728
6729 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6730 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6731 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6732 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6733 let mut labels = BTreeMap::new();
6734 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6735 runtime
6736 .spawn(
6737 SpawnMemberSpec::from_wire(
6738 "worker".to_string(),
6739 "rt:review:singleton:0".to_string(),
6740 Some("You are a hidden wrong projection.".into()),
6741 None,
6742 None,
6743 )
6744 .with_labels(labels),
6745 )
6746 .await
6747 .expect("member spawns");
6748
6749 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6750 crate::identity_first::IdentityRuntimeConfig {
6751 continuity_store: Arc::new(
6752 crate::identity_first::LocalContinuityStore::in_memory()?
6753 ),
6754 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6755 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6756 has_runtime_store: true,
6757 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6758 bridge: None,
6759 default_timeout: None,
6760 },
6761 ));
6762 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6763 identity_runtime
6764 .register(
6765 crate::identity_first::DurableAgentSpec {
6766 identity: identity.clone(),
6767 profile: meerkat_mob::ProfileName::from("worker"),
6768 addressability: crate::identity_first::AgentAddressability::Addressable,
6769 display_name: None,
6770 labels: BTreeMap::new(),
6771 context: None,
6772 additional_instructions: Vec::new(),
6773 initial_message: None,
6774 runtime_mode_override: None,
6775 backend: None,
6776 binding: None,
6777 },
6778 crate::identity_first::IdentityLifecycleState::Active,
6779 Some(crate::identity_first::ContinuityRecord {
6780 identity: identity.clone(),
6781 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6782 "rt:review:singleton:0",
6783 )?,
6784 session_id: SessionId::new(),
6785 generation: crate::identity_first::ContinuityGeneration::new(1),
6786 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6787 }),
6788 Some(crate::identity_first::LeaseGrant {
6789 identity,
6790 fencing_token: crate::identity_first::FencingToken::new(7),
6791 ttl: Duration::from_mins(5),
6792 }),
6793 )
6794 .await;
6795
6796 let aggregator = MobKitConsoleAggregator::in_memory();
6797 aggregator
6798 .inner
6799 .runtimes
6800 .write()
6801 .expect("runtime registry")
6802 .insert(
6803 "runtime-a".to_string(),
6804 RuntimeEntry {
6805 runtime_key: "runtime-a".to_string(),
6806 identity_namespace: String::new(),
6807 runtime: runtime.mob_runtime().clone(),
6808 identity_runtime: Some(identity_runtime),
6809 console_events: runtime.console_events(),
6810 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6811 },
6812 );
6813
6814 let err = aggregator
6815 .inspect_identity("review:singleton")
6816 .await
6817 .expect_err("hidden wrong projection must still fail durable binding validation");
6818 assert!(
6819 err.to_string()
6820 .contains("projects identity other:singleton"),
6821 "unexpected hidden wrong-projection error: {err}"
6822 );
6823 let reserve_err = aggregator
6824 .reserve_identity_first_interaction(
6825 ConsoleSendRequest {
6826 identity: "review:singleton".to_string(),
6827 content: json!("hello"),
6828 origin: "console:test".to_string(),
6829 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6830 handling_mode: Some("queue".to_string()),
6831 },
6832 None,
6833 )
6834 .await
6835 .expect_err("reserve must not hide wrong projection from durable validation");
6836 assert!(
6837 reserve_err
6838 .to_string()
6839 .contains("projects identity other:singleton"),
6840 "unexpected reserve error: {reserve_err}"
6841 );
6842
6843 let _ = runtime.mob_handle().stop().await;
6844 Ok(())
6845 }
6846
6847 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6848 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6849 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6850 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6851 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6852 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6853 crate::identity_first::IdentityRuntimeConfig {
6854 continuity_store: Arc::new(
6855 crate::identity_first::LocalContinuityStore::in_memory()?
6856 ),
6857 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6858 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6859 has_runtime_store: true,
6860 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6861 bridge: None,
6862 default_timeout: None,
6863 },
6864 ));
6865 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6866 crate::identity_first::IdentityRuntimeConfig {
6867 continuity_store: Arc::new(
6868 crate::identity_first::LocalContinuityStore::in_memory()?
6869 ),
6870 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6871 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6872 has_runtime_store: true,
6873 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6874 bridge: None,
6875 default_timeout: None,
6876 },
6877 ));
6878 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6879 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6880 identity_runtime
6881 .register(
6882 crate::identity_first::DurableAgentSpec {
6883 identity: identity.clone(),
6884 profile: meerkat_mob::ProfileName::from("worker"),
6885 addressability: crate::identity_first::AgentAddressability::Addressable,
6886 display_name: None,
6887 labels: BTreeMap::new(),
6888 context: None,
6889 additional_instructions: Vec::new(),
6890 initial_message: None,
6891 runtime_mode_override: None,
6892 backend: None,
6893 binding: None,
6894 },
6895 crate::identity_first::IdentityLifecycleState::Active,
6896 Some(crate::identity_first::ContinuityRecord {
6897 identity: identity.clone(),
6898 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6899 "rt:review:singleton:0",
6900 )?,
6901 session_id: SessionId::new(),
6902 generation: crate::identity_first::ContinuityGeneration::new(1),
6903 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6904 }),
6905 Some(crate::identity_first::LeaseGrant {
6906 identity,
6907 fencing_token: crate::identity_first::FencingToken::new(7),
6908 ttl: Duration::from_mins(5),
6909 }),
6910 )
6911 .await;
6912 }
6913
6914 let aggregator = MobKitConsoleAggregator::in_memory();
6915 {
6916 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6917 runtimes.insert(
6918 "a-hidden".to_string(),
6919 RuntimeEntry {
6920 runtime_key: "a-hidden".to_string(),
6921 identity_namespace: String::new(),
6922 runtime: hidden_runtime.mob_runtime().clone(),
6923 identity_runtime: Some(hidden_identity_runtime),
6924 console_events: hidden_runtime.console_events(),
6925 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6926 },
6927 );
6928 runtimes.insert(
6929 "b-visible".to_string(),
6930 RuntimeEntry {
6931 runtime_key: "b-visible".to_string(),
6932 identity_namespace: String::new(),
6933 runtime: visible_runtime.mob_runtime().clone(),
6934 identity_runtime: Some(visible_identity_runtime),
6935 console_events: visible_runtime.console_events(),
6936 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6937 },
6938 );
6939 }
6940
6941 let inspection = aggregator
6942 .inspect_identity("rt:review:singleton:0")
6943 .await?
6944 .expect("visible runtime-id match should be returned");
6945 assert_eq!(inspection.identity.runtime_key, "b-visible");
6946
6947 let _ = hidden_runtime.mob_handle().stop().await;
6948 let _ = visible_runtime.mob_handle().stop().await;
6949 Ok(())
6950 }
6951
6952 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6953 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6954 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6955 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6956 let mut primary_labels = BTreeMap::new();
6957 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6958 runtime
6959 .spawn(
6960 SpawnMemberSpec::from_wire(
6961 "worker".to_string(),
6962 "rt:review:singleton:0".to_string(),
6963 Some("You are the primary Review Agent.".into()),
6964 None,
6965 None,
6966 )
6967 .with_labels(primary_labels),
6968 )
6969 .await
6970 .expect("primary member spawns");
6971
6972 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6973 let mut delegate_labels = BTreeMap::new();
6974 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6975 delegate_runtime
6976 .spawn(
6977 SpawnMemberSpec::from_wire(
6978 "worker".to_string(),
6979 "rt:review:singleton:0".to_string(),
6980 Some("You are the wrong projected Review Agent.".into()),
6981 None,
6982 None,
6983 )
6984 .with_labels(delegate_labels),
6985 )
6986 .await
6987 .expect("delegate member spawns");
6988 runtime
6989 .mob_runtime()
6990 .agent_mob_mcp_state()
6991 .expect("runtime should expose mob MCP state")
6992 .mob_insert_handle(
6993 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6994 delegate_runtime.mob_handle().clone(),
6995 )
6996 .await;
6997
6998 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6999 crate::identity_first::IdentityRuntimeConfig {
7000 continuity_store: Arc::new(
7001 crate::identity_first::LocalContinuityStore::in_memory()?
7002 ),
7003 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7004 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
7005 has_runtime_store: true,
7006 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7007 bridge: None,
7008 default_timeout: None,
7009 },
7010 ));
7011 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7012 let record = crate::identity_first::ContinuityRecord {
7013 identity: identity.clone(),
7014 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7015 "rt:review:singleton:0",
7016 )?,
7017 session_id: SessionId::new(),
7018 generation: crate::identity_first::ContinuityGeneration::new(1),
7019 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7020 };
7021 identity_runtime
7022 .register(
7023 crate::identity_first::DurableAgentSpec {
7024 identity: identity.clone(),
7025 profile: meerkat_mob::ProfileName::from("worker"),
7026 addressability: crate::identity_first::AgentAddressability::Addressable,
7027 display_name: None,
7028 labels: BTreeMap::new(),
7029 context: None,
7030 additional_instructions: Vec::new(),
7031 initial_message: None,
7032 runtime_mode_override: None,
7033 backend: None,
7034 binding: None,
7035 },
7036 crate::identity_first::IdentityLifecycleState::Active,
7037 Some(record),
7038 Some(crate::identity_first::LeaseGrant {
7039 identity,
7040 fencing_token: crate::identity_first::FencingToken::new(7),
7041 ttl: Duration::from_mins(5),
7042 }),
7043 )
7044 .await;
7045
7046 let aggregator = MobKitConsoleAggregator::in_memory();
7047 aggregator
7048 .inner
7049 .runtimes
7050 .write()
7051 .expect("runtime registry")
7052 .insert(
7053 "runtime-a".to_string(),
7054 RuntimeEntry {
7055 runtime_key: "runtime-a".to_string(),
7056 identity_namespace: String::new(),
7057 runtime: runtime.mob_runtime().clone(),
7058 identity_runtime: Some(identity_runtime),
7059 console_events: runtime.console_events(),
7060 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7061 },
7062 );
7063
7064 let err = aggregator
7065 .inspect_identity("review:singleton")
7066 .await
7067 .expect_err("cross-source wrong projection must not be hidden by primary source");
7068 assert!(
7069 err.to_string()
7070 .contains("projects identity other:singleton")
7071 || err.to_string().contains("ambiguous live identity alias"),
7072 "unexpected error: {err}"
7073 );
7074
7075 let _ = delegate_runtime.mob_handle().stop().await;
7076 let _ = runtime.mob_handle().stop().await;
7077 Ok(())
7078 }
7079
7080 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7081 async fn durable_send_selects_bound_generation_when_stale_generation_is_still_active()
7082 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7083 let runtime = build_empty_runtime("identity-respawn-send-authority-test").await;
7084 let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
7085 for generation in 0..=1 {
7086 runtime
7087 .spawn(
7088 SpawnMemberSpec::from_wire(
7089 "worker".to_string(),
7090 format!("rt:builder:{generation}"),
7091 Some(format!("You are Builder generation {generation}.").into()),
7092 None,
7093 None,
7094 )
7095 .with_labels(labels.clone()),
7096 )
7097 .await
7098 .expect("both live generations spawn");
7099 }
7100 let bound_session_id = runtime
7101 .mob_handle()
7102 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
7103 "rt:builder:1",
7104 ))
7105 .await
7106 .expect("bound generation has a bridge session");
7107
7108 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7109 crate::identity_first::IdentityRuntimeConfig {
7110 continuity_store: Arc::new(
7111 crate::identity_first::LocalContinuityStore::in_memory()?
7112 ),
7113 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7114 runtime_instance_id: "console-aggregator-respawn-send-authority-test".to_string(),
7115 has_runtime_store: true,
7116 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7117 bridge: None,
7118 default_timeout: None,
7119 },
7120 ));
7121 let identity = crate::identity_first::AgentIdentity::parse("builder")?;
7122 identity_runtime
7123 .register(
7124 crate::identity_first::DurableAgentSpec {
7125 identity: identity.clone(),
7126 profile: meerkat_mob::ProfileName::from("worker"),
7127 addressability: crate::identity_first::AgentAddressability::Addressable,
7128 display_name: None,
7129 labels: BTreeMap::new(),
7130 context: None,
7131 additional_instructions: Vec::new(),
7132 initial_message: None,
7133 runtime_mode_override: None,
7134 backend: None,
7135 binding: None,
7136 },
7137 crate::identity_first::IdentityLifecycleState::Active,
7138 Some(crate::identity_first::ContinuityRecord {
7139 identity: identity.clone(),
7140 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse("rt:builder:1")?,
7141 session_id: bound_session_id.clone(),
7142 generation: crate::identity_first::ContinuityGeneration::new(1),
7143 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7144 }),
7145 Some(crate::identity_first::LeaseGrant {
7146 identity,
7147 fencing_token: crate::identity_first::FencingToken::new(9),
7148 ttl: Duration::from_mins(5),
7149 }),
7150 )
7151 .await;
7152
7153 let aggregator = MobKitConsoleAggregator::in_memory();
7154 aggregator
7155 .inner
7156 .runtimes
7157 .write()
7158 .expect("runtime registry")
7159 .insert(
7160 "default".to_string(),
7161 RuntimeEntry {
7162 runtime_key: "default".to_string(),
7163 identity_namespace: String::new(),
7164 runtime: runtime.mob_runtime().clone(),
7165 identity_runtime: Some(identity_runtime),
7166 console_events: runtime.console_events(),
7167 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7168 },
7169 );
7170
7171 let matches = aggregator.resolve_visible_members("builder").await;
7172 assert_eq!(
7173 matches.len(),
7174 2,
7175 "both Active generations remain observable"
7176 );
7177 assert_eq!(
7178 sendable_resolved_member_indices(&matches).len(),
7179 2,
7180 "the stale generation reproduces the pre-fix send ambiguity"
7181 );
7182 let resolved = aggregator
7183 .resolve_send_member("builder")
7184 .await?
7185 .expect("durable builder alias resolves");
7186 assert_eq!(
7187 resolved.runtime_identity, "rt:builder:1",
7188 "continuity binding must shadow the stale Active generation for sends"
7189 );
7190
7191 let inspect_error = aggregator
7192 .inspect_identity("builder")
7193 .await
7194 .expect_err("inspection must keep duplicate live aliases explicit");
7195 assert!(
7196 inspect_error
7197 .to_string()
7198 .contains("ambiguous live identity alias")
7199 );
7200 let retire_error = aggregator
7201 .retire_identity("builder")
7202 .await
7203 .expect_err("control must not silently retire through a durable alias");
7204 assert!(
7205 retire_error
7206 .to_string()
7207 .contains("ambiguous live identity alias")
7208 );
7209
7210 let accepted = aggregator
7211 .send(ConsoleSendRequest {
7212 identity: "builder".to_string(),
7213 content: json!("continue after respawn"),
7214 origin: "console:test".to_string(),
7215 idempotency_key: "respawn-bound-generation-send".to_string(),
7216 handling_mode: Some("queue".to_string()),
7217 })
7218 .await?;
7219 assert_eq!(accepted.session_id, Some(bound_session_id.to_string()));
7220
7221 let _ = runtime.mob_handle().stop().await;
7222 Ok(())
7223 }
7224
7225 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7226 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
7227 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7228 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
7229 let mut labels = BTreeMap::new();
7230 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7231 runtime
7232 .spawn(
7233 SpawnMemberSpec::from_wire(
7234 "worker".to_string(),
7235 "rt:review:singleton:0".to_string(),
7236 Some("You are the first Review Agent.".into()),
7237 None,
7238 None,
7239 )
7240 .with_labels(labels.clone()),
7241 )
7242 .await
7243 .expect("first member spawns");
7244 runtime
7245 .spawn(
7246 SpawnMemberSpec::from_wire(
7247 "worker".to_string(),
7248 "rt:review:singleton:1".to_string(),
7249 Some("You are the second Review Agent.".into()),
7250 None,
7251 None,
7252 )
7253 .with_labels(labels),
7254 )
7255 .await
7256 .expect("second member spawns");
7257
7258 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7259 crate::identity_first::IdentityRuntimeConfig {
7260 continuity_store: Arc::new(
7261 crate::identity_first::LocalContinuityStore::in_memory()?
7262 ),
7263 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7264 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
7265 has_runtime_store: true,
7266 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7267 bridge: None,
7268 default_timeout: None,
7269 },
7270 ));
7271 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7272 identity_runtime
7273 .register(
7274 crate::identity_first::DurableAgentSpec {
7275 identity: identity.clone(),
7276 profile: meerkat_mob::ProfileName::from("worker"),
7277 addressability: crate::identity_first::AgentAddressability::Addressable,
7278 display_name: None,
7279 labels: BTreeMap::new(),
7280 context: None,
7281 additional_instructions: Vec::new(),
7282 initial_message: None,
7283 runtime_mode_override: None,
7284 backend: None,
7285 binding: None,
7286 },
7287 crate::identity_first::IdentityLifecycleState::Active,
7288 Some(crate::identity_first::ContinuityRecord {
7289 identity: identity.clone(),
7290 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7291 "rt:review:singleton:0",
7292 )?,
7293 session_id: SessionId::new(),
7294 generation: crate::identity_first::ContinuityGeneration::new(1),
7295 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7296 }),
7297 Some(crate::identity_first::LeaseGrant {
7298 identity,
7299 fencing_token: crate::identity_first::FencingToken::new(8),
7300 ttl: Duration::from_mins(5),
7301 }),
7302 )
7303 .await;
7304
7305 let aggregator = MobKitConsoleAggregator::in_memory();
7306 aggregator
7307 .inner
7308 .runtimes
7309 .write()
7310 .expect("runtime registry")
7311 .insert(
7312 "runtime-a".to_string(),
7313 RuntimeEntry {
7314 runtime_key: "runtime-a".to_string(),
7315 identity_namespace: String::new(),
7316 runtime: runtime.mob_runtime().clone(),
7317 identity_runtime: Some(identity_runtime),
7318 console_events: runtime.console_events(),
7319 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7320 },
7321 );
7322
7323 let err = aggregator
7324 .inspect_identity("review:singleton")
7325 .await
7326 .expect_err("duplicate live alias must not inspect via durable fast path");
7327 assert!(
7328 err.to_string().contains("ambiguous live identity alias"),
7329 "unexpected error: {err}"
7330 );
7331 let retire_err = aggregator
7332 .retire_identity("review:singleton")
7333 .await
7334 .expect_err("duplicate live alias must not retire via durable fast path");
7335 assert!(
7336 retire_err
7337 .to_string()
7338 .contains("ambiguous live identity alias"),
7339 "unexpected retire error: {retire_err}"
7340 );
7341 let runtime_id_err = aggregator
7342 .inspect_identity("rt:review:singleton:0")
7343 .await
7344 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
7345 assert!(
7346 runtime_id_err
7347 .to_string()
7348 .contains("ambiguous live identity alias"),
7349 "unexpected runtime-id inspect error: {runtime_id_err}"
7350 );
7351 let runtime_id_retire_err = aggregator
7352 .retire_identity("rt:review:singleton:0")
7353 .await
7354 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
7355 assert!(
7356 runtime_id_retire_err
7357 .to_string()
7358 .contains("ambiguous live identity alias")
7359 || runtime_id_retire_err
7360 .to_string()
7361 .contains("stale durable identity alias"),
7362 "unexpected runtime-id retire error: {runtime_id_retire_err}"
7363 );
7364
7365 aggregator
7366 .reserve_identity_first_interaction(
7367 ConsoleSendRequest {
7368 identity: "review:singleton".to_string(),
7369 content: json!("hello"),
7370 origin: "console:test".to_string(),
7371 idempotency_key: "duplicate-live-alias-durable-reserve".to_string(),
7372 handling_mode: Some("queue".to_string()),
7373 },
7374 None,
7375 )
7376 .await
7377 .expect("durable identity binding remains authoritative for interaction reserve");
7378
7379 let _ = runtime.mob_handle().stop().await;
7380 Ok(())
7381 }
7382
7383 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7384 async fn inspect_identity_ignores_hidden_live_alias_candidates()
7385 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7386 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
7387 let mut labels = BTreeMap::new();
7388 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7389 runtime
7390 .spawn(
7391 SpawnMemberSpec::from_wire(
7392 "worker".to_string(),
7393 "rt:review:singleton:0".to_string(),
7394 Some("You are the visible Review Agent.".into()),
7395 None,
7396 None,
7397 )
7398 .with_labels(labels.clone()),
7399 )
7400 .await
7401 .expect("visible member spawns");
7402 runtime
7403 .spawn(
7404 SpawnMemberSpec::from_wire(
7405 "worker".to_string(),
7406 "rt:review:singleton:1".to_string(),
7407 Some("You are hidden maintenance noise.".into()),
7408 None,
7409 None,
7410 )
7411 .with_labels(labels),
7412 )
7413 .await
7414 .expect("hidden member spawns");
7415
7416 let aggregator = MobKitConsoleAggregator::in_memory();
7417 aggregator.register_runtime_handles_with_policy(
7418 "runtime-a",
7419 "",
7420 runtime.mob_runtime().clone(),
7421 None,
7422 runtime.console_events(),
7423 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
7424 );
7425
7426 let inspection = aggregator
7427 .inspect_identity("review:singleton")
7428 .await?
7429 .expect("visible live alias should inspect without hidden ambiguity");
7430 assert_eq!(inspection.identity.identity, "review:singleton");
7431 assert_eq!(
7432 inspection.identity.runtime_member_id,
7433 "rt:review:singleton:0"
7434 );
7435
7436 let retired = aggregator.retire_identity("review:singleton").await?;
7437 assert!(
7438 retired,
7439 "visible live alias should retire without hidden ambiguity"
7440 );
7441
7442 let _ = runtime.mob_handle().stop().await;
7443 Ok(())
7444 }
7445
7446 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7447 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
7448 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7449 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
7450 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
7451 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
7452 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
7453
7454 let aggregator = MobKitConsoleAggregator::in_memory();
7455 aggregator.register_runtime_handles_with_policy(
7456 "a-hidden",
7457 "",
7458 hidden_runtime.mob_runtime().clone(),
7459 Some(hidden_identity_runtime),
7460 hidden_runtime.console_events(),
7461 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
7462 );
7463 aggregator.register_runtime_handles_with_policy(
7464 "b-visible",
7465 "",
7466 visible_runtime.mob_runtime().clone(),
7467 Some(visible_identity_runtime),
7468 visible_runtime.console_events(),
7469 Arc::new(AllowAllConsoleVisibilityPolicy),
7470 );
7471
7472 let accepted = aggregator
7473 .reserve_identity_first_interaction(
7474 ConsoleSendRequest {
7475 identity: "review:singleton".to_string(),
7476 content: json!("hello"),
7477 origin: "console:test".to_string(),
7478 idempotency_key: "visible-after-hidden-reserve".to_string(),
7479 handling_mode: Some("queue".to_string()),
7480 },
7481 None,
7482 )
7483 .await?;
7484 let page = aggregator
7485 .query_timeline(ConsoleTimelineQuery {
7486 identity: Some("review:singleton".to_string()),
7487 ..ConsoleTimelineQuery::default()
7488 })
7489 .await?;
7490 let frame = page
7491 .frames
7492 .iter()
7493 .find(|frame| frame.id == accepted.input_frame_id)
7494 .ok_or("accepted frame missing from timeline")?;
7495 assert_eq!(frame.runtime_key, "b-visible");
7496
7497 let _ = hidden_runtime.mob_handle().stop().await;
7498 let _ = visible_runtime.mob_handle().stop().await;
7499 Ok(())
7500 }
7501
7502 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7503 async fn identity_first_list_identities_projects_cached_topology_peers()
7504 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7505 let runtime = build_empty_runtime("identity-topology-cache-test").await;
7506 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
7507 identity_runtime
7508 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
7509 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
7510 crate::identity_first::AgentIdentity::parse("agent:beta")?,
7511 )?])
7512 .await;
7513
7514 let aggregator = MobKitConsoleAggregator::in_memory();
7515 aggregator.register_runtime_handles_with_policy(
7516 "identity-first",
7517 "",
7518 runtime.mob_runtime().clone(),
7519 Some(identity_runtime),
7520 runtime.console_events(),
7521 Arc::new(AllowAllConsoleVisibilityPolicy),
7522 );
7523
7524 let identities = aggregator.list_identities().await?;
7525 let alpha = identities
7526 .iter()
7527 .find(|record| record.identity == "agent:alpha")
7528 .ok_or("agent:alpha identity missing")?;
7529 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
7530 let inspection = aggregator
7531 .inspect_identity("agent:alpha")
7532 .await?
7533 .ok_or("agent:alpha inspection missing")?;
7534 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
7535
7536 let _ = runtime.mob_handle().stop().await;
7537 Ok(())
7538 }
7539
7540 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7541 async fn identity_first_list_identities_includes_member_only_spawned_workers()
7542 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7543 let runtime = build_empty_runtime("identity-member-only-test").await;
7544 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
7545 let aggregator = MobKitConsoleAggregator::in_memory();
7546 aggregator.register_runtime_handles_with_policy(
7547 "identity-first",
7548 "",
7549 runtime.mob_runtime().clone(),
7550 Some(identity_runtime),
7551 runtime.console_events(),
7552 Arc::new(AllowAllConsoleVisibilityPolicy),
7553 );
7554
7555 runtime
7556 .spawn(SpawnMemberSpec::from_wire(
7557 "worker".to_string(),
7558 "agent:beta".to_string(),
7559 Some("You are beta.".into()),
7560 None,
7561 None,
7562 ))
7563 .await?;
7564
7565 let identities = aggregator.list_identities().await?;
7566 assert!(
7567 identities
7568 .iter()
7569 .any(|record| record.identity == "agent:beta"),
7570 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
7571 );
7572
7573 let _ = runtime.mob_handle().stop().await;
7574 Ok(())
7575 }
7576
7577 #[tokio::test]
7578 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
7579 let aggregator = MobKitConsoleAggregator::in_memory();
7580 let record = identity_record_for_test("agent-cached");
7581 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
7582 aggregator
7583 .inner
7584 .identity_read_model
7585 .primed
7586 .store(true, Ordering::Release);
7587 let _guard = aggregator
7588 .inner
7589 .identity_read_model
7590 .refresh_lock
7591 .clone()
7592 .lock_owned()
7593 .await;
7594
7595 let identities =
7596 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
7597 .await
7598 .expect("hot identity list should not wait for refresh lock")
7599 .expect("identity list succeeds");
7600
7601 assert_eq!(identities, vec![record]);
7602 }
7603
7604 #[tokio::test]
7605 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
7606 let aggregator = MobKitConsoleAggregator::in_memory();
7607 let guard = aggregator
7608 .inner
7609 .identity_read_model
7610 .refresh_lock
7611 .clone()
7612 .lock_owned()
7613 .await;
7614 let waiter = tokio::spawn({
7615 let aggregator = aggregator.clone();
7616 async move { aggregator.list_identities().await }
7617 });
7618
7619 tokio::time::sleep(Duration::from_millis(20)).await;
7620 assert!(
7621 !waiter.is_finished(),
7622 "cold identity list should wait for the in-flight refresh to finish"
7623 );
7624
7625 let record = identity_record_for_test("agent-primed");
7626 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
7627 aggregator
7628 .inner
7629 .identity_read_model
7630 .primed
7631 .store(true, Ordering::Release);
7632 drop(guard);
7633
7634 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
7635 .await
7636 .expect("cold identity list waiter should resume")
7637 .expect("waiter joins")
7638 .expect("identity list succeeds");
7639 assert_eq!(identities, vec![record]);
7640 }
7641
7642 #[tokio::test]
7643 async fn query_timeline_reads_from_aggregate_store() {
7644 let aggregator = MobKitConsoleAggregator::in_memory();
7645 let frame = NewConsoleFrame {
7646 id: None,
7647 dedupe_key: "event-1".to_string(),
7648 timestamp_ms: 1,
7649 runtime_key: "runtime-a".to_string(),
7650 identity: "agent-a".to_string(),
7651 conversation_id: Some("agent-a".to_string()),
7652 session_id: None,
7653 kind: "text_delta".to_string(),
7654 status: ConsoleFrameStatus::Delivered,
7655 payload: json!({ "delta": "hello" }),
7656 source: ConsoleFrameSource {
7657 kind: ConsoleFrameSourceKind::ConsoleEvent,
7658 source_cursor: None,
7659 },
7660 source_event_id: Some("event-1".to_string()),
7661 interaction_id: None,
7662 turn_id: None,
7663 run_id: None,
7664 parent_frame_id: None,
7665 caused_by_frame_id: None,
7666 };
7667 aggregator
7668 .store()
7669 .append_if_absent(frame)
7670 .await
7671 .expect("append frame");
7672
7673 let page = aggregator
7674 .query_timeline(ConsoleTimelineQuery {
7675 identity: Some("agent-a".to_string()),
7676 limit: 10,
7677 ..ConsoleTimelineQuery::default()
7678 })
7679 .await
7680 .expect("query timeline");
7681 assert_eq!(page.frames.len(), 1);
7682 assert_eq!(page.frames[0].kind, "text_delta");
7683 }
7684
7685 #[tokio::test]
7697 async fn system_identity_frames_are_visible_on_global_timeline_queries() {
7698 let aggregator = MobKitConsoleAggregator::in_memory();
7699 let runtime = build_single_member_runtime().await;
7700 aggregator
7704 .inner
7705 .runtimes
7706 .write()
7707 .expect("runtime registry")
7708 .insert(
7709 "runtime-a".to_string(),
7710 runtime_entry_for_test("runtime-a", &runtime),
7711 );
7712 let entry = runtime_entry_for_test("runtime-a", &runtime);
7713
7714 let envelope = |event_id: &str, identity: &str, event_type: &str| {
7715 crate::console_contracts::ConsoleIdentityEventEnvelope {
7716 event_id: event_id.to_string(),
7717 interaction_id: None,
7718 identity: identity.to_string(),
7719 event_type: event_type.to_string(),
7720 timestamp_ms: 1_000,
7721 data: json!({ "realm": "default" }),
7722 }
7723 };
7724
7725 let system_frame = frame_from_console_event(
7726 &entry,
7727 envelope("evt-mem-1", SYSTEM_EVENT_IDENTITY, "memory.dream.completed"),
7728 );
7729 assert_eq!(
7730 system_frame.identity, SYSTEM_EVENT_IDENTITY,
7731 "projection must not namespace the system identity"
7732 );
7733 let ghost_frame = frame_from_console_event(
7734 &entry,
7735 envelope("evt-ghost-1", "ghost-agent", "memory.dream.completed"),
7736 );
7737 assert_eq!(
7738 ghost_frame.identity, "test/ghost-agent",
7739 "ordinary identities keep the namespace"
7740 );
7741 for frame in [system_frame, ghost_frame] {
7742 aggregator
7743 .store()
7744 .append_if_absent(frame)
7745 .await
7746 .expect("append frame");
7747 }
7748
7749 let page = aggregator
7752 .query_timeline(ConsoleTimelineQuery {
7753 identity: None,
7754 limit: 10,
7755 ..ConsoleTimelineQuery::default()
7756 })
7757 .await
7758 .expect("query timeline");
7759 let identities: Vec<&str> = page
7760 .frames
7761 .iter()
7762 .map(|frame| frame.identity.as_str())
7763 .collect();
7764 assert!(
7765 identities.contains(&SYSTEM_EVENT_IDENTITY),
7766 "system-attributed memory event must be globally visible: {identities:?}"
7767 );
7768 assert!(
7769 !identities.contains(&"test/ghost-agent"),
7770 "unknown identities must stay hidden — the exemption is exact: {identities:?}"
7771 );
7772
7773 let _ = runtime.mob_handle().stop().await;
7774 }
7775
7776 #[tokio::test]
7777 async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7778 let aggregator = MobKitConsoleAggregator::in_memory();
7779 let runtime = build_single_member_runtime().await;
7780 let entry = runtime_entry_for_test("runtime-a", &runtime);
7781 for event_type in ["reasoning_delta", "reasoning_complete"] {
7782 let frame = frame_from_console_event(
7783 &entry,
7784 console_envelope_for_test(
7785 event_type,
7786 Some("interaction-a"),
7787 event_type,
7788 json!({
7789 "delta": "I should inspect the file first.",
7790 "text": "I should inspect the file first.",
7791 "turn_id": "turn-a",
7792 }),
7793 ),
7794 );
7795 aggregator
7796 .store()
7797 .append_if_absent(frame)
7798 .await
7799 .expect("append reasoning frame");
7800 }
7801
7802 let page = aggregator
7803 .query_timeline(ConsoleTimelineQuery {
7804 identity: Some("test/agent-a".to_string()),
7805 limit: 10,
7806 ..ConsoleTimelineQuery::default()
7807 })
7808 .await
7809 .expect("query timeline");
7810
7811 assert_eq!(
7812 page.frames.len(),
7813 1,
7814 "identical reasoning re-emissions in one turn should collapse"
7815 );
7816 assert_eq!(page.frames[0].kind, "reasoning_delta");
7817 assert_eq!(
7818 page.frames[0].dedupe_key,
7819 format!(
7820 "console-reasoning:runtime-a:interaction-a:{}",
7821 hash_short("I should inspect the file first.")
7822 )
7823 );
7824 }
7825
7826 #[tokio::test]
7827 async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7828 let aggregator = MobKitConsoleAggregator::in_memory();
7829 let runtime = build_single_member_runtime().await;
7830 let entry = runtime_entry_for_test("runtime-a", &runtime);
7831 for (event_id, text) in [
7832 ("reasoning-1", "I should inspect the file first."),
7833 ("reasoning-2", "Now I should run the focused test."),
7834 ] {
7835 let frame = frame_from_console_event(
7836 &entry,
7837 console_envelope_for_test(
7838 event_id,
7839 Some("interaction-a"),
7840 "reasoning_complete",
7841 json!({
7842 "text": text,
7843 "turn_id": "turn-a",
7844 }),
7845 ),
7846 );
7847 aggregator
7848 .store()
7849 .append_if_absent(frame)
7850 .await
7851 .expect("append reasoning frame");
7852 }
7853
7854 let page = aggregator
7855 .query_timeline(ConsoleTimelineQuery {
7856 identity: Some("test/agent-a".to_string()),
7857 limit: 10,
7858 ..ConsoleTimelineQuery::default()
7859 })
7860 .await
7861 .expect("query timeline");
7862
7863 assert_eq!(
7864 page.frames.len(),
7865 2,
7866 "distinct same-turn reasoning segments should not collapse"
7867 );
7868 assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7869 }
7870
7871 #[tokio::test]
7872 async fn identity_recent_anchor_respects_query_limit() {
7873 let aggregator = MobKitConsoleAggregator::in_memory();
7874 aggregator
7875 .store()
7876 .append_if_absent(NewConsoleFrame {
7877 id: None,
7878 dedupe_key: "anchored-user-input".to_string(),
7879 timestamp_ms: 1,
7880 runtime_key: "runtime-a".to_string(),
7881 identity: "agent-a".to_string(),
7882 conversation_id: Some("agent-a".to_string()),
7883 session_id: None,
7884 kind: "user_input".to_string(),
7885 status: ConsoleFrameStatus::Delivered,
7886 payload: json!({ "content": "anchor me" }),
7887 source: ConsoleFrameSource {
7888 kind: ConsoleFrameSourceKind::Synthetic,
7889 source_cursor: None,
7890 },
7891 source_event_id: Some("anchored-user-input".to_string()),
7892 interaction_id: Some("turn-a".to_string()),
7893 turn_id: None,
7894 run_id: None,
7895 parent_frame_id: None,
7896 caused_by_frame_id: None,
7897 })
7898 .await
7899 .expect("append anchor");
7900 for idx in 2..=40 {
7901 aggregator
7902 .store()
7903 .append_if_absent(NewConsoleFrame {
7904 id: None,
7905 dedupe_key: format!("noisy-tail-{idx}"),
7906 timestamp_ms: idx,
7907 runtime_key: "runtime-a".to_string(),
7908 identity: "agent-a".to_string(),
7909 conversation_id: Some("agent-a".to_string()),
7910 session_id: None,
7911 kind: "reasoning_delta".to_string(),
7912 status: ConsoleFrameStatus::Delivered,
7913 payload: json!({ "delta": idx }),
7914 source: ConsoleFrameSource {
7915 kind: ConsoleFrameSourceKind::ConsoleEvent,
7916 source_cursor: None,
7917 },
7918 source_event_id: Some(format!("noisy-tail-{idx}")),
7919 interaction_id: Some("turn-a".to_string()),
7920 turn_id: None,
7921 run_id: None,
7922 parent_frame_id: None,
7923 caused_by_frame_id: None,
7924 })
7925 .await
7926 .expect("append noisy tail");
7927 }
7928
7929 let page = aggregator
7930 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7931 identity: Some("agent-a".to_string()),
7932 mode: ConsoleTimelineMode::Recent,
7933 limit: 5,
7934 ..ConsoleTimelineWindowQuery::default()
7935 })
7936 .await
7937 .expect("query recent identity timeline");
7938
7939 assert_eq!(
7940 page.frames.len(),
7941 5,
7942 "identity anchor merge must not exceed the requested limit"
7943 );
7944 assert!(
7945 page.frames
7946 .iter()
7947 .any(|frame| frame.dedupe_key == "anchored-user-input"),
7948 "the bounded result should still retain the useful turn anchor: {:#?}",
7949 page.frames
7950 );
7951 assert_eq!(
7952 page.frames.last().and_then(|frame| frame.cursor.seq()),
7953 Some(40)
7954 );
7955 }
7956
7957 #[tokio::test]
7958 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7959 let aggregator = MobKitConsoleAggregator::in_memory();
7960 let runtime = build_single_member_runtime().await;
7961 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7962 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7963 aggregator
7964 .inner
7965 .runtimes
7966 .write()
7967 .expect("runtime registry")
7968 .insert("runtime-a".to_string(), entry);
7969
7970 for idx in 0..1_500 {
7971 aggregator
7972 .store()
7973 .append_if_absent(NewConsoleFrame {
7974 id: None,
7975 dedupe_key: format!("hidden-gap-{idx}"),
7976 timestamp_ms: idx,
7977 runtime_key: "runtime-a".to_string(),
7978 identity: "agent-a".to_string(),
7979 conversation_id: Some("agent-a".to_string()),
7980 session_id: None,
7981 kind: "hidden_noise".to_string(),
7982 status: ConsoleFrameStatus::Delivered,
7983 payload: json!({ "delta": idx }),
7984 source: ConsoleFrameSource {
7985 kind: ConsoleFrameSourceKind::ConsoleEvent,
7986 source_cursor: None,
7987 },
7988 source_event_id: Some(format!("hidden-gap-{idx}")),
7989 interaction_id: None,
7990 turn_id: None,
7991 run_id: None,
7992 parent_frame_id: None,
7993 caused_by_frame_id: None,
7994 })
7995 .await
7996 .expect("append hidden frame");
7997 }
7998 aggregator
7999 .store()
8000 .append_if_absent(NewConsoleFrame {
8001 id: None,
8002 dedupe_key: "visible-after-gap".to_string(),
8003 timestamp_ms: 1_501,
8004 runtime_key: "runtime-a".to_string(),
8005 identity: "agent-a".to_string(),
8006 conversation_id: Some("agent-a".to_string()),
8007 session_id: None,
8008 kind: "interaction_complete".to_string(),
8009 status: ConsoleFrameStatus::Delivered,
8010 payload: json!({ "text": "visible after hidden gap" }),
8011 source: ConsoleFrameSource {
8012 kind: ConsoleFrameSourceKind::ConsoleEvent,
8013 source_cursor: None,
8014 },
8015 source_event_id: Some("visible-after-gap".to_string()),
8016 interaction_id: Some("turn-visible".to_string()),
8017 turn_id: None,
8018 run_id: None,
8019 parent_frame_id: None,
8020 caused_by_frame_id: None,
8021 })
8022 .await
8023 .expect("append visible frame");
8024
8025 let page = aggregator
8026 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8027 identity: Some("agent-a".to_string()),
8028 mode: ConsoleTimelineMode::Since,
8029 limit: 1,
8030 ..ConsoleTimelineWindowQuery::default()
8031 })
8032 .await
8033 .expect("query timeline");
8034 assert_eq!(page.frames.len(), 1);
8035 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
8036 assert_eq!(
8037 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8038 Some(1_501)
8039 );
8040 let _ = runtime.mob_handle().stop().await;
8041 }
8042
8043 #[tokio::test]
8044 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
8045 let aggregator = MobKitConsoleAggregator::in_memory();
8046 for idx in 1..=300 {
8047 aggregator
8048 .store()
8049 .append_if_absent(NewConsoleFrame {
8050 id: None,
8051 dedupe_key: format!("visible-{idx}"),
8052 timestamp_ms: idx,
8053 runtime_key: "runtime-a".to_string(),
8054 identity: "agent-a".to_string(),
8055 conversation_id: Some("agent-a".to_string()),
8056 session_id: None,
8057 kind: "interaction_complete".to_string(),
8058 status: ConsoleFrameStatus::Delivered,
8059 payload: json!({ "text": format!("visible {idx}") }),
8060 source: ConsoleFrameSource {
8061 kind: ConsoleFrameSourceKind::ConsoleEvent,
8062 source_cursor: None,
8063 },
8064 source_event_id: Some(format!("visible-{idx}")),
8065 interaction_id: Some(format!("turn-{idx}")),
8066 turn_id: None,
8067 run_id: None,
8068 parent_frame_id: None,
8069 caused_by_frame_id: None,
8070 })
8071 .await
8072 .expect("append visible frame");
8073 }
8074
8075 let first = aggregator
8076 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8077 identity: Some("agent-a".to_string()),
8078 mode: ConsoleTimelineMode::Since,
8079 limit: 10,
8080 ..ConsoleTimelineWindowQuery::default()
8081 })
8082 .await
8083 .expect("query first page");
8084 assert_eq!(first.frames.len(), 10);
8085 assert_eq!(
8086 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8087 Some(10)
8088 );
8089
8090 let second = aggregator
8091 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8092 identity: Some("agent-a".to_string()),
8093 mode: ConsoleTimelineMode::Since,
8094 after: first.next_cursor,
8095 limit: 10,
8096 ..ConsoleTimelineWindowQuery::default()
8097 })
8098 .await
8099 .expect("query second page");
8100 assert_eq!(second.frames.len(), 10);
8101 assert_eq!(second.frames[0].dedupe_key, "visible-11");
8102 assert_eq!(
8103 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8104 Some(20)
8105 );
8106 }
8107
8108 #[tokio::test]
8109 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
8110 let store = Arc::new(CountingConsoleLogStore::new());
8111 let aggregator = MobKitConsoleAggregator::new(store.clone());
8112 let runtime = build_single_member_runtime().await;
8113 aggregator
8114 .inner
8115 .runtimes
8116 .write()
8117 .expect("runtime registry")
8118 .insert(
8119 "runtime-a".to_string(),
8120 runtime_entry_for_test("runtime-a", &runtime),
8121 );
8122 let inserted = store
8123 .append_if_absent(NewConsoleFrame {
8124 id: None,
8125 dedupe_key: "event-1".to_string(),
8126 timestamp_ms: 1,
8127 runtime_key: "runtime-a".to_string(),
8128 identity: "agent-a".to_string(),
8129 conversation_id: Some("agent-a".to_string()),
8130 session_id: None,
8131 kind: "text_delta".to_string(),
8132 status: ConsoleFrameStatus::Delivered,
8133 payload: json!({ "delta": "hello" }),
8134 source: ConsoleFrameSource {
8135 kind: ConsoleFrameSourceKind::ConsoleEvent,
8136 source_cursor: None,
8137 },
8138 source_event_id: Some("event-1".to_string()),
8139 interaction_id: None,
8140 turn_id: None,
8141 run_id: None,
8142 parent_frame_id: None,
8143 caused_by_frame_id: None,
8144 })
8145 .await
8146 .expect("append frame");
8147
8148 let page = aggregator
8149 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8150 identity: Some("agent-a".to_string()),
8151 mode: ConsoleTimelineMode::Since,
8152 after: Some(inserted.frame.cursor),
8153 limit: 10,
8154 ..ConsoleTimelineWindowQuery::default()
8155 })
8156 .await
8157 .expect("query continuation");
8158
8159 assert!(page.frames.is_empty());
8160 assert_eq!(
8161 store.source_watermark_calls(),
8162 0,
8163 "empty since continuation must not synchronously force session-history backfill"
8164 );
8165 let _ = runtime.mob_handle().stop().await;
8166 }
8167
8168 #[test]
8169 fn legacy_timeline_struct_literals_remain_source_compatible() {
8170 let query = ConsoleTimelineQuery {
8171 identity: Some("agent-a".to_string()),
8172 conversation_id: None,
8173 after: Some(ConsoleCursor::from("console:1")),
8174 limit: 10,
8175 };
8176 let page = ConsoleTimelinePage {
8177 frames: Vec::new(),
8178 next_cursor: query.after.clone(),
8179 };
8180
8181 assert_eq!(query.limit, 10);
8182 assert_eq!(
8183 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8184 Some(1)
8185 );
8186 }
8187
8188 #[tokio::test]
8189 async fn query_timeline_is_store_local_for_registered_runtimes() {
8190 let store = Arc::new(CountingConsoleLogStore::new());
8191 let aggregator = MobKitConsoleAggregator::new(store.clone());
8192 let runtime = build_single_member_runtime().await;
8193 aggregator
8194 .inner
8195 .runtimes
8196 .write()
8197 .expect("runtime registry")
8198 .insert(
8199 "runtime-a".to_string(),
8200 runtime_entry_for_test("runtime-a", &runtime),
8201 );
8202 store
8203 .append_if_absent(NewConsoleFrame {
8204 id: None,
8205 dedupe_key: "event-1".to_string(),
8206 timestamp_ms: 1,
8207 runtime_key: "runtime-a".to_string(),
8208 identity: "agent-a".to_string(),
8209 conversation_id: Some("agent-a".to_string()),
8210 session_id: None,
8211 kind: "text_delta".to_string(),
8212 status: ConsoleFrameStatus::Delivered,
8213 payload: json!({ "delta": "hello" }),
8214 source: ConsoleFrameSource {
8215 kind: ConsoleFrameSourceKind::ConsoleEvent,
8216 source_cursor: None,
8217 },
8218 source_event_id: Some("event-1".to_string()),
8219 interaction_id: None,
8220 turn_id: None,
8221 run_id: None,
8222 parent_frame_id: None,
8223 caused_by_frame_id: None,
8224 })
8225 .await
8226 .expect("append frame");
8227
8228 let page = tokio::time::timeout(
8229 Duration::from_millis(250),
8230 aggregator.query_timeline(ConsoleTimelineQuery {
8231 identity: Some("agent-a".to_string()),
8232 limit: 10,
8233 ..ConsoleTimelineQuery::default()
8234 }),
8235 )
8236 .await
8237 .expect("timeline query should not wait for session history")
8238 .expect("timeline query succeeds");
8239
8240 assert_eq!(page.frames.len(), 1);
8241 assert_eq!(
8242 store.source_watermark_calls(),
8243 0,
8244 "query_timeline must not synchronously touch session-history watermarks"
8245 );
8246 let _ = runtime.mob_handle().stop().await;
8247 }
8248
8249 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8250 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
8251 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
8252 delay: Duration::from_secs(2),
8253 }))
8254 .await;
8255 let aggregator = MobKitConsoleAggregator::in_memory();
8256 aggregator
8257 .inner
8258 .runtimes
8259 .write()
8260 .expect("runtime registry")
8261 .insert(
8262 "runtime-a".to_string(),
8263 runtime_entry_for_test("runtime-a", &runtime),
8264 );
8265
8266 let start = Instant::now();
8267 let accepted = tokio::time::timeout(
8268 Duration::from_millis(300),
8269 aggregator.send(ConsoleSendRequest {
8270 identity: "test/agent-a".to_string(),
8271 content: json!("hello slow agent"),
8272 origin: "console:test".to_string(),
8273 idempotency_key: "nonblocking-send".to_string(),
8274 handling_mode: Some("queue".to_string()),
8275 }),
8276 )
8277 .await
8278 .expect("console send should return once the input is accepted")
8279 .expect("send succeeds");
8280
8281 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
8282 assert!(
8283 start.elapsed() < Duration::from_secs(1),
8284 "console send should not wait for the delayed LLM turn"
8285 );
8286
8287 wait_for_session_history_text(
8288 &aggregator,
8289 "test/agent-a",
8290 "slow ok",
8291 Duration::from_secs(5),
8292 )
8293 .await
8294 .expect("background dispatch should still complete and project history");
8295 let _ = runtime.mob_handle().stop().await;
8296 }
8297
8298 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8299 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
8300 {
8301 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
8302 let aggregator = MobKitConsoleAggregator::in_memory();
8303 aggregator.register_runtime(ConsoleRuntimeRegistration {
8304 runtime_key: "runtime-late".to_string(),
8305 runtime: runtime.clone(),
8306 identity_namespace: "late".to_string(),
8307 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8308 });
8309
8310 runtime
8311 .spawn(SpawnMemberSpec::from_wire(
8312 "worker".to_string(),
8313 "agent-late".to_string(),
8314 Some("You are agent-late.".into()),
8315 None,
8316 None,
8317 ))
8318 .await
8319 .expect("late member spawns");
8320 let session_id = send_message_on_mob_with_mode(
8321 &runtime.mob_handle(),
8322 "agent-late",
8323 ContentInput::Text("hello after registration".to_string()),
8324 meerkat_core::types::HandlingMode::Queue,
8325 )
8326 .await
8327 .expect("direct member send succeeds");
8328 wait_for_identity_record(
8329 &aggregator,
8330 "late/agent-late",
8331 Some(session_id.as_str()),
8332 Duration::from_secs(5),
8333 )
8334 .await?;
8335
8336 wait_for_session_history_text(
8337 &aggregator,
8338 "late/agent-late",
8339 "You are agent-late.",
8340 Duration::from_secs(5),
8341 )
8342 .await?;
8343 let _ = runtime.mob_handle().stop().await;
8344 Ok(())
8345 }
8346
8347 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8348 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
8349 -> Result<(), String> {
8350 let runtime = build_single_member_runtime().await;
8351 let entry = runtime_entry_for_test("runtime-a", &runtime);
8352 let resolved = member_sources_for_entry(&entry)
8353 .await
8354 .into_iter()
8355 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
8356 .expect("agent-a member exists");
8357 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
8358 .await
8359 .expect("identity record exists");
8360 let session_id = record.session_id.expect("agent-a has a session");
8361 wait_for_runtime_session_history_text(
8362 &runtime,
8363 &session_id,
8364 "You are agent-a.",
8365 Duration::from_secs(5),
8366 )
8367 .await?;
8368
8369 let aggregator = MobKitConsoleAggregator::in_memory();
8370 aggregator
8371 .inner
8372 .runtimes
8373 .write()
8374 .expect("runtime registry")
8375 .insert("runtime-a".to_string(), entry);
8376 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
8377 aggregator
8378 .store()
8379 .record_source_watermark(
8380 &watermark_key,
8381 ConsoleFrameSourceKind::SessionHistory,
8382 &format_session_history_watermark(&session_id, 0, current_time_ms()),
8383 )
8384 .await
8385 .expect("record fresh empty watermark");
8386
8387 let page = tokio::time::timeout(
8391 Duration::from_mins(1),
8392 aggregator.query_timeline(ConsoleTimelineQuery {
8393 identity: Some("test/agent-a".to_string()),
8394 limit: 20,
8395 ..ConsoleTimelineQuery::default()
8396 }),
8397 )
8398 .await
8399 .expect("explicit identity query should not stall")
8400 .expect("query succeeds");
8401
8402 if !page.frames.is_empty() {
8403 assert!(
8404 page.frames.iter().any(|frame| {
8405 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8406 && session_history_content_text(frame).as_deref()
8407 == Some("You are agent-a.")
8408 }),
8409 "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
8410 page.frames
8411 );
8412 }
8413 wait_for_session_history_text(
8414 &aggregator,
8415 "test/agent-a",
8416 "You are agent-a.",
8417 Duration::from_secs(5),
8418 )
8419 .await?;
8420 let _ = runtime.mob_handle().stop().await;
8421 Ok(())
8422 }
8423
8424 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8425 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
8426 {
8427 let store = Arc::new(CountingConsoleLogStore::new());
8428 let aggregator = MobKitConsoleAggregator::new(store.clone());
8429 let runtime = build_single_member_runtime().await;
8430 let entry = runtime_entry_for_test("runtime-a", &runtime);
8431 let resolved = member_sources_for_entry(&entry)
8432 .await
8433 .into_iter()
8434 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
8435 .expect("agent-a member exists");
8436 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
8437 .await
8438 .expect("identity record exists");
8439 let session_id = record.session_id.expect("agent-a has a session");
8440 wait_for_runtime_session_history_text(
8441 &runtime,
8442 &session_id,
8443 "You are agent-a.",
8444 Duration::from_secs(5),
8445 )
8446 .await?;
8447 aggregator
8448 .inner
8449 .runtimes
8450 .write()
8451 .expect("runtime registry")
8452 .insert("runtime-a".to_string(), entry);
8453
8454 store
8455 .append_if_absent(NewConsoleFrame {
8456 id: None,
8457 dedupe_key: "stale-session-history-agent-a".to_string(),
8458 timestamp_ms: 10,
8459 runtime_key: "runtime-a".to_string(),
8460 identity: "test/agent-a".to_string(),
8461 conversation_id: Some("test/agent-a".to_string()),
8462 session_id: Some(session_id.clone()),
8463 kind: "user_input".to_string(),
8464 status: ConsoleFrameStatus::Delivered,
8465 payload: json!({
8466 "text": "stale projected history",
8467 "type": "user_input",
8468 }),
8469 source: ConsoleFrameSource {
8470 kind: ConsoleFrameSourceKind::SessionHistory,
8471 source_cursor: Some("stale-session-history-agent-a".to_string()),
8472 },
8473 source_event_id: Some("stale-session-history-agent-a".to_string()),
8474 interaction_id: None,
8475 turn_id: None,
8476 run_id: None,
8477 parent_frame_id: None,
8478 caused_by_frame_id: None,
8479 })
8480 .await
8481 .expect("append stale history frame");
8482
8483 let fresh_prompt = "fresh prompt after stale history";
8484 let sent_session_id = send_message_on_mob_with_mode(
8485 &runtime.mob_handle(),
8486 "agent-a",
8487 ContentInput::Text(fresh_prompt.to_string()),
8488 meerkat_core::types::HandlingMode::Queue,
8489 )
8490 .await
8491 .expect("direct member send succeeds");
8492 assert_eq!(sent_session_id, session_id);
8493 wait_for_runtime_session_history_text(
8494 &runtime,
8495 &session_id,
8496 fresh_prompt,
8497 Duration::from_secs(5),
8498 )
8499 .await?;
8500
8501 let page = aggregator
8502 .query_timeline(ConsoleTimelineQuery {
8503 identity: Some("test/agent-a".to_string()),
8504 limit: 50,
8505 ..ConsoleTimelineQuery::default()
8506 })
8507 .await
8508 .expect("query child timeline");
8509
8510 assert!(
8511 page.frames.iter().any(|frame| {
8512 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8513 && session_history_content_text(frame).as_deref()
8514 == Some("stale projected history")
8515 }),
8516 "explicit identity query should return existing history before async refresh; frames: {:#?}",
8517 page.frames
8518 );
8519 wait_for_session_history_text(
8520 &aggregator,
8521 "test/agent-a",
8522 fresh_prompt,
8523 Duration::from_secs(5),
8524 )
8525 .await?;
8526 assert!(
8527 store.source_watermark_calls() > 0,
8528 "stale existing session history should force a targeted source refresh"
8529 );
8530 let _ = runtime.mob_handle().stop().await;
8531 Ok(())
8532 }
8533
8534 async fn wait_for_session_history_text(
8535 aggregator: &MobKitConsoleAggregator,
8536 identity: &str,
8537 expected: &str,
8538 timeout: Duration,
8539 ) -> Result<(), String> {
8540 let deadline = Instant::now() + timeout;
8541 let mut observed = Vec::new();
8542 while Instant::now() < deadline {
8543 let _ = aggregator.list_identities().await;
8544 let page = aggregator
8545 .query_timeline(ConsoleTimelineQuery {
8546 identity: Some(identity.to_string()),
8547 limit: 20,
8548 ..ConsoleTimelineQuery::default()
8549 })
8550 .await
8551 .expect("query timeline");
8552 observed = page.frames;
8553 if observed.iter().any(|frame| {
8554 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8555 && session_history_content_text(frame).as_deref() == Some(expected)
8556 }) {
8557 return Ok(());
8558 }
8559 tokio::time::sleep(Duration::from_millis(25)).await;
8560 }
8561
8562 Err(format!(
8563 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
8564 ))
8565 }
8566
8567 async fn wait_for_session_backfill_target(
8568 aggregator: &MobKitConsoleAggregator,
8569 identity: &str,
8570 timeout: Duration,
8571 ) -> Result<String, String> {
8572 let deadline = Instant::now() + timeout;
8573 while Instant::now() < deadline {
8574 if let Some(target) =
8575 session_backfill_target_for_identity(&aggregator.inner, identity).await
8576 {
8577 return Ok(target.session_id);
8578 }
8579 tokio::time::sleep(Duration::from_millis(25)).await;
8580 }
8581 Err(format!(
8582 "session backfill target for {identity:?} was not resolvable"
8583 ))
8584 }
8585
8586 async fn wait_for_runtime_session_history_text(
8587 runtime: &UnifiedRuntime,
8588 session_id: &str,
8589 expected: &str,
8590 timeout: Duration,
8591 ) -> Result<(), String> {
8592 let deadline = Instant::now() + timeout;
8593 let mut observed = Vec::new();
8594 while Instant::now() < deadline {
8595 let page = runtime
8596 .mob_runtime()
8597 .read_session_history(session_id, 0, Some(20))
8598 .await
8599 .map_err(|err| err.to_string())?;
8600 observed = page.messages;
8601 if observed.iter().enumerate().any(|(idx, message)| {
8602 let Some(message) = serde_json::to_value(message).ok() else {
8603 return false;
8604 };
8605 frames_from_session_history_message(
8606 "runtime-a",
8607 "test/agent-a",
8608 session_id,
8609 idx,
8610 message,
8611 )
8612 .iter()
8613 .any(|frame| {
8614 matches!(
8615 frame.kind.as_str(),
8616 "user_input" | "system_notice" | "interaction_complete"
8617 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
8618 })
8619 }) {
8620 return Ok(());
8621 }
8622 tokio::time::sleep(Duration::from_millis(25)).await;
8623 }
8624
8625 Err(format!(
8626 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
8627 ))
8628 }
8629
8630 async fn wait_for_identity_record(
8631 aggregator: &MobKitConsoleAggregator,
8632 identity: &str,
8633 session_id: Option<&str>,
8634 timeout: Duration,
8635 ) -> Result<(), String> {
8636 let deadline = Instant::now() + timeout;
8637 let mut observed = Vec::new();
8638 while Instant::now() < deadline {
8639 observed = aggregator
8640 .list_identities()
8641 .await
8642 .map_err(|err| err.to_string())?;
8643 if observed.iter().any(|record| {
8644 record.identity == identity && record.session_id.as_deref() == session_id
8645 }) {
8646 return Ok(());
8647 }
8648 tokio::time::sleep(Duration::from_millis(25)).await;
8649 }
8650
8651 Err(format!(
8652 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
8653 ))
8654 }
8655
8656 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
8657 session_history_payload_text(&frame.payload)
8658 }
8659
8660 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
8661 session_history_payload_text(&frame.payload)
8662 }
8663
8664 fn session_history_payload_text(payload: &Value) -> Option<String> {
8665 if let Some(text) = payload.get("text").and_then(Value::as_str) {
8666 return Some(text.to_string());
8667 }
8668 if let Some(text) = payload.get("result").and_then(Value::as_str) {
8669 return Some(text.to_string());
8670 }
8671 if let Some(text) = payload.get("body").and_then(Value::as_str) {
8672 return Some(text.to_string());
8673 }
8674 match payload.get("content")? {
8675 Value::String(text) => Some(text.clone()),
8676 Value::Array(blocks) => Some(
8677 blocks
8678 .iter()
8679 .filter_map(|block| block.get("text").and_then(Value::as_str))
8680 .collect::<Vec<_>>()
8681 .join(""),
8682 ),
8683 _ => None,
8684 }
8685 }
8686
8687 #[tokio::test]
8688 async fn query_timeline_handles_large_store_without_backfill_calls() {
8689 let store = Arc::new(CountingConsoleLogStore::new());
8690 let aggregator = MobKitConsoleAggregator::new(store.clone());
8691 for idx in 0..5_000 {
8692 store
8693 .append_if_absent(NewConsoleFrame {
8694 id: None,
8695 dedupe_key: format!("event-{idx}"),
8696 timestamp_ms: idx,
8697 runtime_key: "runtime-a".to_string(),
8698 identity: "agent-a".to_string(),
8699 conversation_id: Some("agent-a".to_string()),
8700 session_id: Some("session-a".to_string()),
8701 kind: "text_delta".to_string(),
8702 status: ConsoleFrameStatus::Delivered,
8703 payload: json!({ "delta": idx }),
8704 source: ConsoleFrameSource {
8705 kind: ConsoleFrameSourceKind::ConsoleEvent,
8706 source_cursor: None,
8707 },
8708 source_event_id: Some(format!("event-{idx}")),
8709 interaction_id: None,
8710 turn_id: None,
8711 run_id: None,
8712 parent_frame_id: None,
8713 caused_by_frame_id: None,
8714 })
8715 .await
8716 .expect("append frame");
8717 }
8718
8719 let page = aggregator
8720 .query_timeline(ConsoleTimelineQuery {
8721 identity: Some("agent-a".to_string()),
8722 limit: 1_000,
8723 ..ConsoleTimelineQuery::default()
8724 })
8725 .await
8726 .expect("large query");
8727
8728 assert_eq!(page.frames.len(), 1_000);
8729 assert_eq!(store.source_watermark_calls(), 0);
8730 }
8731
8732 #[tokio::test]
8733 async fn query_timeline_rejects_future_cursor_after_store_reset() {
8734 let aggregator = MobKitConsoleAggregator::in_memory();
8735 let err = aggregator
8736 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8737 after: Some(ConsoleCursor::from("console:99")),
8738 limit: 10,
8739 ..ConsoleTimelineWindowQuery::default()
8740 })
8741 .await
8742 .expect_err("future cursor on empty/reset store must be replay-unavailable");
8743
8744 assert!(
8745 err.to_string()
8746 .contains("beyond the current store frontier"),
8747 "unexpected error: {err}"
8748 );
8749 }
8750
8751 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8752 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
8753 let store = Arc::new(CountingConsoleLogStore::new());
8754 let aggregator = MobKitConsoleAggregator::new(store.clone());
8755 let runtime = build_single_member_runtime().await;
8756 aggregator
8757 .inner
8758 .runtimes
8759 .write()
8760 .expect("runtime registry")
8761 .insert(
8762 "runtime-a".to_string(),
8763 runtime_entry_for_test("runtime-a", &runtime),
8764 );
8765 let session_id =
8766 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8767 .await
8768 .expect("spawned member is resolvable for targeted backfill");
8769 wait_for_runtime_session_history_text(
8770 &runtime,
8771 &session_id,
8772 "You are agent-a.",
8773 Duration::from_secs(5),
8774 )
8775 .await
8776 .expect("spawned member kickoff is readable before live-frame race");
8777 store
8778 .append_if_absent(NewConsoleFrame {
8779 id: None,
8780 dedupe_key: "live-tool-before-history".to_string(),
8781 timestamp_ms: 10,
8782 runtime_key: "runtime-a".to_string(),
8783 identity: "test/agent-a".to_string(),
8784 conversation_id: Some("test/agent-a".to_string()),
8785 session_id: None,
8786 kind: "tool_execution_started".to_string(),
8787 status: ConsoleFrameStatus::Delivered,
8788 payload: json!({
8789 "id": "call-live",
8790 "name": "king_search",
8791 "source_event_type": "tool_execution_started",
8792 "type": "tool_execution_started",
8793 }),
8794 source: ConsoleFrameSource {
8795 kind: ConsoleFrameSourceKind::ConsoleEvent,
8796 source_cursor: Some("live-tool-before-history".to_string()),
8797 },
8798 source_event_id: Some("live-tool-before-history".to_string()),
8799 interaction_id: None,
8800 turn_id: None,
8801 run_id: None,
8802 parent_frame_id: None,
8803 caused_by_frame_id: None,
8804 })
8805 .await
8806 .expect("append live tool frame");
8807
8808 let page = aggregator
8809 .query_timeline(ConsoleTimelineQuery {
8810 identity: Some("test/agent-a".to_string()),
8811 limit: 20,
8812 ..ConsoleTimelineQuery::default()
8813 })
8814 .await
8815 .expect("query child timeline");
8816
8817 assert!(
8818 page.frames
8819 .iter()
8820 .any(|frame| frame.kind == "tool_execution_started"),
8821 "initial explicit child timeline should return existing store frames without waiting for session history"
8822 );
8823
8824 let mut observed_backfill = false;
8825 for _ in 0..80 {
8826 let page = aggregator
8827 .query_timeline(ConsoleTimelineQuery {
8828 identity: Some("test/agent-a".to_string()),
8829 limit: 20,
8830 ..ConsoleTimelineQuery::default()
8831 })
8832 .await
8833 .expect("query child timeline after scheduled backfill");
8834 if page.frames.iter().any(|frame| {
8835 frame.kind == "user_input"
8836 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8837 && frame.payload.to_string().contains("You are agent-a.")
8838 }) {
8839 observed_backfill = true;
8840 break;
8841 }
8842 tokio::time::sleep(Duration::from_millis(25)).await;
8843 }
8844 assert!(
8845 observed_backfill,
8846 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8847 );
8848 let _ = runtime.mob_handle().stop().await;
8849 }
8850
8851 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8852 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8853 let store = Arc::new(CountingConsoleLogStore::new());
8854 let aggregator = MobKitConsoleAggregator::new(store.clone());
8855 let (_temp, runtime, _delayed_service) =
8856 build_stress_runtime(64, Duration::from_millis(0)).await;
8857 aggregator
8858 .inner
8859 .runtimes
8860 .write()
8861 .expect("runtime registry")
8862 .insert(
8863 "runtime-a".to_string(),
8864 runtime_entry_for_test("runtime-a", runtime.as_ref()),
8865 );
8866 for idx in 0..1_000 {
8867 store
8868 .append_if_absent(NewConsoleFrame {
8869 id: None,
8870 dedupe_key: format!("event-{idx}"),
8871 timestamp_ms: idx,
8872 runtime_key: "runtime-a".to_string(),
8873 identity: "test/agent-0".to_string(),
8874 conversation_id: Some("test/agent-0".to_string()),
8875 session_id: Some("session-a".to_string()),
8876 kind: "text_delta".to_string(),
8877 status: ConsoleFrameStatus::Delivered,
8878 payload: json!({ "delta": idx }),
8879 source: ConsoleFrameSource {
8880 kind: ConsoleFrameSourceKind::ConsoleEvent,
8881 source_cursor: None,
8882 },
8883 source_event_id: Some(format!("event-{idx}")),
8884 interaction_id: None,
8885 turn_id: None,
8886 run_id: None,
8887 parent_frame_id: None,
8888 caused_by_frame_id: None,
8889 })
8890 .await
8891 .expect("append frame");
8892 }
8893
8894 let page = tokio::time::timeout(
8901 Duration::from_mins(1),
8902 aggregator.query_timeline(ConsoleTimelineQuery {
8903 identity: Some("test/agent-0".to_string()),
8904 limit: 1_000,
8905 ..ConsoleTimelineQuery::default()
8906 }),
8907 )
8908 .await
8909 .expect("identity timeline query should not hang or rediscover the roster per frame")
8910 .expect("timeline query succeeds");
8911
8912 assert_eq!(page.frames.len(), 1_000);
8913 assert_eq!(store.source_watermark_calls(), 0);
8914 let _ = runtime.mob_handle().stop().await;
8915 }
8916
8917 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8918 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8919 const MEMBER_COUNT: usize = 32;
8920 let (_temp, runtime, delayed_service) =
8921 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8922 let aggregator = MobKitConsoleAggregator::in_memory();
8923 aggregator
8924 .inner
8925 .runtimes
8926 .write()
8927 .expect("runtime registry")
8928 .insert(
8929 "runtime-stress".to_string(),
8930 runtime_entry_for_test("runtime-stress", &runtime),
8931 );
8932
8933 let started = Instant::now();
8934 aggregator
8935 .refresh_session_history()
8936 .await
8937 .expect("stress refresh");
8938 let elapsed = started.elapsed();
8939
8940 assert!(
8941 delayed_service.read_calls() >= MEMBER_COUNT,
8942 "expected at least one history read per member, saw {}",
8943 delayed_service.read_calls()
8944 );
8945 assert!(
8946 delayed_service.max_active_reads() > 1,
8947 "session history backfill should fan out instead of reading members serially"
8948 );
8949 assert!(
8950 delayed_service.max_active_reads()
8951 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8952 "session history backfill should respect the default concurrency limit"
8953 );
8954 assert!(
8955 elapsed < Duration::from_millis(600),
8956 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8957 MEMBER_COUNT * 40
8958 );
8959 let _ = runtime.mob_handle().stop().await;
8960 }
8961
8962 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8963 async fn session_history_backfill_respects_configured_concurrency_limit() {
8964 const MEMBER_COUNT: usize = 16;
8965 let (_temp, runtime, delayed_service) =
8966 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8967 let aggregator =
8968 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8969 max_concurrent_session_backfills: 4,
8970 ..ConsoleAggregatorOptions::default()
8971 });
8972 aggregator
8973 .inner
8974 .runtimes
8975 .write()
8976 .expect("runtime registry")
8977 .insert(
8978 "runtime-stress".to_string(),
8979 runtime_entry_for_test("runtime-stress", &runtime),
8980 );
8981
8982 aggregator
8983 .refresh_session_history()
8984 .await
8985 .expect("stress refresh");
8986
8987 assert!(
8988 delayed_service.max_active_reads() <= 4,
8989 "configured concurrency limit should bound session history reads, saw {}",
8990 delayed_service.max_active_reads()
8991 );
8992 let _ = runtime.mob_handle().stop().await;
8993 }
8994
8995 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8996 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8997 const MEMBER_COUNT: usize = 24;
8998 const LIVE_EVENT_COUNT: usize = 2_048;
8999 let (_temp, runtime, _delayed_service) =
9000 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
9001 let aggregator = MobKitConsoleAggregator::in_memory();
9002 aggregator.register_runtime(ConsoleRuntimeRegistration {
9003 runtime_key: "runtime-burst".to_string(),
9004 runtime: runtime.clone(),
9005 identity_namespace: "stress".to_string(),
9006 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
9007 });
9008 let console_events = runtime.console_events();
9009 for idx in 0..LIVE_EVENT_COUNT {
9010 console_events
9011 .append(
9012 "agent-0",
9013 Some("burst-turn".to_string()),
9014 "text_delta",
9015 json!({ "delta": format!("frame-{idx}") }),
9016 )
9017 .await;
9018 }
9019
9020 let deadline = Instant::now() + Duration::from_secs(30);
9025 let mut observed = 0;
9026 while Instant::now() < deadline {
9027 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
9028 if observed >= LIVE_EVENT_COUNT {
9029 break;
9030 }
9031 tokio::time::sleep(Duration::from_millis(25)).await;
9032 }
9033
9034 assert_eq!(
9035 observed, LIVE_EVENT_COUNT,
9036 "live pump should not drop frames while slow background backfill is running"
9037 );
9038 let _ = runtime.mob_handle().stop().await;
9039 }
9040
9041 async fn count_console_event_frames(
9042 aggregator: &MobKitConsoleAggregator,
9043 identity: &str,
9044 ) -> usize {
9045 let mut after = None;
9046 let mut count = 0;
9047 loop {
9048 let page = aggregator
9049 .query_timeline(ConsoleTimelineQuery {
9050 identity: Some(identity.to_string()),
9051 after,
9052 limit: 1_000,
9053 ..ConsoleTimelineQuery::default()
9054 })
9055 .await
9056 .expect("query burst timeline");
9057 if page.frames.is_empty() {
9058 break;
9059 }
9060 count += page
9061 .frames
9062 .iter()
9063 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
9064 .count();
9065 after = page.next_cursor;
9066 if after.is_none() {
9067 break;
9068 }
9069 }
9070 count
9071 }
9072
9073 #[tokio::test]
9074 async fn status_updates_get_replayable_aggregate_cursors() {
9075 let aggregator = MobKitConsoleAggregator::in_memory();
9076 let frame = NewConsoleFrame {
9077 id: None,
9078 dedupe_key: "send-1".to_string(),
9079 timestamp_ms: 1,
9080 runtime_key: "runtime-a".to_string(),
9081 identity: "agent-a".to_string(),
9082 conversation_id: Some("agent-a".to_string()),
9083 session_id: Some("session-1".to_string()),
9084 kind: "user_input".to_string(),
9085 status: ConsoleFrameStatus::Accepted,
9086 payload: json!({ "content": "hello" }),
9087 source: ConsoleFrameSource {
9088 kind: ConsoleFrameSourceKind::Send,
9089 source_cursor: None,
9090 },
9091 source_event_id: None,
9092 interaction_id: Some("interaction-1".to_string()),
9093 turn_id: None,
9094 run_id: None,
9095 parent_frame_id: None,
9096 caused_by_frame_id: None,
9097 };
9098 let inserted = aggregator
9099 .store()
9100 .append_if_absent(frame)
9101 .await
9102 .expect("append frame");
9103
9104 update_frame_status_and_emit(
9105 &aggregator.inner,
9106 &inserted.frame.id,
9107 ConsoleFrameStatus::Delivered,
9108 )
9109 .await
9110 .expect("update status");
9111
9112 let page = aggregator
9113 .query_timeline(ConsoleTimelineQuery {
9114 identity: Some("agent-a".to_string()),
9115 after: Some(inserted.frame.cursor.clone()),
9116 limit: 10,
9117 ..ConsoleTimelineQuery::default()
9118 })
9119 .await
9120 .expect("query timeline");
9121 assert_eq!(page.frames.len(), 1);
9122 assert_eq!(page.frames[0].kind, "frame_updated");
9123 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
9124 assert_eq!(
9125 page.frames[0]
9126 .payload
9127 .get("frame")
9128 .and_then(|frame| frame.get("status"))
9129 .and_then(Value::as_str),
9130 Some("delivered")
9131 );
9132 }
9133
9134 #[tokio::test]
9135 async fn steer_delivery_appends_terminal_control_frame() {
9136 let aggregator = MobKitConsoleAggregator::in_memory();
9137 let frame = NewConsoleFrame {
9138 id: None,
9139 dedupe_key: "send-steer-1".to_string(),
9140 timestamp_ms: 1,
9141 runtime_key: "runtime-a".to_string(),
9142 identity: "agent-a".to_string(),
9143 conversation_id: Some("agent-a".to_string()),
9144 session_id: Some("session-1".to_string()),
9145 kind: "user_input".to_string(),
9146 status: ConsoleFrameStatus::Delivered,
9147 payload: json!({
9148 "content": "operator steer",
9149 "handling_mode": "steer",
9150 }),
9151 source: ConsoleFrameSource {
9152 kind: ConsoleFrameSourceKind::Send,
9153 source_cursor: None,
9154 },
9155 source_event_id: None,
9156 interaction_id: Some("interaction-steer-1".to_string()),
9157 turn_id: None,
9158 run_id: None,
9159 parent_frame_id: None,
9160 caused_by_frame_id: None,
9161 };
9162 let inserted = aggregator
9163 .store()
9164 .append_if_absent(frame)
9165 .await
9166 .expect("append steer input");
9167
9168 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
9169 .await
9170 .expect("append steer terminal");
9171
9172 let page = aggregator
9173 .query_timeline(ConsoleTimelineQuery {
9174 identity: Some("agent-a".to_string()),
9175 after: Some(inserted.frame.cursor.clone()),
9176 limit: 10,
9177 ..ConsoleTimelineQuery::default()
9178 })
9179 .await
9180 .expect("query timeline");
9181
9182 assert_eq!(page.frames.len(), 1);
9183 let terminal = &page.frames[0];
9184 assert_eq!(terminal.kind, "interaction_complete");
9185 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
9186 assert_eq!(
9187 terminal.interaction_id.as_deref(),
9188 Some("interaction-steer-1")
9189 );
9190 assert_eq!(
9191 terminal.parent_frame_id.as_deref(),
9192 Some(inserted.frame.id.as_str())
9193 );
9194 assert_eq!(
9195 terminal.payload.get("reason").and_then(Value::as_str),
9196 Some("steer_delivered")
9197 );
9198 }
9199
9200 #[test]
9201 fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
9202 let live = transcript_fingerprint(
9203 "reasoning_complete",
9204 &json!({ "text": "I should inspect the file first." }),
9205 );
9206 let replay = transcript_fingerprint(
9207 "reasoning_complete",
9208 &json!({ "summary": "I should inspect the file first." }),
9209 );
9210 let delta = transcript_fingerprint(
9211 "reasoning_delta",
9212 &json!({ "delta": "I should inspect the file first." }),
9213 );
9214
9215 assert_eq!(live.as_deref(), Some("I should inspect the file first."));
9216 assert_eq!(live, replay);
9217 assert_eq!(live, delta);
9218 }
9219
9220 #[tokio::test]
9221 async fn history_counterpart_scan_is_not_capped_to_one_page() {
9222 let aggregator = MobKitConsoleAggregator::in_memory();
9223 for idx in 0..1_005 {
9224 aggregator
9225 .store()
9226 .append_if_absent(NewConsoleFrame {
9227 id: None,
9228 dedupe_key: format!("filler-{idx}"),
9229 timestamp_ms: idx,
9230 runtime_key: "runtime-a".to_string(),
9231 identity: "agent-a".to_string(),
9232 conversation_id: Some("agent-a".to_string()),
9233 session_id: Some("session-a".to_string()),
9234 kind: "text_delta".to_string(),
9235 status: ConsoleFrameStatus::Completed,
9236 payload: json!({ "delta": idx }),
9237 source: ConsoleFrameSource {
9238 kind: ConsoleFrameSourceKind::ConsoleEvent,
9239 source_cursor: None,
9240 },
9241 source_event_id: Some(format!("filler-{idx}")),
9242 interaction_id: None,
9243 turn_id: None,
9244 run_id: None,
9245 parent_frame_id: None,
9246 caused_by_frame_id: None,
9247 })
9248 .await
9249 .expect("append filler");
9250 }
9251 aggregator
9252 .store()
9253 .append_if_absent(NewConsoleFrame {
9254 id: None,
9255 dedupe_key: "live-user-input".to_string(),
9256 timestamp_ms: 2_000,
9257 runtime_key: "runtime-a".to_string(),
9258 identity: "agent-a".to_string(),
9259 conversation_id: Some("agent-a".to_string()),
9260 session_id: Some("session-a".to_string()),
9261 kind: "user_input".to_string(),
9262 status: ConsoleFrameStatus::Delivered,
9263 payload: json!({ "content": "already here" }),
9264 source: ConsoleFrameSource {
9265 kind: ConsoleFrameSourceKind::ConsoleEvent,
9266 source_cursor: None,
9267 },
9268 source_event_id: Some("live-user-input".to_string()),
9269 interaction_id: None,
9270 turn_id: None,
9271 run_id: None,
9272 parent_frame_id: None,
9273 caused_by_frame_id: None,
9274 })
9275 .await
9276 .expect("append live input");
9277
9278 let history = NewConsoleFrame {
9279 id: None,
9280 dedupe_key: "history-user-input".to_string(),
9281 timestamp_ms: 3_000,
9282 runtime_key: "runtime-a".to_string(),
9283 identity: "agent-a".to_string(),
9284 conversation_id: Some("agent-a".to_string()),
9285 session_id: Some("session-a".to_string()),
9286 kind: "user_input".to_string(),
9287 status: ConsoleFrameStatus::Completed,
9288 payload: json!({ "content": "already here" }),
9289 source: ConsoleFrameSource {
9290 kind: ConsoleFrameSourceKind::SessionHistory,
9291 source_cursor: Some("session-a:1006".to_string()),
9292 },
9293 source_event_id: None,
9294 interaction_id: None,
9295 turn_id: None,
9296 run_id: None,
9297 parent_frame_id: None,
9298 caused_by_frame_id: None,
9299 };
9300
9301 assert!(
9302 history_frame_has_existing_counterpart(&aggregator.inner, &history)
9303 .await
9304 .expect("counterpart scan")
9305 );
9306 }
9307
9308 #[tokio::test]
9309 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
9310 let aggregator = MobKitConsoleAggregator::in_memory();
9311 aggregator
9312 .store()
9313 .append_if_absent(NewConsoleFrame {
9314 id: None,
9315 dedupe_key: "live-user-input".to_string(),
9316 timestamp_ms: 2_000,
9317 runtime_key: "runtime-a".to_string(),
9318 identity: "agent-a".to_string(),
9319 conversation_id: Some("agent-a".to_string()),
9320 session_id: Some("session-a".to_string()),
9321 kind: "user_input".to_string(),
9322 status: ConsoleFrameStatus::Delivered,
9323 payload: json!({ "content": "hello from operator" }),
9324 source: ConsoleFrameSource {
9325 kind: ConsoleFrameSourceKind::ConsoleEvent,
9326 source_cursor: None,
9327 },
9328 source_event_id: Some("live-user-input".to_string()),
9329 interaction_id: None,
9330 turn_id: None,
9331 run_id: None,
9332 parent_frame_id: None,
9333 caused_by_frame_id: None,
9334 })
9335 .await
9336 .expect("append live input");
9337
9338 let history = NewConsoleFrame {
9339 id: None,
9340 dedupe_key: "history-user-input".to_string(),
9341 timestamp_ms: 3_000,
9342 runtime_key: "runtime-a".to_string(),
9343 identity: "agent-a".to_string(),
9344 conversation_id: Some("agent-a".to_string()),
9345 session_id: Some("session-a".to_string()),
9346 kind: "user_input".to_string(),
9347 status: ConsoleFrameStatus::Completed,
9348 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
9349 source: ConsoleFrameSource {
9350 kind: ConsoleFrameSourceKind::SessionHistory,
9351 source_cursor: Some("session-a:2".to_string()),
9352 },
9353 source_event_id: None,
9354 interaction_id: None,
9355 turn_id: None,
9356 run_id: None,
9357 parent_frame_id: None,
9358 caused_by_frame_id: None,
9359 };
9360
9361 assert!(
9362 history_frame_has_existing_counterpart(&aggregator.inner, &history)
9363 .await
9364 .expect("counterpart scan")
9365 );
9366 }
9367
9368 #[tokio::test]
9369 async fn history_counterpart_scan_matches_content_block_user_prompts() {
9370 let aggregator = MobKitConsoleAggregator::in_memory();
9371 aggregator
9372 .store()
9373 .append_if_absent(NewConsoleFrame {
9374 id: None,
9375 dedupe_key: "live-user-input".to_string(),
9376 timestamp_ms: 2_000,
9377 runtime_key: "runtime-a".to_string(),
9378 identity: "agent-a".to_string(),
9379 conversation_id: Some("agent-a".to_string()),
9380 session_id: Some("session-a".to_string()),
9381 kind: "user_input".to_string(),
9382 status: ConsoleFrameStatus::Delivered,
9383 payload: json!({ "content": "hello from operator" }),
9384 source: ConsoleFrameSource {
9385 kind: ConsoleFrameSourceKind::ConsoleEvent,
9386 source_cursor: None,
9387 },
9388 source_event_id: Some("live-user-input".to_string()),
9389 interaction_id: None,
9390 turn_id: None,
9391 run_id: None,
9392 parent_frame_id: None,
9393 caused_by_frame_id: None,
9394 })
9395 .await
9396 .expect("append live input");
9397
9398 let history = NewConsoleFrame {
9399 id: None,
9400 dedupe_key: "history-user-input".to_string(),
9401 timestamp_ms: 3_000,
9402 runtime_key: "runtime-a".to_string(),
9403 identity: "agent-a".to_string(),
9404 conversation_id: Some("agent-a".to_string()),
9405 session_id: Some("session-a".to_string()),
9406 kind: "user_input".to_string(),
9407 status: ConsoleFrameStatus::Completed,
9408 payload: json!({
9409 "content": [{ "type": "text", "text": "hello from operator" }]
9410 }),
9411 source: ConsoleFrameSource {
9412 kind: ConsoleFrameSourceKind::SessionHistory,
9413 source_cursor: Some("session-a:2".to_string()),
9414 },
9415 source_event_id: None,
9416 interaction_id: None,
9417 turn_id: None,
9418 run_id: None,
9419 parent_frame_id: None,
9420 caused_by_frame_id: None,
9421 };
9422
9423 assert!(
9424 history_frame_has_existing_counterpart(&aggregator.inner, &history)
9425 .await
9426 .expect("counterpart scan")
9427 );
9428 }
9429
9430 #[tokio::test]
9431 async fn history_counterpart_scan_matches_live_tool_results() {
9432 let aggregator = MobKitConsoleAggregator::in_memory();
9433 aggregator
9434 .store()
9435 .append_if_absent(NewConsoleFrame {
9436 id: None,
9437 dedupe_key: "live-tool-result".to_string(),
9438 timestamp_ms: 2_000,
9439 runtime_key: "runtime-a".to_string(),
9440 identity: "agent-a".to_string(),
9441 conversation_id: Some("agent-a".to_string()),
9442 session_id: Some("session-a".to_string()),
9443 kind: "tool_execution_completed".to_string(),
9444 status: ConsoleFrameStatus::Delivered,
9445 payload: json!({
9446 "id": "call-1",
9447 "tool_call_id": "call-1",
9448 "result": "{ \"count\": 70 }"
9449 }),
9450 source: ConsoleFrameSource {
9451 kind: ConsoleFrameSourceKind::ConsoleEvent,
9452 source_cursor: None,
9453 },
9454 source_event_id: Some("live-tool-result".to_string()),
9455 interaction_id: None,
9456 turn_id: None,
9457 run_id: None,
9458 parent_frame_id: None,
9459 caused_by_frame_id: None,
9460 })
9461 .await
9462 .expect("append live tool result");
9463
9464 let history = NewConsoleFrame {
9465 id: None,
9466 dedupe_key: "history-tool-result".to_string(),
9467 timestamp_ms: 3_000,
9468 runtime_key: "runtime-a".to_string(),
9469 identity: "agent-a".to_string(),
9470 conversation_id: Some("agent-a".to_string()),
9471 session_id: Some("session-a".to_string()),
9472 kind: "tool_execution_completed".to_string(),
9473 status: ConsoleFrameStatus::Completed,
9474 payload: json!({
9475 "id": "call-1",
9476 "tool_call_id": "call-1",
9477 "result": "{ \"count\": 70 }",
9478 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
9479 }),
9480 source: ConsoleFrameSource {
9481 kind: ConsoleFrameSourceKind::SessionHistory,
9482 source_cursor: Some("session-a:3:0".to_string()),
9483 },
9484 source_event_id: None,
9485 interaction_id: None,
9486 turn_id: None,
9487 run_id: None,
9488 parent_frame_id: None,
9489 caused_by_frame_id: None,
9490 };
9491
9492 assert!(
9493 history_frame_has_existing_counterpart(&aggregator.inner, &history)
9494 .await
9495 .expect("counterpart scan")
9496 );
9497 }
9498
9499 #[tokio::test]
9500 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
9501 let aggregator = MobKitConsoleAggregator::in_memory();
9502 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
9503 aggregator
9504 .store()
9505 .append_if_absent(NewConsoleFrame {
9506 id: None,
9507 dedupe_key: format!("live-delta-{idx}"),
9508 timestamp_ms: 2_000 + idx as u64,
9509 runtime_key: "runtime-a".to_string(),
9510 identity: "agent-a".to_string(),
9511 conversation_id: Some("agent-a".to_string()),
9512 session_id: Some("session-a".to_string()),
9513 kind: "text_delta".to_string(),
9514 status: ConsoleFrameStatus::Delivered,
9515 payload: json!({ "delta": delta }),
9516 source: ConsoleFrameSource {
9517 kind: ConsoleFrameSourceKind::ConsoleEvent,
9518 source_cursor: None,
9519 },
9520 source_event_id: Some(format!("live-delta-{idx}")),
9521 interaction_id: Some("turn-a".to_string()),
9522 turn_id: None,
9523 run_id: None,
9524 parent_frame_id: None,
9525 caused_by_frame_id: None,
9526 })
9527 .await
9528 .expect("append live delta");
9529 }
9530
9531 let history = NewConsoleFrame {
9532 id: None,
9533 dedupe_key: "history-assistant-complete".to_string(),
9534 timestamp_ms: 3_000,
9535 runtime_key: "runtime-a".to_string(),
9536 identity: "agent-a".to_string(),
9537 conversation_id: Some("agent-a".to_string()),
9538 session_id: Some("session-a".to_string()),
9539 kind: "interaction_complete".to_string(),
9540 status: ConsoleFrameStatus::Completed,
9541 payload: json!({ "result": "Ready and standing by." }),
9542 source: ConsoleFrameSource {
9543 kind: ConsoleFrameSourceKind::SessionHistory,
9544 source_cursor: Some("session-a:3".to_string()),
9545 },
9546 source_event_id: None,
9547 interaction_id: None,
9548 turn_id: None,
9549 run_id: None,
9550 parent_frame_id: None,
9551 caused_by_frame_id: None,
9552 };
9553
9554 assert!(
9555 history_frame_has_existing_counterpart(&aggregator.inner, &history)
9556 .await
9557 .expect("counterpart scan")
9558 );
9559 }
9560
9561 #[test]
9562 fn session_history_watermark_key_is_session_scoped() {
9563 assert_ne!(
9564 session_history_watermark_runtime_key("runtime-a", "session-1"),
9565 session_history_watermark_runtime_key("runtime-a", "session-2")
9566 );
9567 }
9568
9569 #[test]
9570 fn backfill_offset_advances_only_on_strict_forward_progress() {
9571 assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
9573 assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
9575 assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
9577 assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
9579
9580 assert_eq!(advance_backfill_offset(500, 0, 500), None); assert_eq!(advance_backfill_offset(500, 0, 400), None); assert_eq!(advance_backfill_offset(1000, 500, 500), None); assert_eq!(advance_backfill_offset(10, 10, 0), None); assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
9589 }
9590
9591 #[test]
9592 fn session_history_watermarks_are_cursor_and_ttl_aware() {
9593 let legacy = "session:with:colon:42";
9594 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
9595 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
9596
9597 assert_eq!(
9598 parse_session_history_watermark(legacy, "session:with:colon"),
9599 Some(42)
9600 );
9601 assert_eq!(
9602 parse_session_history_watermark(&checked, "session:with:colon"),
9603 Some(43)
9604 );
9605 assert!(session_history_watermark_is_fresh(
9606 &checked,
9607 "session:with:colon",
9608 1_500
9609 ));
9610 assert!(!session_history_watermark_is_fresh(
9611 &checked,
9612 "session:with:colon",
9613 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
9614 ));
9615 assert!(session_history_watermark_is_fresh(
9616 &empty_checked,
9617 "session:with:colon",
9618 1_500
9619 ));
9620 assert!(!session_history_watermark_is_fresh(
9621 &empty_checked,
9622 "session:with:colon",
9623 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
9624 ));
9625 }
9626
9627 #[test]
9628 fn session_history_messages_project_to_renderable_frames() {
9629 let user = frame_from_session_history_message(
9630 "runtime-a",
9631 "agent-a",
9632 "session-a",
9633 0,
9634 json!({
9635 "role": "user",
9636 "content": "hello",
9637 "timestamp_ms": 10
9638 }),
9639 )
9640 .expect("user history frame");
9641 let assistant = frame_from_session_history_message(
9646 "runtime-a",
9647 "agent-a",
9648 "session-a",
9649 1,
9650 json!({
9651 "role": "block_assistant",
9652 "blocks": [
9653 { "block_type": "text", "data": { "text": "hi there" } }
9654 ],
9655 "stop_reason": "end_turn",
9656 "timestamp_ms": 11
9657 }),
9658 )
9659 .expect("assistant history frame");
9660
9661 assert_eq!(user.kind, "user_input");
9662 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
9663 assert_eq!(
9664 user.payload["content"],
9665 json!([{ "type": "text", "text": "hello" }])
9666 );
9667 assert_eq!(assistant.kind, "interaction_complete");
9668 assert_eq!(assistant.payload["text"], json!("hi there"));
9669 assert!(
9670 assistant
9671 .dedupe_key
9672 .starts_with("session-history:runtime-a:session-a:1:")
9673 );
9674 }
9675
9676 #[test]
9677 fn session_history_projection_filters_scaffold_user_messages() {
9678 let spawn_notice = frame_from_session_history_message(
9679 "runtime-a",
9680 "agent-a",
9681 "session-a",
9682 0,
9683 json!({
9684 "role": "user",
9685 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
9686 "timestamp_ms": 10
9687 }),
9688 );
9689 let peer_update = frame_from_session_history_message(
9690 "runtime-a",
9691 "agent-a",
9692 "session-a",
9693 1,
9694 json!({
9695 "role": "user",
9696 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
9697 "timestamp_ms": 11
9698 }),
9699 );
9700 let real_user = frame_from_session_history_message(
9701 "runtime-a",
9702 "agent-a",
9703 "session-a",
9704 2,
9705 json!({
9706 "role": "user",
9707 "content": "Please review the incident notes.",
9708 "timestamp_ms": 12
9709 }),
9710 );
9711
9712 assert!(spawn_notice.is_none());
9713 assert!(peer_update.is_none());
9714 assert!(real_user.is_some());
9715 }
9716
9717 #[test]
9718 fn session_history_projection_skips_non_transcript_messages() {
9719 let skipped = frame_from_session_history_message(
9720 "runtime-a",
9721 "agent-a",
9722 "session-a",
9723 0,
9724 json!({
9725 "content": "internal system prompt"
9726 }),
9727 );
9728 assert!(skipped.is_none());
9729 }
9730
9731 #[test]
9732 fn session_history_projection_extracts_assistant_blocks() {
9733 let frame = frame_from_session_history_message(
9734 "runtime-a",
9735 "agent-a",
9736 "session-a",
9737 0,
9738 json!({
9739 "role": "block_assistant",
9740 "blocks": [
9741 { "block_type": "text", "data": { "text": "hello " } },
9742 { "block_type": "text", "data": { "text": "there" } }
9743 ],
9744 "stop_reason": "end_turn"
9745 }),
9746 )
9747 .expect("assistant block history frame");
9748 assert_eq!(frame.payload["text"], json!("hello there"));
9749 }
9750
9751 #[test]
9752 fn session_history_projection_extracts_nested_text_block_data() {
9753 let frame = frame_from_session_history_message(
9754 "runtime-a",
9755 "agent-a",
9756 "session-a",
9757 0,
9758 json!({
9759 "role": "block_assistant",
9760 "blocks": [
9761 {
9762 "block_type": "text",
9763 "data": { "text": "Ready and standing by." }
9764 }
9765 ],
9766 "stop_reason": "end_turn",
9767 "created_at": "1970-01-01T00:00:00.010Z"
9768 }),
9769 )
9770 .expect("assistant block history frame");
9771
9772 assert_eq!(frame.kind, "interaction_complete");
9773 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9774 }
9775
9776 #[test]
9777 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9778 let frame = frame_from_session_history_message(
9779 "runtime-a",
9780 "agent-a",
9781 "session-a",
9782 0,
9783 json!({
9784 "role": "block_assistant",
9785 "blocks": [
9786 {
9787 "block_type": "reasoning",
9788 "data": { "text": "**Planning**\nI should not be rendered." }
9789 },
9790 {
9791 "block_type": "text",
9792 "data": { "text": "Visible answer." }
9793 }
9794 ],
9795 "stop_reason": "end_turn",
9796 "created_at": "1970-01-01T00:00:00.010Z"
9797 }),
9798 )
9799 .expect("assistant block history frame");
9800
9801 assert_eq!(frame.kind, "interaction_complete");
9802 assert_eq!(frame.payload["result"], json!("Visible answer."));
9803 assert_eq!(frame.payload["text"], json!("Visible answer."));
9804 }
9805
9806 #[test]
9807 fn session_history_projection_leaves_reasoning_only_result_empty() {
9808 let frame = frame_from_session_history_message(
9809 "runtime-a",
9810 "agent-a",
9811 "session-a",
9812 0,
9813 json!({
9814 "role": "block_assistant",
9815 "blocks": [
9816 {
9817 "block_type": "reasoning",
9818 "data": { "text": "Private planning text." }
9819 },
9820 {
9821 "block_type": "tool_use",
9822 "data": { "id": "toolu-1", "name": "peers", "args": {} }
9823 }
9824 ],
9825 "stop_reason": "end_turn",
9826 "created_at": "1970-01-01T00:00:00.010Z"
9827 }),
9828 )
9829 .expect("assistant block history frame");
9830
9831 assert_eq!(frame.kind, "interaction_complete");
9832 assert_eq!(frame.payload["result"], json!(""));
9833 assert_eq!(frame.payload["text"], json!(""));
9834 }
9835
9836 #[test]
9837 fn session_history_projection_preserves_tool_results() {
9838 let frames = frames_from_session_history_message(
9839 "runtime-a",
9840 "agent-a",
9841 "session-a",
9842 5,
9843 json!({
9844 "role": "tool_results",
9845 "results": [
9846 {
9847 "tool_use_id": "call-peers",
9848 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9849 "is_error": false
9850 }
9851 ],
9852 "created_at": "1970-01-01T00:00:00.050Z"
9853 }),
9854 );
9855
9856 assert_eq!(frames.len(), 1);
9857 let frame = &frames[0];
9858 assert_eq!(frame.kind, "tool_execution_completed");
9859 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9860 assert_eq!(
9861 frame.payload["result"],
9862 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9863 );
9864 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9865 assert_eq!(frame.timestamp_ms, 50);
9866 }
9867
9868 #[test]
9869 fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9870 let image_result = json!({
9877 "operation_id": "00000000-0000-0000-0000-0000000000aa",
9878 "terminal": { "terminal": "generated" },
9879 "images": [
9880 {
9881 "image_id": "00000000-0000-0000-0000-000000000051",
9882 "blob_ref": {
9883 "blob_id": "sha256:test-generated-image",
9884 "media_type": "image/png"
9885 },
9886 "media_type": "image/png",
9887 "width": 1024,
9888 "height": 768
9889 }
9890 ],
9891 "provider_text": { "disposition": "not_emitted" },
9892 "revised_prompt": { "disposition": "not_requested" },
9893 "native_metadata": { "provider": "not_emitted" }
9894 });
9895 serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9898 image_result.clone(),
9899 )
9900 .expect("fixture must be a valid ImageGenerationToolResult");
9901 let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9902
9903 let frames = frames_from_session_history_message(
9904 "runtime-a",
9905 "artist:singleton",
9906 "session-a",
9907 6,
9908 json!({
9909 "role": "tool_results",
9910 "results": [
9911 {
9912 "tool_use_id": "call-image",
9913 "content": result_text,
9914 "is_error": false
9915 }
9916 ],
9917 "created_at": "1970-01-01T00:00:00.060Z"
9918 }),
9919 );
9920
9921 assert!(
9923 frames
9924 .iter()
9925 .any(|frame| frame.kind == "tool_execution_completed"),
9926 "tool_execution_completed frame must still be present: {frames:#?}"
9927 );
9928 let image_frame = frames
9930 .iter()
9931 .find(|frame| frame.kind == "assistant_image")
9932 .expect("backfill must derive an assistant_image frame");
9933 assert_eq!(
9934 image_frame.payload["blob_id"],
9935 json!("sha256:test-generated-image")
9936 );
9937 assert_eq!(
9938 image_frame.payload["image_id"],
9939 json!("00000000-0000-0000-0000-000000000051")
9940 );
9941 assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9942 assert_eq!(image_frame.payload["width"], json!(1024));
9943 assert_eq!(image_frame.payload["height"], json!(768));
9944 assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9945 assert_eq!(
9946 image_frame.source.kind,
9947 ConsoleFrameSourceKind::SessionHistory
9948 );
9949 assert_eq!(image_frame.timestamp_ms, 60);
9950 }
9951
9952 #[test]
9953 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9954 let frames = frames_from_session_history_message(
9955 "runtime-a",
9956 "review:singleton",
9957 "session-a",
9958 7,
9959 json!({
9960 "role": "block_assistant",
9961 "blocks": [
9962 {
9963 "block_type": "tool_use",
9964 "data": {
9965 "id": "call-spawn",
9966 "name": "mob_spawn_member",
9967 "args": {
9968 "member_id": "review-worker-alpha",
9969 "profile": "review-worker",
9970 "initial_message": "Review Initiative Alpha and save the result."
9971 }
9972 }
9973 }
9974 ],
9975 "stop_reason": "tool_use",
9976 "created_at": "1970-01-01T00:00:00.070Z"
9977 }),
9978 );
9979
9980 assert_eq!(frames.len(), 2);
9981 let child = frames
9982 .iter()
9983 .find(|frame| frame.identity == "review-worker-alpha")
9984 .expect("spawned member initial message frame");
9985 assert_eq!(child.kind, "user_input");
9986 assert_eq!(child.timestamp_ms, 70);
9987 assert_eq!(
9988 child.payload["message"]["content"],
9989 "Review Initiative Alpha and save the result."
9990 );
9991 assert_eq!(child.payload["parent_identity"], "review:singleton");
9992 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9993 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9994 }
9995
9996 #[test]
9997 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9998 let frames = frames_from_session_history_message(
9999 "runtime-a",
10000 "review:singleton",
10001 "session-a",
10002 8,
10003 json!({
10004 "role": "block_assistant",
10005 "blocks": [
10006 {
10007 "block_type": "tool_use",
10008 "data": {
10009 "id": "call-spawn-many",
10010 "name": "mob_spawn_member",
10011 "args": {
10012 "specs": [
10013 {
10014 "agent_identity": "review-worker-alpha",
10015 "profile": "review-worker",
10016 "initial_message": "Review Initiative Alpha."
10017 },
10018 {
10019 "agent_identity": "review-worker-beta",
10020 "profile": "review-worker",
10021 "initial_message": "Review Initiative Beta."
10022 }
10023 ]
10024 }
10025 }
10026 }
10027 ],
10028 "stop_reason": "tool_use",
10029 "created_at": "1970-01-01T00:00:00.080Z"
10030 }),
10031 );
10032
10033 assert_eq!(frames.len(), 3);
10034 let child_messages = frames
10035 .iter()
10036 .filter(|frame| frame.kind == "user_input")
10037 .map(|frame| {
10038 (
10039 frame.identity.as_str(),
10040 frame.payload["message"]["content"]
10041 .as_str()
10042 .unwrap_or_default(),
10043 )
10044 })
10045 .collect::<Vec<_>>();
10046 assert_eq!(
10047 child_messages,
10048 vec![
10049 ("review-worker-alpha", "Review Initiative Alpha."),
10050 ("review-worker-beta", "Review Initiative Beta."),
10051 ]
10052 );
10053 }
10054
10055 #[tokio::test]
10056 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
10057 let aggregator = MobKitConsoleAggregator::in_memory();
10058 let frames = frames_from_session_history_message(
10059 "runtime-a",
10060 "review:singleton",
10061 "session-a",
10062 9,
10063 json!({
10064 "role": "block_assistant",
10065 "blocks": [
10066 {
10067 "block_type": "tool_use",
10068 "data": {
10069 "id": "call-spawn-many",
10070 "name": "mob_spawn_member",
10071 "args": {
10072 "specs": [
10073 {
10074 "agent_identity": "review-worker-alpha",
10075 "profile": "review-worker",
10076 "initial_message": "Review Initiative Alpha."
10077 },
10078 {
10079 "agent_identity": "review-worker-beta",
10080 "profile": "review-worker",
10081 "initial_message": "Review Initiative Beta."
10082 }
10083 ]
10084 }
10085 }
10086 }
10087 ],
10088 "stop_reason": "tool_use",
10089 "created_at": "1970-01-01T00:00:00.090Z"
10090 }),
10091 );
10092 for frame in frames {
10093 aggregator
10094 .store()
10095 .append_if_absent(frame)
10096 .await
10097 .expect("append projected session-history frame");
10098 }
10099
10100 let child_page = aggregator
10101 .query_timeline(ConsoleTimelineQuery {
10102 identity: Some("review-worker-alpha".to_string()),
10103 limit: 20,
10104 ..ConsoleTimelineQuery::default()
10105 })
10106 .await
10107 .expect("query child timeline");
10108 assert_eq!(child_page.frames.len(), 1);
10109 assert_eq!(child_page.frames[0].kind, "user_input");
10110 assert_eq!(
10111 child_page.frames[0].payload["message"]["content"],
10112 "Review Initiative Alpha."
10113 );
10114 assert_eq!(
10115 child_page.frames[0].payload["parent_identity"],
10116 "review:singleton"
10117 );
10118
10119 let parent_page = aggregator
10120 .query_timeline(ConsoleTimelineQuery {
10121 identity: Some("review:singleton".to_string()),
10122 limit: 20,
10123 ..ConsoleTimelineQuery::default()
10124 })
10125 .await
10126 .expect("query parent timeline");
10127 assert!(
10128 parent_page
10129 .frames
10130 .iter()
10131 .all(|frame| frame.identity == "review:singleton"),
10132 "child initial messages must not leak into parent timeline: {:#?}",
10133 parent_page.frames
10134 );
10135 }
10136
10137 #[test]
10138 fn session_history_frames_carry_transcript_interaction_identity() {
10139 let interaction = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
10143 let run = "886313e1-3b8a-5372-9b90-0c9aee199e5d";
10144 let frames = frames_from_session_history_message_with_namespace(
10145 "runtime-a",
10146 "helper",
10147 "",
10148 "session-a",
10149 3,
10150 json!({
10151 "role": "user",
10152 "content": "please ack",
10153 "identity": { "interaction_id": interaction, "run_id": run },
10154 "created_at": "1970-01-01T00:00:00.100Z"
10155 }),
10156 );
10157 assert_eq!(frames.len(), 1, "{frames:#?}");
10158 assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
10159 assert_eq!(frames[0].run_id.as_deref(), Some(run));
10160
10161 let frames = frames_from_session_history_message_with_namespace(
10162 "runtime-a",
10163 "helper",
10164 "",
10165 "session-a",
10166 4,
10167 json!({
10168 "role": "block_assistant",
10169 "blocks": [ { "block_type": "text", "data": { "text": "ACK" } } ],
10170 "identity": { "interaction_id": interaction },
10171 "stop_reason": "end_turn",
10172 "created_at": "1970-01-01T00:00:00.200Z"
10173 }),
10174 );
10175 assert!(!frames.is_empty(), "assistant history frame expected");
10176 assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
10177 assert_eq!(frames[0].run_id, None);
10178
10179 let frames = frames_from_session_history_message_with_namespace(
10182 "runtime-a",
10183 "helper",
10184 "",
10185 "session-a",
10186 5,
10187 json!({
10188 "role": "user",
10189 "content": "legacy row",
10190 "created_at": "1970-01-01T00:00:00.300Z"
10191 }),
10192 );
10193 assert_eq!(frames.len(), 1);
10194 assert_eq!(frames[0].interaction_id, None);
10195 }
10196
10197 #[test]
10198 fn identity_first_interaction_ids_are_deterministic_uuids() {
10199 let a = identity_first_interaction_uuid("send:key:1");
10200 let b = identity_first_interaction_uuid("send:key:1");
10201 let c = identity_first_interaction_uuid("send:key:2");
10202 assert_eq!(a, b, "idempotent retries must mint the same id");
10203 assert_ne!(a, c);
10204 assert!(
10205 a.parse::<uuid::Uuid>().is_ok(),
10206 "identity-first interaction ids must be UUID-form (the console's \
10207 authoritative twin-identity scheme): {a}"
10208 );
10209 }
10210
10211 #[test]
10212 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
10213 let frames = frames_from_session_history_message_with_namespace(
10214 "runtime-a",
10215 "test/review:singleton",
10216 "test",
10217 "session-a",
10218 10,
10219 json!({
10220 "role": "block_assistant",
10221 "blocks": [
10222 {
10223 "block_type": "tool_use",
10224 "data": {
10225 "id": "call-spawn-many",
10226 "name": "mob_spawn_member",
10227 "args": {
10228 "specs": [
10229 {
10230 "agent_identity": "review-worker-alpha",
10231 "profile": "review-worker",
10232 "initial_message": "Review Initiative Alpha."
10233 },
10234 {
10235 "agent_identity": "test/review-worker-beta",
10236 "profile": "review-worker",
10237 "initial_message": "Review Initiative Beta."
10238 }
10239 ]
10240 }
10241 }
10242 }
10243 ],
10244 "stop_reason": "tool_use",
10245 "created_at": "1970-01-01T00:00:00.100Z"
10246 }),
10247 );
10248
10249 let child_identities = frames
10250 .iter()
10251 .filter(|frame| frame.kind == "user_input")
10252 .map(|frame| frame.identity.as_str())
10253 .collect::<Vec<_>>();
10254 assert_eq!(
10255 child_identities,
10256 vec!["test/review-worker-alpha", "test/review-worker-beta"]
10257 );
10258 }
10259
10260 #[tokio::test]
10261 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
10262 let aggregator = MobKitConsoleAggregator::in_memory();
10263 aggregator
10264 .inner
10265 .identity_read_model
10266 .replace(vec![
10267 identity_record_for_test("test/review:singleton"),
10268 identity_record_for_test("test/review-worker-alpha"),
10269 ])
10270 .await;
10271 let frames = frames_from_session_history_message_with_namespace(
10272 "runtime-a",
10273 "test/review:singleton",
10274 "test",
10275 "session-a",
10276 11,
10277 json!({
10278 "role": "block_assistant",
10279 "blocks": [
10280 {
10281 "block_type": "tool_use",
10282 "data": {
10283 "id": "call-spawn",
10284 "name": "mob_spawn_member",
10285 "args": {
10286 "agent_identity": "review-worker-alpha",
10287 "profile": "review-worker",
10288 "initial_message": "Review Initiative Alpha."
10289 }
10290 }
10291 }
10292 ],
10293 "stop_reason": "tool_use",
10294 "created_at": "1970-01-01T00:00:00.110Z"
10295 }),
10296 );
10297 for frame in frames {
10298 aggregator
10299 .store()
10300 .append_if_absent(frame)
10301 .await
10302 .expect("append projected namespaced session-history frame");
10303 }
10304
10305 let namespaced_child_page = aggregator
10306 .query_timeline(ConsoleTimelineQuery {
10307 identity: Some("test/review-worker-alpha".to_string()),
10308 limit: 20,
10309 ..ConsoleTimelineQuery::default()
10310 })
10311 .await
10312 .expect("query namespaced child timeline");
10313 assert_eq!(namespaced_child_page.frames.len(), 1);
10314 assert_eq!(
10315 namespaced_child_page.frames[0].identity,
10316 "test/review-worker-alpha"
10317 );
10318 assert_eq!(
10319 namespaced_child_page.frames[0].payload["message"]["content"],
10320 "Review Initiative Alpha."
10321 );
10322
10323 let raw_child_page = aggregator
10324 .query_timeline(ConsoleTimelineQuery {
10325 identity: Some("review-worker-alpha".to_string()),
10326 limit: 20,
10327 ..ConsoleTimelineQuery::default()
10328 })
10329 .await
10330 .expect("query raw child timeline");
10331 assert!(
10332 raw_child_page.frames.is_empty(),
10333 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
10334 raw_child_page.frames
10335 );
10336 }
10337
10338 #[test]
10339 fn namespace_helpers_preserve_namespace_named_member_identity() {
10340 assert_eq!(apply_namespace("test", "test"), "test/test");
10341 assert_eq!(
10342 strip_namespace("test/test", "test").as_deref(),
10343 Some("test")
10344 );
10345 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
10346 assert_eq!(
10347 strip_namespace("test/worker", "test").as_deref(),
10348 Some("worker")
10349 );
10350 assert_eq!(strip_namespace("test", "test"), None);
10351 }
10352
10353 #[test]
10354 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
10355 assert!(!member_id_matches_durable_identity(
10356 "rt:review:singleton:0",
10357 "review:singleton"
10358 ));
10359 assert!(!member_id_matches_durable_identity(
10360 "review:singleton:gen1",
10361 "review:singleton"
10362 ));
10363 assert!(!member_id_matches_durable_identity(
10364 "review:singleton:1",
10365 "review:singleton"
10366 ));
10367 assert!(!member_id_matches_durable_identity(
10368 "rt:review:singleton:qa:0",
10369 "review:singleton"
10370 ));
10371 assert!(!member_id_matches_durable_identity(
10372 "review:singleton:qa",
10373 "review:singleton"
10374 ));
10375 }
10376
10377 #[test]
10378 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
10379 let frame = frame_from_session_history_message(
10380 "runtime-a",
10381 "agent-a",
10382 "session-a",
10383 0,
10384 json!({
10385 "role": "user",
10386 "content": "hello",
10387 "created_at": "2026-05-12T05:00:06.564227Z"
10388 }),
10389 )
10390 .expect("user history frame");
10391
10392 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
10393 }
10394
10395 async fn wait_for_kickoff_frames(
10396 aggregator: &MobKitConsoleAggregator,
10397 identity: &str,
10398 timeout: Duration,
10399 ) -> Vec<ConsoleFrame> {
10400 let deadline = Instant::now() + timeout;
10401 let mut observed = Vec::new();
10402 while Instant::now() < deadline {
10403 let page = aggregator
10404 .query_timeline(ConsoleTimelineQuery {
10405 identity: Some(identity.to_string()),
10406 limit: 50,
10407 ..ConsoleTimelineQuery::default()
10408 })
10409 .await
10410 .expect("query timeline");
10411 observed = page.frames;
10412 if observed.iter().any(|frame| frame.kind == "user_input") {
10413 break;
10414 }
10415 tokio::time::sleep(Duration::from_millis(25)).await;
10416 }
10417 observed
10418 }
10419
10420 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10421 async fn spawn_registry_primes_access_lineage() {
10422 let runtime = build_empty_runtime("spawn-access-lineage-test").await;
10423 let sink = runtime
10424 .mob_runtime()
10425 .console_spawn_sink()
10426 .expect("unified runtime installs the console spawn sink at bootstrap");
10427
10428 runtime
10429 .spawn(SpawnMemberSpec::from_wire(
10430 "worker".to_string(),
10431 "worker-3".to_string(),
10432 None,
10433 None,
10434 None,
10435 ))
10436 .await
10437 .expect("member spawns");
10438 sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
10439 mob_id: Some("spawn-access-lineage-test".to_string()),
10440 member_id: "worker-3".to_string(),
10441 identity: "worker-3".to_string(),
10442 initial_message: None,
10443 labels: BTreeMap::new(),
10444 spawned_by: Some("ops-lead".to_string()),
10445 via_tool: "mob_spawn_member".to_string(),
10446 })
10447 .await;
10448
10449 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
10450 enabled: true,
10451 admins: vec!["root@example.test".to_string()],
10452 groups: BTreeMap::new(),
10453 rules: vec![crate::access::AccessRule {
10454 id: "bob-ops-lead".to_string(),
10455 subjects: vec!["bob@example.test".to_string()],
10456 actions: vec!["agent.view".to_string()],
10457 agents: vec!["ops-lead".to_string()],
10458 ..crate::access::AccessRule::default()
10459 }],
10460 })
10461 .expect("controller");
10462
10463 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
10464 .await;
10465
10466 let bob = controller.view_for_subject(Some("bob@example.test"));
10467 assert!(
10468 bob.can_view_agent("worker-3"),
10469 "spawn registry lineage must let bob see the member ops-lead spawned"
10470 );
10471 assert!(
10472 !bob.can_view_agent("unrelated-agent"),
10473 "agents outside the granted lineage stay denied"
10474 );
10475
10476 let _ = runtime.mob_handle().stop().await;
10477 }
10478
10479 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10480 async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
10481 let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
10482
10483 let mut imposter = SpawnMemberSpec::from_wire(
10486 "worker".to_string(),
10487 "imposter".to_string(),
10488 None,
10489 None,
10490 None,
10491 );
10492 imposter.labels = Some(BTreeMap::from([(
10493 "spawned_by".to_string(),
10494 "ops-lead".to_string(),
10495 )]));
10496 runtime.spawn(imposter).await.expect("imposter spawns");
10497
10498 let mut evader = SpawnMemberSpec::from_wire(
10499 "worker".to_string(),
10500 "evader".to_string(),
10501 None,
10502 None,
10503 None,
10504 );
10505 evader.labels = Some(BTreeMap::from([(
10506 "spawned_by".to_string(),
10507 "secret-lead".to_string(),
10508 )]));
10509 runtime.spawn(evader).await.expect("evader spawns");
10510
10511 let aggregator = MobKitConsoleAggregator::in_memory();
10513 aggregator.register_runtime_handles_with_policy(
10514 "runtime-a",
10515 "",
10516 runtime.mob_runtime().clone(),
10517 None,
10518 runtime.console_events(),
10519 Arc::new(AllowAllConsoleVisibilityPolicy),
10520 );
10521 let identities = aggregator
10522 .list_identities_fresh()
10523 .await
10524 .expect("list identities");
10525 let imposter_record = identities
10526 .iter()
10527 .find(|record| record.identity == "imposter")
10528 .expect("imposter listed");
10529 assert!(
10530 !imposter_record.labels.contains_key("spawned_by"),
10531 "identity records must drop roster-claimed lineage when the spawn registry has none"
10532 );
10533
10534 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
10535 enabled: true,
10536 admins: vec!["root@example.test".to_string()],
10537 groups: BTreeMap::new(),
10538 rules: vec![
10539 crate::access::AccessRule {
10540 id: "bob-ops-lead".to_string(),
10541 subjects: vec!["bob@example.test".to_string()],
10542 actions: vec!["agent.view".to_string()],
10543 agents: vec!["ops-lead".to_string()],
10544 ..crate::access::AccessRule::default()
10545 },
10546 crate::access::AccessRule {
10547 id: "carol-view-all".to_string(),
10548 subjects: vec!["carol@example.test".to_string()],
10549 actions: vec!["agent.view".to_string()],
10550 agents: vec!["*".to_string()],
10551 ..crate::access::AccessRule::default()
10552 },
10553 crate::access::AccessRule {
10554 id: "hide-secret-lead".to_string(),
10555 effect: crate::access::AccessEffect::Deny,
10556 actions: vec!["agent.*".to_string()],
10557 agents: vec!["secret-lead".to_string()],
10558 ..crate::access::AccessRule::default()
10559 },
10560 ],
10561 })
10562 .expect("controller");
10563
10564 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
10565 .await;
10566
10567 let bob = controller.view_for_subject(Some("bob@example.test"));
10568 assert!(
10569 !bob.can_view_agent("imposter"),
10570 "a roster-claimed spawned_by must not mint inherited visibility"
10571 );
10572 let carol = controller.view_for_subject(Some("carol@example.test"));
10573 assert!(
10574 carol.can_view_agent("evader"),
10575 "a roster-claimed spawned_by must not let a member hide behind a denied parent"
10576 );
10577
10578 let _ = runtime.mob_handle().stop().await;
10579 }
10580
10581 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10582 async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
10583 let runtime = build_empty_runtime("agent-spawn-console-test").await;
10584
10585 let sink = runtime
10589 .mob_runtime()
10590 .console_spawn_sink()
10591 .expect("unified runtime installs the console spawn sink at bootstrap");
10592
10593 runtime
10597 .spawn(SpawnMemberSpec::from_wire(
10598 "worker".to_string(),
10599 "worker-3".to_string(),
10600 None,
10601 None,
10602 None,
10603 ))
10604 .await
10605 .expect("member spawns");
10606
10607 let seed = crate::console_spawn::ConsoleSpawnSeed {
10608 mob_id: Some("agent-spawn-console-test".to_string()),
10609 member_id: "worker-3".to_string(),
10610 identity: "worker-3".to_string(),
10611 initial_message: Some(json!("Kick off the person hunt")),
10612 labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
10613 spawned_by: Some("ops-lead".to_string()),
10614 via_tool: "mob_spawn_member".to_string(),
10615 };
10616 sink.project_spawned_member(&seed).await;
10617 sink.project_spawned_member(&seed).await;
10619
10620 let aggregator = MobKitConsoleAggregator::in_memory();
10621 aggregator.register_runtime_handles_with_policy(
10622 "runtime-a",
10623 "test",
10624 runtime.mob_runtime().clone(),
10625 None,
10626 runtime.console_events(),
10627 Arc::new(AllowAllConsoleVisibilityPolicy),
10628 );
10629
10630 let frames =
10631 wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
10632 let kickoffs = frames
10633 .iter()
10634 .filter(|frame| frame.kind == "user_input")
10635 .collect::<Vec<_>>();
10636 assert_eq!(
10637 kickoffs.len(),
10638 1,
10639 "member timeline must contain exactly one kickoff frame; got {frames:#?}"
10640 );
10641 assert_eq!(
10642 kickoffs[0].payload["content"][0]["text"],
10643 "Kick off the person hunt"
10644 );
10645
10646 runtime
10649 .project_console_event_from_unified(&crate::types::EventEnvelope {
10650 event_id: "evt-live-worker-3".to_string(),
10651 source: "test".to_string(),
10652 timestamp_ms: current_time_ms(),
10653 event: crate::types::UnifiedEvent::Agent {
10654 agent_id: "worker-3:0".to_string(),
10655 event_type: "text_delta".to_string(),
10656 payload: Some(json!({ "delta": "on it" })),
10657 },
10658 })
10659 .await;
10660
10661 let deadline = Instant::now() + Duration::from_secs(5);
10662 let mut live_frame = None;
10663 while Instant::now() < deadline && live_frame.is_none() {
10664 let page = aggregator
10665 .query_timeline(ConsoleTimelineQuery {
10666 identity: Some("test/worker-3".to_string()),
10667 limit: 50,
10668 ..ConsoleTimelineQuery::default()
10669 })
10670 .await
10671 .expect("query timeline");
10672 live_frame = page
10673 .frames
10674 .into_iter()
10675 .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
10676 if live_frame.is_none() {
10677 tokio::time::sleep(Duration::from_millis(25)).await;
10678 }
10679 }
10680 let live_frame = live_frame.expect("live runtime event lands in the member chat");
10681 assert_eq!(live_frame.identity, "test/worker-3");
10682
10683 let identities = aggregator
10686 .list_identities_fresh()
10687 .await
10688 .expect("list identities");
10689 let record = identities
10690 .iter()
10691 .find(|record| record.identity == "test/worker-3")
10692 .unwrap_or_else(|| {
10693 panic!("spawned member missing from identity list: {identities:#?}")
10694 });
10695 assert_eq!(
10696 record.labels.get("group").map(String::as_str),
10697 Some("workers")
10698 );
10699 assert_eq!(
10700 record.labels.get("spawned_by").map(String::as_str),
10701 Some("ops-lead")
10702 );
10703
10704 let _ = runtime.mob_handle().stop().await;
10705 }
10706}