1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images,
24 is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32 SourceIngestionTransition,
33};
34pub use store::{
35 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36 SqliteConsoleLogStore,
37};
38pub use types::{
39 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59 CachedOnly,
60 IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65 inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70 pub session_history_backfill_enabled: bool,
71 pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75 fn default() -> Self {
76 Self {
77 session_history_backfill_enabled: true,
78 max_concurrent_session_backfills: 16,
79 }
80 }
81}
82
83struct AggregatorInner {
84 store: Arc<dyn ConsoleLogStore>,
85 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89 session_backfill_permits: Arc<Semaphore>,
90 identity_read_model: ConsoleIdentityReadModel,
91 options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97 refresh_lock: Arc<tokio::sync::Mutex<()>>,
98 primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102 fn default() -> Self {
103 Self {
104 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106 primed: Arc::new(AtomicBool::new(false)),
107 }
108 }
109}
110
111impl ConsoleIdentityReadModel {
112 async fn snapshot(
113 &self,
114 inner: Arc<AggregatorInner>,
115 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116 if !self.primed.load(Ordering::Acquire) {
117 Box::pin(self.prime_now(inner)).await?;
118 }
119 Ok(self.inner.read().await.clone())
120 }
121
122 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123 self.inner.read().await.clone()
124 }
125
126 async fn refresh_now_if_idle(
127 &self,
128 inner: Arc<AggregatorInner>,
129 mode: IdentityCollectionMode,
130 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132 return None;
133 };
134 let result = Box::pin(collect_identity_records(&inner, mode)).await;
135 drop(guard);
136 match result {
137 Ok(identities) => {
138 self.replace(identities.clone()).await;
139 Some(Ok(identities))
140 }
141 Err(err) => Some(Err(err)),
142 }
143 }
144
145 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146 if self.primed.load(Ordering::Acquire) {
147 return Ok(());
148 }
149 let _guard = self.refresh_lock.clone().lock_owned().await;
150 if self.primed.load(Ordering::Acquire) {
151 return Ok(());
152 }
153 let identities = Box::pin(collect_identity_records(
154 &inner,
155 IdentityCollectionMode::CachedOnly,
156 ))
157 .await?;
158 *self.inner.write().await = identities;
159 self.primed.store(true, Ordering::Release);
160 Ok(())
161 }
162
163 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
164 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
165 return;
166 };
167 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
168 return;
169 };
170 let read_model = self.clone();
171 runtime_handle.spawn(async move {
172 let _guard = guard;
173 match Box::pin(collect_identity_records(
174 &inner,
175 IdentityCollectionMode::IncludeLiveMembers,
176 ))
177 .await
178 {
179 Ok(identities) => {
180 *read_model.inner.write().await = identities;
181 read_model.primed.store(true, Ordering::Release);
182 }
183 Err(err) => {
184 tracing::warn!(error = %err, "console identity read-model refresh failed");
185 }
186 }
187 });
188 }
189
190 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
191 *self.inner.write().await = identities;
192 self.primed.store(true, Ordering::Release);
193 }
194}
195
196#[derive(Clone)]
197struct RuntimeEntry {
198 runtime_key: String,
199 identity_namespace: String,
200 runtime: MobRuntime,
201 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
202 console_events: ConsoleEventStore,
203 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
204}
205
206#[derive(Clone)]
207struct ResolvedConsoleMember {
208 entry: RuntimeEntry,
209 handle: MobHandle,
210 member: MobMemberListEntry,
211 source_mob_id: String,
212 runtime_identity: String,
213}
214
215fn console_member_for_resolved_member(
216 resolved: &ResolvedConsoleMember,
217 record: &ConsoleIdentityRecord,
218) -> ConsoleMember {
219 ConsoleMember {
220 agent_identity: crate::member_comms_id::runtime_alias_str(
221 resolved.member.agent_identity.as_str(),
222 )
223 .into_owned(),
224 role: resolved.member.role.to_string(),
225 state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
226 model_capabilities: Default::default(),
227 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
228 session_id: record.session_id.clone(),
229 wired_to: resolved
230 .member
231 .wired_to
232 .iter()
233 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
234 .collect(),
235 labels: record.labels.clone(),
236 }
237}
238
239async fn resolved_member_visible(
240 resolved: &ResolvedConsoleMember,
241 record: &ConsoleIdentityRecord,
242) -> bool {
243 if !raw_resolved_member_visible(resolved, record) {
244 return false;
245 }
246 !Box::pin(live_record_shadowed_by_hidden_durable_binding(
247 resolved, record,
248 ))
249 .await
250}
251
252fn raw_resolved_member_visible(
253 resolved: &ResolvedConsoleMember,
254 record: &ConsoleIdentityRecord,
255) -> bool {
256 resolved.entry.visibility_policy.identity_visible(record)
257 && resolved
258 .entry
259 .visibility_policy
260 .member_visible(&console_member_for_resolved_member(resolved, record))
261}
262
263pub trait ConsoleVisibilityPolicy: Send + Sync {
264 fn include_implicit_delegate_members(&self) -> bool {
265 true
266 }
267
268 fn member_visible(&self, _member: &ConsoleMember) -> bool {
269 true
270 }
271
272 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
273 true
274 }
275
276 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
277 true
278 }
279
280 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
281 None
282 }
283}
284
285#[derive(Debug, Default)]
286pub struct AllowAllConsoleVisibilityPolicy;
287
288impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
289
290#[derive(Debug, Default)]
291pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
292
293impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
294 fn include_implicit_delegate_members(&self) -> bool {
295 false
296 }
297
298 fn member_visible(&self, member: &ConsoleMember) -> bool {
299 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
300 }
301
302 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
303 !is_implicit_delegate_member(
304 record
305 .labels
306 .get("role")
307 .map(String::as_str)
308 .unwrap_or_default(),
309 &record.labels,
310 )
311 }
312}
313
314#[derive(Clone)]
315pub struct ConsoleRuntimeRegistration {
316 pub runtime_key: String,
317 pub runtime: Arc<UnifiedRuntime>,
318 pub identity_namespace: String,
319 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
320}
321
322impl MobKitConsoleAggregator {
323 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
324 Self::new_with_options(store, ConsoleAggregatorOptions::default())
325 }
326
327 pub fn new_with_options(
328 store: Arc<dyn ConsoleLogStore>,
329 mut options: ConsoleAggregatorOptions,
330 ) -> Self {
331 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
332 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
333 Self {
334 inner: Arc::new(AggregatorInner {
335 store,
336 runtimes: RwLock::new(BTreeMap::new()),
337 event_tx,
338 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
339 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
340 session_backfill_permits: Arc::new(Semaphore::new(
341 options.max_concurrent_session_backfills,
342 )),
343 identity_read_model: ConsoleIdentityReadModel::default(),
344 options,
345 }),
346 }
347 }
348
349 pub fn in_memory() -> Self {
350 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
351 }
352
353 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
354 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
355 }
356
357 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
358 self.inner.event_tx.subscribe()
359 }
360
361 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
362 self.inner.store.clone()
363 }
364
365 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
366 let identity_runtime = registration.runtime.identity_runtime().cloned();
367 self.register_runtime_handles_with_policy(
368 registration.runtime_key,
369 registration.identity_namespace,
370 registration.runtime.mob_runtime().clone(),
371 identity_runtime,
372 registration.runtime.console_events(),
373 registration.visibility_policy,
374 );
375 }
376
377 pub(crate) fn register_runtime_handles_with_policy(
378 &self,
379 runtime_key: impl Into<String>,
380 identity_namespace: impl Into<String>,
381 runtime: MobRuntime,
382 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
383 console_events: ConsoleEventStore,
384 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
385 ) {
386 let runtime_key = runtime_key.into();
387 let identity_namespace = identity_namespace.into();
388 let entry = RuntimeEntry {
389 runtime_key: runtime_key.clone(),
390 identity_namespace,
391 runtime,
392 identity_runtime,
393 console_events: console_events.clone(),
394 visibility_policy,
395 };
396 if let Ok(mut runtimes) = self.inner.runtimes.write() {
397 runtimes.insert(runtime_key.clone(), entry);
398 }
399 self.inner
400 .identity_read_model
401 .refresh_soon(self.inner.clone());
402 let inner = self.inner.clone();
403 let events_for_live = console_events.clone();
404 let events_for_live_recovery = console_events.clone();
405 let runtime_key_for_live = runtime_key.clone();
406 tokio::spawn(async move {
407 let mut rx = events_for_live.subscribe();
408 loop {
409 match rx.recv().await {
410 Ok(envelope) => {
411 let _ =
412 project_console_event(inner.clone(), &runtime_key_for_live, envelope)
413 .await;
414 }
415 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
416 let _ = recover_lagged_source_events(
417 inner.clone(),
418 &runtime_key_for_live,
419 &events_for_live_recovery,
420 )
421 .await;
422 }
423 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
424 }
425 }
426 });
427
428 let inner = self.inner.clone();
429 let events_for_replay = console_events;
430 let runtime_key_for_replay = runtime_key;
431 tokio::spawn(async move {
432 let mut ingestion_state = SourceIngestionState::Registered;
433 if let Ok((next, _effects)) =
434 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
435 {
436 ingestion_state = next;
437 }
438 if let Ok(events) = events_for_replay.replay_all(None).await {
439 for envelope in events {
440 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
441 .await;
442 }
443 }
444 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
445 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
446 if let Ok((next, _effects)) =
447 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
448 {
449 ingestion_state = next;
450 }
451 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
452 });
453 }
454
455 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
456 if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
457 self.inner.clone(),
458 IdentityCollectionMode::IncludeLiveMembers,
459 ))
460 .await
461 {
462 let identities = identities?;
463 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464 return Ok(identities);
465 }
466 self.inner
467 .identity_read_model
468 .refresh_soon(self.inner.clone());
469 let identities =
470 Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
471 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
472 Ok(identities)
473 }
474
475 #[cfg(test)]
476 pub(crate) async fn list_identities_fresh(
477 &self,
478 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
479 let _guard = self
480 .inner
481 .identity_read_model
482 .refresh_lock
483 .clone()
484 .lock_owned()
485 .await;
486 let identities =
487 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
488 .await?;
489 self.inner
490 .identity_read_model
491 .replace(identities.clone())
492 .await;
493 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
494 Ok(identities)
495 }
496
497 pub async fn inspect_identity(
498 &self,
499 identity: &str,
500 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
501 let entries = self
502 .inner
503 .runtimes
504 .read()
505 .map_err(|_| runtime_registry_lock_error())?
506 .values()
507 .cloned()
508 .collect::<Vec<_>>();
509 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
510 let mut durable_matches = Vec::new();
511 if !requested_identity_has_runtime_member_alias(identity, &entries) {
512 for entry in &entries {
513 let Some(identity_runtime) = entry.identity_runtime.clone() else {
514 continue;
515 };
516 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
517 {
518 let Ok(parsed_identity) =
519 crate::identity_first::AgentIdentity::parse(&raw_identity)
520 else {
521 continue;
522 };
523 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
524 continue;
525 };
526 let mut record = identity_record_for_status(entry, &status);
527 let topology_peers =
528 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
529 record.topology_peers = topology_peers
530 .get(&record.identity)
531 .cloned()
532 .unwrap_or_default();
533 if Box::pin(console_identity_record_visible(entry, &record)).await {
534 durable_matches.push((
535 entry.clone(),
536 identity_runtime.clone(),
537 parsed_identity,
538 record,
539 ));
540 }
541 }
542 }
543 }
544 if durable_matches.len() > 1 {
545 let candidates = durable_matches
546 .iter()
547 .map(|(_, _, _, record)| record.runtime_member_id.clone())
548 .collect::<Vec<_>>()
549 .join(", ");
550 return Err(format!(
551 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
552 )
553 .into());
554 }
555 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
556 durable_matches.into_iter().next()
557 {
558 let durable_live_records = Box::pin(self.live_records_for_durable_record(
559 &entry,
560 identity,
561 &record,
562 &live_records,
563 ))
564 .await;
565 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
566 &entry,
567 &durable_live_records,
568 ))
569 .await;
570 if let Some(ambiguous_error) =
571 ambiguous_live_alias_error(identity, &visible_durable_live_records)
572 {
573 return Err(ambiguous_error.into());
574 }
575 if let Some(rebound_record) = self_heal_durable_session_mismatch(
576 &entry,
577 identity_runtime.as_ref(),
578 &parsed_identity,
579 identity,
580 &record,
581 &durable_live_records,
582 )
583 .await
584 .map_err(ConsoleLogError::from)?
585 {
586 record = rebound_record;
587 }
588 if let Some(stale_error) =
589 stale_durable_record_error(identity, &record, &durable_live_records)
590 {
591 return Err(stale_error.into());
592 }
593 if let Some(session_id) = record.session_id.clone() {
594 spawn_session_history_backfill_target(
595 self.inner.clone(),
596 SessionBackfillTarget {
597 entry,
598 record: record.clone(),
599 session_id,
600 },
601 false,
602 );
603 }
604 return Ok(Some(ConsoleIdentityInspection {
605 peers: record.topology_peers.clone(),
606 identity: record,
607 }));
608 }
609 let mut runtime_id_matches = Vec::new();
610 for entry in &entries {
611 let Some(identity_runtime) = entry.identity_runtime.clone() else {
612 continue;
613 };
614 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
615 continue;
616 };
617 let Some(status) = identity_runtime
618 .statuses()
619 .await
620 .into_iter()
621 .find(|status| {
622 status
623 .agent_runtime_id
624 .as_ref()
625 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
626 })
627 else {
628 continue;
629 };
630 let mut record = identity_record_for_status(entry, &status);
631 let topology_peers =
632 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
633 record.topology_peers = topology_peers
634 .get(&record.identity)
635 .cloned()
636 .unwrap_or_default();
637 if Box::pin(console_identity_record_visible(entry, &record)).await {
638 runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
639 }
640 }
641 if runtime_id_matches.len() > 1 {
642 let candidates = runtime_id_matches
643 .iter()
644 .map(|(_, _, _, record)| record.identity.clone())
645 .collect::<Vec<_>>()
646 .join(", ");
647 return Err(format!(
648 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
649 )
650 .into());
651 }
652 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
653 runtime_id_matches.into_iter().next()
654 {
655 let durable_live_records = Box::pin(self.live_records_for_durable_record(
656 &entry,
657 identity,
658 &record,
659 &live_records,
660 ))
661 .await;
662 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
663 &entry,
664 &durable_live_records,
665 ))
666 .await;
667 if let Some(ambiguous_error) =
668 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
669 {
670 return Err(ambiguous_error.into());
671 }
672 if let Some(rebound_record) = self_heal_durable_session_mismatch(
673 &entry,
674 identity_runtime.as_ref(),
675 &parsed_identity,
676 identity,
677 &record,
678 &durable_live_records,
679 )
680 .await
681 .map_err(ConsoleLogError::from)?
682 {
683 record = rebound_record;
684 }
685 if let Some(stale_error) =
686 stale_durable_record_error(identity, &record, &durable_live_records)
687 {
688 return Err(stale_error.into());
689 }
690 if let Some(session_id) = record.session_id.clone() {
691 spawn_session_history_backfill_target(
692 self.inner.clone(),
693 SessionBackfillTarget {
694 entry: entry.clone(),
695 record: record.clone(),
696 session_id,
697 },
698 false,
699 );
700 }
701 return Ok(Some(ConsoleIdentityInspection {
702 peers: record.topology_peers.clone(),
703 identity: record,
704 }));
705 }
706
707 let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
708 if live_matches.len() > 1 {
709 let candidates = live_matches
710 .iter()
711 .map(|resolved| resolved.runtime_identity.clone())
712 .collect::<Vec<_>>()
713 .join(", ");
714 return Err(format!(
715 "ambiguous live identity alias {identity}: candidates [{candidates}]"
716 )
717 .into());
718 }
719 if let Some(resolved) = live_matches.into_iter().next() {
720 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
721 return Ok(None);
722 };
723 if let Some(stale_error) =
724 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
725 {
726 return Err(stale_error.into());
727 }
728 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
729 && let Ok(parsed_identity) =
730 crate::identity_first::AgentIdentity::parse(&record.identity)
731 && let Ok(status) = identity_runtime.status(&parsed_identity).await
732 {
733 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
734 let durable_live_records = Box::pin(self.live_records_for_durable_record(
735 &resolved.entry,
736 identity,
737 &durable_record,
738 &live_records,
739 ))
740 .await;
741 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
742 &resolved.entry,
743 &durable_live_records,
744 ))
745 .await;
746 if let Some(ambiguous_error) =
747 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
748 {
749 return Err(ambiguous_error.into());
750 }
751 if let Some(rebound_record) = self_heal_durable_session_mismatch(
752 &resolved.entry,
753 identity_runtime.as_ref(),
754 &parsed_identity,
755 identity,
756 &durable_record,
757 &durable_live_records,
758 )
759 .await
760 .map_err(ConsoleLogError::from)?
761 {
762 durable_record = rebound_record;
763 }
764 if let Some(stale_error) =
765 stale_durable_record_error(identity, &durable_record, &durable_live_records)
766 {
767 return Err(stale_error.into());
768 }
769 if status
770 .agent_runtime_id
771 .as_ref()
772 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
773 {
774 return Ok(None);
775 }
776 }
777 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
778 return Ok(None);
779 }
780 if let Some(session_id) = record.session_id.clone() {
781 spawn_session_history_backfill_target(
782 self.inner.clone(),
783 SessionBackfillTarget {
784 entry: resolved.entry.clone(),
785 record: record.clone(),
786 session_id,
787 },
788 false,
789 );
790 }
791 let peers = resolved
792 .member
793 .wired_to
794 .iter()
795 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
796 .collect();
797 return Ok(Some(ConsoleIdentityInspection {
798 identity: record,
799 peers,
800 }));
801 }
802
803 Ok(None)
804 }
805
806 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
807 let entries = self
808 .inner
809 .runtimes
810 .read()
811 .map_err(|_| runtime_registry_lock_error())?
812 .clone();
813 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
814 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
815 let mut durable_matches = Vec::new();
816 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
817 for entry in entries.values() {
818 let Some(identity_runtime) = entry.identity_runtime.clone() else {
819 continue;
820 };
821 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
822 {
823 let Ok(parsed_identity) =
824 crate::identity_first::AgentIdentity::parse(&raw_identity)
825 else {
826 continue;
827 };
828 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
829 continue;
830 };
831 let record = identity_record_for_status(entry, &status);
832 if !Box::pin(console_identity_record_visible(entry, &record)).await {
833 continue;
834 }
835 durable_matches.push((
836 entry.clone(),
837 identity_runtime.clone(),
838 parsed_identity,
839 record,
840 ));
841 }
842 }
843 }
844 if durable_matches.len() > 1 {
845 let candidates = durable_matches
846 .iter()
847 .map(|(_, _, identity, _)| identity.as_str().to_string())
848 .collect::<Vec<_>>()
849 .join(", ");
850 return Err(format!(
851 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
852 )
853 .into());
854 }
855 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
856 durable_matches.into_iter().next()
857 {
858 let durable_live_records = Box::pin(self.live_records_for_durable_record(
859 &entry,
860 identity,
861 &record,
862 &live_records,
863 ))
864 .await;
865 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
866 &entry,
867 &durable_live_records,
868 ))
869 .await;
870 if let Some(ambiguous_error) =
871 ambiguous_live_alias_error(identity, &visible_durable_live_records)
872 {
873 return Err(ambiguous_error.into());
874 }
875 if let Some(rebound_record) = self_heal_durable_session_mismatch(
876 &entry,
877 identity_runtime.as_ref(),
878 &parsed_identity,
879 identity,
880 &record,
881 &durable_live_records,
882 )
883 .await
884 .map_err(ConsoleLogError::from)?
885 {
886 record = rebound_record;
887 }
888 if let Some(stale_error) =
889 stale_durable_record_error(identity, &record, &durable_live_records)
890 {
891 return Err(stale_error.into());
892 }
893 identity_runtime
894 .retire(&parsed_identity)
895 .await
896 .map_err(|err| -> ConsoleLogError {
897 format!("retire failed for {identity}: {err}").into()
898 })?;
899 {
900 let mut durable_identities =
901 namespace_match_candidates(identity, &entry.identity_namespace);
902 durable_identities.push(parsed_identity.as_str().to_string());
903 durable_identities.sort();
904 durable_identities.dedup();
905 if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
906 &entry,
907 &durable_identities,
908 ))
909 .await
910 {
911 tracing::warn!(
912 identity,
913 error = %err,
914 "stale console member cleanup failed after durable identity retire"
915 );
916 }
917 }
918 return Ok(true);
919 }
920 let matches = Box::pin(self.resolve_visible_members(identity)).await;
921 if matches.len() > 1 {
922 let candidates = matches
923 .iter()
924 .map(|resolved| resolved.runtime_identity.clone())
925 .collect::<Vec<_>>()
926 .join(", ");
927 return Err(format!(
928 "ambiguous live identity alias {identity}: candidates [{candidates}]"
929 )
930 .into());
931 }
932 let mut live_retired_any = false;
933 for resolved in matches {
934 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
935 continue;
936 };
937 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
938 continue;
939 }
940 if let Some(stale_error) =
941 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
942 {
943 return Err(stale_error.into());
944 }
945 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
946 && let Ok(parsed_identity) =
947 crate::identity_first::AgentIdentity::parse(&record.identity)
948 && let Ok(status) = identity_runtime.status(&parsed_identity).await
949 {
950 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
951 let durable_live_records = Box::pin(self.live_records_for_durable_record(
952 &resolved.entry,
953 identity,
954 &durable_record,
955 &live_records,
956 ))
957 .await;
958 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
959 &resolved.entry,
960 &durable_live_records,
961 ))
962 .await;
963 if let Some(ambiguous_error) =
964 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
965 {
966 return Err(ambiguous_error.into());
967 }
968 if let Some(rebound_record) = self_heal_durable_session_mismatch(
969 &resolved.entry,
970 identity_runtime.as_ref(),
971 &parsed_identity,
972 identity,
973 &durable_record,
974 &durable_live_records,
975 )
976 .await
977 .map_err(ConsoleLogError::from)?
978 {
979 durable_record = rebound_record;
980 }
981 if let Some(stale_error) =
982 stale_durable_record_error(identity, &durable_record, &durable_live_records)
983 {
984 return Err(stale_error.into());
985 }
986 if status
987 .agent_runtime_id
988 .as_ref()
989 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
990 {
991 continue;
992 }
993 }
994 resolved
995 .handle
996 .retire(crate::member_comms_id::mob_member_id(
997 resolved.runtime_identity.as_str(),
998 ))
999 .await
1000 .map_err(|err| -> ConsoleLogError {
1001 format!("retire failed for {identity}: {err}").into()
1002 })
1003 .or_else(|err| {
1004 if lifecycle_archive_cleanup_completed(&err.to_string()) {
1005 Ok(())
1006 } else {
1007 Err(err)
1008 }
1009 })?;
1010 live_retired_any = true;
1011 }
1012 Ok(live_retired_any)
1013 }
1014
1015 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1016 self.inner.store.clear_frames().await
1017 }
1018
1019 pub async fn query_timeline(
1020 &self,
1021 query: ConsoleTimelineQuery,
1022 ) -> ConsoleLogResult<ConsoleTimelinePage> {
1023 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1024 Ok(ConsoleTimelinePage {
1025 frames: page.frames,
1026 next_cursor: page.next_cursor,
1027 })
1028 }
1029
1030 pub async fn query_timeline_windowed(
1031 &self,
1032 query: ConsoleTimelineWindowQuery,
1033 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1034 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1035 .await?;
1036 if query.after.is_none()
1037 && query.before.is_none()
1038 && let Some(identity) = query.identity.clone()
1039 {
1040 let mut probe_query = query.clone();
1041 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1042 let page = self.inner.store.query_windowed_frames(probe_query).await?;
1043 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1044 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1045 }
1046 }
1047 Box::pin(self.query_timeline_visible(query)).await
1048 }
1049
1050 async fn reject_after_cursor_beyond_store_frontier(
1051 &self,
1052 after: Option<&ConsoleCursor>,
1053 ) -> ConsoleLogResult<()> {
1054 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1055 return Ok(());
1056 };
1057 let latest_seq = self
1058 .inner
1059 .store
1060 .latest_cursor()
1061 .await?
1062 .and_then(|cursor| cursor.seq())
1063 .unwrap_or(0);
1064 if after_seq > latest_seq {
1065 return Err(std::io::Error::other(
1066 "timeline replay cursor is beyond the current store frontier",
1067 )
1068 .into());
1069 }
1070 Ok(())
1071 }
1072
1073 async fn query_timeline_visible(
1074 &self,
1075 query: ConsoleTimelineWindowQuery,
1076 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1077 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1078 let mut scan_query = query.clone();
1079 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1080 let mut visible_frames = Vec::with_capacity(requested_limit);
1081 let mut anchor_frames = Vec::new();
1082 let mut identity_visibility_cache = HashMap::new();
1083 let identity_records = self.inner.identity_read_model.current().await;
1084 let mut next_cursor = query.after.clone();
1085 let mut since_last_scanned_cursor = query.after.clone();
1086 let mut since_last_delivered_cursor = None;
1087 let mut since_stopped_at_visible_limit = false;
1088 let mut latest_cursor = None;
1089 let mut exhausted = false;
1090 let mut scanned = 0usize;
1091
1092 loop {
1093 let page = self
1094 .inner
1095 .store
1096 .query_windowed_frames(scan_query.clone())
1097 .await?;
1098 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1099 if page.frames.is_empty() {
1100 exhausted = true;
1101 break;
1102 }
1103 let raw_len = page.frames.len();
1104 scanned = scanned.saturating_add(raw_len);
1105 match query.mode {
1106 ConsoleTimelineMode::Since => {
1107 if let Some(cursor) = page.next_cursor.clone() {
1108 since_last_scanned_cursor = Some(cursor.clone());
1109 scan_query.after = Some(cursor);
1110 }
1111 }
1112 ConsoleTimelineMode::Recent => {
1113 if let Some(first) = page.frames.first() {
1114 scan_query.before = Some(first.cursor.clone());
1115 }
1116 next_cursor = page.next_cursor.clone().or(next_cursor);
1117 }
1118 }
1119
1120 for frame in page.frames {
1121 let allow_historical_identity =
1122 query.identity.as_deref() == Some(frame.identity.as_str());
1123 if Box::pin(frame_is_visible_cached(
1124 &self.inner,
1125 &frame,
1126 allow_historical_identity,
1127 &mut identity_visibility_cache,
1128 &identity_records,
1129 ))
1130 .await
1131 .unwrap_or(false)
1132 {
1133 if query.mode == ConsoleTimelineMode::Recent
1134 && query.identity.is_some()
1135 && is_identity_timeline_anchor_frame(&frame)
1136 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1137 {
1138 anchor_frames.push(frame.clone());
1139 }
1140 match query.mode {
1141 ConsoleTimelineMode::Since => {
1142 if visible_frames.len() >= requested_limit {
1143 since_stopped_at_visible_limit = true;
1144 break;
1145 }
1146 since_last_delivered_cursor = Some(frame.cursor.clone());
1147 visible_frames.push(frame);
1148 if visible_frames.len() >= requested_limit {
1149 since_stopped_at_visible_limit = true;
1150 exhausted = false;
1151 break;
1152 }
1153 }
1154 ConsoleTimelineMode::Recent => {
1155 visible_frames.push(frame);
1156 }
1157 }
1158 }
1159 }
1160 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1161 && query.identity.is_some()
1162 && anchor_frames.is_empty()
1163 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1164 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1165 break;
1166 }
1167 if since_stopped_at_visible_limit {
1168 break;
1169 }
1170 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1171 exhausted = true;
1172 break;
1173 }
1174 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1175 break;
1176 }
1177 }
1178
1179 if query.mode == ConsoleTimelineMode::Recent {
1180 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1181 if visible_frames.len() > requested_limit {
1182 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1183 }
1184 if !anchor_frames.is_empty() {
1185 let anchor_cursors = anchor_frames
1186 .iter()
1187 .map(|frame| frame.cursor.clone())
1188 .collect::<Vec<_>>();
1189 let mut merged = anchor_frames;
1190 for frame in visible_frames {
1191 if !merged
1192 .iter()
1193 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1194 {
1195 merged.push(frame);
1196 }
1197 }
1198 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1199 visible_frames = merged;
1200 if visible_frames.len() > requested_limit {
1201 let mut anchors = Vec::new();
1202 let mut tail = Vec::new();
1203 for frame in visible_frames {
1204 if anchor_cursors.contains(&frame.cursor) {
1205 anchors.push(frame);
1206 } else {
1207 tail.push(frame);
1208 }
1209 }
1210 visible_frames = if anchors.len() >= requested_limit {
1211 anchors.split_off(anchors.len() - requested_limit)
1212 } else {
1213 let keep_tail = requested_limit.saturating_sub(anchors.len());
1214 if tail.len() > keep_tail {
1215 tail = tail.split_off(tail.len() - keep_tail);
1216 }
1217 anchors.extend(tail);
1218 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1219 anchors
1220 };
1221 }
1222 }
1223 }
1224
1225 Ok(ConsoleTimelineWindowPage {
1226 frames: visible_frames,
1227 next_cursor: match query.mode {
1228 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1229 since_last_delivered_cursor.or(since_last_scanned_cursor)
1230 }
1231 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1232 ConsoleTimelineMode::Recent => next_cursor,
1233 },
1234 latest_cursor,
1235 exhausted,
1236 })
1237 }
1238
1239 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1240 let runtime_keys = self
1241 .inner
1242 .runtimes
1243 .read()
1244 .map_err(|_| runtime_registry_lock_error())?
1245 .keys()
1246 .cloned()
1247 .collect::<Vec<_>>();
1248 let results =
1249 join_all(runtime_keys.into_iter().map(|runtime_key| {
1250 backfill_session_history(self.inner.clone(), runtime_key, true)
1251 }))
1252 .await;
1253 for result in results {
1254 result?;
1255 }
1256 Ok(())
1257 }
1258
1259 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1260 self.inner.store.latest_cursor().await
1261 }
1262
1263 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1264 match event {
1265 ConsoleTimelineEvent::ConsoleFrame { frame }
1266 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1267 let identity_records = self.inner.identity_read_model.current().await;
1268 Box::pin(frame_is_visible(
1269 &self.inner,
1270 frame,
1271 false,
1272 &identity_records,
1273 ))
1274 .await
1275 .unwrap_or(false)
1276 }
1277 ConsoleTimelineEvent::SnapshotStarted { .. }
1278 | ConsoleTimelineEvent::SnapshotComplete { .. }
1279 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1280 }
1281 }
1282
1283 pub async fn timeline_frame_visible_for_query(
1284 &self,
1285 frame: &ConsoleFrame,
1286 identity: Option<&str>,
1287 ) -> bool {
1288 let allow_historical_identity = identity == Some(frame.identity.as_str());
1289 let identity_records = self.inner.identity_read_model.current().await;
1290 Box::pin(frame_is_visible(
1291 &self.inner,
1292 frame,
1293 allow_historical_identity,
1294 &identity_records,
1295 ))
1296 .await
1297 .unwrap_or(false)
1298 }
1299
1300 pub async fn send(
1301 &self,
1302 request: ConsoleSendRequest,
1303 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1304 validate_send_request(&request)?;
1305 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1306 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1307 };
1308 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1309 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1310 };
1311 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1312 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1313 }
1314 if !member_is_addressable(&resolved.member) {
1315 return Err(ConsoleSendError::NotAddressable(request.identity));
1316 }
1317 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1318 return Err(ConsoleSendError::Retired(request.identity));
1319 }
1320
1321 let content = content_input_from_value(&request.content)?;
1322 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1323 assert_member_accepts_images(
1324 &resolved.handle,
1325 resolved.entry.runtime.session_service(),
1326 &resolved.runtime_identity,
1327 &content,
1328 )
1329 .await
1330 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1331
1332 let dedupe_key = send_dedupe_key(
1333 &resolved.entry.runtime_key,
1334 &request.identity,
1335 &request.origin,
1336 &request.idempotency_key,
1337 );
1338 let handling_mode_value = request
1339 .handling_mode
1340 .as_deref()
1341 .unwrap_or("queue")
1342 .to_string();
1343 let request_fingerprint =
1344 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1345 if let Some(existing) = self
1346 .inner
1347 .store
1348 .frame_by_dedupe_key(&dedupe_key)
1349 .await
1350 .map_err(ConsoleSendError::Log)?
1351 {
1352 let same_request = existing.source.source_cursor.as_deref()
1353 == Some(request_fingerprint.as_str())
1354 || existing.source.source_cursor.is_none()
1355 && existing.payload.get("origin").and_then(Value::as_str)
1356 == Some(request.origin.as_str())
1357 && existing.payload.get("content") == Some(&request.content)
1358 && existing
1359 .payload
1360 .get("handling_mode")
1361 .and_then(Value::as_str)
1362 == Some(handling_mode_value.as_str());
1363 if !same_request {
1364 return Err(ConsoleSendError::IdempotencyConflict(
1365 request.idempotency_key,
1366 ));
1367 }
1368 return Ok(accepted_from_frame(&existing));
1369 }
1370
1371 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1372 resolved
1373 .entry
1374 .console_events
1375 .reserve_interaction_value(
1376 &resolved.runtime_identity,
1377 Some(resolved.runtime_identity.as_str()),
1378 &interaction_id,
1379 &request.origin,
1380 request.content.clone(),
1381 )
1382 .await
1383 .map_err(ConsoleSendError::State)?;
1384 let session_id = resolved
1385 .handle
1386 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1387 resolved.runtime_identity.as_str(),
1388 ))
1389 .await
1390 .map(|sid| sid.to_string());
1391 let mut new_frame = NewConsoleFrame {
1392 id: None,
1393 dedupe_key,
1394 timestamp_ms: current_time_ms(),
1395 runtime_key: resolved.entry.runtime_key.clone(),
1396 identity: request.identity.clone(),
1397 conversation_id: Some(request.identity.clone()),
1398 session_id: session_id.clone(),
1399 kind: "user_input".to_string(),
1400 status: ConsoleFrameStatus::Accepted,
1401 payload: json!({
1402 "content": request.content,
1403 "origin": request.origin,
1404 "idempotency_key": request.idempotency_key,
1405 "handling_mode": handling_mode_value,
1406 }),
1407 source: ConsoleFrameSource {
1408 kind: ConsoleFrameSourceKind::Send,
1409 source_cursor: Some(request_fingerprint),
1410 },
1411 source_event_id: None,
1412 interaction_id: Some(interaction_id.clone()),
1413 turn_id: None,
1414 run_id: None,
1415 parent_frame_id: None,
1416 caused_by_frame_id: None,
1417 };
1418 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1419 new_frame.payload = redacted;
1420 new_frame.status = ConsoleFrameStatus::Redacted;
1421 }
1422
1423 let _ = SendState::Requested
1424 .apply(SendTransition::PersistAccepted)
1425 .map_err(ConsoleSendError::State)?;
1426 let outcome = self
1427 .inner
1428 .store
1429 .append_if_absent(new_frame)
1430 .await
1431 .map_err(ConsoleSendError::Log)?;
1432 if outcome.disposition == AppendDisposition::Existing {
1433 return Ok(accepted_from_frame(&outcome.frame));
1434 }
1435 let _ = self
1436 .inner
1437 .event_tx
1438 .send(ConsoleTimelineEvent::ConsoleFrame {
1439 frame: outcome.frame.clone(),
1440 });
1441 let accepted = accepted_from_frame(&outcome.frame);
1442
1443 let (dispatching, _effects) = SendState::AcceptedPersisted
1444 .apply(SendTransition::StartDispatch)
1445 .map_err(ConsoleSendError::State)?;
1446 update_frame_status_and_emit(
1447 &self.inner,
1448 &outcome.frame.id,
1449 ConsoleFrameStatus::Dispatching,
1450 )
1451 .await
1452 .map_err(ConsoleSendError::Log)?;
1453
1454 spawn_console_send_dispatch(
1455 self.inner.clone(),
1456 resolved,
1457 content,
1458 handling_mode,
1459 dispatching,
1460 outcome.frame,
1461 interaction_id,
1462 );
1463 Ok(accepted)
1464 }
1465
1466 pub async fn reserve_identity_first_interaction(
1467 &self,
1468 request: ConsoleSendRequest,
1469 session_id: Option<&str>,
1470 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1471 validate_send_request(&request)?;
1472 let _content = content_input_from_value(&request.content)?;
1473 let runtime_key =
1474 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1475 let handling_mode_value = request
1476 .handling_mode
1477 .as_deref()
1478 .unwrap_or("queue")
1479 .to_string();
1480 let dedupe_key = send_dedupe_key(
1481 &runtime_key,
1482 &request.identity,
1483 &request.origin,
1484 &request.idempotency_key,
1485 );
1486 let request_fingerprint =
1487 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1488 if let Some(existing) = self
1489 .inner
1490 .store
1491 .frame_by_dedupe_key(&dedupe_key)
1492 .await
1493 .map_err(ConsoleSendError::Log)?
1494 {
1495 let same_request = existing.source.source_cursor.as_deref()
1496 == Some(request_fingerprint.as_str())
1497 || existing.source.source_cursor.is_none()
1498 && existing.payload.get("origin").and_then(Value::as_str)
1499 == Some(request.origin.as_str())
1500 && existing.payload.get("content") == Some(&request.content)
1501 && existing
1502 .payload
1503 .get("handling_mode")
1504 .and_then(Value::as_str)
1505 == Some(handling_mode_value.as_str());
1506 if !same_request {
1507 return Err(ConsoleSendError::IdempotencyConflict(
1508 request.idempotency_key,
1509 ));
1510 }
1511 return Ok(accepted_from_frame(&existing));
1512 }
1513
1514 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1515 let new_frame = NewConsoleFrame {
1516 id: None,
1517 dedupe_key,
1518 timestamp_ms: current_time_ms(),
1519 runtime_key,
1520 identity: request.identity.clone(),
1521 conversation_id: Some(request.identity.clone()),
1522 session_id: session_id.map(ToString::to_string),
1523 kind: "user_input".to_string(),
1524 status: ConsoleFrameStatus::Accepted,
1525 payload: json!({
1526 "content": request.content,
1527 "origin": request.origin,
1528 "idempotency_key": request.idempotency_key,
1529 "handling_mode": handling_mode_value,
1530 }),
1531 source: ConsoleFrameSource {
1532 kind: ConsoleFrameSourceKind::Send,
1533 source_cursor: Some(request_fingerprint),
1534 },
1535 source_event_id: None,
1536 interaction_id: Some(interaction_id),
1537 turn_id: None,
1538 run_id: None,
1539 parent_frame_id: None,
1540 caused_by_frame_id: None,
1541 };
1542 let outcome = self
1543 .inner
1544 .store
1545 .append_if_absent(new_frame)
1546 .await
1547 .map_err(ConsoleSendError::Log)?;
1548 let _ = self
1549 .inner
1550 .event_tx
1551 .send(ConsoleTimelineEvent::ConsoleFrame {
1552 frame: outcome.frame.clone(),
1553 });
1554 Ok(accepted_from_frame(&outcome.frame))
1555 }
1556
1557 async fn runtime_key_for_identity_first_send(
1558 &self,
1559 identity: &str,
1560 ) -> Result<String, ConsoleSendError> {
1561 let entries = self
1562 .inner
1563 .runtimes
1564 .read()
1565 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1566 .clone();
1567 if entries.is_empty() {
1568 return Ok("identity-first".to_string());
1569 }
1570
1571 let mut identity_runtime_keys = Vec::new();
1572 let mut hidden_durable_match = false;
1573 let mut durable_matches = Vec::new();
1574 for entry in entries.values() {
1575 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1576 continue;
1577 };
1578 identity_runtime_keys.push(entry.runtime_key.clone());
1579 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1580 continue;
1581 }
1582 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1583 {
1584 let Ok(parsed_identity) =
1585 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1586 else {
1587 continue;
1588 };
1589 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1590 let record = identity_record_for_status(entry, &status);
1591 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1592 hidden_durable_match = true;
1593 continue;
1594 }
1595 durable_matches.push((
1596 entry.clone(),
1597 identity_runtime.clone(),
1598 parsed_identity,
1599 record,
1600 ));
1601 }
1602 }
1603 }
1604 if durable_matches.len() > 1 {
1605 let candidates = durable_matches
1606 .iter()
1607 .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1608 .collect::<Vec<_>>()
1609 .join(", ");
1610 return Err(ConsoleSendError::InvalidRequest(format!(
1611 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1612 )));
1613 }
1614 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1615 durable_matches.into_iter().next()
1616 {
1617 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1618 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1619 &entry,
1620 identity,
1621 &record,
1622 &live_records,
1623 ))
1624 .await;
1625 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1626 &entry,
1627 &durable_live_records,
1628 ))
1629 .await;
1630 if let Some(ambiguous_error) =
1631 ambiguous_live_alias_error(identity, &visible_durable_live_records)
1632 {
1633 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1634 }
1635 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1636 &entry,
1637 identity_runtime.as_ref(),
1638 &parsed_identity,
1639 identity,
1640 &record,
1641 &durable_live_records,
1642 )
1643 .await
1644 .map_err(ConsoleSendError::InvalidRequest)?
1645 {
1646 record = rebound_record;
1647 }
1648 if let Some(stale_error) =
1649 stale_durable_record_error(identity, &record, &durable_live_records)
1650 {
1651 return Err(ConsoleSendError::InvalidRequest(stale_error));
1652 }
1653 return Ok(entry.runtime_key);
1654 }
1655 if entries.values().any(|entry| {
1656 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1657 }) {
1658 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1659 }
1660 if hidden_durable_match {
1661 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1662 }
1663
1664 match identity_runtime_keys.as_slice() {
1665 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1666 [runtime_key] => Ok(runtime_key.clone()),
1667 _ => Err(ConsoleSendError::InvalidRequest(format!(
1668 "identity-first send for '{identity}' did not match exactly one registered runtime"
1669 ))),
1670 }
1671 }
1672
1673 pub async fn mark_interaction_delivery_failed(
1674 &self,
1675 input_frame_id: &str,
1676 ) -> Result<(), ConsoleSendError> {
1677 update_frame_status_and_emit(
1678 &self.inner,
1679 input_frame_id,
1680 ConsoleFrameStatus::DeliveryFailed,
1681 )
1682 .await
1683 .map_err(ConsoleSendError::Log)?;
1684 Ok(())
1685 }
1686
1687 pub async fn mark_interaction_delivered(
1688 &self,
1689 input_frame_id: &str,
1690 ) -> Result<(), ConsoleSendError> {
1691 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1692 .await
1693 .map_err(ConsoleSendError::Log)?;
1694 Ok(())
1695 }
1696
1697 pub async fn mark_steer_interaction_delivered(
1698 &self,
1699 input_frame_id: &str,
1700 interaction_id: &str,
1701 ) -> Result<(), ConsoleSendError> {
1702 let Some(updated) = update_frame_status_and_emit(
1703 &self.inner,
1704 input_frame_id,
1705 ConsoleFrameStatus::Delivered,
1706 )
1707 .await
1708 .map_err(ConsoleSendError::Log)?
1709 else {
1710 return Ok(());
1711 };
1712 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1713 .await
1714 .map_err(ConsoleSendError::Log)?;
1715 Ok(())
1716 }
1717
1718 pub async fn binary_blob_store_for_identity(
1719 &self,
1720 identity: &str,
1721 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1722 if identity.trim().is_empty() {
1723 return Err(ConsoleSendError::InvalidRequest(
1724 "identity must be non-empty".to_string(),
1725 ));
1726 }
1727 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1728 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1729 };
1730 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1731 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1732 };
1733 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1734 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1735 }
1736 if !member_is_addressable(&resolved.member) {
1737 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1738 }
1739 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1740 return Err(ConsoleSendError::Retired(identity.to_string()));
1741 }
1742 Ok(resolved.entry.runtime.binary_blob_store())
1743 }
1744
1745 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1746 self.inner
1747 .runtimes
1748 .read()
1749 .map(|entries| {
1750 entries
1751 .values()
1752 .filter_map(|entry| entry.runtime.binary_blob_store())
1753 .collect()
1754 })
1755 .unwrap_or_default()
1756 }
1757
1758 async fn resolve_send_member(
1759 &self,
1760 identity: &str,
1761 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1762 let matches = Box::pin(self.resolve_visible_members(identity)).await;
1763 if matches.len() > 1 {
1764 let candidates = matches
1765 .iter()
1766 .map(|resolved| resolved.runtime_identity.clone())
1767 .collect::<Vec<_>>()
1768 .join(", ");
1769 return Err(ConsoleSendError::AmbiguousIdentity {
1770 identity: identity.to_string(),
1771 candidates,
1772 });
1773 }
1774 let Some(resolved) = matches.into_iter().next() else {
1775 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1776 return Err(ConsoleSendError::InvalidRequest(stale_error));
1777 }
1778 return Ok(None);
1779 };
1780 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1781 && let Some(stale_error) =
1782 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1783 {
1784 return Err(ConsoleSendError::InvalidRequest(stale_error));
1785 }
1786 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1787 return Err(ConsoleSendError::InvalidRequest(stale_error));
1788 }
1789 Ok(Some(resolved))
1790 }
1791
1792 async fn durable_send_binding_error(
1793 &self,
1794 identity: &str,
1795 ) -> Result<Option<String>, ConsoleSendError> {
1796 let entries = self
1797 .inner
1798 .runtimes
1799 .read()
1800 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1801 .clone();
1802 if entries.values().any(|entry| {
1803 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1804 }) {
1805 return Ok(None);
1806 }
1807 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1808 let mut durable_matches = Vec::new();
1809 for entry in entries.values() {
1810 let Some(identity_runtime) = entry.identity_runtime.clone() else {
1811 continue;
1812 };
1813 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1814 let Ok(parsed_identity) =
1815 crate::identity_first::AgentIdentity::parse(&raw_identity)
1816 else {
1817 continue;
1818 };
1819 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1820 continue;
1821 };
1822 let record = identity_record_for_status(entry, &status);
1823 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1824 continue;
1825 }
1826 durable_matches.push((
1827 entry.clone(),
1828 identity_runtime.clone(),
1829 parsed_identity,
1830 record,
1831 ));
1832 }
1833 }
1834 if durable_matches.len() > 1 {
1835 let candidates = durable_matches
1836 .iter()
1837 .map(|(_, _, _, record)| record.runtime_member_id.clone())
1838 .collect::<Vec<_>>()
1839 .join(", ");
1840 return Err(ConsoleSendError::AmbiguousIdentity {
1841 identity: identity.to_string(),
1842 candidates,
1843 });
1844 }
1845 let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1846 durable_matches.into_iter().next()
1847 else {
1848 return Ok(None);
1849 };
1850 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1851 &entry,
1852 identity,
1853 &durable_record,
1854 &live_records,
1855 ))
1856 .await;
1857 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1858 &entry,
1859 &durable_live_records,
1860 ))
1861 .await;
1862 if let Some(ambiguous_error) =
1863 ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1864 {
1865 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1866 }
1867 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1868 &entry,
1869 identity_runtime.as_ref(),
1870 &parsed_identity,
1871 identity,
1872 &durable_record,
1873 &durable_live_records,
1874 )
1875 .await
1876 .map_err(ConsoleSendError::InvalidRequest)?
1877 {
1878 durable_record = rebound_record;
1879 }
1880 Ok(stale_durable_record_error(
1881 identity,
1882 &durable_record,
1883 &durable_live_records,
1884 ))
1885 }
1886
1887 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1888 let entries = self
1889 .inner
1890 .runtimes
1891 .read()
1892 .ok()
1893 .map(|entries| entries.clone())
1894 .unwrap_or_default();
1895 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1896 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1897 for entry in entries.values() {
1898 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1899 if raw_identities.is_empty() {
1900 continue;
1901 }
1902 let mids = raw_identities
1903 .iter()
1904 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
1905 .collect::<Vec<_>>();
1906 for resolved in Box::pin(member_sources_for_entry(entry)).await {
1907 let session_id = resolved
1908 .handle
1909 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1910 .await
1911 .map(|sid| sid.to_string())
1912 .unwrap_or_default();
1913 if mids.contains(&resolved.member.agent_identity) {
1914 exact_matches.push((session_id, resolved));
1915 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1916 {
1917 label_matches.push((session_id, resolved));
1918 }
1919 }
1920 }
1921 let mut matches = exact_matches;
1922 matches.extend(label_matches);
1923 let mut seen_members = BTreeSet::new();
1924 matches.retain(|(session_id, resolved)| {
1925 seen_members.insert((
1926 resolved.entry.runtime_key.clone(),
1927 resolved.source_mob_id.clone(),
1928 session_id.clone(),
1929 resolved.runtime_identity.clone(),
1930 ))
1931 });
1932 matches.sort_by(|left, right| right.0.cmp(&left.0));
1933 matches.into_iter().map(|(_, resolved)| resolved).collect()
1934 }
1935
1936 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1937 let mut visible = Vec::new();
1938 for resolved in Box::pin(self.resolve_members(identity)).await {
1939 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1940 continue;
1941 };
1942 if Box::pin(resolved_member_visible(&resolved, &record)).await {
1943 visible.push(resolved);
1944 }
1945 }
1946 visible
1947 }
1948
1949 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1950 let mut records = Vec::new();
1951 for resolved in Box::pin(self.resolve_members(identity)).await {
1952 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1953 continue;
1954 };
1955 if Box::pin(resolved_member_visible(&resolved, &record)).await {
1956 records.push(record);
1957 }
1958 }
1959 records
1960 }
1961
1962 async fn live_records_for_durable_record(
1963 &self,
1964 entry: &RuntimeEntry,
1965 requested_identity: &str,
1966 durable: &ConsoleIdentityRecord,
1967 requested_live_records: &[ConsoleIdentityRecord],
1968 ) -> Vec<ConsoleIdentityRecord> {
1969 let mut records = if durable.identity == requested_identity {
1970 requested_live_records.to_vec()
1971 } else {
1972 Box::pin(self.live_records_for_identity(&durable.identity)).await
1973 };
1974 for runtime_record in Box::pin(live_records_for_runtime_member(
1975 entry,
1976 &durable.runtime_member_id,
1977 ))
1978 .await
1979 {
1980 if records.iter().any(|record| {
1981 record.runtime_key == runtime_record.runtime_key
1982 && record.runtime_member_id == runtime_record.runtime_member_id
1983 && record.session_id == runtime_record.session_id
1984 && record.identity == runtime_record.identity
1985 }) {
1986 continue;
1987 }
1988 records.push(runtime_record);
1989 }
1990 records
1991 }
1992}
1993
1994async fn console_identity_record_visible(
1995 entry: &RuntimeEntry,
1996 record: &ConsoleIdentityRecord,
1997) -> bool {
1998 if !entry.visibility_policy.identity_visible(record) {
1999 return false;
2000 }
2001 let mut bound_live_member_seen = false;
2002 let mut wrong_live_projection_seen = false;
2003 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2004 if resolved.runtime_identity != record.runtime_member_id {
2005 continue;
2006 }
2007 bound_live_member_seen = true;
2008 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2009 continue;
2010 };
2011 if live_record.identity != record.identity {
2012 wrong_live_projection_seen = true;
2013 continue;
2014 }
2015 if raw_resolved_member_visible(&resolved, &live_record) {
2016 return true;
2017 }
2018 }
2019 if wrong_live_projection_seen {
2020 return true;
2021 }
2022 !bound_live_member_seen
2023}
2024
2025async fn live_record_shadowed_by_hidden_durable_binding(
2026 resolved: &ResolvedConsoleMember,
2027 record: &ConsoleIdentityRecord,
2028) -> bool {
2029 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2030 return false;
2031 };
2032 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2033 return false;
2034 };
2035 let Ok(status) = identity_runtime.status(&identity).await else {
2036 return false;
2037 };
2038 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2039 return false;
2040 };
2041 if bound_runtime_id.as_str() == resolved.runtime_identity {
2042 return false;
2043 }
2044 for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2045 if candidate.runtime_identity != bound_runtime_id.as_str() {
2046 continue;
2047 }
2048 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2049 continue;
2050 };
2051 if bound_record.identity == record.identity
2052 && !raw_resolved_member_visible(&candidate, &bound_record)
2053 {
2054 return true;
2055 }
2056 }
2057 false
2058}
2059
2060async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2061 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2062 .unwrap_or_else(|| frame.identity.clone());
2063 let frame_session_id = frame.session_id.as_deref();
2064 let mut saw_hidden = false;
2065 let mut saw_visible = false;
2066 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2067 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2068 continue;
2069 };
2070 let session_matches =
2071 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2072 let runtime_matches = record.runtime_member_id == runtime_member_id;
2073 let identity_matches = record.identity == frame.identity;
2074 if !session_matches && !runtime_matches && !identity_matches {
2075 continue;
2076 }
2077 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2078 return true;
2079 }
2080 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2081 saw_visible = true;
2082 } else {
2083 saw_hidden = true;
2084 }
2085 }
2086 saw_hidden && !saw_visible
2087}
2088
2089async fn live_records_for_runtime_member(
2090 entry: &RuntimeEntry,
2091 runtime_member_id: &str,
2092) -> Vec<ConsoleIdentityRecord> {
2093 let mut records = Vec::new();
2094 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2095 if resolved.runtime_identity != runtime_member_id {
2096 continue;
2097 }
2098 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2099 continue;
2100 };
2101 records.push(record);
2102 }
2103 records
2104}
2105
2106async fn visible_live_records_for_entry(
2107 entry: &RuntimeEntry,
2108 records: &[ConsoleIdentityRecord],
2109) -> Vec<ConsoleIdentityRecord> {
2110 let mut visible = Vec::new();
2111 for record in records {
2112 if record.runtime_key != entry.runtime_key
2113 || Box::pin(console_identity_record_visible(entry, record)).await
2114 {
2115 visible.push(record.clone());
2116 }
2117 }
2118 visible
2119}
2120
2121async fn reconcile_stale_live_record_binding(
2122 entry: &RuntimeEntry,
2123 requested_identity: &str,
2124 live_record: &ConsoleIdentityRecord,
2125) -> Option<String> {
2126 let identity_runtime = entry.identity_runtime.as_ref()?;
2127 for status in identity_runtime.statuses().await {
2128 let matches_runtime = status
2129 .agent_runtime_id
2130 .as_ref()
2131 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2132 if !matches_runtime {
2133 continue;
2134 }
2135 let durable_record = identity_record_for_status(entry, &status);
2136 if let Some(session_mismatch) =
2137 stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2138 {
2139 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2140 continue;
2141 };
2142 let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2143 return Some(format!(
2144 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2145 ));
2146 };
2147 return match identity_runtime
2148 .rebind_session_after_live_respawn(&status.identity, session_id)
2149 .await
2150 {
2151 Ok(_) => {
2152 tracing::warn!(
2153 requested_identity,
2154 identity = %durable_record.identity,
2155 runtime_member_id = %durable_record.runtime_member_id,
2156 old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2157 new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2158 "self-healed stale durable identity alias session mismatch"
2159 );
2160 None
2161 }
2162 Err(err) => Some(format!(
2163 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2164 durable_record.identity,
2165 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2166 )),
2167 };
2168 }
2169 if let Some(stale_error) = stale_durable_record_error(
2170 requested_identity,
2171 &durable_record,
2172 std::slice::from_ref(live_record),
2173 ) {
2174 return Some(stale_error);
2175 }
2176 }
2177 None
2178}
2179
2180async fn self_heal_durable_session_mismatch(
2181 entry: &RuntimeEntry,
2182 identity_runtime: &crate::identity_first::IdentityRuntime,
2183 parsed_identity: &crate::identity_first::AgentIdentity,
2184 requested_identity: &str,
2185 durable: &ConsoleIdentityRecord,
2186 live_records: &[ConsoleIdentityRecord],
2187) -> Result<Option<ConsoleIdentityRecord>, String> {
2188 let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2189 return Ok(None);
2190 };
2191 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2192 return Ok(None);
2193 };
2194 let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2195 format!(
2196 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2197 )
2198 })?;
2199 identity_runtime
2200 .rebind_session_after_live_respawn(parsed_identity, session_id)
2201 .await
2202 .map_err(|err| {
2203 format!(
2204 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2205 durable.identity, live_session_id
2206 )
2207 })?;
2208 tracing::warn!(
2209 requested_identity,
2210 identity = %durable.identity,
2211 runtime_member_id = %durable.runtime_member_id,
2212 old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2213 new_session_id = live_session_id,
2214 "self-healed stale durable identity alias session mismatch"
2215 );
2216 let status = identity_runtime
2217 .status(parsed_identity)
2218 .await
2219 .map_err(|err| {
2220 format!(
2221 "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2222 durable.identity
2223 )
2224 })?;
2225 Ok(Some(identity_record_for_status(entry, &status)))
2226}
2227
2228fn stale_durable_session_mismatch<'a>(
2229 durable: &ConsoleIdentityRecord,
2230 live_records: &'a [ConsoleIdentityRecord],
2231) -> Option<&'a ConsoleIdentityRecord> {
2232 live_records.iter().find(|record| {
2233 record.identity == durable.identity
2234 && record.runtime_member_id == durable.runtime_member_id
2235 && durable.session_id.is_some()
2236 && record.session_id.is_some()
2237 && durable.session_id != record.session_id
2238 })
2239}
2240
2241fn stale_durable_record_error(
2242 requested_identity: &str,
2243 durable: &ConsoleIdentityRecord,
2244 live_records: &[ConsoleIdentityRecord],
2245) -> Option<String> {
2246 let matching_live = live_records
2247 .iter()
2248 .filter(|record| record.identity == durable.identity)
2249 .collect::<Vec<_>>();
2250 if let Some(wrong_projection) = live_records.iter().find(|record| {
2251 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2252 }) {
2253 return Some(format!(
2254 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2255 durable.identity, durable.runtime_member_id, wrong_projection.identity
2256 ));
2257 }
2258 if matching_live.is_empty() {
2259 return None;
2260 }
2261 if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2262 return Some(format!(
2263 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2264 durable.identity,
2265 durable.runtime_member_id,
2266 durable.session_id.as_deref().unwrap_or("<none>"),
2267 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2268 ));
2269 }
2270 if matching_live
2271 .iter()
2272 .any(|record| record.runtime_member_id == durable.runtime_member_id)
2273 {
2274 return None;
2275 }
2276 let live_candidates = matching_live
2277 .iter()
2278 .map(|record| record.runtime_member_id.as_str())
2279 .collect::<Vec<_>>()
2280 .join(", ");
2281 Some(format!(
2282 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2283 durable.identity, durable.runtime_member_id, live_candidates
2284 ))
2285}
2286
2287fn ambiguous_live_alias_error(
2288 requested_identity: &str,
2289 live_records: &[ConsoleIdentityRecord],
2290) -> Option<String> {
2291 if live_records.len() <= 1 {
2292 return None;
2293 }
2294 let candidates = live_records
2295 .iter()
2296 .map(|record| {
2297 format!(
2298 "{}@{}",
2299 record.runtime_member_id,
2300 record
2301 .labels
2302 .get("source_mob_id")
2303 .map(String::as_str)
2304 .unwrap_or(record.runtime_key.as_str())
2305 )
2306 })
2307 .collect::<Vec<_>>()
2308 .join(", ");
2309 Some(format!(
2310 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2311 ))
2312}
2313
2314fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2315 crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2318}
2319
2320async fn retire_stale_console_members_for_identity(
2321 handle: &MobHandle,
2322 durable_identities: &[String],
2323 keep_runtime_member_id: Option<&str>,
2324) -> Result<(), String> {
2325 let stale_members = handle
2326 .list_members_including_retiring()
2327 .await
2328 .into_iter()
2329 .filter(|member| {
2330 (durable_identities.iter().any(|durable_identity| {
2331 member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2332 }) || member.labels.get("agent_identity").is_some_and(|identity| {
2333 durable_identities
2334 .iter()
2335 .any(|durable_identity| identity == durable_identity)
2336 })) && keep_runtime_member_id
2337 .map(|keep| {
2338 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2340 != keep
2341 })
2342 .unwrap_or(true)
2343 })
2344 .map(|member| member.agent_identity)
2345 .collect::<Vec<_>>();
2346 for member_id in stale_members {
2347 match handle.retire(member_id).await {
2348 Ok(()) => {}
2349 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2350 Err(err) => return Err(err.to_string()),
2351 }
2352 }
2353 Ok(())
2354}
2355
2356async fn retire_stale_console_members_for_runtime_entry(
2357 entry: &RuntimeEntry,
2358 durable_identities: &[String],
2359) -> Result<(), String> {
2360 let primary_handle = entry.runtime.handle();
2361 let primary_mob_id = primary_handle.mob_id().to_string();
2362 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2363 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2364 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2365 .await
2366 .unwrap_or_default()
2367 {
2368 if mob_id.as_str() != primary_mob_id {
2369 handles.push((mob_id.to_string(), handle));
2370 }
2371 }
2372 }
2373 let mut seen_handles = BTreeSet::new();
2374 for (mob_id, handle) in handles {
2375 if !seen_handles.insert(mob_id) {
2376 continue;
2377 }
2378 retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2379 }
2380 Ok(())
2381}
2382
2383fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2384 is_recoverable_lifecycle_cleanup_error(error)
2385}
2386
2387fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2388 if frames.is_empty() {
2389 return true;
2390 }
2391 let latest_session_history_timestamp_ms = frames
2392 .iter()
2393 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2394 .map(|frame| frame.timestamp_ms)
2395 .max();
2396 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2397 return current_time_ms().saturating_sub(latest_timestamp_ms)
2398 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2399 }
2400 frames.iter().any(|frame| {
2401 matches!(
2402 frame.kind.as_str(),
2403 "turn_started"
2404 | "run_started"
2405 | "reasoning_delta"
2406 | "reasoning_complete"
2407 | "tool_call_requested"
2408 | "tool_call"
2409 | "tool_execution_started"
2410 | "tool_execution_completed"
2411 | "tool_result_received"
2412 )
2413 })
2414}
2415
2416fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2417 match frame.kind.as_str() {
2418 "user_input" | "run_started" => true,
2419 "interaction_started" => frame
2420 .payload
2421 .get("content")
2422 .or_else(|| frame.payload.get("prompt"))
2423 .is_some(),
2424 _ => false,
2425 }
2426}
2427
2428fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2429 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2430 for record in records {
2431 by_identity
2432 .entry(identity_record_dedupe_key(&record))
2433 .and_modify(|current| {
2434 if identity_record_prefer(&record, current) {
2435 *current = record.clone();
2436 }
2437 })
2438 .or_insert(record);
2439 }
2440 by_identity.into_values().collect()
2441}
2442
2443fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2444 format!(
2445 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2446 record.identity,
2447 record.runtime_key,
2448 record
2449 .labels
2450 .get("source_mob_id")
2451 .map(String::as_str)
2452 .unwrap_or(""),
2453 record.runtime_member_id,
2454 record.session_id.as_deref().unwrap_or("")
2455 )
2456}
2457
2458fn identity_record_prefer(
2459 candidate: &ConsoleIdentityRecord,
2460 current: &ConsoleIdentityRecord,
2461) -> bool {
2462 let candidate_is_live_label_projection =
2463 candidate
2464 .labels
2465 .get("agent_identity")
2466 .is_some_and(|identity| {
2467 identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2468 || candidate.runtime_member_id != candidate.identity
2469 });
2470 let current_is_live_label_projection =
2471 current
2472 .labels
2473 .get("agent_identity")
2474 .is_some_and(|identity| {
2475 identity == &strip_namespace(¤t.identity, "").unwrap_or_default()
2476 || current.runtime_member_id != current.identity
2477 });
2478 if candidate_is_live_label_projection != current_is_live_label_projection {
2479 return candidate_is_live_label_projection;
2480 }
2481 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2482 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2483 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2484 return candidate_has_distinct_runtime_binding;
2485 }
2486 let candidate_live = candidate.addressable && candidate.health != "retired";
2487 let current_live = current.addressable && current.health != "retired";
2488 if candidate_live != current_live {
2489 return candidate_live;
2490 }
2491 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2492}
2493
2494async fn collect_identity_records(
2495 inner: &Arc<AggregatorInner>,
2496 mode: IdentityCollectionMode,
2497) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2498 let entries = inner
2499 .runtimes
2500 .read()
2501 .map_err(|_| runtime_registry_lock_error())?
2502 .clone();
2503 let mut identities = Vec::new();
2504 for entry in entries.values() {
2505 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2506 let topology_peers =
2507 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2508 for status in identity_runtime.statuses().await {
2509 let mut record = identity_record_for_status(entry, &status);
2510 record.topology_peers = topology_peers
2511 .get(&record.identity)
2512 .cloned()
2513 .unwrap_or_default();
2514 if Box::pin(console_identity_record_visible(entry, &record)).await {
2515 identities.push(record);
2516 }
2517 }
2518 if mode == IdentityCollectionMode::CachedOnly {
2519 continue;
2520 }
2521 }
2522 let members = if entry.identity_runtime.is_some() {
2523 match Box::pin(tokio::time::timeout(
2524 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2525 member_sources_for_entry(entry),
2526 ))
2527 .await
2528 {
2529 Ok(members) => members,
2530 Err(_) => {
2531 tracing::warn!(
2532 runtime_key = %entry.runtime_key,
2533 "identity-first live member refresh timed out; keeping cached identity read model"
2534 );
2535 Vec::new()
2536 }
2537 }
2538 } else {
2539 Box::pin(member_sources_for_entry(entry)).await
2540 };
2541 for resolved in members {
2542 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2543 && Box::pin(resolved_member_visible(&resolved, &record)).await
2544 {
2545 identities.push(record);
2546 }
2547 }
2548 }
2549 Ok(dedupe_identity_records(identities))
2550}
2551
2552async fn identity_runtime_topology_peers(
2553 entry: &RuntimeEntry,
2554 identity_runtime: &crate::identity_first::IdentityRuntime,
2555) -> BTreeMap<String, Vec<String>> {
2556 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2557 for edge in identity_runtime.desired_peer_edges().await {
2558 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2559 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2560 peers.entry(a.clone()).or_default().insert(b.clone());
2561 peers.entry(b).or_default().insert(a);
2562 }
2563 peers
2564 .into_iter()
2565 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2566 .collect()
2567}
2568
2569fn spawn_identity_backfills_for_records(
2570 inner: Arc<AggregatorInner>,
2571 records: &[ConsoleIdentityRecord],
2572) {
2573 if !inner.options.session_history_backfill_enabled {
2574 return;
2575 }
2576 let entries = match inner.runtimes.read() {
2577 Ok(entries) => entries.clone(),
2578 Err(_) => return,
2579 };
2580 for record in records {
2581 if record.health == "dormant" || record.health == "uninitialized" {
2582 continue;
2583 }
2584 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2585 continue;
2586 };
2587 let Some(session_id) = record.session_id.clone() else {
2588 continue;
2589 };
2590 spawn_session_history_backfill_target(
2591 inner.clone(),
2592 SessionBackfillTarget {
2593 entry,
2594 record: record.clone(),
2595 session_id,
2596 },
2597 false,
2598 );
2599 }
2600}
2601
2602async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2603 let mut resolved = Vec::new();
2604 let primary_handle = entry.runtime.handle();
2605 let primary_mob_id = primary_handle.mob_id().to_string();
2606 for member in primary_handle.list_members_observation_snapshot().await {
2607 resolved.push(ResolvedConsoleMember {
2608 entry: entry.clone(),
2609 handle: primary_handle.clone(),
2610 runtime_identity: crate::member_comms_id::runtime_alias_str(
2613 member.agent_identity.as_str(),
2614 )
2615 .into_owned(),
2616 source_mob_id: primary_mob_id.clone(),
2617 member,
2618 });
2619 }
2620
2621 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2622 return resolved;
2623 };
2624 if !entry.visibility_policy.include_implicit_delegate_members() {
2625 return resolved;
2626 }
2627 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2628 .await
2629 .unwrap_or_default()
2630 {
2631 if mob_id.as_str() == primary_mob_id {
2632 continue;
2633 }
2634 for member in handle.list_members_observation_snapshot().await {
2635 resolved.push(ResolvedConsoleMember {
2636 entry: entry.clone(),
2637 handle: handle.clone(),
2638 runtime_identity: crate::member_comms_id::runtime_alias_str(
2641 member.agent_identity.as_str(),
2642 )
2643 .into_owned(),
2644 source_mob_id: mob_id.to_string(),
2645 member,
2646 });
2647 }
2648 }
2649 resolved
2650}
2651
2652async fn dispatch_message_to_resolved_member(
2653 resolved: &ResolvedConsoleMember,
2654 content: ContentInput,
2655 handling_mode: meerkat_core::types::HandlingMode,
2656) -> Result<String, String> {
2657 let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2658 match send_message_on_mob_with_mode(
2659 &resolved.handle,
2660 &resolved.runtime_identity,
2661 content.clone(),
2662 handling_mode,
2663 )
2664 .await
2665 {
2666 Ok(session_id) => Ok(session_id),
2667 Err(err) if is_not_externally_addressable(&err) => {
2668 let member = resolved
2669 .handle
2670 .member(&mid)
2671 .await
2672 .map_err(|err| err.to_string())?;
2673 let _receipt = member
2674 .internal_turn(content)
2675 .await
2676 .map_err(|err| err.to_string())?;
2677 resolved
2678 .handle
2679 .resolve_bridge_session_id_observation(&mid)
2680 .await
2681 .map(|sid| sid.to_string())
2682 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2683 }
2684 Err(err) => Err(err.to_string()),
2685 }
2686}
2687
2688fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2689 matches!(
2690 err,
2691 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2692 )
2693}
2694
2695fn spawn_console_send_dispatch(
2696 inner: Arc<AggregatorInner>,
2697 resolved: ResolvedConsoleMember,
2698 content: ContentInput,
2699 handling_mode: meerkat_core::types::HandlingMode,
2700 dispatching: SendState,
2701 user_frame: ConsoleFrame,
2702 interaction_id: String,
2703) {
2704 tokio::spawn(async move {
2705 match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2706 Ok(_) => {
2707 let _ = dispatching.apply(SendTransition::MarkDelivered);
2708 if let Err(err) = update_frame_status_and_emit(
2709 &inner,
2710 &user_frame.id,
2711 ConsoleFrameStatus::Delivered,
2712 )
2713 .await
2714 {
2715 tracing::warn!(
2716 frame_id = %user_frame.id,
2717 error = %err,
2718 "failed to update console send delivery status"
2719 );
2720 }
2721 if handling_mode == HandlingMode::Steer
2722 && let Err(err) =
2723 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2724 {
2725 tracing::warn!(
2726 frame_id = %user_frame.id,
2727 interaction_id = %interaction_id,
2728 error = %err,
2729 "failed to append console steer terminal frame"
2730 );
2731 }
2732 }
2733 Err(err) => {
2734 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2735 if let Err(update_err) = update_frame_status_and_emit(
2736 &inner,
2737 &user_frame.id,
2738 ConsoleFrameStatus::DeliveryFailed,
2739 )
2740 .await
2741 {
2742 tracing::warn!(
2743 frame_id = %user_frame.id,
2744 error = %update_err,
2745 "failed to update console send failure status"
2746 );
2747 }
2748 let failure_frame = NewConsoleFrame {
2749 id: None,
2750 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2751 timestamp_ms: current_time_ms(),
2752 runtime_key: user_frame.runtime_key,
2753 identity: user_frame.identity,
2754 conversation_id: user_frame.conversation_id,
2755 session_id: user_frame.session_id,
2756 kind: "message_delivery_failed".to_string(),
2757 status: ConsoleFrameStatus::DeliveryFailed,
2758 payload: json!({ "reason": err }),
2759 source: ConsoleFrameSource {
2760 kind: ConsoleFrameSourceKind::Synthetic,
2761 source_cursor: None,
2762 },
2763 source_event_id: None,
2764 interaction_id: Some(interaction_id),
2765 turn_id: None,
2766 run_id: None,
2767 parent_frame_id: Some(user_frame.id.clone()),
2768 caused_by_frame_id: Some(user_frame.id),
2769 };
2770 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2771 tracing::warn!(
2772 error = %append_err,
2773 "failed to append console send failure frame"
2774 );
2775 }
2776 }
2777 }
2778 });
2779}
2780
2781async fn append_steer_delivery_terminal(
2782 inner: &AggregatorInner,
2783 user_frame: &ConsoleFrame,
2784 interaction_id: &str,
2785) -> ConsoleLogResult<AppendOutcome> {
2786 append_and_emit(
2787 inner,
2788 NewConsoleFrame {
2789 id: None,
2790 dedupe_key: format!("steer-delivered:{}", user_frame.id),
2791 timestamp_ms: current_time_ms(),
2792 runtime_key: user_frame.runtime_key.clone(),
2793 identity: user_frame.identity.clone(),
2794 conversation_id: user_frame.conversation_id.clone(),
2795 session_id: user_frame.session_id.clone(),
2796 kind: "interaction_complete".to_string(),
2797 status: ConsoleFrameStatus::Completed,
2798 payload: json!({
2799 "reason": "steer_delivered",
2800 "handling_mode": "steer",
2801 }),
2802 source: ConsoleFrameSource {
2803 kind: ConsoleFrameSourceKind::Synthetic,
2804 source_cursor: None,
2805 },
2806 source_event_id: None,
2807 interaction_id: Some(interaction_id.to_string()),
2808 turn_id: None,
2809 run_id: None,
2810 parent_frame_id: Some(user_frame.id.clone()),
2811 caused_by_frame_id: Some(user_frame.id.clone()),
2812 },
2813 )
2814 .await
2815}
2816
2817#[derive(Debug)]
2818pub enum ConsoleSendError {
2819 UnknownIdentity(String),
2820 AmbiguousIdentity {
2821 identity: String,
2822 candidates: String,
2823 },
2824 NotAddressable(String),
2825 Retired(String),
2826 InvalidContent(String),
2827 InvalidHandlingMode(String),
2828 InvalidRequest(String),
2829 IdempotencyConflict(String),
2830 State(&'static str),
2831 Dispatch(String),
2832 Log(ConsoleLogError),
2833}
2834
2835impl std::fmt::Display for ConsoleSendError {
2836 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2837 match self {
2838 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2839 Self::AmbiguousIdentity {
2840 identity,
2841 candidates,
2842 } => write!(
2843 f,
2844 "ambiguous live identity alias {identity}: candidates [{candidates}]"
2845 ),
2846 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2847 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2848 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2849 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2850 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2851 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2852 Self::State(message) => write!(f, "console send state error: {message}"),
2853 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2854 Self::Log(err) => write!(f, "console log error: {err}"),
2855 }
2856 }
2857}
2858
2859impl std::error::Error for ConsoleSendError {}
2860
2861async fn backfill_session_history(
2862 inner: Arc<AggregatorInner>,
2863 runtime_key: String,
2864 force_refresh: bool,
2865) -> ConsoleLogResult<()> {
2866 if !inner.options.session_history_backfill_enabled {
2867 return Ok(());
2868 }
2869 let Some(entry) = inner
2870 .runtimes
2871 .read()
2872 .ok()
2873 .and_then(|entries| entries.get(&runtime_key).cloned())
2874 else {
2875 return Ok(());
2876 };
2877 let members = Box::pin(member_sources_for_entry(&entry)).await;
2878 let mut targets = Vec::new();
2879 for resolved in members {
2880 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2881 continue;
2882 };
2883 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
2884 continue;
2885 }
2886 let Some(session_id) = record.session_id.clone() else {
2887 continue;
2888 };
2889 targets.push(SessionBackfillTarget {
2890 entry: entry.clone(),
2891 record,
2892 session_id,
2893 });
2894 }
2895 backfill_session_history_targets(inner, targets, force_refresh).await
2896}
2897
2898#[derive(Clone)]
2899struct SessionBackfillTarget {
2900 entry: RuntimeEntry,
2901 record: ConsoleIdentityRecord,
2902 session_id: String,
2903}
2904
2905async fn backfill_session_history_targets(
2906 inner: Arc<AggregatorInner>,
2907 targets: Vec<SessionBackfillTarget>,
2908 force_refresh: bool,
2909) -> ConsoleLogResult<()> {
2910 let mut tasks = tokio::task::JoinSet::new();
2911 for target in targets {
2912 tasks.spawn(backfill_one_session_history(
2913 inner.clone(),
2914 target,
2915 force_refresh,
2916 ));
2917 }
2918 let mut first_error = None;
2919 while let Some(result) = tasks.join_next().await {
2920 match result {
2921 Ok(Ok(())) => {}
2922 Ok(Err(err)) => {
2923 if first_error.is_none() {
2924 first_error = Some(err);
2925 }
2926 }
2927 Err(err) => {
2928 if first_error.is_none() {
2929 first_error = Some(Box::new(std::io::Error::other(format!(
2930 "session backfill task failed: {err}"
2931 ))) as ConsoleLogError);
2932 }
2933 }
2934 }
2935 }
2936 if let Some(err) = first_error {
2937 Err(err)
2938 } else {
2939 Ok(())
2940 }
2941}
2942
2943async fn backfill_one_session_history(
2944 inner: Arc<AggregatorInner>,
2945 target: SessionBackfillTarget,
2946 force_refresh: bool,
2947) -> ConsoleLogResult<()> {
2948 let _permit = inner
2949 .session_backfill_permits
2950 .clone()
2951 .acquire_owned()
2952 .await
2953 .map_err(|err| -> ConsoleLogError {
2954 Box::new(std::io::Error::other(format!(
2955 "session backfill limiter closed: {err}"
2956 )))
2957 })?;
2958 let SessionBackfillTarget {
2959 entry,
2960 record,
2961 session_id,
2962 } = target;
2963 let watermark_runtime_key =
2964 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2965 let watermark = inner
2966 .store
2967 .source_watermark(
2968 &watermark_runtime_key,
2969 ConsoleFrameSourceKind::SessionHistory,
2970 )
2971 .await?;
2972 let now_ms = current_time_ms();
2973 let mut offset = watermark
2974 .as_deref()
2975 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2976 .unwrap_or(0);
2977 if !force_refresh
2978 && watermark.as_deref().is_some_and(|watermark| {
2979 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2980 })
2981 {
2982 return Ok(());
2983 }
2984 loop {
2985 let page = match entry
2986 .runtime
2987 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2988 .await
2989 {
2990 Ok(page) => page,
2991 Err(err) => {
2992 record_session_history_backfill_failure(
2993 &inner,
2994 &watermark_runtime_key,
2995 &entry.runtime_key,
2996 &record.identity,
2997 &session_id,
2998 offset,
2999 err.to_string(),
3000 )
3001 .await?;
3002 break;
3003 }
3004 };
3005 let page_value = match serde_json::to_value(page) {
3006 Ok(value) => value,
3007 Err(err) => {
3008 record_session_history_backfill_failure(
3009 &inner,
3010 &watermark_runtime_key,
3011 &entry.runtime_key,
3012 &record.identity,
3013 &session_id,
3014 offset,
3015 err.to_string(),
3016 )
3017 .await?;
3018 break;
3019 }
3020 };
3021 let base_offset = page_value
3022 .get("offset")
3023 .and_then(Value::as_u64)
3024 .unwrap_or(offset as u64) as usize;
3025 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3026 record_session_history_backfill_failure(
3027 &inner,
3028 &watermark_runtime_key,
3029 &entry.runtime_key,
3030 &record.identity,
3031 &session_id,
3032 offset,
3033 "session history page missing messages".to_string(),
3034 )
3035 .await?;
3036 break;
3037 };
3038 if messages.is_empty() {
3039 if offset > 0 {
3040 record_session_history_watermark(
3041 &inner,
3042 &watermark_runtime_key,
3043 &session_id,
3044 offset,
3045 )
3046 .await?;
3047 }
3048 break;
3049 }
3050 for (idx, message) in messages.iter().enumerate() {
3051 let absolute_offset = base_offset + idx;
3052 let frames = frames_from_session_history_message_with_namespace(
3053 &entry.runtime_key,
3054 &record.identity,
3055 &entry.identity_namespace,
3056 &session_id,
3057 absolute_offset,
3058 message.clone(),
3059 );
3060 for mut frame in frames {
3061 if history_frame_has_existing_counterpart(&inner, &frame).await? {
3062 continue;
3063 }
3064 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3065 frame.payload = redacted;
3066 frame.status = ConsoleFrameStatus::Redacted;
3067 }
3068 append_and_emit(&inner, frame).await?;
3069 }
3070 }
3071 let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3072 tracing::warn!(
3079 session_id = %session_id,
3080 requested_offset = offset,
3081 page_base_offset = base_offset,
3082 page_len = messages.len(),
3083 "session history backfill made no forward progress; stopping to avoid a livelock"
3084 );
3085 break;
3086 };
3087 offset = next_offset;
3088 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3089 .await?;
3090 let has_more = page_value
3091 .get("has_more")
3092 .and_then(Value::as_bool)
3093 .unwrap_or(false);
3094 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3095 break;
3096 }
3097 }
3098 Ok(())
3099}
3100
3101fn advance_backfill_offset(
3106 requested_offset: usize,
3107 base_offset: usize,
3108 page_len: usize,
3109) -> Option<usize> {
3110 let next = base_offset.saturating_add(page_len);
3111 (next > requested_offset).then_some(next)
3112}
3113
3114async fn record_session_history_watermark(
3115 inner: &AggregatorInner,
3116 watermark_runtime_key: &str,
3117 session_id: &str,
3118 offset: usize,
3119) -> ConsoleLogResult<()> {
3120 inner
3121 .store
3122 .record_source_watermark(
3123 watermark_runtime_key,
3124 ConsoleFrameSourceKind::SessionHistory,
3125 &format_session_history_watermark(session_id, offset, current_time_ms()),
3126 )
3127 .await
3128}
3129
3130fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3131 if !inner.options.session_history_backfill_enabled {
3132 return;
3133 }
3134 tokio::spawn(async move {
3135 {
3136 let mut active = inner.active_session_backfills.lock().await;
3137 if !active.insert(runtime_key.clone()) {
3138 return;
3139 }
3140 }
3141 let result = Box::pin(backfill_session_history(
3142 inner.clone(),
3143 runtime_key.clone(),
3144 false,
3145 ))
3146 .await;
3147 let mut active = inner.active_session_backfills.lock().await;
3148 active.remove(&runtime_key);
3149 drop(active);
3150 if let Err(err) = result {
3151 tracing::warn!(
3152 runtime_key = %runtime_key,
3153 error = %err,
3154 "console session-history backfill failed"
3155 );
3156 }
3157 });
3158}
3159
3160fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3161 if !inner.options.session_history_backfill_enabled {
3162 return;
3163 }
3164 tokio::spawn(async move {
3165 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3166 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3167 loop {
3168 interval.tick().await;
3169 let runtime_still_registered = inner
3170 .runtimes
3171 .read()
3172 .ok()
3173 .is_some_and(|entries| entries.contains_key(&runtime_key));
3174 if !runtime_still_registered {
3175 break;
3176 }
3177 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3178 }
3179 });
3180}
3181
3182fn spawn_session_history_backfill_target(
3183 inner: Arc<AggregatorInner>,
3184 target: SessionBackfillTarget,
3185 force_refresh: bool,
3186) {
3187 if !inner.options.session_history_backfill_enabled {
3188 return;
3189 }
3190 tokio::spawn(async move {
3191 let active_key = targeted_session_history_active_key(&target, force_refresh);
3192 let result =
3193 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3194 if let Err(err) = result {
3195 tracing::warn!(
3196 active_key = %active_key,
3197 error = %err,
3198 "console targeted session-history backfill failed"
3199 );
3200 }
3201 });
3202}
3203
3204async fn run_targeted_session_history_backfill(
3205 inner: Arc<AggregatorInner>,
3206 target: SessionBackfillTarget,
3207 force_refresh: bool,
3208) -> ConsoleLogResult<()> {
3209 let active_key = targeted_session_history_active_key(&target, force_refresh);
3210 {
3211 let mut active = inner.active_session_backfills.lock().await;
3212 if !active.insert(active_key.clone()) {
3213 return Ok(());
3214 }
3215 }
3216 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3217 let mut active = inner.active_session_backfills.lock().await;
3218 active.remove(&active_key);
3219 result
3220}
3221
3222fn targeted_session_history_active_key(
3223 target: &SessionBackfillTarget,
3224 force_refresh: bool,
3225) -> String {
3226 let mode = if force_refresh { "force" } else { "refresh" };
3227 format!(
3228 "{}:session-history:{}:{mode}",
3229 target.entry.runtime_key, target.session_id
3230 )
3231}
3232
3233fn spawn_session_history_backfill_for_identity(
3234 inner: Arc<AggregatorInner>,
3235 identity: String,
3236 force_refresh: bool,
3237) {
3238 if !inner.options.session_history_backfill_enabled {
3239 return;
3240 }
3241 tokio::spawn(async move {
3242 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3243 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3244 }
3245 });
3246}
3247
3248fn spawn_opportunistic_session_history_backfill_for_identity(
3249 inner: Arc<AggregatorInner>,
3250 identity: String,
3251) {
3252 if !inner.options.session_history_backfill_enabled {
3253 return;
3254 }
3255 tokio::spawn(async move {
3256 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3257 let active_key = format!(
3258 "{}:session-history:{}",
3259 target.entry.runtime_key, target.session_id
3260 );
3261 {
3262 let mut seen = inner.opportunistic_session_backfills.lock().await;
3263 if !seen.insert(active_key) {
3264 continue;
3265 }
3266 }
3267 spawn_session_history_backfill_target(inner.clone(), target, false);
3268 }
3269 });
3270}
3271
3272#[cfg(test)]
3273async fn session_backfill_target_for_identity(
3274 inner: &AggregatorInner,
3275 identity: &str,
3276) -> Option<SessionBackfillTarget> {
3277 session_backfill_targets_for_identity(inner, identity)
3278 .await
3279 .into_iter()
3280 .next()
3281}
3282
3283async fn session_backfill_targets_for_identity(
3284 inner: &AggregatorInner,
3285 identity: &str,
3286) -> Vec<SessionBackfillTarget> {
3287 let entries = inner
3288 .runtimes
3289 .read()
3290 .ok()
3291 .map(|entries| entries.clone())
3292 .unwrap_or_default();
3293 let mut targets = Vec::new();
3294 for entry in entries.values() {
3295 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3296 if raw_identities.is_empty() {
3297 continue;
3298 }
3299 let mids = raw_identities
3300 .iter()
3301 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3302 .collect::<Vec<_>>();
3303 for resolved in Box::pin(member_sources_for_entry(entry))
3304 .await
3305 .into_iter()
3306 .filter(|candidate| {
3307 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3308 })
3309 {
3310 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3311 continue;
3312 };
3313 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3314 continue;
3315 }
3316 let Some(session_id) = record.session_id.clone() else {
3317 continue;
3318 };
3319 targets.push(SessionBackfillTarget {
3320 entry: entry.clone(),
3321 record,
3322 session_id,
3323 });
3324 }
3325 }
3326 targets
3327}
3328
3329fn resolved_member_matches_raw_identities(
3330 resolved: &ResolvedConsoleMember,
3331 raw_identities: &[String],
3332 mids: &[AgentIdentity],
3333) -> bool {
3334 mids.contains(&resolved.member.agent_identity)
3335 || resolved
3336 .member
3337 .labels
3338 .get("agent_identity")
3339 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3340}
3341
3342async fn recover_lagged_source_events(
3343 inner: Arc<AggregatorInner>,
3344 runtime_key: &str,
3345 console_events: &ConsoleEventStore,
3346) -> ConsoleLogResult<()> {
3347 let watermark = inner
3348 .store
3349 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3350 .await?;
3351 match console_events.replay_all(watermark.as_deref()).await {
3352 Ok(events) => {
3353 for envelope in events {
3354 project_console_event(inner.clone(), runtime_key, envelope).await?;
3355 }
3356 }
3357 Err(err) => {
3358 append_source_gap(
3359 &inner,
3360 runtime_key,
3361 format!(
3362 "{}:{}:{}",
3363 err.error, err.stream, err.requested_last_event_id
3364 ),
3365 )
3366 .await?;
3367 }
3368 }
3369 Ok(())
3370}
3371
3372async fn append_source_gap(
3373 inner: &AggregatorInner,
3374 runtime_key: &str,
3375 reason: String,
3376) -> ConsoleLogResult<()> {
3377 append_and_emit(
3378 inner,
3379 NewConsoleFrame {
3380 id: None,
3381 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3382 timestamp_ms: current_time_ms(),
3383 runtime_key: runtime_key.to_string(),
3384 identity: "__console__".to_string(),
3385 conversation_id: None,
3386 session_id: None,
3387 kind: "replay_unavailable".to_string(),
3388 status: ConsoleFrameStatus::DeliveryFailed,
3389 payload: json!({
3390 "reason": reason,
3391 "source_kind": "console_event",
3392 }),
3393 source: ConsoleFrameSource {
3394 kind: ConsoleFrameSourceKind::Synthetic,
3395 source_cursor: None,
3396 },
3397 source_event_id: None,
3398 interaction_id: None,
3399 turn_id: None,
3400 run_id: None,
3401 parent_frame_id: None,
3402 caused_by_frame_id: None,
3403 },
3404 )
3405 .await?;
3406 let _ = inner
3407 .event_tx
3408 .send(ConsoleTimelineEvent::ReplayUnavailable {
3409 requested_cursor: format!("source-gap:{runtime_key}"),
3410 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3411 });
3412 Ok(())
3413}
3414
3415async fn record_session_history_backfill_failure(
3420 inner: &AggregatorInner,
3421 watermark_runtime_key: &str,
3422 runtime_key: &str,
3423 identity: &str,
3424 session_id: &str,
3425 offset: usize,
3426 reason: String,
3427) -> ConsoleLogResult<()> {
3428 record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3429 append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3430}
3431
3432fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3435 format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3436}
3437
3438async fn append_backfill_gap(
3439 inner: &AggregatorInner,
3440 runtime_key: &str,
3441 identity: &str,
3442 session_id: &str,
3443 reason: String,
3444) -> ConsoleLogResult<()> {
3445 append_and_emit(
3446 inner,
3447 NewConsoleFrame {
3448 id: None,
3449 dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3454 timestamp_ms: current_time_ms(),
3455 runtime_key: runtime_key.to_string(),
3456 identity: identity.to_string(),
3457 conversation_id: Some(identity.to_string()),
3458 session_id: Some(session_id.to_string()),
3459 kind: "replay_unavailable".to_string(),
3460 status: ConsoleFrameStatus::DeliveryFailed,
3461 payload: json!({
3462 "reason": reason,
3463 "source_kind": "session_history",
3464 }),
3465 source: ConsoleFrameSource {
3466 kind: ConsoleFrameSourceKind::Synthetic,
3467 source_cursor: None,
3468 },
3469 source_event_id: None,
3470 interaction_id: None,
3471 turn_id: None,
3472 run_id: None,
3473 parent_frame_id: None,
3474 caused_by_frame_id: None,
3475 },
3476 )
3477 .await?;
3478 Ok(())
3479}
3480
3481async fn project_console_event(
3482 inner: Arc<AggregatorInner>,
3483 runtime_key: &str,
3484 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3485) -> ConsoleLogResult<()> {
3486 let Some(entry) = inner
3487 .runtimes
3488 .read()
3489 .ok()
3490 .and_then(|entries| entries.get(runtime_key).cloned())
3491 else {
3492 return Ok(());
3493 };
3494 let mut frame = frame_from_console_event(&entry, envelope);
3495 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3496 frame.payload = redacted;
3497 frame.status = ConsoleFrameStatus::Redacted;
3498 }
3499 let source_cursor = frame
3500 .source_event_id
3501 .clone()
3502 .unwrap_or_else(|| frame.dedupe_key.clone());
3503 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3504 Some(frame.identity.clone())
3505 } else {
3506 None
3507 };
3508 let opportunistic_refresh_identity = if refresh_identity.is_none()
3509 && console_event_should_start_session_history_backfill(&frame)
3510 {
3511 Some(frame.identity.clone())
3512 } else {
3513 None
3514 };
3515 append_and_emit(&inner, frame).await?;
3516 inner
3517 .store
3518 .record_source_watermark(
3519 &entry.runtime_key,
3520 ConsoleFrameSourceKind::ConsoleEvent,
3521 &source_cursor,
3522 )
3523 .await?;
3524 if let Some(identity) = refresh_identity {
3525 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3526 } else if let Some(identity) = opportunistic_refresh_identity {
3527 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3528 }
3529 Ok(())
3530}
3531
3532fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3533 matches!(
3534 frame.kind.as_str(),
3535 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3536 ) || frame.session_id.is_some()
3537}
3538
3539fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3540 if frame.identity == SYSTEM_EVENT_IDENTITY {
3541 return false;
3542 }
3543 matches!(
3544 frame.kind.as_str(),
3545 "turn_started"
3546 | "run_started"
3547 | "reasoning_complete"
3548 | "tool_call_requested"
3549 | "tool_call"
3550 | "tool_execution_started"
3551 | "text_delta"
3552 | "system_notice"
3553 )
3554}
3555
3556async fn append_and_emit(
3557 inner: &AggregatorInner,
3558 frame: NewConsoleFrame,
3559) -> ConsoleLogResult<AppendOutcome> {
3560 let outcome = inner.store.append_if_absent(frame).await?;
3561 if outcome.disposition == AppendDisposition::Inserted {
3562 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3563 frame: outcome.frame.clone(),
3564 });
3565 }
3566 Ok(outcome)
3567}
3568
3569async fn update_frame_status_and_emit(
3570 inner: &AggregatorInner,
3571 frame_id: &str,
3572 status: ConsoleFrameStatus,
3573) -> ConsoleLogResult<Option<ConsoleFrame>> {
3574 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3575 return Ok(None);
3576 };
3577 let update_marker = NewConsoleFrame {
3578 id: None,
3579 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3580 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3581 runtime_key: updated.runtime_key.clone(),
3582 identity: updated.identity.clone(),
3583 conversation_id: updated.conversation_id.clone(),
3584 session_id: updated.session_id.clone(),
3585 kind: "frame_updated".to_string(),
3586 status: updated.status,
3587 payload: json!({ "frame": updated.clone() }),
3588 source: ConsoleFrameSource {
3589 kind: ConsoleFrameSourceKind::Synthetic,
3590 source_cursor: None,
3591 },
3592 source_event_id: None,
3593 interaction_id: updated.interaction_id.clone(),
3594 turn_id: updated.turn_id.clone(),
3595 run_id: updated.run_id.clone(),
3596 parent_frame_id: Some(updated.id.clone()),
3597 caused_by_frame_id: Some(updated.id.clone()),
3598 };
3599 let outcome = inner.store.append_if_absent(update_marker).await?;
3600 if outcome.disposition == AppendDisposition::Inserted {
3601 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3602 frame: outcome.frame,
3603 });
3604 }
3605 Ok(Some(updated))
3606}
3607
3608fn frame_from_console_event(
3609 entry: &RuntimeEntry,
3610 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3611) -> NewConsoleFrame {
3612 let event_id = envelope.event_id;
3613 let interaction_id = envelope.interaction_id;
3614 let event_type = envelope.event_type;
3615 let payload = envelope.data;
3616 let turn_id = payload
3617 .get("turn_id")
3618 .and_then(Value::as_str)
3619 .map(ToString::to_string);
3620 let run_id = payload
3621 .get("run_id")
3622 .and_then(Value::as_str)
3623 .map(ToString::to_string);
3624 let status = match event_type.as_str() {
3625 "interaction_started" => ConsoleFrameStatus::Accepted,
3626 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3627 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3628 _ => ConsoleFrameStatus::Delivered,
3629 };
3630 let identity = if envelope.identity == SYSTEM_EVENT_IDENTITY {
3635 envelope.identity
3636 } else {
3637 apply_namespace(&envelope.identity, &entry.identity_namespace)
3638 };
3639 let dedupe_key = reasoning_payload_text(&event_type, &payload)
3640 .map(|reasoning_text| {
3641 let turn_scope = interaction_id
3644 .as_deref()
3645 .or(turn_id.as_deref())
3646 .or(run_id.as_deref())
3647 .unwrap_or(event_id.as_str());
3648 format!(
3649 "console-reasoning:{}:{}:{}",
3650 entry.runtime_key,
3651 turn_scope,
3652 hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3653 )
3654 })
3655 .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3656 NewConsoleFrame {
3657 id: Some(event_id.clone()),
3658 dedupe_key,
3659 timestamp_ms: envelope.timestamp_ms,
3660 runtime_key: entry.runtime_key.clone(),
3661 identity: identity.clone(),
3662 conversation_id: Some(identity),
3663 session_id: payload
3664 .get("session_id")
3665 .and_then(Value::as_str)
3666 .map(ToString::to_string),
3667 kind: event_type,
3668 status,
3669 payload,
3670 source: ConsoleFrameSource {
3671 kind: ConsoleFrameSourceKind::ConsoleEvent,
3672 source_cursor: None,
3673 },
3674 source_event_id: Some(event_id),
3675 interaction_id,
3676 turn_id,
3677 run_id,
3678 parent_frame_id: None,
3679 caused_by_frame_id: None,
3680 }
3681}
3682
3683#[cfg(test)]
3684fn frame_from_session_history_message(
3685 runtime_key: &str,
3686 identity: &str,
3687 session_id: &str,
3688 offset: usize,
3689 message: Value,
3690) -> Option<NewConsoleFrame> {
3691 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3692 .into_iter()
3693 .next()
3694}
3695
3696#[cfg(test)]
3697fn frames_from_session_history_message(
3698 runtime_key: &str,
3699 identity: &str,
3700 session_id: &str,
3701 offset: usize,
3702 message: Value,
3703) -> Vec<NewConsoleFrame> {
3704 frames_from_session_history_message_with_namespace(
3705 runtime_key,
3706 identity,
3707 "",
3708 session_id,
3709 offset,
3710 message,
3711 )
3712}
3713
3714#[allow(clippy::too_many_arguments)]
3722fn session_history_assistant_image_frames(
3723 runtime_key: &str,
3724 identity: &str,
3725 session_id: &str,
3726 offset: usize,
3727 result_idx: usize,
3728 timestamp_ms: u64,
3729 tool_use_id: &str,
3730 result_text: &str,
3731) -> Vec<NewConsoleFrame> {
3732 let Ok(image_result) = serde_json::from_str::<
3733 meerkat_core::image_generation::ImageGenerationToolResult,
3734 >(result_text) else {
3735 return Vec::new();
3736 };
3737 image_result
3738 .images
3739 .iter()
3740 .enumerate()
3741 .map(|(image_idx, image)| NewConsoleFrame {
3742 id: None,
3743 dedupe_key: format!(
3746 "console-assistant-image:{runtime_key}:{}",
3747 image.blob_ref.blob_id
3748 ),
3749 timestamp_ms,
3750 runtime_key: runtime_key.to_string(),
3751 identity: identity.to_string(),
3752 conversation_id: Some(identity.to_string()),
3753 session_id: Some(session_id.to_string()),
3754 kind: "assistant_image".to_string(),
3755 status: ConsoleFrameStatus::Completed,
3756 payload: json!({
3757 "source_event_type": "session_history",
3758 "tool_call_id": tool_use_id,
3759 "image_id": image.image_id.0.to_string(),
3760 "blob_id": image.blob_ref.blob_id,
3761 "media_type": image.media_type.as_str(),
3762 "width": image.width,
3763 "height": image.height,
3764 "revised_prompt": image_result.revised_prompt.clone(),
3765 "type": "session_history",
3766 }),
3767 source: ConsoleFrameSource {
3768 kind: ConsoleFrameSourceKind::SessionHistory,
3769 source_cursor: Some(format!(
3770 "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3771 )),
3772 },
3773 source_event_id: None,
3774 interaction_id: None,
3775 turn_id: None,
3776 run_id: None,
3777 parent_frame_id: None,
3778 caused_by_frame_id: None,
3779 })
3780 .collect()
3781}
3782
3783fn frames_from_session_history_message_with_namespace(
3784 runtime_key: &str,
3785 identity: &str,
3786 identity_namespace: &str,
3787 session_id: &str,
3788 offset: usize,
3789 message: Value,
3790) -> Vec<NewConsoleFrame> {
3791 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3792 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3793 return Vec::new();
3794 };
3795 if let Message::ToolResults {
3796 results,
3797 created_at,
3798 } = parsed
3799 {
3800 return results
3801 .into_iter()
3802 .enumerate()
3803 .flat_map(|(idx, result)| {
3804 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3805 let result_text = result.text_content();
3806 let tool_use_id = result.tool_use_id.clone();
3807 let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3808 let mut frames = vec![NewConsoleFrame {
3809 id: None,
3810 dedupe_key: format!(
3811 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3812 ),
3813 timestamp_ms,
3814 runtime_key: runtime_key.to_string(),
3815 identity: identity.to_string(),
3816 conversation_id: Some(identity.to_string()),
3817 session_id: Some(session_id.to_string()),
3818 kind: "tool_execution_completed".to_string(),
3819 status: ConsoleFrameStatus::Completed,
3820 payload: json!({
3821 "id": tool_use_id,
3822 "tool_call_id": tool_use_id,
3823 "result": result_text,
3824 "content": content,
3825 "is_error": result.is_error,
3826 "source_event_type": "session_history",
3827 "type": "session_history",
3828 }),
3829 source: ConsoleFrameSource {
3830 kind: ConsoleFrameSourceKind::SessionHistory,
3831 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3832 },
3833 source_event_id: None,
3834 interaction_id: None,
3835 turn_id: None,
3836 run_id: None,
3837 parent_frame_id: None,
3838 caused_by_frame_id: None,
3839 }];
3840 frames.extend(session_history_assistant_image_frames(
3848 runtime_key,
3849 identity,
3850 session_id,
3851 offset,
3852 idx,
3853 timestamp_ms,
3854 &tool_use_id,
3855 &result_text,
3856 ));
3857 frames
3858 })
3859 .collect();
3860 }
3861 let (kind, timestamp_ms, payload) = match &parsed {
3862 Message::User(user) => {
3863 if session_history_user_message_is_scaffold(&message) {
3864 return Vec::new();
3865 }
3866 (
3867 "user_input",
3868 user.created_at.timestamp_millis().max(0) as u64,
3869 json!({
3870 "content": user.content,
3871 "message": message,
3872 }),
3873 )
3874 }
3875 Message::BlockAssistant(assistant) => {
3878 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3879 (
3880 "interaction_complete",
3881 assistant.created_at.timestamp_millis().max(0) as u64,
3882 json!({
3883 "result": text,
3884 "text": text,
3885 "message": message,
3886 "source_event_type": "session_history",
3887 "type": "session_history",
3888 }),
3889 )
3890 }
3891 Message::SystemNotice(notice) => (
3892 "system_notice",
3893 notice.created_at.timestamp_millis().max(0) as u64,
3894 json!({
3895 "message": message,
3896 "kind": notice.kind,
3897 "render_class": notice.kind.render_class(),
3898 "body": notice.body,
3899 "blocks": notice.blocks,
3900 "source_event_type": "session_history",
3901 "type": "session_history",
3902 }),
3903 ),
3904 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3905 };
3906 let mut frames = vec![NewConsoleFrame {
3907 id: None,
3908 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3909 timestamp_ms,
3910 runtime_key: runtime_key.to_string(),
3911 identity: identity.to_string(),
3912 conversation_id: Some(identity.to_string()),
3913 session_id: Some(session_id.to_string()),
3914 kind: kind.to_string(),
3915 status: ConsoleFrameStatus::Completed,
3916 payload,
3917 source: ConsoleFrameSource {
3918 kind: ConsoleFrameSourceKind::SessionHistory,
3919 source_cursor: Some(format!("{session_id}:{offset}")),
3920 },
3921 source_event_id: None,
3922 interaction_id: None,
3923 turn_id: None,
3924 run_id: None,
3925 parent_frame_id: None,
3926 caused_by_frame_id: None,
3927 }];
3928 if let Message::BlockAssistant(assistant) = &parsed {
3929 frames.extend(spawn_initial_message_frames_from_assistant(
3930 runtime_key,
3931 identity,
3932 identity_namespace,
3933 session_id,
3934 offset,
3935 assistant,
3936 &payload_hash,
3937 ));
3938 }
3939 frames
3940}
3941
3942fn spawn_initial_message_frames_from_assistant(
3943 runtime_key: &str,
3944 parent_identity: &str,
3945 identity_namespace: &str,
3946 session_id: &str,
3947 offset: usize,
3948 assistant: &meerkat_core::types::BlockAssistantMessage,
3949 payload_hash: &str,
3950) -> Vec<NewConsoleFrame> {
3951 let mut frames = Vec::new();
3952 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3953 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3954 continue;
3955 }
3956 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3957 continue;
3958 };
3959 for (spawn_idx, (target_identity, initial_message)) in
3960 spawn_initial_messages_from_tool_args(&args)
3961 .into_iter()
3962 .enumerate()
3963 {
3964 let target_identity = apply_namespace(&target_identity, identity_namespace);
3965 let message_content = match &initial_message {
3966 Value::String(text) => json!([
3967 {
3968 "type": "text",
3969 "text": text,
3970 }
3971 ]),
3972 other => other.clone(),
3973 };
3974 let message = match &initial_message {
3975 Value::String(text) => json!({
3976 "role": "user",
3977 "content": text,
3978 "created_at": assistant.created_at,
3979 }),
3980 other => json!({
3981 "role": "user",
3982 "content": other,
3983 "created_at": assistant.created_at,
3984 }),
3985 };
3986 frames.push(NewConsoleFrame {
3987 id: None,
3988 dedupe_key: format!(
3989 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3990 ),
3991 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3992 runtime_key: runtime_key.to_string(),
3993 identity: target_identity.clone(),
3994 conversation_id: Some(target_identity),
3995 session_id: None,
3996 kind: "user_input".to_string(),
3997 status: ConsoleFrameStatus::Completed,
3998 payload: json!({
3999 "content": message_content,
4000 "message": message,
4001 "source_event_type": "session_history_spawn_initial_message",
4002 "type": "session_history",
4003 "tool_call_id": tool.id,
4004 "parent_identity": parent_identity,
4005 "via_tool": tool.name,
4006 }),
4007 source: ConsoleFrameSource {
4008 kind: ConsoleFrameSourceKind::SessionHistory,
4009 source_cursor: Some(format!(
4010 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4011 )),
4012 },
4013 source_event_id: None,
4014 interaction_id: None,
4015 turn_id: None,
4016 run_id: None,
4017 parent_frame_id: None,
4018 caused_by_frame_id: None,
4019 });
4020 }
4021 }
4022 frames
4023}
4024
4025fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4026 let mut messages = Vec::new();
4027 if let Some(message) = spawn_initial_message_from_record(args) {
4028 messages.push(message);
4029 }
4030 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4031 for spec in specs {
4032 if let Some(message) = spawn_initial_message_from_record(spec) {
4033 messages.push(message);
4034 }
4035 }
4036 }
4037 messages
4038}
4039
4040fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4041 let target_identity = record
4042 .get("member_id")
4043 .or_else(|| record.get("agent_identity"))
4044 .and_then(Value::as_str)?
4045 .trim();
4046 if target_identity.is_empty() {
4047 return None;
4048 }
4049 let initial_message = record
4050 .get("initial_message")
4051 .or_else(|| record.get("task"))?
4052 .clone();
4053 Some((target_identity.to_string(), initial_message))
4054}
4055
4056fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4057 message
4058 .get("content")
4059 .is_some_and(scaffold_message_content_is_noise)
4060}
4061
4062fn scaffold_message_content_is_noise(value: &Value) -> bool {
4063 match value {
4064 Value::String(text) => scaffold_message_text_is_noise(text),
4065 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4066 Value::Object(map) => ["text", "content", "message"]
4067 .iter()
4068 .filter_map(|key| map.get(*key))
4069 .any(scaffold_message_content_is_noise),
4070 _ => false,
4071 }
4072}
4073
4074fn scaffold_message_text_is_noise(text: &str) -> bool {
4075 let trimmed = text.trim_start();
4076 trimmed.starts_with("[PEER UPDATE]")
4077 || trimmed
4078 .to_ascii_lowercase()
4079 .starts_with("you have been spawned")
4080}
4081
4082fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4083 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4084 rest.split(':').next()?.parse().ok()
4085}
4086
4087fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4088 format!("{session_id}:{offset}:{checked_at_ms}")
4089}
4090
4091fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4092 let Some(checked_at_ms) = watermark
4093 .rsplit_once(':')
4094 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4095 else {
4096 return false;
4097 };
4098 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4099 let ttl_ms = if offset > 0 {
4100 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4101 } else {
4102 SESSION_HISTORY_REFRESH_TTL_MS
4103 };
4104 now_ms.saturating_sub(checked_at_ms) < ttl_ms
4105}
4106
4107async fn history_frame_has_existing_counterpart(
4108 inner: &AggregatorInner,
4109 frame: &NewConsoleFrame,
4110) -> ConsoleLogResult<bool> {
4111 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4112 let Some(fingerprint) = fingerprint else {
4113 return Ok(false);
4114 };
4115 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4116 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4117 let mut after = None;
4118 loop {
4119 let page = inner
4120 .store
4121 .query_frames(ConsoleTimelineQuery {
4122 identity: Some(frame.identity.clone()),
4123 conversation_id: frame.conversation_id.clone(),
4124 after,
4125 limit: 1_000,
4126 })
4127 .await?;
4128 for existing in &page.frames {
4129 let same_session = existing.session_id == frame.session_id
4130 || existing.session_id.is_none()
4131 || frame.session_id.is_none();
4132 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4133 continue;
4134 }
4135 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4136 == Some(&fingerprint)
4137 {
4138 return Ok(true);
4139 }
4140 if assistant_terminal
4141 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4142 {
4143 let turn_key = existing
4144 .interaction_id
4145 .as_deref()
4146 .or(existing.turn_id.as_deref())
4147 .or(existing.run_id.as_deref())
4148 .unwrap_or("session");
4149 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4150 aggregated.push_str(delta);
4151 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4152 return Ok(true);
4153 }
4154 }
4155 }
4156 if page.frames.is_empty() || page.next_cursor.is_none() {
4157 return Ok(false);
4158 }
4159 after = page.next_cursor;
4160 }
4161}
4162
4163fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4164 format!("{runtime_key}:session-history:{session_id}")
4165}
4166
4167fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4168 match kind {
4169 "user_input" | "interaction_started" => payload
4170 .get("content")
4171 .map(stable_value_fingerprint)
4172 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4173 "tool_execution_completed" => tool_result_fingerprint(payload),
4174 "text_delta" => {
4175 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4176 }
4177 kind if is_reasoning_event_kind(kind) => {
4178 reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4179 }
4180 "text_complete" | "interaction_complete" | "run_completed" => {
4181 assistant_terminal_fingerprint(kind, payload)
4182 }
4183 _ => None,
4184 }
4185}
4186
4187fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4188 let id = payload
4189 .get("tool_call_id")
4190 .or_else(|| payload.get("id"))
4191 .and_then(Value::as_str)
4192 .unwrap_or("");
4193 let result = payload
4194 .get("result")
4195 .or_else(|| payload.get("content"))
4196 .map(stable_value_fingerprint)?;
4197 Some(format!("{id}:{result}"))
4198}
4199
4200fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4201 match kind {
4202 "text_complete" | "interaction_complete" | "run_completed" => payload
4203 .get("text")
4204 .or_else(|| payload.get("result"))
4205 .or_else(|| payload.get("content"))
4206 .map(stable_value_fingerprint),
4207 _ => None,
4208 }
4209}
4210
4211fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4212 if kind != "text_delta" {
4213 return None;
4214 }
4215 payload
4216 .get("delta")
4217 .or_else(|| payload.get("text"))
4218 .or_else(|| payload.get("content"))
4219 .and_then(Value::as_str)
4220 .or_else(|| payload.as_str())
4221}
4222
4223fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4224 if !is_reasoning_event_kind(kind) {
4225 return None;
4226 }
4227 if kind == "reasoning_delta" {
4228 return payload
4229 .get("delta")
4230 .or_else(|| payload.get("text"))
4231 .or_else(|| payload.get("content"))
4232 .or_else(|| payload.get("summary"))
4233 .or_else(|| payload.get("reasoning"))
4234 .and_then(Value::as_str)
4235 .or_else(|| payload.as_str());
4236 }
4237 payload
4238 .get("summary")
4239 .or_else(|| payload.get("text"))
4240 .or_else(|| payload.get("content"))
4241 .or_else(|| payload.get("delta"))
4242 .or_else(|| payload.get("reasoning"))
4243 .and_then(Value::as_str)
4244 .or_else(|| payload.as_str())
4245}
4246
4247fn is_reasoning_event_kind(kind: &str) -> bool {
4248 matches!(
4249 kind,
4250 "reasoning_delta"
4251 | "reasoning_complete"
4252 | "reasoning_summary"
4253 | "reasoning_summary_delta"
4254 | "reasoning_summary_complete"
4255 )
4256}
4257
4258fn stable_value_fingerprint(value: &Value) -> String {
4259 if let Some(text) = content_value_text(value) {
4260 return normalize_transcript_fingerprint_text(&text);
4261 }
4262 match value {
4263 Value::String(text) => normalize_transcript_fingerprint_text(text),
4264 other => serde_json::to_string(other).unwrap_or_default(),
4265 }
4266}
4267
4268fn content_value_text(value: &Value) -> Option<String> {
4269 match value {
4270 Value::String(text) => Some(text.clone()),
4271 Value::Array(items) => {
4272 let text = items
4273 .iter()
4274 .filter_map(content_value_text)
4275 .collect::<String>();
4276 (!text.is_empty()).then_some(text)
4277 }
4278 Value::Object(map) => ["text", "content", "message", "blocks"]
4279 .iter()
4280 .filter_map(|key| map.get(*key))
4281 .find_map(content_value_text),
4282 _ => None,
4283 }
4284}
4285
4286fn normalize_transcript_fingerprint_text(text: &str) -> String {
4287 let trimmed = text.trim();
4288 trimmed
4289 .strip_prefix("[EVENT via rpc] ")
4290 .unwrap_or(trimmed)
4291 .trim()
4292 .to_string()
4293}
4294
4295#[derive(Debug, Clone, Copy)]
4296enum CachedIdentityVisibility {
4297 Visible,
4298 Hidden,
4299 Missing,
4300}
4301
4302async fn frame_is_visible(
4303 inner: &AggregatorInner,
4304 frame: &ConsoleFrame,
4305 allow_historical_identity: bool,
4306 identity_records: &[ConsoleIdentityRecord],
4307) -> ConsoleLogResult<bool> {
4308 let mut identity_visibility_cache = HashMap::new();
4309 Box::pin(frame_is_visible_cached(
4310 inner,
4311 frame,
4312 allow_historical_identity,
4313 &mut identity_visibility_cache,
4314 identity_records,
4315 ))
4316 .await
4317}
4318
4319async fn frame_is_visible_cached(
4320 inner: &AggregatorInner,
4321 frame: &ConsoleFrame,
4322 allow_historical_identity: bool,
4323 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4324 identity_records: &[ConsoleIdentityRecord],
4325) -> ConsoleLogResult<bool> {
4326 let entry = {
4327 let entries = inner
4328 .runtimes
4329 .read()
4330 .map_err(|_| runtime_registry_lock_error())?;
4331 if entries.is_empty() {
4332 return Ok(true);
4333 }
4334 let Some(entry) = entries.get(&frame.runtime_key) else {
4335 return Ok(false);
4336 };
4337 entry.clone()
4338 };
4339 if frame.identity != "__console__" && frame.identity != SYSTEM_EVENT_IDENTITY {
4347 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4348 let identity_visibility =
4349 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4350 cached
4351 } else {
4352 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4353 .unwrap_or_else(|| frame.identity.clone());
4354 let visibility = match identity_records.iter().find(|record| {
4355 record.runtime_key == frame.runtime_key
4356 && (record.identity == frame.identity
4357 || record.runtime_member_id == runtime_member_id)
4358 }) {
4359 Some(record) => {
4360 if Box::pin(console_identity_record_visible(&entry, record)).await {
4361 CachedIdentityVisibility::Visible
4362 } else {
4363 CachedIdentityVisibility::Hidden
4364 }
4365 }
4366 None => CachedIdentityVisibility::Missing,
4367 };
4368 identity_visibility_cache.insert(cache_key, visibility);
4369 visibility
4370 };
4371 match identity_visibility {
4372 CachedIdentityVisibility::Visible => {
4373 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4374 return Ok(false);
4375 }
4376 }
4377 CachedIdentityVisibility::Hidden => return Ok(false),
4378 CachedIdentityVisibility::Missing => {
4379 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4380 return Ok(false);
4381 }
4382 return Ok(
4383 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4384 );
4385 }
4386 }
4387 }
4388 Ok(entry.visibility_policy.frame_visible(frame))
4389}
4390
4391async fn identity_record_for_member(
4392 entry: &RuntimeEntry,
4393 handle: &MobHandle,
4394 member: &MobMemberListEntry,
4395) -> Option<ConsoleIdentityRecord> {
4396 let runtime_member_id =
4399 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4400 let durable_identity = member
4401 .labels
4402 .get("agent_identity")
4403 .filter(|value| !value.trim().is_empty())
4404 .map_or(runtime_member_id.as_str(), String::as_str);
4405 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4406 let addressable = member_is_addressable(member);
4407 let visibility = match member.status {
4408 meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4409 ConsoleVisibility::RetiredReadable
4410 }
4411 meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4414 ConsoleVisibility::Unreachable
4415 }
4416 _ if addressable => ConsoleVisibility::Addressable,
4417 _ => ConsoleVisibility::Hidden,
4418 };
4419 let session_id = handle
4420 .resolve_bridge_session_id_observation(&member.agent_identity)
4421 .await
4422 .map(|sid| sid.to_string());
4423 let mut labels = member.labels.clone();
4424 labels
4425 .entry("role".to_string())
4426 .or_insert_with(|| member.role.to_string());
4427 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4433 if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4434 crate::console_spawn::merge_registered_labels(&mut labels, ®istered);
4435 }
4436 let display_name = labels
4437 .get("display_name")
4438 .cloned()
4439 .unwrap_or_else(|| runtime_member_id.clone());
4440 Some(ConsoleIdentityRecord {
4441 identity,
4442 display_name,
4443 runtime_key: entry.runtime_key.clone(),
4444 runtime_member_id,
4445 session_id,
4446 visibility,
4447 addressable,
4448 health: if addressable {
4449 "ready"
4450 } else {
4451 "hidden_by_policy"
4452 }
4453 .to_string(),
4454 topology_peers: member
4455 .wired_to
4456 .iter()
4457 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4458 .collect(),
4459 labels,
4460 })
4461}
4462
4463async fn identity_record_for_resolved_member(
4464 resolved: &ResolvedConsoleMember,
4465) -> Option<ConsoleIdentityRecord> {
4466 let mut record =
4467 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4468 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4469 record
4470 .labels
4471 .entry("source_mob_id".to_string())
4472 .or_insert_with(|| resolved.source_mob_id.clone());
4473 }
4474 Some(record)
4475}
4476
4477fn identity_record_for_status(
4478 entry: &RuntimeEntry,
4479 status: &crate::identity_first::IdentityStatus,
4480) -> ConsoleIdentityRecord {
4481 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4482 let runtime_member_id = status
4483 .agent_runtime_id
4484 .as_ref()
4485 .map(crate::identity_first::AgentRuntimeId::as_str)
4486 .unwrap_or_else(|| status.identity.as_str())
4487 .to_string();
4488 let addressable = status.addressability
4489 == crate::identity_first::AgentAddressability::Addressable
4490 && matches!(
4491 status.state,
4492 crate::identity_first::IdentityLifecycleState::Active
4493 | crate::identity_first::IdentityLifecycleState::Dormant
4494 | crate::identity_first::IdentityLifecycleState::Uninitialized
4495 );
4496 let visibility = match status.state {
4497 crate::identity_first::IdentityLifecycleState::Retiring => {
4498 ConsoleVisibility::RetiredReadable
4499 }
4500 crate::identity_first::IdentityLifecycleState::Broken
4501 | crate::identity_first::IdentityLifecycleState::Suspended => {
4502 ConsoleVisibility::Unreachable
4503 }
4504 _ if addressable => ConsoleVisibility::Addressable,
4505 _ => ConsoleVisibility::Hidden,
4506 };
4507 let health = match status.state {
4508 crate::identity_first::IdentityLifecycleState::Active => "ready",
4509 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4510 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4511 crate::identity_first::IdentityLifecycleState::Broken => "broken",
4512 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4513 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4514 }
4515 .to_string();
4516 let display_name = status
4517 .display_name
4518 .as_ref()
4519 .map(crate::identity_first::DisplayName::as_str)
4520 .unwrap_or_else(|| status.identity.as_str())
4521 .to_string();
4522 ConsoleIdentityRecord {
4523 identity,
4524 display_name,
4525 runtime_key: entry.runtime_key.clone(),
4526 runtime_member_id,
4527 session_id: status.session_id.as_ref().map(ToString::to_string),
4528 visibility,
4529 addressable,
4530 health,
4531 topology_peers: Vec::new(),
4532 labels: status.labels.clone(),
4533 }
4534}
4535
4536pub(crate) fn is_implicit_delegate_member(
4537 role: &str,
4538 labels: &std::collections::BTreeMap<String, String>,
4539) -> bool {
4540 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4541}
4542
4543fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4544 member
4545 .labels
4546 .get("addressable")
4547 .map(|value| !value.eq_ignore_ascii_case("false"))
4548 .unwrap_or(true)
4549}
4550
4551fn apply_namespace(identity: &str, namespace: &str) -> String {
4552 let namespace = namespace.trim().trim_matches('/');
4553 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4554 identity.to_string()
4555 } else {
4556 format!("{namespace}/{identity}")
4557 }
4558}
4559
4560fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4561 let namespace = namespace.trim().trim_matches('/');
4562 if namespace.is_empty() {
4563 return Some(identity.to_string());
4564 }
4565 identity
4566 .strip_prefix(namespace)
4567 .and_then(|rest| rest.strip_prefix('/'))
4568 .map(ToString::to_string)
4569}
4570
4571fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4572 let namespace = namespace.trim().trim_matches('/');
4573 if namespace.is_empty() {
4574 return vec![identity.to_string()];
4575 }
4576 let mut candidates = Vec::new();
4577 if let Some(stripped) = strip_namespace(identity, namespace) {
4578 candidates.push(stripped);
4579 candidates.push(identity.to_string());
4580 }
4581 candidates.sort();
4582 candidates.dedup();
4583 candidates
4584}
4585
4586fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4587 namespace_match_candidates(identity, namespace)
4588 .iter()
4589 .any(|candidate| candidate.starts_with("rt:"))
4590}
4591
4592fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4593 entries.iter().any(|entry| {
4594 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4595 })
4596}
4597
4598fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4599 if request.identity.trim().is_empty() {
4600 return Err(ConsoleSendError::InvalidRequest(
4601 "identity must be non-empty".to_string(),
4602 ));
4603 }
4604 if request.origin.trim().is_empty() {
4605 return Err(ConsoleSendError::InvalidRequest(
4606 "origin must be non-empty".to_string(),
4607 ));
4608 }
4609 if request.idempotency_key.trim().is_empty() {
4610 return Err(ConsoleSendError::InvalidRequest(
4611 "idempotency_key must be non-empty".to_string(),
4612 ));
4613 }
4614 Ok(())
4615}
4616
4617fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4618 let content: ContentInput = serde_json::from_value(value.clone())
4619 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4620 match &content {
4621 ContentInput::Text(text) if text.trim().is_empty() => Err(
4622 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4623 ),
4624 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4625 "content blocks must be non-empty".to_string(),
4626 )),
4627 _ => Ok(content),
4628 }
4629}
4630
4631fn parse_handling_mode(
4632 value: Option<&str>,
4633) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4634 match value.unwrap_or("queue") {
4635 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4636 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4637 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4638 }
4639}
4640
4641fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4642 ConsoleInteractionAccepted {
4643 interaction_id: frame
4644 .interaction_id
4645 .clone()
4646 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4647 identity: frame.identity.clone(),
4648 conversation_id: frame.conversation_id.clone(),
4649 session_id: frame.session_id.clone(),
4650 input_frame_id: frame.id.clone(),
4651 cursor: frame.cursor.clone(),
4652 status: frame.status,
4653 }
4654}
4655
4656fn send_dedupe_key(
4657 runtime_key: &str,
4658 identity: &str,
4659 origin: &str,
4660 idempotency_key: &str,
4661) -> String {
4662 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4663}
4664
4665fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4666 let content_json = serde_json::to_string(content).unwrap_or_default();
4667 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4668}
4669
4670fn hash_short(value: &str) -> String {
4671 let mut hasher = Sha256::new();
4672 hasher.update(value.as_bytes());
4673 let digest = hasher.finalize();
4674 to_hex(&digest[..8])
4675}
4676
4677fn to_hex(bytes: &[u8]) -> String {
4678 const HEX: &[u8; 16] = b"0123456789abcdef";
4679 let mut out = String::with_capacity(bytes.len() * 2);
4680 for byte in bytes {
4681 out.push(HEX[(byte >> 4) as usize] as char);
4682 out.push(HEX[(byte & 0x0f) as usize] as char);
4683 }
4684 out
4685}
4686
4687fn current_time_ms() -> u64 {
4688 match SystemTime::now().duration_since(UNIX_EPOCH) {
4689 Ok(duration) => duration.as_millis() as u64,
4690 Err(_) => 0,
4691 }
4692}
4693
4694fn runtime_registry_lock_error() -> ConsoleLogError {
4695 Box::new(std::io::Error::other(
4696 "console runtime registry lock poisoned",
4697 ))
4698}
4699
4700#[cfg(test)]
4701#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4702mod tests {
4703 #[test]
4708 fn backfill_gap_dedupe_key_is_stable_per_session() {
4709 use super::backfill_gap_dedupe_key;
4710 let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4711 let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4712 assert_eq!(a, b, "same target must reuse one gap frame");
4713 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4714 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4715 assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4718 }
4719
4720 use std::sync::Arc;
4721 use std::sync::atomic::{AtomicUsize, Ordering};
4722 use std::time::Duration;
4723 use std::time::Instant;
4724
4725 use futures::StreamExt;
4726 use meerkat::{AgentFactory, Config, build_ephemeral_service};
4727 use meerkat_client::types::LlmStream;
4728 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4729 use meerkat_core::{
4730 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4731 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4732 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4733 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4734 StreamError,
4735 };
4736 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4737 use serde_json::json;
4738
4739 use super::*;
4740 use crate::mob_handle_runtime::MobBootstrapSpec;
4741
4742 #[derive(Debug)]
4743 struct HideHiddenNoiseFrames;
4744
4745 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4746 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4747 frame.kind != "hidden_noise"
4748 }
4749 }
4750
4751 #[derive(Debug)]
4752 struct HideRuntimeMemberIdentity(&'static str);
4753
4754 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4755 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4756 record.runtime_member_id != self.0
4757 }
4758 }
4759
4760 #[derive(Debug)]
4761 struct HideRuntimeMemberOnly(&'static str);
4762
4763 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4764 fn member_visible(&self, member: &ConsoleMember) -> bool {
4765 member.agent_identity != self.0
4766 }
4767 }
4768
4769 #[derive(Debug)]
4770 struct HideConsoleIdentity(&'static str);
4771
4772 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4773 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4774 record.identity != self.0
4775 }
4776 }
4777
4778 struct CountingConsoleLogStore {
4779 inner: InMemoryConsoleLogStore,
4780 source_watermark_calls: AtomicUsize,
4781 record_watermark_calls: AtomicUsize,
4782 }
4783
4784 impl CountingConsoleLogStore {
4785 fn new() -> Self {
4786 Self {
4787 inner: InMemoryConsoleLogStore::new(),
4788 source_watermark_calls: AtomicUsize::new(0),
4789 record_watermark_calls: AtomicUsize::new(0),
4790 }
4791 }
4792
4793 fn source_watermark_calls(&self) -> usize {
4794 self.source_watermark_calls.load(Ordering::SeqCst)
4795 }
4796 }
4797
4798 struct SlowTestClient {
4799 delay: Duration,
4800 }
4801
4802 #[async_trait::async_trait]
4803 impl LlmClient for SlowTestClient {
4804 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4805 Ok(messages.to_vec())
4806 }
4807
4808 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4809 let delay = self.delay;
4810 let delayed_text = futures::stream::once(async move {
4811 tokio::time::sleep(delay).await;
4812 Ok(LlmEvent::TextDelta {
4813 delta: "slow ok".to_string(),
4814 meta: None,
4815 })
4816 });
4817 let done = futures::stream::once(async {
4818 Ok(LlmEvent::Done {
4819 outcome: LlmDoneOutcome::Success {
4820 stop_reason: StopReason::EndTurn,
4821 },
4822 })
4823 });
4824 Box::pin(delayed_text.chain(done))
4825 }
4826
4827 fn provider(&self) -> meerkat_core::Provider {
4828 meerkat_core::Provider::Other
4829 }
4830
4831 async fn health_check(&self) -> Result<(), LlmError> {
4832 Ok(())
4833 }
4834 }
4835
4836 #[derive(Clone)]
4837 struct DelayedHistorySessionService {
4838 inner: Arc<dyn MobSessionService>,
4839 delay: Duration,
4840 read_calls: Arc<AtomicUsize>,
4841 active_reads: Arc<AtomicUsize>,
4842 max_active_reads: Arc<AtomicUsize>,
4843 }
4844
4845 impl DelayedHistorySessionService {
4846 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4847 Self {
4848 inner,
4849 delay,
4850 read_calls: Arc::new(AtomicUsize::new(0)),
4851 active_reads: Arc::new(AtomicUsize::new(0)),
4852 max_active_reads: Arc::new(AtomicUsize::new(0)),
4853 }
4854 }
4855
4856 fn read_calls(&self) -> usize {
4857 self.read_calls.load(Ordering::SeqCst)
4858 }
4859
4860 fn max_active_reads(&self) -> usize {
4861 self.max_active_reads.load(Ordering::SeqCst)
4862 }
4863 }
4864
4865 #[async_trait::async_trait]
4866 impl SessionService for DelayedHistorySessionService {
4867 async fn create_session(
4868 &self,
4869 req: meerkat_core::CreateSessionRequest,
4870 ) -> Result<RunResult, SessionError> {
4871 self.inner.create_session(req).await
4872 }
4873
4874 async fn start_turn(
4875 &self,
4876 id: &SessionId,
4877 req: StartTurnRequest,
4878 ) -> Result<RunResult, SessionError> {
4879 self.inner.start_turn(id, req).await
4880 }
4881
4882 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4883 self.inner.interrupt(id).await
4884 }
4885
4886 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4887 self.inner.read(id).await
4888 }
4889
4890 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4891 self.inner.list(query).await
4892 }
4893
4894 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4895 self.inner.archive(id).await
4896 }
4897
4898 async fn subscribe_session_events(
4899 &self,
4900 id: &SessionId,
4901 ) -> Result<EventStream, StreamError> {
4902 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4903 }
4904 }
4905
4906 #[async_trait::async_trait]
4907 impl SessionServiceCommsExt for DelayedHistorySessionService {
4908 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4909 self.inner.comms_runtime(session_id).await
4910 }
4911 }
4912
4913 #[async_trait::async_trait]
4914 impl SessionServiceControlExt for DelayedHistorySessionService {
4915 async fn append_system_context(
4916 &self,
4917 id: &SessionId,
4918 req: AppendSystemContextRequest,
4919 ) -> Result<AppendSystemContextResult, SessionControlError> {
4920 self.inner.append_system_context(id, req).await
4921 }
4922 }
4923
4924 #[async_trait::async_trait]
4925 impl SessionServiceHistoryExt for DelayedHistorySessionService {
4926 async fn read_history(
4927 &self,
4928 id: &SessionId,
4929 query: SessionHistoryQuery,
4930 ) -> Result<SessionHistoryPage, SessionError> {
4931 self.read_calls.fetch_add(1, Ordering::SeqCst);
4932 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4933 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4934 tokio::time::sleep(self.delay).await;
4935 let result = self.inner.read_history(id, query).await;
4936 self.active_reads.fetch_sub(1, Ordering::SeqCst);
4937 result
4938 }
4939 }
4940
4941 #[async_trait::async_trait]
4942 impl MobSessionService for DelayedHistorySessionService {
4943 fn supports_persistent_sessions(&self) -> bool {
4944 self.inner.supports_persistent_sessions()
4945 }
4946
4947 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4948 self.inner.runtime_adapter()
4949 }
4950
4951 async fn session_belongs_to_mob(
4952 &self,
4953 session_id: &SessionId,
4954 mob_id: &meerkat_mob::MobId,
4955 ) -> bool {
4956 self.inner.session_belongs_to_mob(session_id, mob_id).await
4957 }
4958
4959 async fn cancel_all_checkpointers(&self) {
4960 self.inner.cancel_all_checkpointers().await;
4961 }
4962
4963 async fn rearm_all_checkpointers(&self) {
4964 self.inner.rearm_all_checkpointers().await;
4965 }
4966 }
4967
4968 #[async_trait::async_trait]
4969 impl ConsoleLogStore for CountingConsoleLogStore {
4970 async fn append_if_absent(
4971 &self,
4972 frame: NewConsoleFrame,
4973 ) -> ConsoleLogResult<AppendOutcome> {
4974 self.inner.append_if_absent(frame).await
4975 }
4976
4977 async fn update_frame_status(
4978 &self,
4979 frame_id: &str,
4980 status: ConsoleFrameStatus,
4981 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4982 self.inner.update_frame_status(frame_id, status).await
4983 }
4984
4985 async fn query_frames(
4986 &self,
4987 query: ConsoleTimelineQuery,
4988 ) -> ConsoleLogResult<ConsoleTimelinePage> {
4989 self.inner.query_frames(query).await
4990 }
4991
4992 async fn query_windowed_frames(
4993 &self,
4994 query: ConsoleTimelineWindowQuery,
4995 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4996 self.inner.query_windowed_frames(query).await
4997 }
4998
4999 async fn frame_by_dedupe_key(
5000 &self,
5001 dedupe_key: &str,
5002 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5003 self.inner.frame_by_dedupe_key(dedupe_key).await
5004 }
5005
5006 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
5007 self.inner.latest_cursor().await
5008 }
5009
5010 async fn clear_frames(&self) -> ConsoleLogResult<()> {
5011 self.inner.clear_frames().await
5012 }
5013
5014 async fn record_source_watermark(
5015 &self,
5016 runtime_key: &str,
5017 source_kind: ConsoleFrameSourceKind,
5018 source_cursor: &str,
5019 ) -> ConsoleLogResult<()> {
5020 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5021 self.inner
5022 .record_source_watermark(runtime_key, source_kind, source_cursor)
5023 .await
5024 }
5025
5026 async fn source_watermark(
5027 &self,
5028 runtime_key: &str,
5029 source_kind: ConsoleFrameSourceKind,
5030 ) -> ConsoleLogResult<Option<String>> {
5031 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5032 self.inner.source_watermark(runtime_key, source_kind).await
5033 }
5034 }
5035
5036 async fn build_single_member_runtime() -> UnifiedRuntime {
5037 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5038 }
5039
5040 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5041 let definition = MobDefinition::from_toml(
5042 r#"
5043[mob]
5044id = "console-aggregator-perf-test"
5045
5046[profiles.worker]
5047model = "gpt-5.5"
5048external_addressable = true
5049
5050[profiles.worker.tools]
5051comms = true
5052"#,
5053 )
5054 .expect("definition parses");
5055 let runtime = UnifiedRuntime::builder()
5056 .definition(definition)
5057 .default_llm_client(client)
5058 .build()
5059 .await
5060 .expect("runtime builds");
5061 runtime
5062 .spawn(SpawnMemberSpec::from_wire(
5063 "worker".to_string(),
5064 "agent-a".to_string(),
5065 Some("You are agent-a.".into()),
5066 None,
5067 None,
5068 ))
5069 .await
5070 .expect("member spawns");
5071 runtime
5072 }
5073
5074 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5075 let definition = MobDefinition::from_toml(&format!(
5076 r#"
5077[mob]
5078id = "{mob_id}"
5079
5080[profiles.worker]
5081model = "gpt-5.5"
5082external_addressable = true
5083
5084[profiles.worker.tools]
5085comms = true
5086"#
5087 ))
5088 .expect("definition parses");
5089 UnifiedRuntime::builder()
5090 .definition(definition)
5091 .default_llm_client(Arc::new(TestClient::default()))
5092 .build()
5093 .await
5094 .expect("runtime builds")
5095 }
5096
5097 async fn build_stress_runtime(
5098 member_count: usize,
5099 history_delay: Duration,
5100 ) -> (
5101 tempfile::TempDir,
5102 Arc<UnifiedRuntime>,
5103 DelayedHistorySessionService,
5104 ) {
5105 let temp_dir = tempfile::tempdir().expect("temp dir");
5106 let session_path = temp_dir.path().join("sessions");
5107 std::fs::create_dir_all(&session_path).expect("session path");
5108 let factory = AgentFactory::new(&session_path).comms(true);
5109 let base_service = Arc::new(build_ephemeral_service(
5110 factory,
5111 Config::default(),
5112 member_count + 8,
5113 ));
5114 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5115 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5116 let definition = MobDefinition::from_toml(
5117 r#"
5118[mob]
5119id = "console-aggregator-stress-test"
5120
5121[profiles.worker]
5122model = "gpt-5.5"
5123external_addressable = true
5124
5125[profiles.worker.tools]
5126comms = true
5127"#,
5128 )
5129 .expect("definition parses");
5130 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5131 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5132 allow_ephemeral_sessions: true,
5133 notify_orchestrator_on_resume: true,
5134 default_llm_client: Some(Arc::new(TestClient::default())),
5135 });
5136 let runtime = Arc::new(
5137 UnifiedRuntime::bootstrap(
5138 spec,
5139 crate::types::MobKitConfig {
5140 modules: Vec::new(),
5141 discovery: crate::types::DiscoverySpec {
5142 namespace: "stress".to_string(),
5143 modules: Vec::new(),
5144 },
5145 pre_spawn: Vec::new(),
5146 },
5147 Duration::from_secs(5),
5148 )
5149 .await
5150 .expect("runtime boots"),
5151 );
5152 for idx in 0..member_count {
5153 runtime
5154 .spawn(SpawnMemberSpec::from_wire(
5155 "worker".to_string(),
5156 format!("agent-{idx}"),
5157 Some(format!("You are agent-{idx}.").into()),
5158 None,
5159 None,
5160 ))
5161 .await
5162 .expect("member spawns");
5163 }
5164 (temp_dir, runtime, delayed_service)
5165 }
5166
5167 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5168 RuntimeEntry {
5169 runtime_key: runtime_key.to_string(),
5170 identity_namespace: "test".to_string(),
5171 runtime: runtime.mob_runtime().clone(),
5172 identity_runtime: runtime.identity_runtime().cloned(),
5173 console_events: runtime.console_events(),
5174 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5175 }
5176 }
5177
5178 fn console_envelope_for_test(
5179 event_id: &str,
5180 interaction_id: Option<&str>,
5181 event_type: &str,
5182 data: Value,
5183 ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5184 crate::console_contracts::ConsoleIdentityEventEnvelope {
5185 event_id: event_id.to_string(),
5186 interaction_id: interaction_id.map(ToString::to_string),
5187 identity: "agent-a".to_string(),
5188 event_type: event_type.to_string(),
5189 timestamp_ms: 1_000,
5190 data,
5191 }
5192 }
5193
5194 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5195 ConsoleIdentityRecord {
5196 identity: identity.to_string(),
5197 display_name: identity.to_string(),
5198 runtime_key: "runtime-cache".to_string(),
5199 runtime_member_id: identity.to_string(),
5200 session_id: Some(format!("session-{identity}")),
5201 visibility: ConsoleVisibility::Addressable,
5202 addressable: true,
5203 health: "ready".to_string(),
5204 topology_peers: Vec::new(),
5205 labels: BTreeMap::new(),
5206 }
5207 }
5208
5209 #[test]
5210 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5211 let mut durable = identity_record_for_test("test/review:singleton");
5212 durable.runtime_member_id = "rt:review:singleton:1".to_string();
5213 durable.session_id = Some("session-stale".to_string());
5214 durable
5215 .labels
5216 .insert("agent_identity".to_string(), "review:singleton".to_string());
5217
5218 let mut live = identity_record_for_test("test/review:singleton");
5219 live.runtime_member_id = "rt:review:singleton:0".to_string();
5220 live.session_id = Some("session-live".to_string());
5221 live.labels
5222 .insert("agent_identity".to_string(), "review:singleton".to_string());
5223
5224 let records = dedupe_identity_records(vec![durable, live]);
5225
5226 assert_eq!(
5227 records.len(),
5228 2,
5229 "list projection must expose stale durable/live alias split-brain instead of picking one record"
5230 );
5231 assert!(
5232 records
5233 .iter()
5234 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5235 );
5236 assert!(
5237 records
5238 .iter()
5239 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5240 );
5241 }
5242
5243 #[test]
5244 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5245 let mut first = identity_record_for_test("test/review:singleton");
5246 first.runtime_member_id = "rt:review:singleton:0".to_string();
5247 first
5248 .labels
5249 .insert("agent_identity".to_string(), "review:singleton".to_string());
5250
5251 let mut second = identity_record_for_test("test/review:singleton");
5252 second.runtime_member_id = "rt:review:singleton:1".to_string();
5253 second
5254 .labels
5255 .insert("agent_identity".to_string(), "review:singleton".to_string());
5256
5257 let records = dedupe_identity_records(vec![first, second]);
5258
5259 assert_eq!(
5260 records.len(),
5261 2,
5262 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5263 );
5264 }
5265
5266 #[test]
5267 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5268 let mut first = identity_record_for_test("test/review:singleton");
5269 first.runtime_member_id = "rt:review:singleton:0".to_string();
5270 first.session_id = None;
5271 first
5272 .labels
5273 .insert("agent_identity".to_string(), "review:singleton".to_string());
5274 first
5275 .labels
5276 .insert("source_mob_id".to_string(), "mob-a".to_string());
5277
5278 let mut second = first.clone();
5279 second
5280 .labels
5281 .insert("source_mob_id".to_string(), "mob-b".to_string());
5282
5283 let records = dedupe_identity_records(vec![first, second]);
5284
5285 assert_eq!(
5286 records.len(),
5287 2,
5288 "same member/session aliases from different source mobs must remain visible"
5289 );
5290 }
5291
5292 #[test]
5293 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5294 let durable = identity_record_for_test("test/review:singleton");
5295 let live = durable.clone();
5296
5297 let records = dedupe_identity_records(vec![durable, live]);
5298
5299 assert_eq!(
5300 records.len(),
5301 1,
5302 "healthy durable and primary live projections should not duplicate the roster"
5303 );
5304 }
5305
5306 #[test]
5307 fn stale_durable_record_error_detects_session_split_brain() {
5308 let mut durable = identity_record_for_test("test/review:singleton");
5309 durable.runtime_member_id = "rt:review:singleton:0".to_string();
5310 durable.session_id = Some("session-durable".to_string());
5311
5312 let mut live = durable.clone();
5313 live.session_id = Some("session-live".to_string());
5314
5315 let err = stale_durable_record_error("review:singleton", &durable, &[live])
5316 .expect("session mismatch must be stale split-brain");
5317
5318 assert!(
5319 err.contains("stale durable identity alias"),
5320 "unexpected error: {err}"
5321 );
5322 assert!(
5323 err.contains("session-durable") && err.contains("session-live"),
5324 "error should name both sessions: {err}"
5325 );
5326 }
5327
5328 async fn identity_runtime_for_test(
5329 identities: &[&str],
5330 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5331 {
5332 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5333 crate::identity_first::IdentityRuntimeConfig {
5334 continuity_store: Arc::new(
5335 crate::identity_first::LocalContinuityStore::in_memory()?
5336 ),
5337 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5338 runtime_instance_id: "console-aggregator-identity-test".to_string(),
5339 has_runtime_store: true,
5340 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5341 bridge: None,
5342 default_timeout: None,
5343 },
5344 ));
5345 for identity in identities {
5346 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5347 let record = crate::identity_first::ContinuityRecord {
5348 identity: identity.clone(),
5349 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5350 "rt:{}:0",
5351 identity.as_str()
5352 ))?,
5353 session_id: SessionId::new(),
5354 generation: crate::identity_first::ContinuityGeneration::new(0),
5355 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5356 };
5357 runtime
5358 .register(
5359 crate::identity_first::DurableAgentSpec {
5360 identity: identity.clone(),
5361 profile: meerkat_mob::ProfileName::from("worker"),
5362 addressability: crate::identity_first::AgentAddressability::Addressable,
5363 display_name: None,
5364 labels: BTreeMap::new(),
5365 context: None,
5366 additional_instructions: Vec::new(),
5367 initial_message: None,
5368 runtime_mode_override: None,
5369 backend: None,
5370 binding: None,
5371 },
5372 crate::identity_first::IdentityLifecycleState::Active,
5373 Some(record),
5374 Some(crate::identity_first::LeaseGrant {
5375 identity,
5376 fencing_token: crate::identity_first::FencingToken::new(1),
5377 ttl: Duration::from_mins(5),
5378 }),
5379 )
5380 .await;
5381 }
5382 Ok(runtime)
5383 }
5384
5385 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5386 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5387 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5388 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5389 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5390 let aggregator = MobKitConsoleAggregator::in_memory();
5391 aggregator.register_runtime_handles_with_policy(
5392 "runtime-a",
5393 "",
5394 runtime.mob_runtime().clone(),
5395 Some(identity_runtime),
5396 runtime.console_events(),
5397 Arc::new(AllowAllConsoleVisibilityPolicy),
5398 );
5399
5400 assert!(
5401 aggregator
5402 .inspect_identity("rt:review:singleton:0")
5403 .await?
5404 .is_none(),
5405 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5406 );
5407 assert!(
5408 !aggregator.retire_identity("rt:review:singleton:0").await?,
5409 "runtime-member-id shaped requests must not retire a durable rt:* identity"
5410 );
5411 let reserve_err = aggregator
5412 .reserve_identity_first_interaction(
5413 ConsoleSendRequest {
5414 identity: "rt:review:singleton:0".to_string(),
5415 content: json!("hello"),
5416 origin: "console:test".to_string(),
5417 idempotency_key: "rt-shadow-reserve".to_string(),
5418 handling_mode: Some("queue".to_string()),
5419 },
5420 None,
5421 )
5422 .await
5423 .expect_err(
5424 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5425 );
5426 assert!(
5427 reserve_err.to_string().contains("unknown identity"),
5428 "unexpected reserve error: {reserve_err}"
5429 );
5430 let send_err = aggregator
5431 .send(ConsoleSendRequest {
5432 identity: "rt:review:singleton:0".to_string(),
5433 content: json!("hello"),
5434 origin: "console:test".to_string(),
5435 idempotency_key: "rt-shadow-send".to_string(),
5436 handling_mode: Some("queue".to_string()),
5437 })
5438 .await
5439 .expect_err(
5440 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5441 );
5442 assert!(
5443 send_err.to_string().contains("unknown identity"),
5444 "unexpected send error: {send_err}"
5445 );
5446 let blob_err = match aggregator
5447 .binary_blob_store_for_identity("rt:review:singleton:0")
5448 .await
5449 {
5450 Ok(_) => panic!(
5451 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5452 ),
5453 Err(err) => err,
5454 };
5455 assert!(
5456 blob_err.to_string().contains("unknown identity"),
5457 "unexpected blob error: {blob_err}"
5458 );
5459
5460 let _ = runtime.mob_handle().stop().await;
5461 Ok(())
5462 }
5463
5464 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5465 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5466 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5467 let runtime = build_empty_runtime("identity-label-test").await;
5468 let mut labels = BTreeMap::new();
5469 labels.insert(
5470 "agent_identity".to_string(),
5471 "channel:C0SMOKEOB3".to_string(),
5472 );
5473 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5474 runtime
5475 .spawn(
5476 SpawnMemberSpec::from_wire(
5477 "worker".to_string(),
5478 "rt:channel:C0SMOKEOB3:0".to_string(),
5479 Some("You are C0SMOKEOB3.".into()),
5480 None,
5481 None,
5482 )
5483 .with_labels(labels),
5484 )
5485 .await
5486 .expect("member spawns");
5487
5488 let entry = RuntimeEntry {
5489 runtime_key: "runtime-a".to_string(),
5490 identity_namespace: String::new(),
5491 runtime: runtime.mob_runtime().clone(),
5492 identity_runtime: runtime.identity_runtime().cloned(),
5493 console_events: runtime.console_events(),
5494 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5495 };
5496 let aggregator = MobKitConsoleAggregator::in_memory();
5497 aggregator
5498 .inner
5499 .runtimes
5500 .write()
5501 .expect("runtime registry")
5502 .insert("runtime-a".to_string(), entry);
5503
5504 let records = aggregator.list_identities_fresh().await?;
5505 let record = records
5506 .iter()
5507 .find(|record| record.identity == "channel:C0SMOKEOB3")
5508 .expect("durable identity is exposed");
5509 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5510
5511 let inspection = aggregator
5512 .inspect_identity("channel:C0SMOKEOB3")
5513 .await?
5514 .expect("durable identity resolves back to runtime member");
5515 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5516 assert_eq!(
5517 inspection.identity.runtime_member_id,
5518 "rt:channel:C0SMOKEOB3:0"
5519 );
5520
5521 let _ = runtime.mob_handle().stop().await;
5522 Ok(())
5523 }
5524
5525 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5526 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
5527 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5528 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
5529 let mut labels = BTreeMap::new();
5530 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5531 runtime
5532 .spawn(
5533 SpawnMemberSpec::from_wire(
5534 "worker".to_string(),
5535 "rt:review:singleton:0".to_string(),
5536 Some("You are the Review Agent.".into()),
5537 None,
5538 None,
5539 )
5540 .with_labels(labels),
5541 )
5542 .await?;
5543 let hidden_session_id = runtime
5544 .mob_handle()
5545 .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
5546 "rt:review:singleton:0",
5547 ))
5548 .await
5549 .map(|sid| sid.to_string());
5550 let mut duplicate_labels = BTreeMap::new();
5551 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5552 runtime
5553 .spawn(
5554 SpawnMemberSpec::from_wire(
5555 "worker".to_string(),
5556 "rt:review:singleton:1".to_string(),
5557 Some("You are a visible duplicate Review Agent.".into()),
5558 None,
5559 None,
5560 )
5561 .with_labels(duplicate_labels),
5562 )
5563 .await?;
5564
5565 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5566 let aggregator = MobKitConsoleAggregator::in_memory();
5567 aggregator.register_runtime_handles_with_policy(
5568 "runtime-a",
5569 "",
5570 runtime.mob_runtime().clone(),
5571 Some(identity_runtime),
5572 runtime.console_events(),
5573 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5574 );
5575
5576 assert!(
5577 aggregator
5578 .inspect_identity("review:singleton")
5579 .await?
5580 .is_none(),
5581 "member-hidden live alias must not inspect through aggregator"
5582 );
5583 let records = aggregator.list_identities_fresh().await?;
5584 assert!(
5585 records
5586 .iter()
5587 .all(|record| record.identity != "review:singleton"),
5588 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5589 );
5590 let appended = aggregator
5591 .store()
5592 .append_if_absent(NewConsoleFrame {
5593 id: None,
5594 dedupe_key: "member-hidden-frame".to_string(),
5595 timestamp_ms: 1,
5596 runtime_key: "runtime-a".to_string(),
5597 identity: "review:singleton".to_string(),
5598 conversation_id: Some("review:singleton".to_string()),
5599 session_id: hidden_session_id,
5600 kind: "text_delta".to_string(),
5601 status: ConsoleFrameStatus::Delivered,
5602 payload: json!({ "delta": "hidden" }),
5603 source: ConsoleFrameSource {
5604 kind: ConsoleFrameSourceKind::ConsoleEvent,
5605 source_cursor: None,
5606 },
5607 source_event_id: Some("member-hidden-frame".to_string()),
5608 interaction_id: None,
5609 turn_id: None,
5610 run_id: None,
5611 parent_frame_id: None,
5612 caused_by_frame_id: None,
5613 })
5614 .await?;
5615 assert!(
5616 !aggregator
5617 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5618 frame: appended.frame.clone()
5619 })
5620 .await,
5621 "member-hidden live alias must not pass timeline event visibility"
5622 );
5623 let identity_only = aggregator
5624 .store()
5625 .append_if_absent(NewConsoleFrame {
5626 id: None,
5627 dedupe_key: "member-hidden-identity-only-frame".to_string(),
5628 timestamp_ms: 2,
5629 runtime_key: "runtime-a".to_string(),
5630 identity: "review:singleton".to_string(),
5631 conversation_id: Some("review:singleton".to_string()),
5632 session_id: None,
5633 kind: "text_delta".to_string(),
5634 status: ConsoleFrameStatus::Delivered,
5635 payload: json!({ "delta": "identity-only-hidden" }),
5636 source: ConsoleFrameSource {
5637 kind: ConsoleFrameSourceKind::ConsoleEvent,
5638 source_cursor: None,
5639 },
5640 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5641 interaction_id: None,
5642 turn_id: None,
5643 run_id: None,
5644 parent_frame_id: None,
5645 caused_by_frame_id: None,
5646 })
5647 .await?;
5648 assert!(
5649 !aggregator
5650 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5651 frame: identity_only.frame.clone()
5652 })
5653 .await,
5654 "member-hidden durable alias must not leak identity-only timeline frames"
5655 );
5656 let page = aggregator
5657 .query_timeline(ConsoleTimelineQuery {
5658 identity: Some("review:singleton".to_string()),
5659 limit: 10,
5660 ..ConsoleTimelineQuery::default()
5661 })
5662 .await?;
5663 assert!(
5664 page.frames.is_empty(),
5665 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5666 );
5667 assert!(
5668 !aggregator.retire_identity("review:singleton").await?,
5669 "member-hidden live alias must not retire through aggregator"
5670 );
5671 let reserve_err = aggregator
5672 .reserve_identity_first_interaction(
5673 ConsoleSendRequest {
5674 identity: "review:singleton".to_string(),
5675 content: json!("hello"),
5676 origin: "console:test".to_string(),
5677 idempotency_key: "member-hidden-reserve".to_string(),
5678 handling_mode: Some("queue".to_string()),
5679 },
5680 None,
5681 )
5682 .await
5683 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5684 assert!(
5685 reserve_err.to_string().contains("unknown identity"),
5686 "unexpected reserve error: {reserve_err}"
5687 );
5688 let send_err = aggregator
5689 .send(ConsoleSendRequest {
5690 identity: "review:singleton".to_string(),
5691 content: json!("hello"),
5692 origin: "console:test".to_string(),
5693 idempotency_key: "member-hidden-send".to_string(),
5694 handling_mode: Some("queue".to_string()),
5695 })
5696 .await
5697 .expect_err("member-hidden live alias must not receive sends");
5698 assert!(
5699 send_err.to_string().contains("unknown identity"),
5700 "unexpected send error: {send_err}"
5701 );
5702 let blob_result = aggregator
5703 .binary_blob_store_for_identity("review:singleton")
5704 .await;
5705 let Err(blob_err) = blob_result else {
5706 panic!("member-hidden live alias must not expose blob store");
5707 };
5708 assert!(
5709 blob_err.to_string().contains("unknown identity"),
5710 "unexpected blob error: {blob_err}"
5711 );
5712
5713 let _ = runtime.mob_handle().stop().await;
5714 Ok(())
5715 }
5716
5717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5718 async fn explicit_session_backfill_target_resolves_durable_identity_label()
5719 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5720 let runtime = build_empty_runtime("identity-label-backfill-test").await;
5721 let mut labels = BTreeMap::new();
5722 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5723 runtime
5724 .spawn(
5725 SpawnMemberSpec::from_wire(
5726 "worker".to_string(),
5727 "rt:review:singleton:0".to_string(),
5728 Some("You are the Review Agent.".into()),
5729 None,
5730 None,
5731 )
5732 .with_labels(labels),
5733 )
5734 .await
5735 .expect("member spawns");
5736
5737 let entry = RuntimeEntry {
5738 runtime_key: "runtime-a".to_string(),
5739 identity_namespace: String::new(),
5740 runtime: runtime.mob_runtime().clone(),
5741 identity_runtime: runtime.identity_runtime().cloned(),
5742 console_events: runtime.console_events(),
5743 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5744 };
5745 let aggregator = MobKitConsoleAggregator::in_memory();
5746 aggregator
5747 .inner
5748 .runtimes
5749 .write()
5750 .expect("runtime registry")
5751 .insert("runtime-a".to_string(), entry);
5752
5753 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5754 .await
5755 .expect("durable label should resolve targeted session backfill");
5756 assert_eq!(target.record.identity, "review:singleton");
5757 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5758
5759 let _ = runtime.mob_handle().stop().await;
5760 Ok(())
5761 }
5762
5763 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5764 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5765 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5766 let runtime = build_empty_runtime("identity-stale-durable-test").await;
5767 let mut labels = BTreeMap::new();
5768 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5769 runtime
5770 .spawn(
5771 SpawnMemberSpec::from_wire(
5772 "worker".to_string(),
5773 "rt:review:singleton:0".to_string(),
5774 Some("You are the live Review Agent.".into()),
5775 None,
5776 None,
5777 )
5778 .with_labels(labels),
5779 )
5780 .await
5781 .expect("member spawns");
5782
5783 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5784 crate::identity_first::IdentityRuntimeConfig {
5785 continuity_store: Arc::new(
5786 crate::identity_first::LocalContinuityStore::in_memory()?
5787 ),
5788 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5789 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5790 has_runtime_store: true,
5791 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5792 bridge: None,
5793 default_timeout: None,
5794 },
5795 ));
5796 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5797 let record = crate::identity_first::ContinuityRecord {
5798 identity: identity.clone(),
5799 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5800 "rt:review:singleton:1",
5801 )?,
5802 session_id: SessionId::new(),
5803 generation: crate::identity_first::ContinuityGeneration::new(1),
5804 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5805 };
5806 identity_runtime
5807 .register(
5808 crate::identity_first::DurableAgentSpec {
5809 identity: identity.clone(),
5810 profile: meerkat_mob::ProfileName::from("worker"),
5811 addressability: crate::identity_first::AgentAddressability::Addressable,
5812 display_name: None,
5813 labels: BTreeMap::new(),
5814 context: None,
5815 additional_instructions: Vec::new(),
5816 initial_message: None,
5817 runtime_mode_override: None,
5818 backend: None,
5819 binding: None,
5820 },
5821 crate::identity_first::IdentityLifecycleState::Active,
5822 Some(record),
5823 Some(crate::identity_first::LeaseGrant {
5824 identity,
5825 fencing_token: crate::identity_first::FencingToken::new(7),
5826 ttl: Duration::from_mins(5),
5827 }),
5828 )
5829 .await;
5830
5831 let aggregator = MobKitConsoleAggregator::in_memory();
5832 aggregator
5833 .inner
5834 .runtimes
5835 .write()
5836 .expect("runtime registry")
5837 .insert(
5838 "runtime-a".to_string(),
5839 RuntimeEntry {
5840 runtime_key: "runtime-a".to_string(),
5841 identity_namespace: String::new(),
5842 runtime: runtime.mob_runtime().clone(),
5843 identity_runtime: Some(identity_runtime),
5844 console_events: runtime.console_events(),
5845 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5846 },
5847 );
5848
5849 let err = aggregator
5850 .inspect_identity("review:singleton")
5851 .await
5852 .expect_err("stale durable binding must not inspect as healthy");
5853 assert!(
5854 err.to_string().contains("stale durable identity alias"),
5855 "unexpected error: {err}"
5856 );
5857 let retire_err = aggregator
5858 .retire_identity("review:singleton")
5859 .await
5860 .expect_err("stale durable binding must not retire as healthy");
5861 assert!(
5862 retire_err
5863 .to_string()
5864 .contains("stale durable identity alias"),
5865 "unexpected retire error: {retire_err}"
5866 );
5867 let send_err = aggregator
5868 .send(ConsoleSendRequest {
5869 identity: "review:singleton".to_string(),
5870 content: json!("hello"),
5871 origin: "console:test".to_string(),
5872 idempotency_key: "stale-durable-send".to_string(),
5873 handling_mode: Some("queue".to_string()),
5874 })
5875 .await
5876 .expect_err("stale durable binding must not send to the wrong live member");
5877 assert!(
5878 send_err
5879 .to_string()
5880 .contains("stale durable identity alias"),
5881 "unexpected send error: {send_err}"
5882 );
5883 let reserve_err = aggregator
5884 .reserve_identity_first_interaction(
5885 ConsoleSendRequest {
5886 identity: "review:singleton".to_string(),
5887 content: json!("hello"),
5888 origin: "console:test".to_string(),
5889 idempotency_key: "stale-durable-reserve".to_string(),
5890 handling_mode: Some("queue".to_string()),
5891 },
5892 None,
5893 )
5894 .await
5895 .expect_err("identity-first reserve must not bypass stale durable binding");
5896 assert!(
5897 reserve_err
5898 .to_string()
5899 .contains("stale durable identity alias"),
5900 "unexpected reserve error: {reserve_err}"
5901 );
5902 let blob_err = match aggregator
5903 .binary_blob_store_for_identity("review:singleton")
5904 .await
5905 {
5906 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5907 Err(err) => err,
5908 };
5909 assert!(
5910 blob_err
5911 .to_string()
5912 .contains("stale durable identity alias"),
5913 "unexpected blob error: {blob_err}"
5914 );
5915 let records = aggregator.list_identities_fresh().await?;
5916 assert!(
5917 records
5918 .iter()
5919 .any(|record| record.identity == "review:singleton"
5920 && record.runtime_member_id == "rt:review:singleton:0"),
5921 "live alias should still be listed; records: {records:#?}"
5922 );
5923 assert!(
5924 records
5925 .iter()
5926 .any(|record| record.identity == "review:singleton"
5927 && record.runtime_member_id == "rt:review:singleton:1"),
5928 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5929 );
5930
5931 let _ = runtime.mob_handle().stop().await;
5932 Ok(())
5933 }
5934
5935 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5936 async fn identity_first_alias_self_heals_session_only_split_brain()
5937 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5938 let runtime = build_empty_runtime("identity-session-rebind-test").await;
5939 let mut labels = BTreeMap::new();
5940 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5941 runtime
5942 .spawn(
5943 SpawnMemberSpec::from_wire(
5944 "worker".to_string(),
5945 "rt:review:singleton:0".to_string(),
5946 Some("You are the live Review Agent.".into()),
5947 None,
5948 None,
5949 )
5950 .with_labels(labels),
5951 )
5952 .await
5953 .expect("member spawns");
5954 let live_session_id = runtime
5955 .mob_handle()
5956 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
5957 "rt:review:singleton:0",
5958 ))
5959 .await
5960 .expect("live member has bridge session");
5961
5962 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5963 crate::identity_first::IdentityRuntimeConfig {
5964 continuity_store: Arc::new(
5965 crate::identity_first::LocalContinuityStore::in_memory()?
5966 ),
5967 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5968 runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
5969 has_runtime_store: true,
5970 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5971 bridge: None,
5972 default_timeout: None,
5973 },
5974 ));
5975 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5976 let stale_session_id = SessionId::new();
5977 assert_ne!(stale_session_id, live_session_id);
5978 let record = crate::identity_first::ContinuityRecord {
5979 identity: identity.clone(),
5980 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5981 "rt:review:singleton:0",
5982 )?,
5983 session_id: stale_session_id,
5984 generation: crate::identity_first::ContinuityGeneration::new(1),
5985 checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
5986 };
5987 identity_runtime
5988 .register(
5989 crate::identity_first::DurableAgentSpec {
5990 identity: identity.clone(),
5991 profile: meerkat_mob::ProfileName::from("worker"),
5992 addressability: crate::identity_first::AgentAddressability::Addressable,
5993 display_name: None,
5994 labels: BTreeMap::new(),
5995 context: None,
5996 additional_instructions: Vec::new(),
5997 initial_message: None,
5998 runtime_mode_override: None,
5999 backend: None,
6000 binding: None,
6001 },
6002 crate::identity_first::IdentityLifecycleState::Active,
6003 Some(record),
6004 Some(crate::identity_first::LeaseGrant {
6005 identity: identity.clone(),
6006 fencing_token: crate::identity_first::FencingToken::new(7),
6007 ttl: Duration::from_mins(5),
6008 }),
6009 )
6010 .await;
6011
6012 let aggregator = MobKitConsoleAggregator::in_memory();
6013 aggregator
6014 .inner
6015 .runtimes
6016 .write()
6017 .expect("runtime registry")
6018 .insert(
6019 "runtime-a".to_string(),
6020 RuntimeEntry {
6021 runtime_key: "runtime-a".to_string(),
6022 identity_namespace: String::new(),
6023 runtime: runtime.mob_runtime().clone(),
6024 identity_runtime: Some(identity_runtime.clone()),
6025 console_events: runtime.console_events(),
6026 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6027 },
6028 );
6029
6030 aggregator
6031 .reserve_identity_first_interaction(
6032 ConsoleSendRequest {
6033 identity: "review:singleton".to_string(),
6034 content: json!("hello"),
6035 origin: "console:test".to_string(),
6036 idempotency_key: "session-rebind-reserve".to_string(),
6037 handling_mode: Some("queue".to_string()),
6038 },
6039 None,
6040 )
6041 .await
6042 .expect("session-only split-brain should self-heal before reserve");
6043
6044 let rebound_status = identity_runtime.status(&identity).await?;
6045 assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6046
6047 let inspection = aggregator
6048 .inspect_identity("review:singleton")
6049 .await?
6050 .expect("rebound identity should inspect");
6051 let live_session_string = live_session_id.to_string();
6052 assert_eq!(
6053 inspection.identity.session_id.as_deref(),
6054 Some(live_session_string.as_str())
6055 );
6056
6057 let _ = runtime.mob_handle().stop().await;
6058 Ok(())
6059 }
6060
6061 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6062 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6063 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6064 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6065 let mut labels = BTreeMap::new();
6066 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6067 runtime
6068 .spawn(
6069 SpawnMemberSpec::from_wire(
6070 "worker".to_string(),
6071 "rt:review:singleton:0".to_string(),
6072 Some("You are the mislabeled Review Agent.".into()),
6073 None,
6074 None,
6075 )
6076 .with_labels(labels),
6077 )
6078 .await
6079 .expect("member spawns");
6080
6081 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6082 crate::identity_first::IdentityRuntimeConfig {
6083 continuity_store: Arc::new(
6084 crate::identity_first::LocalContinuityStore::in_memory()?
6085 ),
6086 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6087 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6088 has_runtime_store: true,
6089 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6090 bridge: None,
6091 default_timeout: None,
6092 },
6093 ));
6094 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6095 let record = crate::identity_first::ContinuityRecord {
6096 identity: identity.clone(),
6097 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6098 "rt:review:singleton:0",
6099 )?,
6100 session_id: SessionId::new(),
6101 generation: crate::identity_first::ContinuityGeneration::new(1),
6102 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6103 };
6104 identity_runtime
6105 .register(
6106 crate::identity_first::DurableAgentSpec {
6107 identity: identity.clone(),
6108 profile: meerkat_mob::ProfileName::from("worker"),
6109 addressability: crate::identity_first::AgentAddressability::Addressable,
6110 display_name: None,
6111 labels: BTreeMap::new(),
6112 context: None,
6113 additional_instructions: Vec::new(),
6114 initial_message: None,
6115 runtime_mode_override: None,
6116 backend: None,
6117 binding: None,
6118 },
6119 crate::identity_first::IdentityLifecycleState::Active,
6120 Some(record),
6121 Some(crate::identity_first::LeaseGrant {
6122 identity,
6123 fencing_token: crate::identity_first::FencingToken::new(7),
6124 ttl: Duration::from_mins(5),
6125 }),
6126 )
6127 .await;
6128
6129 let aggregator = MobKitConsoleAggregator::in_memory();
6130 aggregator
6131 .inner
6132 .runtimes
6133 .write()
6134 .expect("runtime registry")
6135 .insert(
6136 "runtime-a".to_string(),
6137 RuntimeEntry {
6138 runtime_key: "runtime-a".to_string(),
6139 identity_namespace: String::new(),
6140 runtime: runtime.mob_runtime().clone(),
6141 identity_runtime: Some(identity_runtime),
6142 console_events: runtime.console_events(),
6143 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6144 },
6145 );
6146
6147 let err = aggregator
6148 .inspect_identity("review:singleton")
6149 .await
6150 .expect_err("wrong projected live identity must not inspect as healthy");
6151 assert!(
6152 err.to_string()
6153 .contains("projects identity other:singleton"),
6154 "unexpected error: {err}"
6155 );
6156 let retire_err = aggregator
6157 .retire_identity("review:singleton")
6158 .await
6159 .expect_err("wrong projected live identity must not retire as healthy");
6160 assert!(
6161 retire_err
6162 .to_string()
6163 .contains("projects identity other:singleton"),
6164 "unexpected retire error: {retire_err}"
6165 );
6166 let send_err = aggregator
6167 .send(ConsoleSendRequest {
6168 identity: "review:singleton".to_string(),
6169 content: json!("hello"),
6170 origin: "console:test".to_string(),
6171 idempotency_key: "wrong-projection-send".to_string(),
6172 handling_mode: Some("queue".to_string()),
6173 })
6174 .await
6175 .expect_err("wrong projected live identity must not send as unknown");
6176 assert!(
6177 send_err
6178 .to_string()
6179 .contains("projects identity other:singleton"),
6180 "unexpected send error: {send_err}"
6181 );
6182 let reserve_err = aggregator
6183 .reserve_identity_first_interaction(
6184 ConsoleSendRequest {
6185 identity: "review:singleton".to_string(),
6186 content: json!("hello"),
6187 origin: "console:test".to_string(),
6188 idempotency_key: "wrong-projection-reserve".to_string(),
6189 handling_mode: Some("queue".to_string()),
6190 },
6191 None,
6192 )
6193 .await
6194 .expect_err("identity-first reserve must not bypass wrong live projection");
6195 assert!(
6196 reserve_err
6197 .to_string()
6198 .contains("projects identity other:singleton"),
6199 "unexpected reserve error: {reserve_err}"
6200 );
6201 let blob_err = match aggregator
6202 .binary_blob_store_for_identity("review:singleton")
6203 .await
6204 {
6205 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6206 Err(err) => err,
6207 };
6208 assert!(
6209 blob_err
6210 .to_string()
6211 .contains("projects identity other:singleton"),
6212 "unexpected blob error: {blob_err}"
6213 );
6214 let projected_err = aggregator
6215 .inspect_identity("other:singleton")
6216 .await
6217 .expect_err("wrong projected alias must not inspect through live-only fallback");
6218 assert!(
6219 projected_err
6220 .to_string()
6221 .contains("identity runtime binding for review:singleton"),
6222 "unexpected projected-alias error: {projected_err}"
6223 );
6224 let projected_retire_err = aggregator
6225 .retire_identity("other:singleton")
6226 .await
6227 .expect_err("wrong projected alias must not retire through live-only fallback");
6228 assert!(
6229 projected_retire_err
6230 .to_string()
6231 .contains("identity runtime binding for review:singleton"),
6232 "unexpected projected-alias retire error: {projected_retire_err}"
6233 );
6234
6235 let _ = runtime.mob_handle().stop().await;
6236 Ok(())
6237 }
6238
6239 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6240 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6241 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6242 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6243 let mut labels = BTreeMap::new();
6244 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6245 runtime
6246 .spawn(
6247 SpawnMemberSpec::from_wire(
6248 "worker".to_string(),
6249 "rt:review:singleton:0".to_string(),
6250 Some("You are a hidden wrong projection.".into()),
6251 None,
6252 None,
6253 )
6254 .with_labels(labels),
6255 )
6256 .await
6257 .expect("member spawns");
6258
6259 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6260 crate::identity_first::IdentityRuntimeConfig {
6261 continuity_store: Arc::new(
6262 crate::identity_first::LocalContinuityStore::in_memory()?
6263 ),
6264 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6265 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6266 has_runtime_store: true,
6267 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6268 bridge: None,
6269 default_timeout: None,
6270 },
6271 ));
6272 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6273 identity_runtime
6274 .register(
6275 crate::identity_first::DurableAgentSpec {
6276 identity: identity.clone(),
6277 profile: meerkat_mob::ProfileName::from("worker"),
6278 addressability: crate::identity_first::AgentAddressability::Addressable,
6279 display_name: None,
6280 labels: BTreeMap::new(),
6281 context: None,
6282 additional_instructions: Vec::new(),
6283 initial_message: None,
6284 runtime_mode_override: None,
6285 backend: None,
6286 binding: None,
6287 },
6288 crate::identity_first::IdentityLifecycleState::Active,
6289 Some(crate::identity_first::ContinuityRecord {
6290 identity: identity.clone(),
6291 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6292 "rt:review:singleton:0",
6293 )?,
6294 session_id: SessionId::new(),
6295 generation: crate::identity_first::ContinuityGeneration::new(1),
6296 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6297 }),
6298 Some(crate::identity_first::LeaseGrant {
6299 identity,
6300 fencing_token: crate::identity_first::FencingToken::new(7),
6301 ttl: Duration::from_mins(5),
6302 }),
6303 )
6304 .await;
6305
6306 let aggregator = MobKitConsoleAggregator::in_memory();
6307 aggregator
6308 .inner
6309 .runtimes
6310 .write()
6311 .expect("runtime registry")
6312 .insert(
6313 "runtime-a".to_string(),
6314 RuntimeEntry {
6315 runtime_key: "runtime-a".to_string(),
6316 identity_namespace: String::new(),
6317 runtime: runtime.mob_runtime().clone(),
6318 identity_runtime: Some(identity_runtime),
6319 console_events: runtime.console_events(),
6320 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6321 },
6322 );
6323
6324 let err = aggregator
6325 .inspect_identity("review:singleton")
6326 .await
6327 .expect_err("hidden wrong projection must still fail durable binding validation");
6328 assert!(
6329 err.to_string()
6330 .contains("projects identity other:singleton"),
6331 "unexpected hidden wrong-projection error: {err}"
6332 );
6333 let reserve_err = aggregator
6334 .reserve_identity_first_interaction(
6335 ConsoleSendRequest {
6336 identity: "review:singleton".to_string(),
6337 content: json!("hello"),
6338 origin: "console:test".to_string(),
6339 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6340 handling_mode: Some("queue".to_string()),
6341 },
6342 None,
6343 )
6344 .await
6345 .expect_err("reserve must not hide wrong projection from durable validation");
6346 assert!(
6347 reserve_err
6348 .to_string()
6349 .contains("projects identity other:singleton"),
6350 "unexpected reserve error: {reserve_err}"
6351 );
6352
6353 let _ = runtime.mob_handle().stop().await;
6354 Ok(())
6355 }
6356
6357 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6358 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6359 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6360 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6361 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6362 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6363 crate::identity_first::IdentityRuntimeConfig {
6364 continuity_store: Arc::new(
6365 crate::identity_first::LocalContinuityStore::in_memory()?
6366 ),
6367 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6368 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6369 has_runtime_store: true,
6370 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6371 bridge: None,
6372 default_timeout: None,
6373 },
6374 ));
6375 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6376 crate::identity_first::IdentityRuntimeConfig {
6377 continuity_store: Arc::new(
6378 crate::identity_first::LocalContinuityStore::in_memory()?
6379 ),
6380 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6381 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6382 has_runtime_store: true,
6383 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6384 bridge: None,
6385 default_timeout: None,
6386 },
6387 ));
6388 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6389 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6390 identity_runtime
6391 .register(
6392 crate::identity_first::DurableAgentSpec {
6393 identity: identity.clone(),
6394 profile: meerkat_mob::ProfileName::from("worker"),
6395 addressability: crate::identity_first::AgentAddressability::Addressable,
6396 display_name: None,
6397 labels: BTreeMap::new(),
6398 context: None,
6399 additional_instructions: Vec::new(),
6400 initial_message: None,
6401 runtime_mode_override: None,
6402 backend: None,
6403 binding: None,
6404 },
6405 crate::identity_first::IdentityLifecycleState::Active,
6406 Some(crate::identity_first::ContinuityRecord {
6407 identity: identity.clone(),
6408 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6409 "rt:review:singleton:0",
6410 )?,
6411 session_id: SessionId::new(),
6412 generation: crate::identity_first::ContinuityGeneration::new(1),
6413 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6414 }),
6415 Some(crate::identity_first::LeaseGrant {
6416 identity,
6417 fencing_token: crate::identity_first::FencingToken::new(7),
6418 ttl: Duration::from_mins(5),
6419 }),
6420 )
6421 .await;
6422 }
6423
6424 let aggregator = MobKitConsoleAggregator::in_memory();
6425 {
6426 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6427 runtimes.insert(
6428 "a-hidden".to_string(),
6429 RuntimeEntry {
6430 runtime_key: "a-hidden".to_string(),
6431 identity_namespace: String::new(),
6432 runtime: hidden_runtime.mob_runtime().clone(),
6433 identity_runtime: Some(hidden_identity_runtime),
6434 console_events: hidden_runtime.console_events(),
6435 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6436 },
6437 );
6438 runtimes.insert(
6439 "b-visible".to_string(),
6440 RuntimeEntry {
6441 runtime_key: "b-visible".to_string(),
6442 identity_namespace: String::new(),
6443 runtime: visible_runtime.mob_runtime().clone(),
6444 identity_runtime: Some(visible_identity_runtime),
6445 console_events: visible_runtime.console_events(),
6446 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6447 },
6448 );
6449 }
6450
6451 let inspection = aggregator
6452 .inspect_identity("rt:review:singleton:0")
6453 .await?
6454 .expect("visible runtime-id match should be returned");
6455 assert_eq!(inspection.identity.runtime_key, "b-visible");
6456
6457 let _ = hidden_runtime.mob_handle().stop().await;
6458 let _ = visible_runtime.mob_handle().stop().await;
6459 Ok(())
6460 }
6461
6462 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6463 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6464 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6465 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6466 let mut primary_labels = BTreeMap::new();
6467 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6468 runtime
6469 .spawn(
6470 SpawnMemberSpec::from_wire(
6471 "worker".to_string(),
6472 "rt:review:singleton:0".to_string(),
6473 Some("You are the primary Review Agent.".into()),
6474 None,
6475 None,
6476 )
6477 .with_labels(primary_labels),
6478 )
6479 .await
6480 .expect("primary member spawns");
6481
6482 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6483 let mut delegate_labels = BTreeMap::new();
6484 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6485 delegate_runtime
6486 .spawn(
6487 SpawnMemberSpec::from_wire(
6488 "worker".to_string(),
6489 "rt:review:singleton:0".to_string(),
6490 Some("You are the wrong projected Review Agent.".into()),
6491 None,
6492 None,
6493 )
6494 .with_labels(delegate_labels),
6495 )
6496 .await
6497 .expect("delegate member spawns");
6498 runtime
6499 .mob_runtime()
6500 .agent_mob_mcp_state()
6501 .expect("runtime should expose mob MCP state")
6502 .mob_insert_handle(
6503 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6504 delegate_runtime.mob_handle().clone(),
6505 )
6506 .await;
6507
6508 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6509 crate::identity_first::IdentityRuntimeConfig {
6510 continuity_store: Arc::new(
6511 crate::identity_first::LocalContinuityStore::in_memory()?
6512 ),
6513 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6514 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
6515 has_runtime_store: true,
6516 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6517 bridge: None,
6518 default_timeout: None,
6519 },
6520 ));
6521 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6522 let record = crate::identity_first::ContinuityRecord {
6523 identity: identity.clone(),
6524 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6525 "rt:review:singleton:0",
6526 )?,
6527 session_id: SessionId::new(),
6528 generation: crate::identity_first::ContinuityGeneration::new(1),
6529 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6530 };
6531 identity_runtime
6532 .register(
6533 crate::identity_first::DurableAgentSpec {
6534 identity: identity.clone(),
6535 profile: meerkat_mob::ProfileName::from("worker"),
6536 addressability: crate::identity_first::AgentAddressability::Addressable,
6537 display_name: None,
6538 labels: BTreeMap::new(),
6539 context: None,
6540 additional_instructions: Vec::new(),
6541 initial_message: None,
6542 runtime_mode_override: None,
6543 backend: None,
6544 binding: None,
6545 },
6546 crate::identity_first::IdentityLifecycleState::Active,
6547 Some(record),
6548 Some(crate::identity_first::LeaseGrant {
6549 identity,
6550 fencing_token: crate::identity_first::FencingToken::new(7),
6551 ttl: Duration::from_mins(5),
6552 }),
6553 )
6554 .await;
6555
6556 let aggregator = MobKitConsoleAggregator::in_memory();
6557 aggregator
6558 .inner
6559 .runtimes
6560 .write()
6561 .expect("runtime registry")
6562 .insert(
6563 "runtime-a".to_string(),
6564 RuntimeEntry {
6565 runtime_key: "runtime-a".to_string(),
6566 identity_namespace: String::new(),
6567 runtime: runtime.mob_runtime().clone(),
6568 identity_runtime: Some(identity_runtime),
6569 console_events: runtime.console_events(),
6570 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6571 },
6572 );
6573
6574 let err = aggregator
6575 .inspect_identity("review:singleton")
6576 .await
6577 .expect_err("cross-source wrong projection must not be hidden by primary source");
6578 assert!(
6579 err.to_string()
6580 .contains("projects identity other:singleton")
6581 || err.to_string().contains("ambiguous live identity alias"),
6582 "unexpected error: {err}"
6583 );
6584
6585 let _ = delegate_runtime.mob_handle().stop().await;
6586 let _ = runtime.mob_handle().stop().await;
6587 Ok(())
6588 }
6589
6590 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6591 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
6592 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6593 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
6594 let mut labels = BTreeMap::new();
6595 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6596 runtime
6597 .spawn(
6598 SpawnMemberSpec::from_wire(
6599 "worker".to_string(),
6600 "rt:review:singleton:0".to_string(),
6601 Some("You are the first Review Agent.".into()),
6602 None,
6603 None,
6604 )
6605 .with_labels(labels.clone()),
6606 )
6607 .await
6608 .expect("first member spawns");
6609 runtime
6610 .spawn(
6611 SpawnMemberSpec::from_wire(
6612 "worker".to_string(),
6613 "rt:review:singleton:1".to_string(),
6614 Some("You are the second Review Agent.".into()),
6615 None,
6616 None,
6617 )
6618 .with_labels(labels),
6619 )
6620 .await
6621 .expect("second member spawns");
6622
6623 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6624 crate::identity_first::IdentityRuntimeConfig {
6625 continuity_store: Arc::new(
6626 crate::identity_first::LocalContinuityStore::in_memory()?
6627 ),
6628 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6629 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
6630 has_runtime_store: true,
6631 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6632 bridge: None,
6633 default_timeout: None,
6634 },
6635 ));
6636 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6637 identity_runtime
6638 .register(
6639 crate::identity_first::DurableAgentSpec {
6640 identity: identity.clone(),
6641 profile: meerkat_mob::ProfileName::from("worker"),
6642 addressability: crate::identity_first::AgentAddressability::Addressable,
6643 display_name: None,
6644 labels: BTreeMap::new(),
6645 context: None,
6646 additional_instructions: Vec::new(),
6647 initial_message: None,
6648 runtime_mode_override: None,
6649 backend: None,
6650 binding: None,
6651 },
6652 crate::identity_first::IdentityLifecycleState::Active,
6653 Some(crate::identity_first::ContinuityRecord {
6654 identity: identity.clone(),
6655 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6656 "rt:review:singleton:0",
6657 )?,
6658 session_id: SessionId::new(),
6659 generation: crate::identity_first::ContinuityGeneration::new(1),
6660 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6661 }),
6662 Some(crate::identity_first::LeaseGrant {
6663 identity,
6664 fencing_token: crate::identity_first::FencingToken::new(8),
6665 ttl: Duration::from_mins(5),
6666 }),
6667 )
6668 .await;
6669
6670 let aggregator = MobKitConsoleAggregator::in_memory();
6671 aggregator
6672 .inner
6673 .runtimes
6674 .write()
6675 .expect("runtime registry")
6676 .insert(
6677 "runtime-a".to_string(),
6678 RuntimeEntry {
6679 runtime_key: "runtime-a".to_string(),
6680 identity_namespace: String::new(),
6681 runtime: runtime.mob_runtime().clone(),
6682 identity_runtime: Some(identity_runtime),
6683 console_events: runtime.console_events(),
6684 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6685 },
6686 );
6687
6688 let err = aggregator
6689 .inspect_identity("review:singleton")
6690 .await
6691 .expect_err("duplicate live alias must not inspect via durable fast path");
6692 assert!(
6693 err.to_string().contains("ambiguous live identity alias"),
6694 "unexpected error: {err}"
6695 );
6696 let retire_err = aggregator
6697 .retire_identity("review:singleton")
6698 .await
6699 .expect_err("duplicate live alias must not retire via durable fast path");
6700 assert!(
6701 retire_err
6702 .to_string()
6703 .contains("ambiguous live identity alias"),
6704 "unexpected retire error: {retire_err}"
6705 );
6706 let runtime_id_err = aggregator
6707 .inspect_identity("rt:review:singleton:0")
6708 .await
6709 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6710 assert!(
6711 runtime_id_err
6712 .to_string()
6713 .contains("ambiguous live identity alias"),
6714 "unexpected runtime-id inspect error: {runtime_id_err}"
6715 );
6716 let runtime_id_retire_err = aggregator
6717 .retire_identity("rt:review:singleton:0")
6718 .await
6719 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6720 assert!(
6721 runtime_id_retire_err
6722 .to_string()
6723 .contains("ambiguous live identity alias")
6724 || runtime_id_retire_err
6725 .to_string()
6726 .contains("stale durable identity alias"),
6727 "unexpected runtime-id retire error: {runtime_id_retire_err}"
6728 );
6729
6730 let _ = runtime.mob_handle().stop().await;
6731 Ok(())
6732 }
6733
6734 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6735 async fn inspect_identity_ignores_hidden_live_alias_candidates()
6736 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6737 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6738 let mut labels = BTreeMap::new();
6739 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6740 runtime
6741 .spawn(
6742 SpawnMemberSpec::from_wire(
6743 "worker".to_string(),
6744 "rt:review:singleton:0".to_string(),
6745 Some("You are the visible Review Agent.".into()),
6746 None,
6747 None,
6748 )
6749 .with_labels(labels.clone()),
6750 )
6751 .await
6752 .expect("visible member spawns");
6753 runtime
6754 .spawn(
6755 SpawnMemberSpec::from_wire(
6756 "worker".to_string(),
6757 "rt:review:singleton:1".to_string(),
6758 Some("You are hidden maintenance noise.".into()),
6759 None,
6760 None,
6761 )
6762 .with_labels(labels),
6763 )
6764 .await
6765 .expect("hidden member spawns");
6766
6767 let aggregator = MobKitConsoleAggregator::in_memory();
6768 aggregator.register_runtime_handles_with_policy(
6769 "runtime-a",
6770 "",
6771 runtime.mob_runtime().clone(),
6772 None,
6773 runtime.console_events(),
6774 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6775 );
6776
6777 let inspection = aggregator
6778 .inspect_identity("review:singleton")
6779 .await?
6780 .expect("visible live alias should inspect without hidden ambiguity");
6781 assert_eq!(inspection.identity.identity, "review:singleton");
6782 assert_eq!(
6783 inspection.identity.runtime_member_id,
6784 "rt:review:singleton:0"
6785 );
6786
6787 let retired = aggregator.retire_identity("review:singleton").await?;
6788 assert!(
6789 retired,
6790 "visible live alias should retire without hidden ambiguity"
6791 );
6792
6793 let _ = runtime.mob_handle().stop().await;
6794 Ok(())
6795 }
6796
6797 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6798 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6799 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6800 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6801 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6802 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6803 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6804
6805 let aggregator = MobKitConsoleAggregator::in_memory();
6806 aggregator.register_runtime_handles_with_policy(
6807 "a-hidden",
6808 "",
6809 hidden_runtime.mob_runtime().clone(),
6810 Some(hidden_identity_runtime),
6811 hidden_runtime.console_events(),
6812 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6813 );
6814 aggregator.register_runtime_handles_with_policy(
6815 "b-visible",
6816 "",
6817 visible_runtime.mob_runtime().clone(),
6818 Some(visible_identity_runtime),
6819 visible_runtime.console_events(),
6820 Arc::new(AllowAllConsoleVisibilityPolicy),
6821 );
6822
6823 let accepted = aggregator
6824 .reserve_identity_first_interaction(
6825 ConsoleSendRequest {
6826 identity: "review:singleton".to_string(),
6827 content: json!("hello"),
6828 origin: "console:test".to_string(),
6829 idempotency_key: "visible-after-hidden-reserve".to_string(),
6830 handling_mode: Some("queue".to_string()),
6831 },
6832 None,
6833 )
6834 .await?;
6835 let page = aggregator
6836 .query_timeline(ConsoleTimelineQuery {
6837 identity: Some("review:singleton".to_string()),
6838 ..ConsoleTimelineQuery::default()
6839 })
6840 .await?;
6841 let frame = page
6842 .frames
6843 .iter()
6844 .find(|frame| frame.id == accepted.input_frame_id)
6845 .ok_or("accepted frame missing from timeline")?;
6846 assert_eq!(frame.runtime_key, "b-visible");
6847
6848 let _ = hidden_runtime.mob_handle().stop().await;
6849 let _ = visible_runtime.mob_handle().stop().await;
6850 Ok(())
6851 }
6852
6853 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6854 async fn identity_first_list_identities_projects_cached_topology_peers()
6855 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6856 let runtime = build_empty_runtime("identity-topology-cache-test").await;
6857 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6858 identity_runtime
6859 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6860 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6861 crate::identity_first::AgentIdentity::parse("agent:beta")?,
6862 )?])
6863 .await;
6864
6865 let aggregator = MobKitConsoleAggregator::in_memory();
6866 aggregator.register_runtime_handles_with_policy(
6867 "identity-first",
6868 "",
6869 runtime.mob_runtime().clone(),
6870 Some(identity_runtime),
6871 runtime.console_events(),
6872 Arc::new(AllowAllConsoleVisibilityPolicy),
6873 );
6874
6875 let identities = aggregator.list_identities().await?;
6876 let alpha = identities
6877 .iter()
6878 .find(|record| record.identity == "agent:alpha")
6879 .ok_or("agent:alpha identity missing")?;
6880 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6881 let inspection = aggregator
6882 .inspect_identity("agent:alpha")
6883 .await?
6884 .ok_or("agent:alpha inspection missing")?;
6885 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6886
6887 let _ = runtime.mob_handle().stop().await;
6888 Ok(())
6889 }
6890
6891 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6892 async fn identity_first_list_identities_includes_member_only_spawned_workers()
6893 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6894 let runtime = build_empty_runtime("identity-member-only-test").await;
6895 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6896 let aggregator = MobKitConsoleAggregator::in_memory();
6897 aggregator.register_runtime_handles_with_policy(
6898 "identity-first",
6899 "",
6900 runtime.mob_runtime().clone(),
6901 Some(identity_runtime),
6902 runtime.console_events(),
6903 Arc::new(AllowAllConsoleVisibilityPolicy),
6904 );
6905
6906 runtime
6907 .spawn(SpawnMemberSpec::from_wire(
6908 "worker".to_string(),
6909 "agent:beta".to_string(),
6910 Some("You are beta.".into()),
6911 None,
6912 None,
6913 ))
6914 .await?;
6915
6916 let identities = aggregator.list_identities().await?;
6917 assert!(
6918 identities
6919 .iter()
6920 .any(|record| record.identity == "agent:beta"),
6921 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6922 );
6923
6924 let _ = runtime.mob_handle().stop().await;
6925 Ok(())
6926 }
6927
6928 #[tokio::test]
6929 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6930 let aggregator = MobKitConsoleAggregator::in_memory();
6931 let record = identity_record_for_test("agent-cached");
6932 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6933 aggregator
6934 .inner
6935 .identity_read_model
6936 .primed
6937 .store(true, Ordering::Release);
6938 let _guard = aggregator
6939 .inner
6940 .identity_read_model
6941 .refresh_lock
6942 .clone()
6943 .lock_owned()
6944 .await;
6945
6946 let identities =
6947 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6948 .await
6949 .expect("hot identity list should not wait for refresh lock")
6950 .expect("identity list succeeds");
6951
6952 assert_eq!(identities, vec![record]);
6953 }
6954
6955 #[tokio::test]
6956 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6957 let aggregator = MobKitConsoleAggregator::in_memory();
6958 let guard = aggregator
6959 .inner
6960 .identity_read_model
6961 .refresh_lock
6962 .clone()
6963 .lock_owned()
6964 .await;
6965 let waiter = tokio::spawn({
6966 let aggregator = aggregator.clone();
6967 async move { aggregator.list_identities().await }
6968 });
6969
6970 tokio::time::sleep(Duration::from_millis(20)).await;
6971 assert!(
6972 !waiter.is_finished(),
6973 "cold identity list should wait for the in-flight refresh to finish"
6974 );
6975
6976 let record = identity_record_for_test("agent-primed");
6977 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6978 aggregator
6979 .inner
6980 .identity_read_model
6981 .primed
6982 .store(true, Ordering::Release);
6983 drop(guard);
6984
6985 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6986 .await
6987 .expect("cold identity list waiter should resume")
6988 .expect("waiter joins")
6989 .expect("identity list succeeds");
6990 assert_eq!(identities, vec![record]);
6991 }
6992
6993 #[tokio::test]
6994 async fn query_timeline_reads_from_aggregate_store() {
6995 let aggregator = MobKitConsoleAggregator::in_memory();
6996 let frame = NewConsoleFrame {
6997 id: None,
6998 dedupe_key: "event-1".to_string(),
6999 timestamp_ms: 1,
7000 runtime_key: "runtime-a".to_string(),
7001 identity: "agent-a".to_string(),
7002 conversation_id: Some("agent-a".to_string()),
7003 session_id: None,
7004 kind: "text_delta".to_string(),
7005 status: ConsoleFrameStatus::Delivered,
7006 payload: json!({ "delta": "hello" }),
7007 source: ConsoleFrameSource {
7008 kind: ConsoleFrameSourceKind::ConsoleEvent,
7009 source_cursor: None,
7010 },
7011 source_event_id: Some("event-1".to_string()),
7012 interaction_id: None,
7013 turn_id: None,
7014 run_id: None,
7015 parent_frame_id: None,
7016 caused_by_frame_id: None,
7017 };
7018 aggregator
7019 .store()
7020 .append_if_absent(frame)
7021 .await
7022 .expect("append frame");
7023
7024 let page = aggregator
7025 .query_timeline(ConsoleTimelineQuery {
7026 identity: Some("agent-a".to_string()),
7027 limit: 10,
7028 ..ConsoleTimelineQuery::default()
7029 })
7030 .await
7031 .expect("query timeline");
7032 assert_eq!(page.frames.len(), 1);
7033 assert_eq!(page.frames[0].kind, "text_delta");
7034 }
7035
7036 #[tokio::test]
7048 async fn system_identity_frames_are_visible_on_global_timeline_queries() {
7049 let aggregator = MobKitConsoleAggregator::in_memory();
7050 let runtime = build_single_member_runtime().await;
7051 aggregator
7055 .inner
7056 .runtimes
7057 .write()
7058 .expect("runtime registry")
7059 .insert(
7060 "runtime-a".to_string(),
7061 runtime_entry_for_test("runtime-a", &runtime),
7062 );
7063 let entry = runtime_entry_for_test("runtime-a", &runtime);
7064
7065 let envelope = |event_id: &str, identity: &str, event_type: &str| {
7066 crate::console_contracts::ConsoleIdentityEventEnvelope {
7067 event_id: event_id.to_string(),
7068 interaction_id: None,
7069 identity: identity.to_string(),
7070 event_type: event_type.to_string(),
7071 timestamp_ms: 1_000,
7072 data: json!({ "realm": "default" }),
7073 }
7074 };
7075
7076 let system_frame = frame_from_console_event(
7077 &entry,
7078 envelope("evt-mem-1", SYSTEM_EVENT_IDENTITY, "memory.dream.completed"),
7079 );
7080 assert_eq!(
7081 system_frame.identity, SYSTEM_EVENT_IDENTITY,
7082 "projection must not namespace the system identity"
7083 );
7084 let ghost_frame = frame_from_console_event(
7085 &entry,
7086 envelope("evt-ghost-1", "ghost-agent", "memory.dream.completed"),
7087 );
7088 assert_eq!(
7089 ghost_frame.identity, "test/ghost-agent",
7090 "ordinary identities keep the namespace"
7091 );
7092 for frame in [system_frame, ghost_frame] {
7093 aggregator
7094 .store()
7095 .append_if_absent(frame)
7096 .await
7097 .expect("append frame");
7098 }
7099
7100 let page = aggregator
7103 .query_timeline(ConsoleTimelineQuery {
7104 identity: None,
7105 limit: 10,
7106 ..ConsoleTimelineQuery::default()
7107 })
7108 .await
7109 .expect("query timeline");
7110 let identities: Vec<&str> = page
7111 .frames
7112 .iter()
7113 .map(|frame| frame.identity.as_str())
7114 .collect();
7115 assert!(
7116 identities.contains(&SYSTEM_EVENT_IDENTITY),
7117 "system-attributed memory event must be globally visible: {identities:?}"
7118 );
7119 assert!(
7120 !identities.contains(&"test/ghost-agent"),
7121 "unknown identities must stay hidden — the exemption is exact: {identities:?}"
7122 );
7123
7124 let _ = runtime.mob_handle().stop().await;
7125 }
7126
7127 #[tokio::test]
7128 async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7129 let aggregator = MobKitConsoleAggregator::in_memory();
7130 let runtime = build_single_member_runtime().await;
7131 let entry = runtime_entry_for_test("runtime-a", &runtime);
7132 for event_type in ["reasoning_delta", "reasoning_complete"] {
7133 let frame = frame_from_console_event(
7134 &entry,
7135 console_envelope_for_test(
7136 event_type,
7137 Some("interaction-a"),
7138 event_type,
7139 json!({
7140 "delta": "I should inspect the file first.",
7141 "text": "I should inspect the file first.",
7142 "turn_id": "turn-a",
7143 }),
7144 ),
7145 );
7146 aggregator
7147 .store()
7148 .append_if_absent(frame)
7149 .await
7150 .expect("append reasoning frame");
7151 }
7152
7153 let page = aggregator
7154 .query_timeline(ConsoleTimelineQuery {
7155 identity: Some("test/agent-a".to_string()),
7156 limit: 10,
7157 ..ConsoleTimelineQuery::default()
7158 })
7159 .await
7160 .expect("query timeline");
7161
7162 assert_eq!(
7163 page.frames.len(),
7164 1,
7165 "identical reasoning re-emissions in one turn should collapse"
7166 );
7167 assert_eq!(page.frames[0].kind, "reasoning_delta");
7168 assert_eq!(
7169 page.frames[0].dedupe_key,
7170 format!(
7171 "console-reasoning:runtime-a:interaction-a:{}",
7172 hash_short("I should inspect the file first.")
7173 )
7174 );
7175 }
7176
7177 #[tokio::test]
7178 async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7179 let aggregator = MobKitConsoleAggregator::in_memory();
7180 let runtime = build_single_member_runtime().await;
7181 let entry = runtime_entry_for_test("runtime-a", &runtime);
7182 for (event_id, text) in [
7183 ("reasoning-1", "I should inspect the file first."),
7184 ("reasoning-2", "Now I should run the focused test."),
7185 ] {
7186 let frame = frame_from_console_event(
7187 &entry,
7188 console_envelope_for_test(
7189 event_id,
7190 Some("interaction-a"),
7191 "reasoning_complete",
7192 json!({
7193 "text": text,
7194 "turn_id": "turn-a",
7195 }),
7196 ),
7197 );
7198 aggregator
7199 .store()
7200 .append_if_absent(frame)
7201 .await
7202 .expect("append reasoning frame");
7203 }
7204
7205 let page = aggregator
7206 .query_timeline(ConsoleTimelineQuery {
7207 identity: Some("test/agent-a".to_string()),
7208 limit: 10,
7209 ..ConsoleTimelineQuery::default()
7210 })
7211 .await
7212 .expect("query timeline");
7213
7214 assert_eq!(
7215 page.frames.len(),
7216 2,
7217 "distinct same-turn reasoning segments should not collapse"
7218 );
7219 assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7220 }
7221
7222 #[tokio::test]
7223 async fn identity_recent_anchor_respects_query_limit() {
7224 let aggregator = MobKitConsoleAggregator::in_memory();
7225 aggregator
7226 .store()
7227 .append_if_absent(NewConsoleFrame {
7228 id: None,
7229 dedupe_key: "anchored-user-input".to_string(),
7230 timestamp_ms: 1,
7231 runtime_key: "runtime-a".to_string(),
7232 identity: "agent-a".to_string(),
7233 conversation_id: Some("agent-a".to_string()),
7234 session_id: None,
7235 kind: "user_input".to_string(),
7236 status: ConsoleFrameStatus::Delivered,
7237 payload: json!({ "content": "anchor me" }),
7238 source: ConsoleFrameSource {
7239 kind: ConsoleFrameSourceKind::Synthetic,
7240 source_cursor: None,
7241 },
7242 source_event_id: Some("anchored-user-input".to_string()),
7243 interaction_id: Some("turn-a".to_string()),
7244 turn_id: None,
7245 run_id: None,
7246 parent_frame_id: None,
7247 caused_by_frame_id: None,
7248 })
7249 .await
7250 .expect("append anchor");
7251 for idx in 2..=40 {
7252 aggregator
7253 .store()
7254 .append_if_absent(NewConsoleFrame {
7255 id: None,
7256 dedupe_key: format!("noisy-tail-{idx}"),
7257 timestamp_ms: idx,
7258 runtime_key: "runtime-a".to_string(),
7259 identity: "agent-a".to_string(),
7260 conversation_id: Some("agent-a".to_string()),
7261 session_id: None,
7262 kind: "reasoning_delta".to_string(),
7263 status: ConsoleFrameStatus::Delivered,
7264 payload: json!({ "delta": idx }),
7265 source: ConsoleFrameSource {
7266 kind: ConsoleFrameSourceKind::ConsoleEvent,
7267 source_cursor: None,
7268 },
7269 source_event_id: Some(format!("noisy-tail-{idx}")),
7270 interaction_id: Some("turn-a".to_string()),
7271 turn_id: None,
7272 run_id: None,
7273 parent_frame_id: None,
7274 caused_by_frame_id: None,
7275 })
7276 .await
7277 .expect("append noisy tail");
7278 }
7279
7280 let page = aggregator
7281 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7282 identity: Some("agent-a".to_string()),
7283 mode: ConsoleTimelineMode::Recent,
7284 limit: 5,
7285 ..ConsoleTimelineWindowQuery::default()
7286 })
7287 .await
7288 .expect("query recent identity timeline");
7289
7290 assert_eq!(
7291 page.frames.len(),
7292 5,
7293 "identity anchor merge must not exceed the requested limit"
7294 );
7295 assert!(
7296 page.frames
7297 .iter()
7298 .any(|frame| frame.dedupe_key == "anchored-user-input"),
7299 "the bounded result should still retain the useful turn anchor: {:#?}",
7300 page.frames
7301 );
7302 assert_eq!(
7303 page.frames.last().and_then(|frame| frame.cursor.seq()),
7304 Some(40)
7305 );
7306 }
7307
7308 #[tokio::test]
7309 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7310 let aggregator = MobKitConsoleAggregator::in_memory();
7311 let runtime = build_single_member_runtime().await;
7312 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7313 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7314 aggregator
7315 .inner
7316 .runtimes
7317 .write()
7318 .expect("runtime registry")
7319 .insert("runtime-a".to_string(), entry);
7320
7321 for idx in 0..1_500 {
7322 aggregator
7323 .store()
7324 .append_if_absent(NewConsoleFrame {
7325 id: None,
7326 dedupe_key: format!("hidden-gap-{idx}"),
7327 timestamp_ms: idx,
7328 runtime_key: "runtime-a".to_string(),
7329 identity: "agent-a".to_string(),
7330 conversation_id: Some("agent-a".to_string()),
7331 session_id: None,
7332 kind: "hidden_noise".to_string(),
7333 status: ConsoleFrameStatus::Delivered,
7334 payload: json!({ "delta": idx }),
7335 source: ConsoleFrameSource {
7336 kind: ConsoleFrameSourceKind::ConsoleEvent,
7337 source_cursor: None,
7338 },
7339 source_event_id: Some(format!("hidden-gap-{idx}")),
7340 interaction_id: None,
7341 turn_id: None,
7342 run_id: None,
7343 parent_frame_id: None,
7344 caused_by_frame_id: None,
7345 })
7346 .await
7347 .expect("append hidden frame");
7348 }
7349 aggregator
7350 .store()
7351 .append_if_absent(NewConsoleFrame {
7352 id: None,
7353 dedupe_key: "visible-after-gap".to_string(),
7354 timestamp_ms: 1_501,
7355 runtime_key: "runtime-a".to_string(),
7356 identity: "agent-a".to_string(),
7357 conversation_id: Some("agent-a".to_string()),
7358 session_id: None,
7359 kind: "interaction_complete".to_string(),
7360 status: ConsoleFrameStatus::Delivered,
7361 payload: json!({ "text": "visible after hidden gap" }),
7362 source: ConsoleFrameSource {
7363 kind: ConsoleFrameSourceKind::ConsoleEvent,
7364 source_cursor: None,
7365 },
7366 source_event_id: Some("visible-after-gap".to_string()),
7367 interaction_id: Some("turn-visible".to_string()),
7368 turn_id: None,
7369 run_id: None,
7370 parent_frame_id: None,
7371 caused_by_frame_id: None,
7372 })
7373 .await
7374 .expect("append visible frame");
7375
7376 let page = aggregator
7377 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7378 identity: Some("agent-a".to_string()),
7379 mode: ConsoleTimelineMode::Since,
7380 limit: 1,
7381 ..ConsoleTimelineWindowQuery::default()
7382 })
7383 .await
7384 .expect("query timeline");
7385 assert_eq!(page.frames.len(), 1);
7386 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
7387 assert_eq!(
7388 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7389 Some(1_501)
7390 );
7391 let _ = runtime.mob_handle().stop().await;
7392 }
7393
7394 #[tokio::test]
7395 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
7396 let aggregator = MobKitConsoleAggregator::in_memory();
7397 for idx in 1..=300 {
7398 aggregator
7399 .store()
7400 .append_if_absent(NewConsoleFrame {
7401 id: None,
7402 dedupe_key: format!("visible-{idx}"),
7403 timestamp_ms: idx,
7404 runtime_key: "runtime-a".to_string(),
7405 identity: "agent-a".to_string(),
7406 conversation_id: Some("agent-a".to_string()),
7407 session_id: None,
7408 kind: "interaction_complete".to_string(),
7409 status: ConsoleFrameStatus::Delivered,
7410 payload: json!({ "text": format!("visible {idx}") }),
7411 source: ConsoleFrameSource {
7412 kind: ConsoleFrameSourceKind::ConsoleEvent,
7413 source_cursor: None,
7414 },
7415 source_event_id: Some(format!("visible-{idx}")),
7416 interaction_id: Some(format!("turn-{idx}")),
7417 turn_id: None,
7418 run_id: None,
7419 parent_frame_id: None,
7420 caused_by_frame_id: None,
7421 })
7422 .await
7423 .expect("append visible frame");
7424 }
7425
7426 let first = aggregator
7427 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7428 identity: Some("agent-a".to_string()),
7429 mode: ConsoleTimelineMode::Since,
7430 limit: 10,
7431 ..ConsoleTimelineWindowQuery::default()
7432 })
7433 .await
7434 .expect("query first page");
7435 assert_eq!(first.frames.len(), 10);
7436 assert_eq!(
7437 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7438 Some(10)
7439 );
7440
7441 let second = aggregator
7442 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7443 identity: Some("agent-a".to_string()),
7444 mode: ConsoleTimelineMode::Since,
7445 after: first.next_cursor,
7446 limit: 10,
7447 ..ConsoleTimelineWindowQuery::default()
7448 })
7449 .await
7450 .expect("query second page");
7451 assert_eq!(second.frames.len(), 10);
7452 assert_eq!(second.frames[0].dedupe_key, "visible-11");
7453 assert_eq!(
7454 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7455 Some(20)
7456 );
7457 }
7458
7459 #[tokio::test]
7460 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
7461 let store = Arc::new(CountingConsoleLogStore::new());
7462 let aggregator = MobKitConsoleAggregator::new(store.clone());
7463 let runtime = build_single_member_runtime().await;
7464 aggregator
7465 .inner
7466 .runtimes
7467 .write()
7468 .expect("runtime registry")
7469 .insert(
7470 "runtime-a".to_string(),
7471 runtime_entry_for_test("runtime-a", &runtime),
7472 );
7473 let inserted = store
7474 .append_if_absent(NewConsoleFrame {
7475 id: None,
7476 dedupe_key: "event-1".to_string(),
7477 timestamp_ms: 1,
7478 runtime_key: "runtime-a".to_string(),
7479 identity: "agent-a".to_string(),
7480 conversation_id: Some("agent-a".to_string()),
7481 session_id: None,
7482 kind: "text_delta".to_string(),
7483 status: ConsoleFrameStatus::Delivered,
7484 payload: json!({ "delta": "hello" }),
7485 source: ConsoleFrameSource {
7486 kind: ConsoleFrameSourceKind::ConsoleEvent,
7487 source_cursor: None,
7488 },
7489 source_event_id: Some("event-1".to_string()),
7490 interaction_id: None,
7491 turn_id: None,
7492 run_id: None,
7493 parent_frame_id: None,
7494 caused_by_frame_id: None,
7495 })
7496 .await
7497 .expect("append frame");
7498
7499 let page = aggregator
7500 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7501 identity: Some("agent-a".to_string()),
7502 mode: ConsoleTimelineMode::Since,
7503 after: Some(inserted.frame.cursor),
7504 limit: 10,
7505 ..ConsoleTimelineWindowQuery::default()
7506 })
7507 .await
7508 .expect("query continuation");
7509
7510 assert!(page.frames.is_empty());
7511 assert_eq!(
7512 store.source_watermark_calls(),
7513 0,
7514 "empty since continuation must not synchronously force session-history backfill"
7515 );
7516 let _ = runtime.mob_handle().stop().await;
7517 }
7518
7519 #[test]
7520 fn legacy_timeline_struct_literals_remain_source_compatible() {
7521 let query = ConsoleTimelineQuery {
7522 identity: Some("agent-a".to_string()),
7523 conversation_id: None,
7524 after: Some(ConsoleCursor::from("console:1")),
7525 limit: 10,
7526 };
7527 let page = ConsoleTimelinePage {
7528 frames: Vec::new(),
7529 next_cursor: query.after.clone(),
7530 };
7531
7532 assert_eq!(query.limit, 10);
7533 assert_eq!(
7534 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7535 Some(1)
7536 );
7537 }
7538
7539 #[tokio::test]
7540 async fn query_timeline_is_store_local_for_registered_runtimes() {
7541 let store = Arc::new(CountingConsoleLogStore::new());
7542 let aggregator = MobKitConsoleAggregator::new(store.clone());
7543 let runtime = build_single_member_runtime().await;
7544 aggregator
7545 .inner
7546 .runtimes
7547 .write()
7548 .expect("runtime registry")
7549 .insert(
7550 "runtime-a".to_string(),
7551 runtime_entry_for_test("runtime-a", &runtime),
7552 );
7553 store
7554 .append_if_absent(NewConsoleFrame {
7555 id: None,
7556 dedupe_key: "event-1".to_string(),
7557 timestamp_ms: 1,
7558 runtime_key: "runtime-a".to_string(),
7559 identity: "agent-a".to_string(),
7560 conversation_id: Some("agent-a".to_string()),
7561 session_id: None,
7562 kind: "text_delta".to_string(),
7563 status: ConsoleFrameStatus::Delivered,
7564 payload: json!({ "delta": "hello" }),
7565 source: ConsoleFrameSource {
7566 kind: ConsoleFrameSourceKind::ConsoleEvent,
7567 source_cursor: None,
7568 },
7569 source_event_id: Some("event-1".to_string()),
7570 interaction_id: None,
7571 turn_id: None,
7572 run_id: None,
7573 parent_frame_id: None,
7574 caused_by_frame_id: None,
7575 })
7576 .await
7577 .expect("append frame");
7578
7579 let page = tokio::time::timeout(
7580 Duration::from_millis(250),
7581 aggregator.query_timeline(ConsoleTimelineQuery {
7582 identity: Some("agent-a".to_string()),
7583 limit: 10,
7584 ..ConsoleTimelineQuery::default()
7585 }),
7586 )
7587 .await
7588 .expect("timeline query should not wait for session history")
7589 .expect("timeline query succeeds");
7590
7591 assert_eq!(page.frames.len(), 1);
7592 assert_eq!(
7593 store.source_watermark_calls(),
7594 0,
7595 "query_timeline must not synchronously touch session-history watermarks"
7596 );
7597 let _ = runtime.mob_handle().stop().await;
7598 }
7599
7600 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7601 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
7602 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
7603 delay: Duration::from_secs(2),
7604 }))
7605 .await;
7606 let aggregator = MobKitConsoleAggregator::in_memory();
7607 aggregator
7608 .inner
7609 .runtimes
7610 .write()
7611 .expect("runtime registry")
7612 .insert(
7613 "runtime-a".to_string(),
7614 runtime_entry_for_test("runtime-a", &runtime),
7615 );
7616
7617 let start = Instant::now();
7618 let accepted = tokio::time::timeout(
7619 Duration::from_millis(300),
7620 aggregator.send(ConsoleSendRequest {
7621 identity: "test/agent-a".to_string(),
7622 content: json!("hello slow agent"),
7623 origin: "console:test".to_string(),
7624 idempotency_key: "nonblocking-send".to_string(),
7625 handling_mode: Some("queue".to_string()),
7626 }),
7627 )
7628 .await
7629 .expect("console send should return once the input is accepted")
7630 .expect("send succeeds");
7631
7632 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
7633 assert!(
7634 start.elapsed() < Duration::from_secs(1),
7635 "console send should not wait for the delayed LLM turn"
7636 );
7637
7638 wait_for_session_history_text(
7639 &aggregator,
7640 "test/agent-a",
7641 "slow ok",
7642 Duration::from_secs(5),
7643 )
7644 .await
7645 .expect("background dispatch should still complete and project history");
7646 let _ = runtime.mob_handle().stop().await;
7647 }
7648
7649 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7650 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
7651 {
7652 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
7653 let aggregator = MobKitConsoleAggregator::in_memory();
7654 aggregator.register_runtime(ConsoleRuntimeRegistration {
7655 runtime_key: "runtime-late".to_string(),
7656 runtime: runtime.clone(),
7657 identity_namespace: "late".to_string(),
7658 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7659 });
7660
7661 runtime
7662 .spawn(SpawnMemberSpec::from_wire(
7663 "worker".to_string(),
7664 "agent-late".to_string(),
7665 Some("You are agent-late.".into()),
7666 None,
7667 None,
7668 ))
7669 .await
7670 .expect("late member spawns");
7671 let session_id = send_message_on_mob_with_mode(
7672 &runtime.mob_handle(),
7673 "agent-late",
7674 ContentInput::Text("hello after registration".to_string()),
7675 meerkat_core::types::HandlingMode::Queue,
7676 )
7677 .await
7678 .expect("direct member send succeeds");
7679 wait_for_identity_record(
7680 &aggregator,
7681 "late/agent-late",
7682 Some(session_id.as_str()),
7683 Duration::from_secs(5),
7684 )
7685 .await?;
7686
7687 wait_for_session_history_text(
7688 &aggregator,
7689 "late/agent-late",
7690 "You are agent-late.",
7691 Duration::from_secs(5),
7692 )
7693 .await?;
7694 let _ = runtime.mob_handle().stop().await;
7695 Ok(())
7696 }
7697
7698 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
7700 -> Result<(), String> {
7701 let runtime = build_single_member_runtime().await;
7702 let entry = runtime_entry_for_test("runtime-a", &runtime);
7703 let resolved = member_sources_for_entry(&entry)
7704 .await
7705 .into_iter()
7706 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7707 .expect("agent-a member exists");
7708 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7709 .await
7710 .expect("identity record exists");
7711 let session_id = record.session_id.expect("agent-a has a session");
7712 wait_for_runtime_session_history_text(
7713 &runtime,
7714 &session_id,
7715 "You are agent-a.",
7716 Duration::from_secs(5),
7717 )
7718 .await?;
7719
7720 let aggregator = MobKitConsoleAggregator::in_memory();
7721 aggregator
7722 .inner
7723 .runtimes
7724 .write()
7725 .expect("runtime registry")
7726 .insert("runtime-a".to_string(), entry);
7727 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
7728 aggregator
7729 .store()
7730 .record_source_watermark(
7731 &watermark_key,
7732 ConsoleFrameSourceKind::SessionHistory,
7733 &format_session_history_watermark(&session_id, 0, current_time_ms()),
7734 )
7735 .await
7736 .expect("record fresh empty watermark");
7737
7738 let page = tokio::time::timeout(
7742 Duration::from_mins(1),
7743 aggregator.query_timeline(ConsoleTimelineQuery {
7744 identity: Some("test/agent-a".to_string()),
7745 limit: 20,
7746 ..ConsoleTimelineQuery::default()
7747 }),
7748 )
7749 .await
7750 .expect("explicit identity query should not stall")
7751 .expect("query succeeds");
7752
7753 if !page.frames.is_empty() {
7754 assert!(
7755 page.frames.iter().any(|frame| {
7756 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7757 && session_history_content_text(frame).as_deref()
7758 == Some("You are agent-a.")
7759 }),
7760 "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
7761 page.frames
7762 );
7763 }
7764 wait_for_session_history_text(
7765 &aggregator,
7766 "test/agent-a",
7767 "You are agent-a.",
7768 Duration::from_secs(5),
7769 )
7770 .await?;
7771 let _ = runtime.mob_handle().stop().await;
7772 Ok(())
7773 }
7774
7775 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7776 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
7777 {
7778 let store = Arc::new(CountingConsoleLogStore::new());
7779 let aggregator = MobKitConsoleAggregator::new(store.clone());
7780 let runtime = build_single_member_runtime().await;
7781 let entry = runtime_entry_for_test("runtime-a", &runtime);
7782 let resolved = member_sources_for_entry(&entry)
7783 .await
7784 .into_iter()
7785 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7786 .expect("agent-a member exists");
7787 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7788 .await
7789 .expect("identity record exists");
7790 let session_id = record.session_id.expect("agent-a has a session");
7791 wait_for_runtime_session_history_text(
7792 &runtime,
7793 &session_id,
7794 "You are agent-a.",
7795 Duration::from_secs(5),
7796 )
7797 .await?;
7798 aggregator
7799 .inner
7800 .runtimes
7801 .write()
7802 .expect("runtime registry")
7803 .insert("runtime-a".to_string(), entry);
7804
7805 store
7806 .append_if_absent(NewConsoleFrame {
7807 id: None,
7808 dedupe_key: "stale-session-history-agent-a".to_string(),
7809 timestamp_ms: 10,
7810 runtime_key: "runtime-a".to_string(),
7811 identity: "test/agent-a".to_string(),
7812 conversation_id: Some("test/agent-a".to_string()),
7813 session_id: Some(session_id.clone()),
7814 kind: "user_input".to_string(),
7815 status: ConsoleFrameStatus::Delivered,
7816 payload: json!({
7817 "text": "stale projected history",
7818 "type": "user_input",
7819 }),
7820 source: ConsoleFrameSource {
7821 kind: ConsoleFrameSourceKind::SessionHistory,
7822 source_cursor: Some("stale-session-history-agent-a".to_string()),
7823 },
7824 source_event_id: Some("stale-session-history-agent-a".to_string()),
7825 interaction_id: None,
7826 turn_id: None,
7827 run_id: None,
7828 parent_frame_id: None,
7829 caused_by_frame_id: None,
7830 })
7831 .await
7832 .expect("append stale history frame");
7833
7834 let fresh_prompt = "fresh prompt after stale history";
7835 let sent_session_id = send_message_on_mob_with_mode(
7836 &runtime.mob_handle(),
7837 "agent-a",
7838 ContentInput::Text(fresh_prompt.to_string()),
7839 meerkat_core::types::HandlingMode::Queue,
7840 )
7841 .await
7842 .expect("direct member send succeeds");
7843 assert_eq!(sent_session_id, session_id);
7844 wait_for_runtime_session_history_text(
7845 &runtime,
7846 &session_id,
7847 fresh_prompt,
7848 Duration::from_secs(5),
7849 )
7850 .await?;
7851
7852 let page = aggregator
7853 .query_timeline(ConsoleTimelineQuery {
7854 identity: Some("test/agent-a".to_string()),
7855 limit: 50,
7856 ..ConsoleTimelineQuery::default()
7857 })
7858 .await
7859 .expect("query child timeline");
7860
7861 assert!(
7862 page.frames.iter().any(|frame| {
7863 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7864 && session_history_content_text(frame).as_deref()
7865 == Some("stale projected history")
7866 }),
7867 "explicit identity query should return existing history before async refresh; frames: {:#?}",
7868 page.frames
7869 );
7870 wait_for_session_history_text(
7871 &aggregator,
7872 "test/agent-a",
7873 fresh_prompt,
7874 Duration::from_secs(5),
7875 )
7876 .await?;
7877 assert!(
7878 store.source_watermark_calls() > 0,
7879 "stale existing session history should force a targeted source refresh"
7880 );
7881 let _ = runtime.mob_handle().stop().await;
7882 Ok(())
7883 }
7884
7885 async fn wait_for_session_history_text(
7886 aggregator: &MobKitConsoleAggregator,
7887 identity: &str,
7888 expected: &str,
7889 timeout: Duration,
7890 ) -> Result<(), String> {
7891 let deadline = Instant::now() + timeout;
7892 let mut observed = Vec::new();
7893 while Instant::now() < deadline {
7894 let _ = aggregator.list_identities().await;
7895 let page = aggregator
7896 .query_timeline(ConsoleTimelineQuery {
7897 identity: Some(identity.to_string()),
7898 limit: 20,
7899 ..ConsoleTimelineQuery::default()
7900 })
7901 .await
7902 .expect("query timeline");
7903 observed = page.frames;
7904 if observed.iter().any(|frame| {
7905 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7906 && session_history_content_text(frame).as_deref() == Some(expected)
7907 }) {
7908 return Ok(());
7909 }
7910 tokio::time::sleep(Duration::from_millis(25)).await;
7911 }
7912
7913 Err(format!(
7914 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7915 ))
7916 }
7917
7918 async fn wait_for_session_backfill_target(
7919 aggregator: &MobKitConsoleAggregator,
7920 identity: &str,
7921 timeout: Duration,
7922 ) -> Result<String, String> {
7923 let deadline = Instant::now() + timeout;
7924 while Instant::now() < deadline {
7925 if let Some(target) =
7926 session_backfill_target_for_identity(&aggregator.inner, identity).await
7927 {
7928 return Ok(target.session_id);
7929 }
7930 tokio::time::sleep(Duration::from_millis(25)).await;
7931 }
7932 Err(format!(
7933 "session backfill target for {identity:?} was not resolvable"
7934 ))
7935 }
7936
7937 async fn wait_for_runtime_session_history_text(
7938 runtime: &UnifiedRuntime,
7939 session_id: &str,
7940 expected: &str,
7941 timeout: Duration,
7942 ) -> Result<(), String> {
7943 let deadline = Instant::now() + timeout;
7944 let mut observed = Vec::new();
7945 while Instant::now() < deadline {
7946 let page = runtime
7947 .mob_runtime()
7948 .read_session_history(session_id, 0, Some(20))
7949 .await
7950 .map_err(|err| err.to_string())?;
7951 observed = page.messages;
7952 if observed.iter().enumerate().any(|(idx, message)| {
7953 let Some(message) = serde_json::to_value(message).ok() else {
7954 return false;
7955 };
7956 frames_from_session_history_message(
7957 "runtime-a",
7958 "test/agent-a",
7959 session_id,
7960 idx,
7961 message,
7962 )
7963 .iter()
7964 .any(|frame| {
7965 matches!(
7966 frame.kind.as_str(),
7967 "user_input" | "system_notice" | "interaction_complete"
7968 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7969 })
7970 }) {
7971 return Ok(());
7972 }
7973 tokio::time::sleep(Duration::from_millis(25)).await;
7974 }
7975
7976 Err(format!(
7977 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7978 ))
7979 }
7980
7981 async fn wait_for_identity_record(
7982 aggregator: &MobKitConsoleAggregator,
7983 identity: &str,
7984 session_id: Option<&str>,
7985 timeout: Duration,
7986 ) -> Result<(), String> {
7987 let deadline = Instant::now() + timeout;
7988 let mut observed = Vec::new();
7989 while Instant::now() < deadline {
7990 observed = aggregator
7991 .list_identities()
7992 .await
7993 .map_err(|err| err.to_string())?;
7994 if observed.iter().any(|record| {
7995 record.identity == identity && record.session_id.as_deref() == session_id
7996 }) {
7997 return Ok(());
7998 }
7999 tokio::time::sleep(Duration::from_millis(25)).await;
8000 }
8001
8002 Err(format!(
8003 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
8004 ))
8005 }
8006
8007 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
8008 session_history_payload_text(&frame.payload)
8009 }
8010
8011 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
8012 session_history_payload_text(&frame.payload)
8013 }
8014
8015 fn session_history_payload_text(payload: &Value) -> Option<String> {
8016 if let Some(text) = payload.get("text").and_then(Value::as_str) {
8017 return Some(text.to_string());
8018 }
8019 if let Some(text) = payload.get("result").and_then(Value::as_str) {
8020 return Some(text.to_string());
8021 }
8022 if let Some(text) = payload.get("body").and_then(Value::as_str) {
8023 return Some(text.to_string());
8024 }
8025 match payload.get("content")? {
8026 Value::String(text) => Some(text.clone()),
8027 Value::Array(blocks) => Some(
8028 blocks
8029 .iter()
8030 .filter_map(|block| block.get("text").and_then(Value::as_str))
8031 .collect::<Vec<_>>()
8032 .join(""),
8033 ),
8034 _ => None,
8035 }
8036 }
8037
8038 #[tokio::test]
8039 async fn query_timeline_handles_large_store_without_backfill_calls() {
8040 let store = Arc::new(CountingConsoleLogStore::new());
8041 let aggregator = MobKitConsoleAggregator::new(store.clone());
8042 for idx in 0..5_000 {
8043 store
8044 .append_if_absent(NewConsoleFrame {
8045 id: None,
8046 dedupe_key: format!("event-{idx}"),
8047 timestamp_ms: idx,
8048 runtime_key: "runtime-a".to_string(),
8049 identity: "agent-a".to_string(),
8050 conversation_id: Some("agent-a".to_string()),
8051 session_id: Some("session-a".to_string()),
8052 kind: "text_delta".to_string(),
8053 status: ConsoleFrameStatus::Delivered,
8054 payload: json!({ "delta": idx }),
8055 source: ConsoleFrameSource {
8056 kind: ConsoleFrameSourceKind::ConsoleEvent,
8057 source_cursor: None,
8058 },
8059 source_event_id: Some(format!("event-{idx}")),
8060 interaction_id: None,
8061 turn_id: None,
8062 run_id: None,
8063 parent_frame_id: None,
8064 caused_by_frame_id: None,
8065 })
8066 .await
8067 .expect("append frame");
8068 }
8069
8070 let page = aggregator
8071 .query_timeline(ConsoleTimelineQuery {
8072 identity: Some("agent-a".to_string()),
8073 limit: 1_000,
8074 ..ConsoleTimelineQuery::default()
8075 })
8076 .await
8077 .expect("large query");
8078
8079 assert_eq!(page.frames.len(), 1_000);
8080 assert_eq!(store.source_watermark_calls(), 0);
8081 }
8082
8083 #[tokio::test]
8084 async fn query_timeline_rejects_future_cursor_after_store_reset() {
8085 let aggregator = MobKitConsoleAggregator::in_memory();
8086 let err = aggregator
8087 .query_timeline_windowed(ConsoleTimelineWindowQuery {
8088 after: Some(ConsoleCursor::from("console:99")),
8089 limit: 10,
8090 ..ConsoleTimelineWindowQuery::default()
8091 })
8092 .await
8093 .expect_err("future cursor on empty/reset store must be replay-unavailable");
8094
8095 assert!(
8096 err.to_string()
8097 .contains("beyond the current store frontier"),
8098 "unexpected error: {err}"
8099 );
8100 }
8101
8102 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8103 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
8104 let store = Arc::new(CountingConsoleLogStore::new());
8105 let aggregator = MobKitConsoleAggregator::new(store.clone());
8106 let runtime = build_single_member_runtime().await;
8107 aggregator
8108 .inner
8109 .runtimes
8110 .write()
8111 .expect("runtime registry")
8112 .insert(
8113 "runtime-a".to_string(),
8114 runtime_entry_for_test("runtime-a", &runtime),
8115 );
8116 let session_id =
8117 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8118 .await
8119 .expect("spawned member is resolvable for targeted backfill");
8120 wait_for_runtime_session_history_text(
8121 &runtime,
8122 &session_id,
8123 "You are agent-a.",
8124 Duration::from_secs(5),
8125 )
8126 .await
8127 .expect("spawned member kickoff is readable before live-frame race");
8128 store
8129 .append_if_absent(NewConsoleFrame {
8130 id: None,
8131 dedupe_key: "live-tool-before-history".to_string(),
8132 timestamp_ms: 10,
8133 runtime_key: "runtime-a".to_string(),
8134 identity: "test/agent-a".to_string(),
8135 conversation_id: Some("test/agent-a".to_string()),
8136 session_id: None,
8137 kind: "tool_execution_started".to_string(),
8138 status: ConsoleFrameStatus::Delivered,
8139 payload: json!({
8140 "id": "call-live",
8141 "name": "king_search",
8142 "source_event_type": "tool_execution_started",
8143 "type": "tool_execution_started",
8144 }),
8145 source: ConsoleFrameSource {
8146 kind: ConsoleFrameSourceKind::ConsoleEvent,
8147 source_cursor: Some("live-tool-before-history".to_string()),
8148 },
8149 source_event_id: Some("live-tool-before-history".to_string()),
8150 interaction_id: None,
8151 turn_id: None,
8152 run_id: None,
8153 parent_frame_id: None,
8154 caused_by_frame_id: None,
8155 })
8156 .await
8157 .expect("append live tool frame");
8158
8159 let page = aggregator
8160 .query_timeline(ConsoleTimelineQuery {
8161 identity: Some("test/agent-a".to_string()),
8162 limit: 20,
8163 ..ConsoleTimelineQuery::default()
8164 })
8165 .await
8166 .expect("query child timeline");
8167
8168 assert!(
8169 page.frames
8170 .iter()
8171 .any(|frame| frame.kind == "tool_execution_started"),
8172 "initial explicit child timeline should return existing store frames without waiting for session history"
8173 );
8174
8175 let mut observed_backfill = false;
8176 for _ in 0..80 {
8177 let page = aggregator
8178 .query_timeline(ConsoleTimelineQuery {
8179 identity: Some("test/agent-a".to_string()),
8180 limit: 20,
8181 ..ConsoleTimelineQuery::default()
8182 })
8183 .await
8184 .expect("query child timeline after scheduled backfill");
8185 if page.frames.iter().any(|frame| {
8186 frame.kind == "user_input"
8187 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8188 && frame.payload.to_string().contains("You are agent-a.")
8189 }) {
8190 observed_backfill = true;
8191 break;
8192 }
8193 tokio::time::sleep(Duration::from_millis(25)).await;
8194 }
8195 assert!(
8196 observed_backfill,
8197 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8198 );
8199 let _ = runtime.mob_handle().stop().await;
8200 }
8201
8202 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8203 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8204 let store = Arc::new(CountingConsoleLogStore::new());
8205 let aggregator = MobKitConsoleAggregator::new(store.clone());
8206 let (_temp, runtime, _delayed_service) =
8207 build_stress_runtime(64, Duration::from_millis(0)).await;
8208 aggregator
8209 .inner
8210 .runtimes
8211 .write()
8212 .expect("runtime registry")
8213 .insert(
8214 "runtime-a".to_string(),
8215 runtime_entry_for_test("runtime-a", runtime.as_ref()),
8216 );
8217 for idx in 0..1_000 {
8218 store
8219 .append_if_absent(NewConsoleFrame {
8220 id: None,
8221 dedupe_key: format!("event-{idx}"),
8222 timestamp_ms: idx,
8223 runtime_key: "runtime-a".to_string(),
8224 identity: "test/agent-0".to_string(),
8225 conversation_id: Some("test/agent-0".to_string()),
8226 session_id: Some("session-a".to_string()),
8227 kind: "text_delta".to_string(),
8228 status: ConsoleFrameStatus::Delivered,
8229 payload: json!({ "delta": idx }),
8230 source: ConsoleFrameSource {
8231 kind: ConsoleFrameSourceKind::ConsoleEvent,
8232 source_cursor: None,
8233 },
8234 source_event_id: Some(format!("event-{idx}")),
8235 interaction_id: None,
8236 turn_id: None,
8237 run_id: None,
8238 parent_frame_id: None,
8239 caused_by_frame_id: None,
8240 })
8241 .await
8242 .expect("append frame");
8243 }
8244
8245 let page = tokio::time::timeout(
8252 Duration::from_mins(1),
8253 aggregator.query_timeline(ConsoleTimelineQuery {
8254 identity: Some("test/agent-0".to_string()),
8255 limit: 1_000,
8256 ..ConsoleTimelineQuery::default()
8257 }),
8258 )
8259 .await
8260 .expect("identity timeline query should not hang or rediscover the roster per frame")
8261 .expect("timeline query succeeds");
8262
8263 assert_eq!(page.frames.len(), 1_000);
8264 assert_eq!(store.source_watermark_calls(), 0);
8265 let _ = runtime.mob_handle().stop().await;
8266 }
8267
8268 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8269 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8270 const MEMBER_COUNT: usize = 32;
8271 let (_temp, runtime, delayed_service) =
8272 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8273 let aggregator = MobKitConsoleAggregator::in_memory();
8274 aggregator
8275 .inner
8276 .runtimes
8277 .write()
8278 .expect("runtime registry")
8279 .insert(
8280 "runtime-stress".to_string(),
8281 runtime_entry_for_test("runtime-stress", &runtime),
8282 );
8283
8284 let started = Instant::now();
8285 aggregator
8286 .refresh_session_history()
8287 .await
8288 .expect("stress refresh");
8289 let elapsed = started.elapsed();
8290
8291 assert!(
8292 delayed_service.read_calls() >= MEMBER_COUNT,
8293 "expected at least one history read per member, saw {}",
8294 delayed_service.read_calls()
8295 );
8296 assert!(
8297 delayed_service.max_active_reads() > 1,
8298 "session history backfill should fan out instead of reading members serially"
8299 );
8300 assert!(
8301 delayed_service.max_active_reads()
8302 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8303 "session history backfill should respect the default concurrency limit"
8304 );
8305 assert!(
8306 elapsed < Duration::from_millis(600),
8307 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8308 MEMBER_COUNT * 40
8309 );
8310 let _ = runtime.mob_handle().stop().await;
8311 }
8312
8313 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8314 async fn session_history_backfill_respects_configured_concurrency_limit() {
8315 const MEMBER_COUNT: usize = 16;
8316 let (_temp, runtime, delayed_service) =
8317 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8318 let aggregator =
8319 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8320 max_concurrent_session_backfills: 4,
8321 ..ConsoleAggregatorOptions::default()
8322 });
8323 aggregator
8324 .inner
8325 .runtimes
8326 .write()
8327 .expect("runtime registry")
8328 .insert(
8329 "runtime-stress".to_string(),
8330 runtime_entry_for_test("runtime-stress", &runtime),
8331 );
8332
8333 aggregator
8334 .refresh_session_history()
8335 .await
8336 .expect("stress refresh");
8337
8338 assert!(
8339 delayed_service.max_active_reads() <= 4,
8340 "configured concurrency limit should bound session history reads, saw {}",
8341 delayed_service.max_active_reads()
8342 );
8343 let _ = runtime.mob_handle().stop().await;
8344 }
8345
8346 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8347 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8348 const MEMBER_COUNT: usize = 24;
8349 const LIVE_EVENT_COUNT: usize = 2_048;
8350 let (_temp, runtime, _delayed_service) =
8351 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
8352 let aggregator = MobKitConsoleAggregator::in_memory();
8353 aggregator.register_runtime(ConsoleRuntimeRegistration {
8354 runtime_key: "runtime-burst".to_string(),
8355 runtime: runtime.clone(),
8356 identity_namespace: "stress".to_string(),
8357 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8358 });
8359 let console_events = runtime.console_events();
8360 for idx in 0..LIVE_EVENT_COUNT {
8361 console_events
8362 .append(
8363 "agent-0",
8364 Some("burst-turn".to_string()),
8365 "text_delta",
8366 json!({ "delta": format!("frame-{idx}") }),
8367 )
8368 .await;
8369 }
8370
8371 let deadline = Instant::now() + Duration::from_secs(30);
8376 let mut observed = 0;
8377 while Instant::now() < deadline {
8378 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
8379 if observed >= LIVE_EVENT_COUNT {
8380 break;
8381 }
8382 tokio::time::sleep(Duration::from_millis(25)).await;
8383 }
8384
8385 assert_eq!(
8386 observed, LIVE_EVENT_COUNT,
8387 "live pump should not drop frames while slow background backfill is running"
8388 );
8389 let _ = runtime.mob_handle().stop().await;
8390 }
8391
8392 async fn count_console_event_frames(
8393 aggregator: &MobKitConsoleAggregator,
8394 identity: &str,
8395 ) -> usize {
8396 let mut after = None;
8397 let mut count = 0;
8398 loop {
8399 let page = aggregator
8400 .query_timeline(ConsoleTimelineQuery {
8401 identity: Some(identity.to_string()),
8402 after,
8403 limit: 1_000,
8404 ..ConsoleTimelineQuery::default()
8405 })
8406 .await
8407 .expect("query burst timeline");
8408 if page.frames.is_empty() {
8409 break;
8410 }
8411 count += page
8412 .frames
8413 .iter()
8414 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
8415 .count();
8416 after = page.next_cursor;
8417 if after.is_none() {
8418 break;
8419 }
8420 }
8421 count
8422 }
8423
8424 #[tokio::test]
8425 async fn status_updates_get_replayable_aggregate_cursors() {
8426 let aggregator = MobKitConsoleAggregator::in_memory();
8427 let frame = NewConsoleFrame {
8428 id: None,
8429 dedupe_key: "send-1".to_string(),
8430 timestamp_ms: 1,
8431 runtime_key: "runtime-a".to_string(),
8432 identity: "agent-a".to_string(),
8433 conversation_id: Some("agent-a".to_string()),
8434 session_id: Some("session-1".to_string()),
8435 kind: "user_input".to_string(),
8436 status: ConsoleFrameStatus::Accepted,
8437 payload: json!({ "content": "hello" }),
8438 source: ConsoleFrameSource {
8439 kind: ConsoleFrameSourceKind::Send,
8440 source_cursor: None,
8441 },
8442 source_event_id: None,
8443 interaction_id: Some("interaction-1".to_string()),
8444 turn_id: None,
8445 run_id: None,
8446 parent_frame_id: None,
8447 caused_by_frame_id: None,
8448 };
8449 let inserted = aggregator
8450 .store()
8451 .append_if_absent(frame)
8452 .await
8453 .expect("append frame");
8454
8455 update_frame_status_and_emit(
8456 &aggregator.inner,
8457 &inserted.frame.id,
8458 ConsoleFrameStatus::Delivered,
8459 )
8460 .await
8461 .expect("update status");
8462
8463 let page = aggregator
8464 .query_timeline(ConsoleTimelineQuery {
8465 identity: Some("agent-a".to_string()),
8466 after: Some(inserted.frame.cursor.clone()),
8467 limit: 10,
8468 ..ConsoleTimelineQuery::default()
8469 })
8470 .await
8471 .expect("query timeline");
8472 assert_eq!(page.frames.len(), 1);
8473 assert_eq!(page.frames[0].kind, "frame_updated");
8474 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
8475 assert_eq!(
8476 page.frames[0]
8477 .payload
8478 .get("frame")
8479 .and_then(|frame| frame.get("status"))
8480 .and_then(Value::as_str),
8481 Some("delivered")
8482 );
8483 }
8484
8485 #[tokio::test]
8486 async fn steer_delivery_appends_terminal_control_frame() {
8487 let aggregator = MobKitConsoleAggregator::in_memory();
8488 let frame = NewConsoleFrame {
8489 id: None,
8490 dedupe_key: "send-steer-1".to_string(),
8491 timestamp_ms: 1,
8492 runtime_key: "runtime-a".to_string(),
8493 identity: "agent-a".to_string(),
8494 conversation_id: Some("agent-a".to_string()),
8495 session_id: Some("session-1".to_string()),
8496 kind: "user_input".to_string(),
8497 status: ConsoleFrameStatus::Delivered,
8498 payload: json!({
8499 "content": "operator steer",
8500 "handling_mode": "steer",
8501 }),
8502 source: ConsoleFrameSource {
8503 kind: ConsoleFrameSourceKind::Send,
8504 source_cursor: None,
8505 },
8506 source_event_id: None,
8507 interaction_id: Some("interaction-steer-1".to_string()),
8508 turn_id: None,
8509 run_id: None,
8510 parent_frame_id: None,
8511 caused_by_frame_id: None,
8512 };
8513 let inserted = aggregator
8514 .store()
8515 .append_if_absent(frame)
8516 .await
8517 .expect("append steer input");
8518
8519 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
8520 .await
8521 .expect("append steer terminal");
8522
8523 let page = aggregator
8524 .query_timeline(ConsoleTimelineQuery {
8525 identity: Some("agent-a".to_string()),
8526 after: Some(inserted.frame.cursor.clone()),
8527 limit: 10,
8528 ..ConsoleTimelineQuery::default()
8529 })
8530 .await
8531 .expect("query timeline");
8532
8533 assert_eq!(page.frames.len(), 1);
8534 let terminal = &page.frames[0];
8535 assert_eq!(terminal.kind, "interaction_complete");
8536 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
8537 assert_eq!(
8538 terminal.interaction_id.as_deref(),
8539 Some("interaction-steer-1")
8540 );
8541 assert_eq!(
8542 terminal.parent_frame_id.as_deref(),
8543 Some(inserted.frame.id.as_str())
8544 );
8545 assert_eq!(
8546 terminal.payload.get("reason").and_then(Value::as_str),
8547 Some("steer_delivered")
8548 );
8549 }
8550
8551 #[test]
8552 fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
8553 let live = transcript_fingerprint(
8554 "reasoning_complete",
8555 &json!({ "text": "I should inspect the file first." }),
8556 );
8557 let replay = transcript_fingerprint(
8558 "reasoning_complete",
8559 &json!({ "summary": "I should inspect the file first." }),
8560 );
8561 let delta = transcript_fingerprint(
8562 "reasoning_delta",
8563 &json!({ "delta": "I should inspect the file first." }),
8564 );
8565
8566 assert_eq!(live.as_deref(), Some("I should inspect the file first."));
8567 assert_eq!(live, replay);
8568 assert_eq!(live, delta);
8569 }
8570
8571 #[tokio::test]
8572 async fn history_counterpart_scan_is_not_capped_to_one_page() {
8573 let aggregator = MobKitConsoleAggregator::in_memory();
8574 for idx in 0..1_005 {
8575 aggregator
8576 .store()
8577 .append_if_absent(NewConsoleFrame {
8578 id: None,
8579 dedupe_key: format!("filler-{idx}"),
8580 timestamp_ms: idx,
8581 runtime_key: "runtime-a".to_string(),
8582 identity: "agent-a".to_string(),
8583 conversation_id: Some("agent-a".to_string()),
8584 session_id: Some("session-a".to_string()),
8585 kind: "text_delta".to_string(),
8586 status: ConsoleFrameStatus::Completed,
8587 payload: json!({ "delta": idx }),
8588 source: ConsoleFrameSource {
8589 kind: ConsoleFrameSourceKind::ConsoleEvent,
8590 source_cursor: None,
8591 },
8592 source_event_id: Some(format!("filler-{idx}")),
8593 interaction_id: None,
8594 turn_id: None,
8595 run_id: None,
8596 parent_frame_id: None,
8597 caused_by_frame_id: None,
8598 })
8599 .await
8600 .expect("append filler");
8601 }
8602 aggregator
8603 .store()
8604 .append_if_absent(NewConsoleFrame {
8605 id: None,
8606 dedupe_key: "live-user-input".to_string(),
8607 timestamp_ms: 2_000,
8608 runtime_key: "runtime-a".to_string(),
8609 identity: "agent-a".to_string(),
8610 conversation_id: Some("agent-a".to_string()),
8611 session_id: Some("session-a".to_string()),
8612 kind: "user_input".to_string(),
8613 status: ConsoleFrameStatus::Delivered,
8614 payload: json!({ "content": "already here" }),
8615 source: ConsoleFrameSource {
8616 kind: ConsoleFrameSourceKind::ConsoleEvent,
8617 source_cursor: None,
8618 },
8619 source_event_id: Some("live-user-input".to_string()),
8620 interaction_id: None,
8621 turn_id: None,
8622 run_id: None,
8623 parent_frame_id: None,
8624 caused_by_frame_id: None,
8625 })
8626 .await
8627 .expect("append live input");
8628
8629 let history = NewConsoleFrame {
8630 id: None,
8631 dedupe_key: "history-user-input".to_string(),
8632 timestamp_ms: 3_000,
8633 runtime_key: "runtime-a".to_string(),
8634 identity: "agent-a".to_string(),
8635 conversation_id: Some("agent-a".to_string()),
8636 session_id: Some("session-a".to_string()),
8637 kind: "user_input".to_string(),
8638 status: ConsoleFrameStatus::Completed,
8639 payload: json!({ "content": "already here" }),
8640 source: ConsoleFrameSource {
8641 kind: ConsoleFrameSourceKind::SessionHistory,
8642 source_cursor: Some("session-a:1006".to_string()),
8643 },
8644 source_event_id: None,
8645 interaction_id: None,
8646 turn_id: None,
8647 run_id: None,
8648 parent_frame_id: None,
8649 caused_by_frame_id: None,
8650 };
8651
8652 assert!(
8653 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8654 .await
8655 .expect("counterpart scan")
8656 );
8657 }
8658
8659 #[tokio::test]
8660 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
8661 let aggregator = MobKitConsoleAggregator::in_memory();
8662 aggregator
8663 .store()
8664 .append_if_absent(NewConsoleFrame {
8665 id: None,
8666 dedupe_key: "live-user-input".to_string(),
8667 timestamp_ms: 2_000,
8668 runtime_key: "runtime-a".to_string(),
8669 identity: "agent-a".to_string(),
8670 conversation_id: Some("agent-a".to_string()),
8671 session_id: Some("session-a".to_string()),
8672 kind: "user_input".to_string(),
8673 status: ConsoleFrameStatus::Delivered,
8674 payload: json!({ "content": "hello from operator" }),
8675 source: ConsoleFrameSource {
8676 kind: ConsoleFrameSourceKind::ConsoleEvent,
8677 source_cursor: None,
8678 },
8679 source_event_id: Some("live-user-input".to_string()),
8680 interaction_id: None,
8681 turn_id: None,
8682 run_id: None,
8683 parent_frame_id: None,
8684 caused_by_frame_id: None,
8685 })
8686 .await
8687 .expect("append live input");
8688
8689 let history = NewConsoleFrame {
8690 id: None,
8691 dedupe_key: "history-user-input".to_string(),
8692 timestamp_ms: 3_000,
8693 runtime_key: "runtime-a".to_string(),
8694 identity: "agent-a".to_string(),
8695 conversation_id: Some("agent-a".to_string()),
8696 session_id: Some("session-a".to_string()),
8697 kind: "user_input".to_string(),
8698 status: ConsoleFrameStatus::Completed,
8699 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
8700 source: ConsoleFrameSource {
8701 kind: ConsoleFrameSourceKind::SessionHistory,
8702 source_cursor: Some("session-a:2".to_string()),
8703 },
8704 source_event_id: None,
8705 interaction_id: None,
8706 turn_id: None,
8707 run_id: None,
8708 parent_frame_id: None,
8709 caused_by_frame_id: None,
8710 };
8711
8712 assert!(
8713 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8714 .await
8715 .expect("counterpart scan")
8716 );
8717 }
8718
8719 #[tokio::test]
8720 async fn history_counterpart_scan_matches_content_block_user_prompts() {
8721 let aggregator = MobKitConsoleAggregator::in_memory();
8722 aggregator
8723 .store()
8724 .append_if_absent(NewConsoleFrame {
8725 id: None,
8726 dedupe_key: "live-user-input".to_string(),
8727 timestamp_ms: 2_000,
8728 runtime_key: "runtime-a".to_string(),
8729 identity: "agent-a".to_string(),
8730 conversation_id: Some("agent-a".to_string()),
8731 session_id: Some("session-a".to_string()),
8732 kind: "user_input".to_string(),
8733 status: ConsoleFrameStatus::Delivered,
8734 payload: json!({ "content": "hello from operator" }),
8735 source: ConsoleFrameSource {
8736 kind: ConsoleFrameSourceKind::ConsoleEvent,
8737 source_cursor: None,
8738 },
8739 source_event_id: Some("live-user-input".to_string()),
8740 interaction_id: None,
8741 turn_id: None,
8742 run_id: None,
8743 parent_frame_id: None,
8744 caused_by_frame_id: None,
8745 })
8746 .await
8747 .expect("append live input");
8748
8749 let history = NewConsoleFrame {
8750 id: None,
8751 dedupe_key: "history-user-input".to_string(),
8752 timestamp_ms: 3_000,
8753 runtime_key: "runtime-a".to_string(),
8754 identity: "agent-a".to_string(),
8755 conversation_id: Some("agent-a".to_string()),
8756 session_id: Some("session-a".to_string()),
8757 kind: "user_input".to_string(),
8758 status: ConsoleFrameStatus::Completed,
8759 payload: json!({
8760 "content": [{ "type": "text", "text": "hello from operator" }]
8761 }),
8762 source: ConsoleFrameSource {
8763 kind: ConsoleFrameSourceKind::SessionHistory,
8764 source_cursor: Some("session-a:2".to_string()),
8765 },
8766 source_event_id: None,
8767 interaction_id: None,
8768 turn_id: None,
8769 run_id: None,
8770 parent_frame_id: None,
8771 caused_by_frame_id: None,
8772 };
8773
8774 assert!(
8775 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8776 .await
8777 .expect("counterpart scan")
8778 );
8779 }
8780
8781 #[tokio::test]
8782 async fn history_counterpart_scan_matches_live_tool_results() {
8783 let aggregator = MobKitConsoleAggregator::in_memory();
8784 aggregator
8785 .store()
8786 .append_if_absent(NewConsoleFrame {
8787 id: None,
8788 dedupe_key: "live-tool-result".to_string(),
8789 timestamp_ms: 2_000,
8790 runtime_key: "runtime-a".to_string(),
8791 identity: "agent-a".to_string(),
8792 conversation_id: Some("agent-a".to_string()),
8793 session_id: Some("session-a".to_string()),
8794 kind: "tool_execution_completed".to_string(),
8795 status: ConsoleFrameStatus::Delivered,
8796 payload: json!({
8797 "id": "call-1",
8798 "tool_call_id": "call-1",
8799 "result": "{ \"count\": 70 }"
8800 }),
8801 source: ConsoleFrameSource {
8802 kind: ConsoleFrameSourceKind::ConsoleEvent,
8803 source_cursor: None,
8804 },
8805 source_event_id: Some("live-tool-result".to_string()),
8806 interaction_id: None,
8807 turn_id: None,
8808 run_id: None,
8809 parent_frame_id: None,
8810 caused_by_frame_id: None,
8811 })
8812 .await
8813 .expect("append live tool result");
8814
8815 let history = NewConsoleFrame {
8816 id: None,
8817 dedupe_key: "history-tool-result".to_string(),
8818 timestamp_ms: 3_000,
8819 runtime_key: "runtime-a".to_string(),
8820 identity: "agent-a".to_string(),
8821 conversation_id: Some("agent-a".to_string()),
8822 session_id: Some("session-a".to_string()),
8823 kind: "tool_execution_completed".to_string(),
8824 status: ConsoleFrameStatus::Completed,
8825 payload: json!({
8826 "id": "call-1",
8827 "tool_call_id": "call-1",
8828 "result": "{ \"count\": 70 }",
8829 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
8830 }),
8831 source: ConsoleFrameSource {
8832 kind: ConsoleFrameSourceKind::SessionHistory,
8833 source_cursor: Some("session-a:3:0".to_string()),
8834 },
8835 source_event_id: None,
8836 interaction_id: None,
8837 turn_id: None,
8838 run_id: None,
8839 parent_frame_id: None,
8840 caused_by_frame_id: None,
8841 };
8842
8843 assert!(
8844 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8845 .await
8846 .expect("counterpart scan")
8847 );
8848 }
8849
8850 #[tokio::test]
8851 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
8852 let aggregator = MobKitConsoleAggregator::in_memory();
8853 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
8854 aggregator
8855 .store()
8856 .append_if_absent(NewConsoleFrame {
8857 id: None,
8858 dedupe_key: format!("live-delta-{idx}"),
8859 timestamp_ms: 2_000 + idx as u64,
8860 runtime_key: "runtime-a".to_string(),
8861 identity: "agent-a".to_string(),
8862 conversation_id: Some("agent-a".to_string()),
8863 session_id: Some("session-a".to_string()),
8864 kind: "text_delta".to_string(),
8865 status: ConsoleFrameStatus::Delivered,
8866 payload: json!({ "delta": delta }),
8867 source: ConsoleFrameSource {
8868 kind: ConsoleFrameSourceKind::ConsoleEvent,
8869 source_cursor: None,
8870 },
8871 source_event_id: Some(format!("live-delta-{idx}")),
8872 interaction_id: Some("turn-a".to_string()),
8873 turn_id: None,
8874 run_id: None,
8875 parent_frame_id: None,
8876 caused_by_frame_id: None,
8877 })
8878 .await
8879 .expect("append live delta");
8880 }
8881
8882 let history = NewConsoleFrame {
8883 id: None,
8884 dedupe_key: "history-assistant-complete".to_string(),
8885 timestamp_ms: 3_000,
8886 runtime_key: "runtime-a".to_string(),
8887 identity: "agent-a".to_string(),
8888 conversation_id: Some("agent-a".to_string()),
8889 session_id: Some("session-a".to_string()),
8890 kind: "interaction_complete".to_string(),
8891 status: ConsoleFrameStatus::Completed,
8892 payload: json!({ "result": "Ready and standing by." }),
8893 source: ConsoleFrameSource {
8894 kind: ConsoleFrameSourceKind::SessionHistory,
8895 source_cursor: Some("session-a:3".to_string()),
8896 },
8897 source_event_id: None,
8898 interaction_id: None,
8899 turn_id: None,
8900 run_id: None,
8901 parent_frame_id: None,
8902 caused_by_frame_id: None,
8903 };
8904
8905 assert!(
8906 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8907 .await
8908 .expect("counterpart scan")
8909 );
8910 }
8911
8912 #[test]
8913 fn session_history_watermark_key_is_session_scoped() {
8914 assert_ne!(
8915 session_history_watermark_runtime_key("runtime-a", "session-1"),
8916 session_history_watermark_runtime_key("runtime-a", "session-2")
8917 );
8918 }
8919
8920 #[test]
8921 fn backfill_offset_advances_only_on_strict_forward_progress() {
8922 assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
8924 assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
8926 assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
8928 assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
8930
8931 assert_eq!(advance_backfill_offset(500, 0, 500), None); assert_eq!(advance_backfill_offset(500, 0, 400), None); assert_eq!(advance_backfill_offset(1000, 500, 500), None); assert_eq!(advance_backfill_offset(10, 10, 0), None); assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
8940 }
8941
8942 #[test]
8943 fn session_history_watermarks_are_cursor_and_ttl_aware() {
8944 let legacy = "session:with:colon:42";
8945 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8946 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8947
8948 assert_eq!(
8949 parse_session_history_watermark(legacy, "session:with:colon"),
8950 Some(42)
8951 );
8952 assert_eq!(
8953 parse_session_history_watermark(&checked, "session:with:colon"),
8954 Some(43)
8955 );
8956 assert!(session_history_watermark_is_fresh(
8957 &checked,
8958 "session:with:colon",
8959 1_500
8960 ));
8961 assert!(!session_history_watermark_is_fresh(
8962 &checked,
8963 "session:with:colon",
8964 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8965 ));
8966 assert!(session_history_watermark_is_fresh(
8967 &empty_checked,
8968 "session:with:colon",
8969 1_500
8970 ));
8971 assert!(!session_history_watermark_is_fresh(
8972 &empty_checked,
8973 "session:with:colon",
8974 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8975 ));
8976 }
8977
8978 #[test]
8979 fn session_history_messages_project_to_renderable_frames() {
8980 let user = frame_from_session_history_message(
8981 "runtime-a",
8982 "agent-a",
8983 "session-a",
8984 0,
8985 json!({
8986 "role": "user",
8987 "content": "hello",
8988 "timestamp_ms": 10
8989 }),
8990 )
8991 .expect("user history frame");
8992 let assistant = frame_from_session_history_message(
8997 "runtime-a",
8998 "agent-a",
8999 "session-a",
9000 1,
9001 json!({
9002 "role": "block_assistant",
9003 "blocks": [
9004 { "block_type": "text", "data": { "text": "hi there" } }
9005 ],
9006 "stop_reason": "end_turn",
9007 "timestamp_ms": 11
9008 }),
9009 )
9010 .expect("assistant history frame");
9011
9012 assert_eq!(user.kind, "user_input");
9013 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
9014 assert_eq!(
9015 user.payload["content"],
9016 json!([{ "type": "text", "text": "hello" }])
9017 );
9018 assert_eq!(assistant.kind, "interaction_complete");
9019 assert_eq!(assistant.payload["text"], json!("hi there"));
9020 assert!(
9021 assistant
9022 .dedupe_key
9023 .starts_with("session-history:runtime-a:session-a:1:")
9024 );
9025 }
9026
9027 #[test]
9028 fn session_history_projection_filters_scaffold_user_messages() {
9029 let spawn_notice = frame_from_session_history_message(
9030 "runtime-a",
9031 "agent-a",
9032 "session-a",
9033 0,
9034 json!({
9035 "role": "user",
9036 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
9037 "timestamp_ms": 10
9038 }),
9039 );
9040 let peer_update = frame_from_session_history_message(
9041 "runtime-a",
9042 "agent-a",
9043 "session-a",
9044 1,
9045 json!({
9046 "role": "user",
9047 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
9048 "timestamp_ms": 11
9049 }),
9050 );
9051 let real_user = frame_from_session_history_message(
9052 "runtime-a",
9053 "agent-a",
9054 "session-a",
9055 2,
9056 json!({
9057 "role": "user",
9058 "content": "Please review the incident notes.",
9059 "timestamp_ms": 12
9060 }),
9061 );
9062
9063 assert!(spawn_notice.is_none());
9064 assert!(peer_update.is_none());
9065 assert!(real_user.is_some());
9066 }
9067
9068 #[test]
9069 fn session_history_projection_skips_non_transcript_messages() {
9070 let skipped = frame_from_session_history_message(
9071 "runtime-a",
9072 "agent-a",
9073 "session-a",
9074 0,
9075 json!({
9076 "content": "internal system prompt"
9077 }),
9078 );
9079 assert!(skipped.is_none());
9080 }
9081
9082 #[test]
9083 fn session_history_projection_extracts_assistant_blocks() {
9084 let frame = frame_from_session_history_message(
9085 "runtime-a",
9086 "agent-a",
9087 "session-a",
9088 0,
9089 json!({
9090 "role": "block_assistant",
9091 "blocks": [
9092 { "block_type": "text", "data": { "text": "hello " } },
9093 { "block_type": "text", "data": { "text": "there" } }
9094 ],
9095 "stop_reason": "end_turn"
9096 }),
9097 )
9098 .expect("assistant block history frame");
9099 assert_eq!(frame.payload["text"], json!("hello there"));
9100 }
9101
9102 #[test]
9103 fn session_history_projection_extracts_nested_text_block_data() {
9104 let frame = frame_from_session_history_message(
9105 "runtime-a",
9106 "agent-a",
9107 "session-a",
9108 0,
9109 json!({
9110 "role": "block_assistant",
9111 "blocks": [
9112 {
9113 "block_type": "text",
9114 "data": { "text": "Ready and standing by." }
9115 }
9116 ],
9117 "stop_reason": "end_turn",
9118 "created_at": "1970-01-01T00:00:00.010Z"
9119 }),
9120 )
9121 .expect("assistant block history frame");
9122
9123 assert_eq!(frame.kind, "interaction_complete");
9124 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9125 }
9126
9127 #[test]
9128 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9129 let frame = frame_from_session_history_message(
9130 "runtime-a",
9131 "agent-a",
9132 "session-a",
9133 0,
9134 json!({
9135 "role": "block_assistant",
9136 "blocks": [
9137 {
9138 "block_type": "reasoning",
9139 "data": { "text": "**Planning**\nI should not be rendered." }
9140 },
9141 {
9142 "block_type": "text",
9143 "data": { "text": "Visible answer." }
9144 }
9145 ],
9146 "stop_reason": "end_turn",
9147 "created_at": "1970-01-01T00:00:00.010Z"
9148 }),
9149 )
9150 .expect("assistant block history frame");
9151
9152 assert_eq!(frame.kind, "interaction_complete");
9153 assert_eq!(frame.payload["result"], json!("Visible answer."));
9154 assert_eq!(frame.payload["text"], json!("Visible answer."));
9155 }
9156
9157 #[test]
9158 fn session_history_projection_leaves_reasoning_only_result_empty() {
9159 let frame = frame_from_session_history_message(
9160 "runtime-a",
9161 "agent-a",
9162 "session-a",
9163 0,
9164 json!({
9165 "role": "block_assistant",
9166 "blocks": [
9167 {
9168 "block_type": "reasoning",
9169 "data": { "text": "Private planning text." }
9170 },
9171 {
9172 "block_type": "tool_use",
9173 "data": { "id": "toolu-1", "name": "peers", "args": {} }
9174 }
9175 ],
9176 "stop_reason": "end_turn",
9177 "created_at": "1970-01-01T00:00:00.010Z"
9178 }),
9179 )
9180 .expect("assistant block history frame");
9181
9182 assert_eq!(frame.kind, "interaction_complete");
9183 assert_eq!(frame.payload["result"], json!(""));
9184 assert_eq!(frame.payload["text"], json!(""));
9185 }
9186
9187 #[test]
9188 fn session_history_projection_preserves_tool_results() {
9189 let frames = frames_from_session_history_message(
9190 "runtime-a",
9191 "agent-a",
9192 "session-a",
9193 5,
9194 json!({
9195 "role": "tool_results",
9196 "results": [
9197 {
9198 "tool_use_id": "call-peers",
9199 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9200 "is_error": false
9201 }
9202 ],
9203 "created_at": "1970-01-01T00:00:00.050Z"
9204 }),
9205 );
9206
9207 assert_eq!(frames.len(), 1);
9208 let frame = &frames[0];
9209 assert_eq!(frame.kind, "tool_execution_completed");
9210 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9211 assert_eq!(
9212 frame.payload["result"],
9213 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9214 );
9215 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9216 assert_eq!(frame.timestamp_ms, 50);
9217 }
9218
9219 #[test]
9220 fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9221 let image_result = json!({
9228 "operation_id": "00000000-0000-0000-0000-0000000000aa",
9229 "terminal": { "terminal": "generated" },
9230 "images": [
9231 {
9232 "image_id": "00000000-0000-0000-0000-000000000051",
9233 "blob_ref": {
9234 "blob_id": "sha256:test-generated-image",
9235 "media_type": "image/png"
9236 },
9237 "media_type": "image/png",
9238 "width": 1024,
9239 "height": 768
9240 }
9241 ],
9242 "provider_text": { "disposition": "not_emitted" },
9243 "revised_prompt": { "disposition": "not_requested" },
9244 "native_metadata": { "provider": "not_emitted" }
9245 });
9246 serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9249 image_result.clone(),
9250 )
9251 .expect("fixture must be a valid ImageGenerationToolResult");
9252 let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9253
9254 let frames = frames_from_session_history_message(
9255 "runtime-a",
9256 "artist:singleton",
9257 "session-a",
9258 6,
9259 json!({
9260 "role": "tool_results",
9261 "results": [
9262 {
9263 "tool_use_id": "call-image",
9264 "content": result_text,
9265 "is_error": false
9266 }
9267 ],
9268 "created_at": "1970-01-01T00:00:00.060Z"
9269 }),
9270 );
9271
9272 assert!(
9274 frames
9275 .iter()
9276 .any(|frame| frame.kind == "tool_execution_completed"),
9277 "tool_execution_completed frame must still be present: {frames:#?}"
9278 );
9279 let image_frame = frames
9281 .iter()
9282 .find(|frame| frame.kind == "assistant_image")
9283 .expect("backfill must derive an assistant_image frame");
9284 assert_eq!(
9285 image_frame.payload["blob_id"],
9286 json!("sha256:test-generated-image")
9287 );
9288 assert_eq!(
9289 image_frame.payload["image_id"],
9290 json!("00000000-0000-0000-0000-000000000051")
9291 );
9292 assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9293 assert_eq!(image_frame.payload["width"], json!(1024));
9294 assert_eq!(image_frame.payload["height"], json!(768));
9295 assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9296 assert_eq!(
9297 image_frame.source.kind,
9298 ConsoleFrameSourceKind::SessionHistory
9299 );
9300 assert_eq!(image_frame.timestamp_ms, 60);
9301 }
9302
9303 #[test]
9304 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9305 let frames = frames_from_session_history_message(
9306 "runtime-a",
9307 "review:singleton",
9308 "session-a",
9309 7,
9310 json!({
9311 "role": "block_assistant",
9312 "blocks": [
9313 {
9314 "block_type": "tool_use",
9315 "data": {
9316 "id": "call-spawn",
9317 "name": "mob_spawn_member",
9318 "args": {
9319 "member_id": "review-worker-alpha",
9320 "profile": "review-worker",
9321 "initial_message": "Review Initiative Alpha and save the result."
9322 }
9323 }
9324 }
9325 ],
9326 "stop_reason": "tool_use",
9327 "created_at": "1970-01-01T00:00:00.070Z"
9328 }),
9329 );
9330
9331 assert_eq!(frames.len(), 2);
9332 let child = frames
9333 .iter()
9334 .find(|frame| frame.identity == "review-worker-alpha")
9335 .expect("spawned member initial message frame");
9336 assert_eq!(child.kind, "user_input");
9337 assert_eq!(child.timestamp_ms, 70);
9338 assert_eq!(
9339 child.payload["message"]["content"],
9340 "Review Initiative Alpha and save the result."
9341 );
9342 assert_eq!(child.payload["parent_identity"], "review:singleton");
9343 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9344 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9345 }
9346
9347 #[test]
9348 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9349 let frames = frames_from_session_history_message(
9350 "runtime-a",
9351 "review:singleton",
9352 "session-a",
9353 8,
9354 json!({
9355 "role": "block_assistant",
9356 "blocks": [
9357 {
9358 "block_type": "tool_use",
9359 "data": {
9360 "id": "call-spawn-many",
9361 "name": "mob_spawn_member",
9362 "args": {
9363 "specs": [
9364 {
9365 "agent_identity": "review-worker-alpha",
9366 "profile": "review-worker",
9367 "initial_message": "Review Initiative Alpha."
9368 },
9369 {
9370 "agent_identity": "review-worker-beta",
9371 "profile": "review-worker",
9372 "initial_message": "Review Initiative Beta."
9373 }
9374 ]
9375 }
9376 }
9377 }
9378 ],
9379 "stop_reason": "tool_use",
9380 "created_at": "1970-01-01T00:00:00.080Z"
9381 }),
9382 );
9383
9384 assert_eq!(frames.len(), 3);
9385 let child_messages = frames
9386 .iter()
9387 .filter(|frame| frame.kind == "user_input")
9388 .map(|frame| {
9389 (
9390 frame.identity.as_str(),
9391 frame.payload["message"]["content"]
9392 .as_str()
9393 .unwrap_or_default(),
9394 )
9395 })
9396 .collect::<Vec<_>>();
9397 assert_eq!(
9398 child_messages,
9399 vec![
9400 ("review-worker-alpha", "Review Initiative Alpha."),
9401 ("review-worker-beta", "Review Initiative Beta."),
9402 ]
9403 );
9404 }
9405
9406 #[tokio::test]
9407 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
9408 let aggregator = MobKitConsoleAggregator::in_memory();
9409 let frames = frames_from_session_history_message(
9410 "runtime-a",
9411 "review:singleton",
9412 "session-a",
9413 9,
9414 json!({
9415 "role": "block_assistant",
9416 "blocks": [
9417 {
9418 "block_type": "tool_use",
9419 "data": {
9420 "id": "call-spawn-many",
9421 "name": "mob_spawn_member",
9422 "args": {
9423 "specs": [
9424 {
9425 "agent_identity": "review-worker-alpha",
9426 "profile": "review-worker",
9427 "initial_message": "Review Initiative Alpha."
9428 },
9429 {
9430 "agent_identity": "review-worker-beta",
9431 "profile": "review-worker",
9432 "initial_message": "Review Initiative Beta."
9433 }
9434 ]
9435 }
9436 }
9437 }
9438 ],
9439 "stop_reason": "tool_use",
9440 "created_at": "1970-01-01T00:00:00.090Z"
9441 }),
9442 );
9443 for frame in frames {
9444 aggregator
9445 .store()
9446 .append_if_absent(frame)
9447 .await
9448 .expect("append projected session-history frame");
9449 }
9450
9451 let child_page = aggregator
9452 .query_timeline(ConsoleTimelineQuery {
9453 identity: Some("review-worker-alpha".to_string()),
9454 limit: 20,
9455 ..ConsoleTimelineQuery::default()
9456 })
9457 .await
9458 .expect("query child timeline");
9459 assert_eq!(child_page.frames.len(), 1);
9460 assert_eq!(child_page.frames[0].kind, "user_input");
9461 assert_eq!(
9462 child_page.frames[0].payload["message"]["content"],
9463 "Review Initiative Alpha."
9464 );
9465 assert_eq!(
9466 child_page.frames[0].payload["parent_identity"],
9467 "review:singleton"
9468 );
9469
9470 let parent_page = aggregator
9471 .query_timeline(ConsoleTimelineQuery {
9472 identity: Some("review:singleton".to_string()),
9473 limit: 20,
9474 ..ConsoleTimelineQuery::default()
9475 })
9476 .await
9477 .expect("query parent timeline");
9478 assert!(
9479 parent_page
9480 .frames
9481 .iter()
9482 .all(|frame| frame.identity == "review:singleton"),
9483 "child initial messages must not leak into parent timeline: {:#?}",
9484 parent_page.frames
9485 );
9486 }
9487
9488 #[test]
9489 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
9490 let frames = frames_from_session_history_message_with_namespace(
9491 "runtime-a",
9492 "test/review:singleton",
9493 "test",
9494 "session-a",
9495 10,
9496 json!({
9497 "role": "block_assistant",
9498 "blocks": [
9499 {
9500 "block_type": "tool_use",
9501 "data": {
9502 "id": "call-spawn-many",
9503 "name": "mob_spawn_member",
9504 "args": {
9505 "specs": [
9506 {
9507 "agent_identity": "review-worker-alpha",
9508 "profile": "review-worker",
9509 "initial_message": "Review Initiative Alpha."
9510 },
9511 {
9512 "agent_identity": "test/review-worker-beta",
9513 "profile": "review-worker",
9514 "initial_message": "Review Initiative Beta."
9515 }
9516 ]
9517 }
9518 }
9519 }
9520 ],
9521 "stop_reason": "tool_use",
9522 "created_at": "1970-01-01T00:00:00.100Z"
9523 }),
9524 );
9525
9526 let child_identities = frames
9527 .iter()
9528 .filter(|frame| frame.kind == "user_input")
9529 .map(|frame| frame.identity.as_str())
9530 .collect::<Vec<_>>();
9531 assert_eq!(
9532 child_identities,
9533 vec!["test/review-worker-alpha", "test/review-worker-beta"]
9534 );
9535 }
9536
9537 #[tokio::test]
9538 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
9539 let aggregator = MobKitConsoleAggregator::in_memory();
9540 aggregator
9541 .inner
9542 .identity_read_model
9543 .replace(vec![
9544 identity_record_for_test("test/review:singleton"),
9545 identity_record_for_test("test/review-worker-alpha"),
9546 ])
9547 .await;
9548 let frames = frames_from_session_history_message_with_namespace(
9549 "runtime-a",
9550 "test/review:singleton",
9551 "test",
9552 "session-a",
9553 11,
9554 json!({
9555 "role": "block_assistant",
9556 "blocks": [
9557 {
9558 "block_type": "tool_use",
9559 "data": {
9560 "id": "call-spawn",
9561 "name": "mob_spawn_member",
9562 "args": {
9563 "agent_identity": "review-worker-alpha",
9564 "profile": "review-worker",
9565 "initial_message": "Review Initiative Alpha."
9566 }
9567 }
9568 }
9569 ],
9570 "stop_reason": "tool_use",
9571 "created_at": "1970-01-01T00:00:00.110Z"
9572 }),
9573 );
9574 for frame in frames {
9575 aggregator
9576 .store()
9577 .append_if_absent(frame)
9578 .await
9579 .expect("append projected namespaced session-history frame");
9580 }
9581
9582 let namespaced_child_page = aggregator
9583 .query_timeline(ConsoleTimelineQuery {
9584 identity: Some("test/review-worker-alpha".to_string()),
9585 limit: 20,
9586 ..ConsoleTimelineQuery::default()
9587 })
9588 .await
9589 .expect("query namespaced child timeline");
9590 assert_eq!(namespaced_child_page.frames.len(), 1);
9591 assert_eq!(
9592 namespaced_child_page.frames[0].identity,
9593 "test/review-worker-alpha"
9594 );
9595 assert_eq!(
9596 namespaced_child_page.frames[0].payload["message"]["content"],
9597 "Review Initiative Alpha."
9598 );
9599
9600 let raw_child_page = aggregator
9601 .query_timeline(ConsoleTimelineQuery {
9602 identity: Some("review-worker-alpha".to_string()),
9603 limit: 20,
9604 ..ConsoleTimelineQuery::default()
9605 })
9606 .await
9607 .expect("query raw child timeline");
9608 assert!(
9609 raw_child_page.frames.is_empty(),
9610 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
9611 raw_child_page.frames
9612 );
9613 }
9614
9615 #[test]
9616 fn namespace_helpers_preserve_namespace_named_member_identity() {
9617 assert_eq!(apply_namespace("test", "test"), "test/test");
9618 assert_eq!(
9619 strip_namespace("test/test", "test").as_deref(),
9620 Some("test")
9621 );
9622 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
9623 assert_eq!(
9624 strip_namespace("test/worker", "test").as_deref(),
9625 Some("worker")
9626 );
9627 assert_eq!(strip_namespace("test", "test"), None);
9628 }
9629
9630 #[test]
9631 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
9632 assert!(!member_id_matches_durable_identity(
9633 "rt:review:singleton:0",
9634 "review:singleton"
9635 ));
9636 assert!(!member_id_matches_durable_identity(
9637 "review:singleton:gen1",
9638 "review:singleton"
9639 ));
9640 assert!(!member_id_matches_durable_identity(
9641 "review:singleton:1",
9642 "review:singleton"
9643 ));
9644 assert!(!member_id_matches_durable_identity(
9645 "rt:review:singleton:qa:0",
9646 "review:singleton"
9647 ));
9648 assert!(!member_id_matches_durable_identity(
9649 "review:singleton:qa",
9650 "review:singleton"
9651 ));
9652 }
9653
9654 #[test]
9655 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
9656 let frame = frame_from_session_history_message(
9657 "runtime-a",
9658 "agent-a",
9659 "session-a",
9660 0,
9661 json!({
9662 "role": "user",
9663 "content": "hello",
9664 "created_at": "2026-05-12T05:00:06.564227Z"
9665 }),
9666 )
9667 .expect("user history frame");
9668
9669 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
9670 }
9671
9672 async fn wait_for_kickoff_frames(
9673 aggregator: &MobKitConsoleAggregator,
9674 identity: &str,
9675 timeout: Duration,
9676 ) -> Vec<ConsoleFrame> {
9677 let deadline = Instant::now() + timeout;
9678 let mut observed = Vec::new();
9679 while Instant::now() < deadline {
9680 let page = aggregator
9681 .query_timeline(ConsoleTimelineQuery {
9682 identity: Some(identity.to_string()),
9683 limit: 50,
9684 ..ConsoleTimelineQuery::default()
9685 })
9686 .await
9687 .expect("query timeline");
9688 observed = page.frames;
9689 if observed.iter().any(|frame| frame.kind == "user_input") {
9690 break;
9691 }
9692 tokio::time::sleep(Duration::from_millis(25)).await;
9693 }
9694 observed
9695 }
9696
9697 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9698 async fn spawn_registry_primes_access_lineage() {
9699 let runtime = build_empty_runtime("spawn-access-lineage-test").await;
9700 let sink = runtime
9701 .mob_runtime()
9702 .console_spawn_sink()
9703 .expect("unified runtime installs the console spawn sink at bootstrap");
9704
9705 runtime
9706 .spawn(SpawnMemberSpec::from_wire(
9707 "worker".to_string(),
9708 "worker-3".to_string(),
9709 None,
9710 None,
9711 None,
9712 ))
9713 .await
9714 .expect("member spawns");
9715 sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
9716 mob_id: Some("spawn-access-lineage-test".to_string()),
9717 member_id: "worker-3".to_string(),
9718 identity: "worker-3".to_string(),
9719 initial_message: None,
9720 labels: BTreeMap::new(),
9721 spawned_by: Some("ops-lead".to_string()),
9722 via_tool: "mob_spawn_member".to_string(),
9723 })
9724 .await;
9725
9726 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9727 enabled: true,
9728 admins: vec!["root@example.test".to_string()],
9729 groups: BTreeMap::new(),
9730 rules: vec![crate::access::AccessRule {
9731 id: "bob-ops-lead".to_string(),
9732 subjects: vec!["bob@example.test".to_string()],
9733 actions: vec!["agent.view".to_string()],
9734 agents: vec!["ops-lead".to_string()],
9735 ..crate::access::AccessRule::default()
9736 }],
9737 })
9738 .expect("controller");
9739
9740 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9741 .await;
9742
9743 let bob = controller.view_for_subject(Some("bob@example.test"));
9744 assert!(
9745 bob.can_view_agent("worker-3"),
9746 "spawn registry lineage must let bob see the member ops-lead spawned"
9747 );
9748 assert!(
9749 !bob.can_view_agent("unrelated-agent"),
9750 "agents outside the granted lineage stay denied"
9751 );
9752
9753 let _ = runtime.mob_handle().stop().await;
9754 }
9755
9756 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9757 async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
9758 let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
9759
9760 let mut imposter = SpawnMemberSpec::from_wire(
9763 "worker".to_string(),
9764 "imposter".to_string(),
9765 None,
9766 None,
9767 None,
9768 );
9769 imposter.labels = Some(BTreeMap::from([(
9770 "spawned_by".to_string(),
9771 "ops-lead".to_string(),
9772 )]));
9773 runtime.spawn(imposter).await.expect("imposter spawns");
9774
9775 let mut evader = SpawnMemberSpec::from_wire(
9776 "worker".to_string(),
9777 "evader".to_string(),
9778 None,
9779 None,
9780 None,
9781 );
9782 evader.labels = Some(BTreeMap::from([(
9783 "spawned_by".to_string(),
9784 "secret-lead".to_string(),
9785 )]));
9786 runtime.spawn(evader).await.expect("evader spawns");
9787
9788 let aggregator = MobKitConsoleAggregator::in_memory();
9790 aggregator.register_runtime_handles_with_policy(
9791 "runtime-a",
9792 "",
9793 runtime.mob_runtime().clone(),
9794 None,
9795 runtime.console_events(),
9796 Arc::new(AllowAllConsoleVisibilityPolicy),
9797 );
9798 let identities = aggregator
9799 .list_identities_fresh()
9800 .await
9801 .expect("list identities");
9802 let imposter_record = identities
9803 .iter()
9804 .find(|record| record.identity == "imposter")
9805 .expect("imposter listed");
9806 assert!(
9807 !imposter_record.labels.contains_key("spawned_by"),
9808 "identity records must drop roster-claimed lineage when the spawn registry has none"
9809 );
9810
9811 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9812 enabled: true,
9813 admins: vec!["root@example.test".to_string()],
9814 groups: BTreeMap::new(),
9815 rules: vec![
9816 crate::access::AccessRule {
9817 id: "bob-ops-lead".to_string(),
9818 subjects: vec!["bob@example.test".to_string()],
9819 actions: vec!["agent.view".to_string()],
9820 agents: vec!["ops-lead".to_string()],
9821 ..crate::access::AccessRule::default()
9822 },
9823 crate::access::AccessRule {
9824 id: "carol-view-all".to_string(),
9825 subjects: vec!["carol@example.test".to_string()],
9826 actions: vec!["agent.view".to_string()],
9827 agents: vec!["*".to_string()],
9828 ..crate::access::AccessRule::default()
9829 },
9830 crate::access::AccessRule {
9831 id: "hide-secret-lead".to_string(),
9832 effect: crate::access::AccessEffect::Deny,
9833 actions: vec!["agent.*".to_string()],
9834 agents: vec!["secret-lead".to_string()],
9835 ..crate::access::AccessRule::default()
9836 },
9837 ],
9838 })
9839 .expect("controller");
9840
9841 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9842 .await;
9843
9844 let bob = controller.view_for_subject(Some("bob@example.test"));
9845 assert!(
9846 !bob.can_view_agent("imposter"),
9847 "a roster-claimed spawned_by must not mint inherited visibility"
9848 );
9849 let carol = controller.view_for_subject(Some("carol@example.test"));
9850 assert!(
9851 carol.can_view_agent("evader"),
9852 "a roster-claimed spawned_by must not let a member hide behind a denied parent"
9853 );
9854
9855 let _ = runtime.mob_handle().stop().await;
9856 }
9857
9858 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9859 async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
9860 let runtime = build_empty_runtime("agent-spawn-console-test").await;
9861
9862 let sink = runtime
9866 .mob_runtime()
9867 .console_spawn_sink()
9868 .expect("unified runtime installs the console spawn sink at bootstrap");
9869
9870 runtime
9874 .spawn(SpawnMemberSpec::from_wire(
9875 "worker".to_string(),
9876 "worker-3".to_string(),
9877 None,
9878 None,
9879 None,
9880 ))
9881 .await
9882 .expect("member spawns");
9883
9884 let seed = crate::console_spawn::ConsoleSpawnSeed {
9885 mob_id: Some("agent-spawn-console-test".to_string()),
9886 member_id: "worker-3".to_string(),
9887 identity: "worker-3".to_string(),
9888 initial_message: Some(json!("Kick off the person hunt")),
9889 labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
9890 spawned_by: Some("ops-lead".to_string()),
9891 via_tool: "mob_spawn_member".to_string(),
9892 };
9893 sink.project_spawned_member(&seed).await;
9894 sink.project_spawned_member(&seed).await;
9896
9897 let aggregator = MobKitConsoleAggregator::in_memory();
9898 aggregator.register_runtime_handles_with_policy(
9899 "runtime-a",
9900 "test",
9901 runtime.mob_runtime().clone(),
9902 None,
9903 runtime.console_events(),
9904 Arc::new(AllowAllConsoleVisibilityPolicy),
9905 );
9906
9907 let frames =
9908 wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
9909 let kickoffs = frames
9910 .iter()
9911 .filter(|frame| frame.kind == "user_input")
9912 .collect::<Vec<_>>();
9913 assert_eq!(
9914 kickoffs.len(),
9915 1,
9916 "member timeline must contain exactly one kickoff frame; got {frames:#?}"
9917 );
9918 assert_eq!(
9919 kickoffs[0].payload["content"][0]["text"],
9920 "Kick off the person hunt"
9921 );
9922
9923 runtime
9926 .project_console_event_from_unified(&crate::types::EventEnvelope {
9927 event_id: "evt-live-worker-3".to_string(),
9928 source: "test".to_string(),
9929 timestamp_ms: current_time_ms(),
9930 event: crate::types::UnifiedEvent::Agent {
9931 agent_id: "worker-3:0".to_string(),
9932 event_type: "text_delta".to_string(),
9933 payload: Some(json!({ "delta": "on it" })),
9934 },
9935 })
9936 .await;
9937
9938 let deadline = Instant::now() + Duration::from_secs(5);
9939 let mut live_frame = None;
9940 while Instant::now() < deadline && live_frame.is_none() {
9941 let page = aggregator
9942 .query_timeline(ConsoleTimelineQuery {
9943 identity: Some("test/worker-3".to_string()),
9944 limit: 50,
9945 ..ConsoleTimelineQuery::default()
9946 })
9947 .await
9948 .expect("query timeline");
9949 live_frame = page
9950 .frames
9951 .into_iter()
9952 .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
9953 if live_frame.is_none() {
9954 tokio::time::sleep(Duration::from_millis(25)).await;
9955 }
9956 }
9957 let live_frame = live_frame.expect("live runtime event lands in the member chat");
9958 assert_eq!(live_frame.identity, "test/worker-3");
9959
9960 let identities = aggregator
9963 .list_identities_fresh()
9964 .await
9965 .expect("list identities");
9966 let record = identities
9967 .iter()
9968 .find(|record| record.identity == "test/worker-3")
9969 .unwrap_or_else(|| {
9970 panic!("spawned member missing from identity list: {identities:#?}")
9971 });
9972 assert_eq!(
9973 record.labels.get("group").map(String::as_str),
9974 Some("workers")
9975 );
9976 assert_eq!(
9977 record.labels.get("spawned_by").map(String::as_str),
9978 Some("ops-lead")
9979 );
9980
9981 let _ = runtime.mob_handle().stop().await;
9982 }
9983}