1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23 MobRuntime, MobRuntimeError, assert_member_accepts_images,
24 is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30 ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31 SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32 SourceIngestionTransition,
33};
34pub use store::{
35 ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36 SqliteConsoleLogStore,
37};
38pub use types::{
39 AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40 ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41 ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42 ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43 ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59 CachedOnly,
60 IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65 inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70 pub session_history_backfill_enabled: bool,
71 pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75 fn default() -> Self {
76 Self {
77 session_history_backfill_enabled: true,
78 max_concurrent_session_backfills: 16,
79 }
80 }
81}
82
83struct AggregatorInner {
84 store: Arc<dyn ConsoleLogStore>,
85 runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86 event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87 active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88 opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89 session_backfill_permits: Arc<Semaphore>,
90 identity_read_model: ConsoleIdentityReadModel,
91 options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96 inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97 refresh_lock: Arc<tokio::sync::Mutex<()>>,
98 primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102 fn default() -> Self {
103 Self {
104 inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106 primed: Arc::new(AtomicBool::new(false)),
107 }
108 }
109}
110
111impl ConsoleIdentityReadModel {
112 async fn snapshot(
113 &self,
114 inner: Arc<AggregatorInner>,
115 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116 if !self.primed.load(Ordering::Acquire) {
117 Box::pin(self.prime_now(inner)).await?;
118 }
119 Ok(self.inner.read().await.clone())
120 }
121
122 async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123 self.inner.read().await.clone()
124 }
125
126 async fn refresh_now_if_idle(
127 &self,
128 inner: Arc<AggregatorInner>,
129 mode: IdentityCollectionMode,
130 ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132 return None;
133 };
134 let result = Box::pin(collect_identity_records(&inner, mode)).await;
135 drop(guard);
136 match result {
137 Ok(identities) => {
138 self.replace(identities.clone()).await;
139 Some(Ok(identities))
140 }
141 Err(err) => Some(Err(err)),
142 }
143 }
144
145 async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146 if self.primed.load(Ordering::Acquire) {
147 return Ok(());
148 }
149 let _guard = self.refresh_lock.clone().lock_owned().await;
150 if self.primed.load(Ordering::Acquire) {
151 return Ok(());
152 }
153 let identities = Box::pin(collect_identity_records(
154 &inner,
155 IdentityCollectionMode::CachedOnly,
156 ))
157 .await?;
158 *self.inner.write().await = identities;
159 self.primed.store(true, Ordering::Release);
160 Ok(())
161 }
162
163 fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
164 let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
165 return;
166 };
167 let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
168 return;
169 };
170 let read_model = self.clone();
171 runtime_handle.spawn(async move {
172 let _guard = guard;
173 match Box::pin(collect_identity_records(
174 &inner,
175 IdentityCollectionMode::IncludeLiveMembers,
176 ))
177 .await
178 {
179 Ok(identities) => {
180 *read_model.inner.write().await = identities;
181 read_model.primed.store(true, Ordering::Release);
182 }
183 Err(err) => {
184 tracing::warn!(error = %err, "console identity read-model refresh failed");
185 }
186 }
187 });
188 }
189
190 async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
191 *self.inner.write().await = identities;
192 self.primed.store(true, Ordering::Release);
193 }
194}
195
196#[derive(Clone)]
197struct RuntimeEntry {
198 runtime_key: String,
199 identity_namespace: String,
200 runtime: MobRuntime,
201 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
202 console_events: ConsoleEventStore,
203 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
204}
205
206#[derive(Clone)]
207struct ResolvedConsoleMember {
208 entry: RuntimeEntry,
209 handle: MobHandle,
210 member: MobMemberListEntry,
211 source_mob_id: String,
212 runtime_identity: String,
213}
214
215fn console_member_for_resolved_member(
216 resolved: &ResolvedConsoleMember,
217 record: &ConsoleIdentityRecord,
218) -> ConsoleMember {
219 ConsoleMember {
220 agent_identity: crate::member_comms_id::runtime_alias_str(
221 resolved.member.agent_identity.as_str(),
222 )
223 .into_owned(),
224 role: resolved.member.role.to_string(),
225 state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
226 model_capabilities: Default::default(),
227 runtime_mode: Some(resolved.member.runtime_mode.to_string()),
228 session_id: record.session_id.clone(),
229 wired_to: resolved
230 .member
231 .wired_to
232 .iter()
233 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
234 .collect(),
235 labels: record.labels.clone(),
236 }
237}
238
239async fn resolved_member_visible(
240 resolved: &ResolvedConsoleMember,
241 record: &ConsoleIdentityRecord,
242) -> bool {
243 if !raw_resolved_member_visible(resolved, record) {
244 return false;
245 }
246 !Box::pin(live_record_shadowed_by_hidden_durable_binding(
247 resolved, record,
248 ))
249 .await
250}
251
252fn raw_resolved_member_visible(
253 resolved: &ResolvedConsoleMember,
254 record: &ConsoleIdentityRecord,
255) -> bool {
256 resolved.entry.visibility_policy.identity_visible(record)
257 && resolved
258 .entry
259 .visibility_policy
260 .member_visible(&console_member_for_resolved_member(resolved, record))
261}
262
263pub trait ConsoleVisibilityPolicy: Send + Sync {
264 fn include_implicit_delegate_members(&self) -> bool {
265 true
266 }
267
268 fn member_visible(&self, _member: &ConsoleMember) -> bool {
269 true
270 }
271
272 fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
273 true
274 }
275
276 fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
277 true
278 }
279
280 fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
281 None
282 }
283}
284
285#[derive(Debug, Default)]
286pub struct AllowAllConsoleVisibilityPolicy;
287
288impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
289
290#[derive(Debug, Default)]
291pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
292
293impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
294 fn include_implicit_delegate_members(&self) -> bool {
295 false
296 }
297
298 fn member_visible(&self, member: &ConsoleMember) -> bool {
299 !is_implicit_delegate_member(member.role.as_str(), &member.labels)
300 }
301
302 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
303 !is_implicit_delegate_member(
304 record
305 .labels
306 .get("role")
307 .map(String::as_str)
308 .unwrap_or_default(),
309 &record.labels,
310 )
311 }
312}
313
314#[derive(Clone)]
315pub struct ConsoleRuntimeRegistration {
316 pub runtime_key: String,
317 pub runtime: Arc<UnifiedRuntime>,
318 pub identity_namespace: String,
319 pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
320}
321
322impl MobKitConsoleAggregator {
323 pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
324 Self::new_with_options(store, ConsoleAggregatorOptions::default())
325 }
326
327 pub fn new_with_options(
328 store: Arc<dyn ConsoleLogStore>,
329 mut options: ConsoleAggregatorOptions,
330 ) -> Self {
331 options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
332 let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
333 Self {
334 inner: Arc::new(AggregatorInner {
335 store,
336 runtimes: RwLock::new(BTreeMap::new()),
337 event_tx,
338 active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
339 opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
340 session_backfill_permits: Arc::new(Semaphore::new(
341 options.max_concurrent_session_backfills,
342 )),
343 identity_read_model: ConsoleIdentityReadModel::default(),
344 options,
345 }),
346 }
347 }
348
349 pub fn in_memory() -> Self {
350 Self::new(Arc::new(InMemoryConsoleLogStore::new()))
351 }
352
353 pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
354 Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
355 }
356
357 pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
358 self.inner.event_tx.subscribe()
359 }
360
361 pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
362 self.inner.store.clone()
363 }
364
365 pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
366 let identity_runtime = registration.runtime.identity_runtime().cloned();
367 self.register_runtime_handles_with_policy(
368 registration.runtime_key,
369 registration.identity_namespace,
370 registration.runtime.mob_runtime().clone(),
371 identity_runtime,
372 registration.runtime.console_events(),
373 registration.visibility_policy,
374 );
375 }
376
377 pub(crate) fn register_runtime_handles_with_policy(
378 &self,
379 runtime_key: impl Into<String>,
380 identity_namespace: impl Into<String>,
381 runtime: MobRuntime,
382 identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
383 console_events: ConsoleEventStore,
384 visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
385 ) {
386 let runtime_key = runtime_key.into();
387 let identity_namespace = identity_namespace.into();
388 let entry = RuntimeEntry {
389 runtime_key: runtime_key.clone(),
390 identity_namespace,
391 runtime,
392 identity_runtime,
393 console_events: console_events.clone(),
394 visibility_policy,
395 };
396 if let Ok(mut runtimes) = self.inner.runtimes.write() {
397 runtimes.insert(runtime_key.clone(), entry);
398 }
399 self.inner
400 .identity_read_model
401 .refresh_soon(self.inner.clone());
402 let inner = self.inner.clone();
403 let events_for_live = console_events.clone();
404 let events_for_live_recovery = console_events.clone();
405 let runtime_key_for_live = runtime_key.clone();
406 tokio::spawn(async move {
407 let mut rx = events_for_live.subscribe();
408 loop {
409 match rx.recv().await {
410 Ok(envelope) => {
411 let _ =
412 project_console_event(inner.clone(), &runtime_key_for_live, envelope)
413 .await;
414 }
415 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
416 let _ = recover_lagged_source_events(
417 inner.clone(),
418 &runtime_key_for_live,
419 &events_for_live_recovery,
420 )
421 .await;
422 }
423 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
424 }
425 }
426 });
427
428 let inner = self.inner.clone();
429 let events_for_replay = console_events;
430 let runtime_key_for_replay = runtime_key;
431 tokio::spawn(async move {
432 let mut ingestion_state = SourceIngestionState::Registered;
433 if let Ok((next, _effects)) =
434 ingestion_state.apply(SourceIngestionTransition::StartBackfill)
435 {
436 ingestion_state = next;
437 }
438 if let Ok(events) = events_for_replay.replay_all(None).await {
439 for envelope in events {
440 let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
441 .await;
442 }
443 }
444 spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
445 spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
446 if let Ok((next, _effects)) =
447 ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
448 {
449 ingestion_state = next;
450 }
451 let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
452 });
453 }
454
455 pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
456 if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
457 self.inner.clone(),
458 IdentityCollectionMode::IncludeLiveMembers,
459 ))
460 .await
461 {
462 let identities = identities?;
463 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464 return Ok(identities);
465 }
466 self.inner
467 .identity_read_model
468 .refresh_soon(self.inner.clone());
469 let identities =
470 Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
471 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
472 Ok(identities)
473 }
474
475 #[cfg(test)]
476 pub(crate) async fn list_identities_fresh(
477 &self,
478 ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
479 let _guard = self
480 .inner
481 .identity_read_model
482 .refresh_lock
483 .clone()
484 .lock_owned()
485 .await;
486 let identities =
487 collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
488 .await?;
489 self.inner
490 .identity_read_model
491 .replace(identities.clone())
492 .await;
493 spawn_identity_backfills_for_records(self.inner.clone(), &identities);
494 Ok(identities)
495 }
496
497 pub async fn inspect_identity(
498 &self,
499 identity: &str,
500 ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
501 let entries = self
502 .inner
503 .runtimes
504 .read()
505 .map_err(|_| runtime_registry_lock_error())?
506 .values()
507 .cloned()
508 .collect::<Vec<_>>();
509 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
510 let mut durable_matches = Vec::new();
511 if !requested_identity_has_runtime_member_alias(identity, &entries) {
512 for entry in &entries {
513 let Some(identity_runtime) = entry.identity_runtime.clone() else {
514 continue;
515 };
516 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
517 {
518 let Ok(parsed_identity) =
519 crate::identity_first::AgentIdentity::parse(&raw_identity)
520 else {
521 continue;
522 };
523 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
524 continue;
525 };
526 let mut record = identity_record_for_status(entry, &status);
527 let topology_peers =
528 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
529 record.topology_peers = topology_peers
530 .get(&record.identity)
531 .cloned()
532 .unwrap_or_default();
533 if Box::pin(console_identity_record_visible(entry, &record)).await {
534 durable_matches.push((
535 entry.clone(),
536 identity_runtime.clone(),
537 parsed_identity,
538 record,
539 ));
540 }
541 }
542 }
543 }
544 if durable_matches.len() > 1 {
545 let candidates = durable_matches
546 .iter()
547 .map(|(_, _, _, record)| record.runtime_member_id.clone())
548 .collect::<Vec<_>>()
549 .join(", ");
550 return Err(format!(
551 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
552 )
553 .into());
554 }
555 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
556 durable_matches.into_iter().next()
557 {
558 let durable_live_records = Box::pin(self.live_records_for_durable_record(
559 &entry,
560 identity,
561 &record,
562 &live_records,
563 ))
564 .await;
565 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
566 &entry,
567 &durable_live_records,
568 ))
569 .await;
570 if let Some(ambiguous_error) =
571 ambiguous_live_alias_error(identity, &visible_durable_live_records)
572 {
573 return Err(ambiguous_error.into());
574 }
575 if let Some(rebound_record) = self_heal_durable_session_mismatch(
576 &entry,
577 identity_runtime.as_ref(),
578 &parsed_identity,
579 identity,
580 &record,
581 &durable_live_records,
582 )
583 .await
584 .map_err(ConsoleLogError::from)?
585 {
586 record = rebound_record;
587 }
588 if let Some(stale_error) =
589 stale_durable_record_error(identity, &record, &durable_live_records)
590 {
591 return Err(stale_error.into());
592 }
593 if let Some(session_id) = record.session_id.clone() {
594 spawn_session_history_backfill_target(
595 self.inner.clone(),
596 SessionBackfillTarget {
597 entry,
598 record: record.clone(),
599 session_id,
600 },
601 false,
602 );
603 }
604 return Ok(Some(ConsoleIdentityInspection {
605 peers: record.topology_peers.clone(),
606 identity: record,
607 }));
608 }
609 let mut runtime_id_matches = Vec::new();
610 for entry in &entries {
611 let Some(identity_runtime) = entry.identity_runtime.clone() else {
612 continue;
613 };
614 let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
615 continue;
616 };
617 let Some(status) = identity_runtime
618 .statuses()
619 .await
620 .into_iter()
621 .find(|status| {
622 status
623 .agent_runtime_id
624 .as_ref()
625 .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
626 })
627 else {
628 continue;
629 };
630 let mut record = identity_record_for_status(entry, &status);
631 let topology_peers =
632 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
633 record.topology_peers = topology_peers
634 .get(&record.identity)
635 .cloned()
636 .unwrap_or_default();
637 if Box::pin(console_identity_record_visible(entry, &record)).await {
638 runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
639 }
640 }
641 if runtime_id_matches.len() > 1 {
642 let candidates = runtime_id_matches
643 .iter()
644 .map(|(_, _, _, record)| record.identity.clone())
645 .collect::<Vec<_>>()
646 .join(", ");
647 return Err(format!(
648 "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
649 )
650 .into());
651 }
652 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
653 runtime_id_matches.into_iter().next()
654 {
655 let durable_live_records = Box::pin(self.live_records_for_durable_record(
656 &entry,
657 identity,
658 &record,
659 &live_records,
660 ))
661 .await;
662 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
663 &entry,
664 &durable_live_records,
665 ))
666 .await;
667 if let Some(ambiguous_error) =
668 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
669 {
670 return Err(ambiguous_error.into());
671 }
672 if let Some(rebound_record) = self_heal_durable_session_mismatch(
673 &entry,
674 identity_runtime.as_ref(),
675 &parsed_identity,
676 identity,
677 &record,
678 &durable_live_records,
679 )
680 .await
681 .map_err(ConsoleLogError::from)?
682 {
683 record = rebound_record;
684 }
685 if let Some(stale_error) =
686 stale_durable_record_error(identity, &record, &durable_live_records)
687 {
688 return Err(stale_error.into());
689 }
690 if let Some(session_id) = record.session_id.clone() {
691 spawn_session_history_backfill_target(
692 self.inner.clone(),
693 SessionBackfillTarget {
694 entry: entry.clone(),
695 record: record.clone(),
696 session_id,
697 },
698 false,
699 );
700 }
701 return Ok(Some(ConsoleIdentityInspection {
702 peers: record.topology_peers.clone(),
703 identity: record,
704 }));
705 }
706
707 let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
708 if live_matches.len() > 1 {
709 let candidates = live_matches
710 .iter()
711 .map(|resolved| resolved.runtime_identity.clone())
712 .collect::<Vec<_>>()
713 .join(", ");
714 return Err(format!(
715 "ambiguous live identity alias {identity}: candidates [{candidates}]"
716 )
717 .into());
718 }
719 if let Some(resolved) = live_matches.into_iter().next() {
720 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
721 return Ok(None);
722 };
723 if let Some(stale_error) =
724 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
725 {
726 return Err(stale_error.into());
727 }
728 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
729 && let Ok(parsed_identity) =
730 crate::identity_first::AgentIdentity::parse(&record.identity)
731 && let Ok(status) = identity_runtime.status(&parsed_identity).await
732 {
733 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
734 let durable_live_records = Box::pin(self.live_records_for_durable_record(
735 &resolved.entry,
736 identity,
737 &durable_record,
738 &live_records,
739 ))
740 .await;
741 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
742 &resolved.entry,
743 &durable_live_records,
744 ))
745 .await;
746 if let Some(ambiguous_error) =
747 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
748 {
749 return Err(ambiguous_error.into());
750 }
751 if let Some(rebound_record) = self_heal_durable_session_mismatch(
752 &resolved.entry,
753 identity_runtime.as_ref(),
754 &parsed_identity,
755 identity,
756 &durable_record,
757 &durable_live_records,
758 )
759 .await
760 .map_err(ConsoleLogError::from)?
761 {
762 durable_record = rebound_record;
763 }
764 if let Some(stale_error) =
765 stale_durable_record_error(identity, &durable_record, &durable_live_records)
766 {
767 return Err(stale_error.into());
768 }
769 if status
770 .agent_runtime_id
771 .as_ref()
772 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
773 {
774 return Ok(None);
775 }
776 }
777 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
778 return Ok(None);
779 }
780 if let Some(session_id) = record.session_id.clone() {
781 spawn_session_history_backfill_target(
782 self.inner.clone(),
783 SessionBackfillTarget {
784 entry: resolved.entry.clone(),
785 record: record.clone(),
786 session_id,
787 },
788 false,
789 );
790 }
791 let peers = resolved
792 .member
793 .wired_to
794 .iter()
795 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
796 .collect();
797 return Ok(Some(ConsoleIdentityInspection {
798 identity: record,
799 peers,
800 }));
801 }
802
803 Ok(None)
804 }
805
806 pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
807 let entries = self
808 .inner
809 .runtimes
810 .read()
811 .map_err(|_| runtime_registry_lock_error())?
812 .clone();
813 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
814 let entries_vec = entries.values().cloned().collect::<Vec<_>>();
815 let mut durable_matches = Vec::new();
816 if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
817 for entry in entries.values() {
818 let Some(identity_runtime) = entry.identity_runtime.clone() else {
819 continue;
820 };
821 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
822 {
823 let Ok(parsed_identity) =
824 crate::identity_first::AgentIdentity::parse(&raw_identity)
825 else {
826 continue;
827 };
828 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
829 continue;
830 };
831 let record = identity_record_for_status(entry, &status);
832 if !Box::pin(console_identity_record_visible(entry, &record)).await {
833 continue;
834 }
835 durable_matches.push((
836 entry.clone(),
837 identity_runtime.clone(),
838 parsed_identity,
839 record,
840 ));
841 }
842 }
843 }
844 if durable_matches.len() > 1 {
845 let candidates = durable_matches
846 .iter()
847 .map(|(_, _, identity, _)| identity.as_str().to_string())
848 .collect::<Vec<_>>()
849 .join(", ");
850 return Err(format!(
851 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
852 )
853 .into());
854 }
855 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
856 durable_matches.into_iter().next()
857 {
858 let durable_live_records = Box::pin(self.live_records_for_durable_record(
859 &entry,
860 identity,
861 &record,
862 &live_records,
863 ))
864 .await;
865 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
866 &entry,
867 &durable_live_records,
868 ))
869 .await;
870 if let Some(ambiguous_error) =
871 ambiguous_live_alias_error(identity, &visible_durable_live_records)
872 {
873 return Err(ambiguous_error.into());
874 }
875 if let Some(rebound_record) = self_heal_durable_session_mismatch(
876 &entry,
877 identity_runtime.as_ref(),
878 &parsed_identity,
879 identity,
880 &record,
881 &durable_live_records,
882 )
883 .await
884 .map_err(ConsoleLogError::from)?
885 {
886 record = rebound_record;
887 }
888 if let Some(stale_error) =
889 stale_durable_record_error(identity, &record, &durable_live_records)
890 {
891 return Err(stale_error.into());
892 }
893 identity_runtime
894 .retire(&parsed_identity)
895 .await
896 .map_err(|err| -> ConsoleLogError {
897 format!("retire failed for {identity}: {err}").into()
898 })?;
899 {
900 let mut durable_identities =
901 namespace_match_candidates(identity, &entry.identity_namespace);
902 durable_identities.push(parsed_identity.as_str().to_string());
903 durable_identities.sort();
904 durable_identities.dedup();
905 if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
906 &entry,
907 &durable_identities,
908 ))
909 .await
910 {
911 tracing::warn!(
912 identity,
913 error = %err,
914 "stale console member cleanup failed after durable identity retire"
915 );
916 }
917 }
918 return Ok(true);
919 }
920 let matches = Box::pin(self.resolve_visible_members(identity)).await;
921 if matches.len() > 1 {
922 let candidates = matches
923 .iter()
924 .map(|resolved| resolved.runtime_identity.clone())
925 .collect::<Vec<_>>()
926 .join(", ");
927 return Err(format!(
928 "ambiguous live identity alias {identity}: candidates [{candidates}]"
929 )
930 .into());
931 }
932 let mut live_retired_any = false;
933 for resolved in matches {
934 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
935 continue;
936 };
937 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
938 continue;
939 }
940 if let Some(stale_error) =
941 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
942 {
943 return Err(stale_error.into());
944 }
945 if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
946 && let Ok(parsed_identity) =
947 crate::identity_first::AgentIdentity::parse(&record.identity)
948 && let Ok(status) = identity_runtime.status(&parsed_identity).await
949 {
950 let mut durable_record = identity_record_for_status(&resolved.entry, &status);
951 let durable_live_records = Box::pin(self.live_records_for_durable_record(
952 &resolved.entry,
953 identity,
954 &durable_record,
955 &live_records,
956 ))
957 .await;
958 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
959 &resolved.entry,
960 &durable_live_records,
961 ))
962 .await;
963 if let Some(ambiguous_error) =
964 ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
965 {
966 return Err(ambiguous_error.into());
967 }
968 if let Some(rebound_record) = self_heal_durable_session_mismatch(
969 &resolved.entry,
970 identity_runtime.as_ref(),
971 &parsed_identity,
972 identity,
973 &durable_record,
974 &durable_live_records,
975 )
976 .await
977 .map_err(ConsoleLogError::from)?
978 {
979 durable_record = rebound_record;
980 }
981 if let Some(stale_error) =
982 stale_durable_record_error(identity, &durable_record, &durable_live_records)
983 {
984 return Err(stale_error.into());
985 }
986 if status
987 .agent_runtime_id
988 .as_ref()
989 .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
990 {
991 continue;
992 }
993 }
994 resolved
995 .handle
996 .retire(crate::member_comms_id::mob_member_id(
997 resolved.runtime_identity.as_str(),
998 ))
999 .await
1000 .map_err(|err| -> ConsoleLogError {
1001 format!("retire failed for {identity}: {err}").into()
1002 })
1003 .or_else(|err| {
1004 if lifecycle_archive_cleanup_completed(&err.to_string()) {
1005 Ok(())
1006 } else {
1007 Err(err)
1008 }
1009 })?;
1010 live_retired_any = true;
1011 }
1012 Ok(live_retired_any)
1013 }
1014
1015 pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1016 self.inner.store.clear_frames().await
1017 }
1018
1019 pub async fn query_timeline(
1020 &self,
1021 query: ConsoleTimelineQuery,
1022 ) -> ConsoleLogResult<ConsoleTimelinePage> {
1023 let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1024 Ok(ConsoleTimelinePage {
1025 frames: page.frames,
1026 next_cursor: page.next_cursor,
1027 })
1028 }
1029
1030 pub async fn query_timeline_windowed(
1031 &self,
1032 query: ConsoleTimelineWindowQuery,
1033 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1034 self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1035 .await?;
1036 if query.after.is_none()
1037 && query.before.is_none()
1038 && let Some(identity) = query.identity.clone()
1039 {
1040 let mut probe_query = query.clone();
1041 probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1042 let page = self.inner.store.query_windowed_frames(probe_query).await?;
1043 if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1044 spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1045 }
1046 }
1047 Box::pin(self.query_timeline_visible(query)).await
1048 }
1049
1050 async fn reject_after_cursor_beyond_store_frontier(
1051 &self,
1052 after: Option<&ConsoleCursor>,
1053 ) -> ConsoleLogResult<()> {
1054 let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1055 return Ok(());
1056 };
1057 let latest_seq = self
1058 .inner
1059 .store
1060 .latest_cursor()
1061 .await?
1062 .and_then(|cursor| cursor.seq())
1063 .unwrap_or(0);
1064 if after_seq > latest_seq {
1065 return Err(std::io::Error::other(
1066 "timeline replay cursor is beyond the current store frontier",
1067 )
1068 .into());
1069 }
1070 Ok(())
1071 }
1072
1073 async fn query_timeline_visible(
1074 &self,
1075 query: ConsoleTimelineWindowQuery,
1076 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1077 let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1078 let mut scan_query = query.clone();
1079 scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1080 let mut visible_frames = Vec::with_capacity(requested_limit);
1081 let mut anchor_frames = Vec::new();
1082 let mut identity_visibility_cache = HashMap::new();
1083 let identity_records = self.inner.identity_read_model.current().await;
1084 let mut next_cursor = query.after.clone();
1085 let mut since_last_scanned_cursor = query.after.clone();
1086 let mut since_last_delivered_cursor = None;
1087 let mut since_stopped_at_visible_limit = false;
1088 let mut latest_cursor = None;
1089 let mut exhausted = false;
1090 let mut scanned = 0usize;
1091
1092 loop {
1093 let page = self
1094 .inner
1095 .store
1096 .query_windowed_frames(scan_query.clone())
1097 .await?;
1098 latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1099 if page.frames.is_empty() {
1100 exhausted = true;
1101 break;
1102 }
1103 let raw_len = page.frames.len();
1104 scanned = scanned.saturating_add(raw_len);
1105 match query.mode {
1106 ConsoleTimelineMode::Since => {
1107 if let Some(cursor) = page.next_cursor.clone() {
1108 since_last_scanned_cursor = Some(cursor.clone());
1109 scan_query.after = Some(cursor);
1110 }
1111 }
1112 ConsoleTimelineMode::Recent => {
1113 if let Some(first) = page.frames.first() {
1114 scan_query.before = Some(first.cursor.clone());
1115 }
1116 next_cursor = page.next_cursor.clone().or(next_cursor);
1117 }
1118 }
1119
1120 for frame in page.frames {
1121 let allow_historical_identity =
1122 query.identity.as_deref() == Some(frame.identity.as_str());
1123 if Box::pin(frame_is_visible_cached(
1124 &self.inner,
1125 &frame,
1126 allow_historical_identity,
1127 &mut identity_visibility_cache,
1128 &identity_records,
1129 ))
1130 .await
1131 .unwrap_or(false)
1132 {
1133 if query.mode == ConsoleTimelineMode::Recent
1134 && query.identity.is_some()
1135 && is_identity_timeline_anchor_frame(&frame)
1136 && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1137 {
1138 anchor_frames.push(frame.clone());
1139 }
1140 match query.mode {
1141 ConsoleTimelineMode::Since => {
1142 if visible_frames.len() >= requested_limit {
1143 since_stopped_at_visible_limit = true;
1144 break;
1145 }
1146 since_last_delivered_cursor = Some(frame.cursor.clone());
1147 visible_frames.push(frame);
1148 if visible_frames.len() >= requested_limit {
1149 since_stopped_at_visible_limit = true;
1150 exhausted = false;
1151 break;
1152 }
1153 }
1154 ConsoleTimelineMode::Recent => {
1155 visible_frames.push(frame);
1156 }
1157 }
1158 }
1159 }
1160 let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1161 && query.identity.is_some()
1162 && anchor_frames.is_empty()
1163 && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1164 if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1165 break;
1166 }
1167 if since_stopped_at_visible_limit {
1168 break;
1169 }
1170 if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1171 exhausted = true;
1172 break;
1173 }
1174 if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1175 break;
1176 }
1177 }
1178
1179 if query.mode == ConsoleTimelineMode::Recent {
1180 visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1181 if visible_frames.len() > requested_limit {
1182 visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1183 }
1184 if !anchor_frames.is_empty() {
1185 let anchor_cursors = anchor_frames
1186 .iter()
1187 .map(|frame| frame.cursor.clone())
1188 .collect::<Vec<_>>();
1189 let mut merged = anchor_frames;
1190 for frame in visible_frames {
1191 if !merged
1192 .iter()
1193 .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1194 {
1195 merged.push(frame);
1196 }
1197 }
1198 merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1199 visible_frames = merged;
1200 if visible_frames.len() > requested_limit {
1201 let mut anchors = Vec::new();
1202 let mut tail = Vec::new();
1203 for frame in visible_frames {
1204 if anchor_cursors.contains(&frame.cursor) {
1205 anchors.push(frame);
1206 } else {
1207 tail.push(frame);
1208 }
1209 }
1210 visible_frames = if anchors.len() >= requested_limit {
1211 anchors.split_off(anchors.len() - requested_limit)
1212 } else {
1213 let keep_tail = requested_limit.saturating_sub(anchors.len());
1214 if tail.len() > keep_tail {
1215 tail = tail.split_off(tail.len() - keep_tail);
1216 }
1217 anchors.extend(tail);
1218 anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1219 anchors
1220 };
1221 }
1222 }
1223 }
1224
1225 Ok(ConsoleTimelineWindowPage {
1226 frames: visible_frames,
1227 next_cursor: match query.mode {
1228 ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1229 since_last_delivered_cursor.or(since_last_scanned_cursor)
1230 }
1231 ConsoleTimelineMode::Since => since_last_scanned_cursor,
1232 ConsoleTimelineMode::Recent => next_cursor,
1233 },
1234 latest_cursor,
1235 exhausted,
1236 })
1237 }
1238
1239 pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1240 let runtime_keys = self
1241 .inner
1242 .runtimes
1243 .read()
1244 .map_err(|_| runtime_registry_lock_error())?
1245 .keys()
1246 .cloned()
1247 .collect::<Vec<_>>();
1248 let results =
1249 join_all(runtime_keys.into_iter().map(|runtime_key| {
1250 backfill_session_history(self.inner.clone(), runtime_key, true)
1251 }))
1252 .await;
1253 for result in results {
1254 result?;
1255 }
1256 Ok(())
1257 }
1258
1259 pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1260 self.inner.store.latest_cursor().await
1261 }
1262
1263 pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1264 match event {
1265 ConsoleTimelineEvent::ConsoleFrame { frame }
1266 | ConsoleTimelineEvent::FrameUpdated { frame } => {
1267 let identity_records = self.inner.identity_read_model.current().await;
1268 Box::pin(frame_is_visible(
1269 &self.inner,
1270 frame,
1271 false,
1272 &identity_records,
1273 ))
1274 .await
1275 .unwrap_or(false)
1276 }
1277 ConsoleTimelineEvent::SnapshotStarted { .. }
1278 | ConsoleTimelineEvent::SnapshotComplete { .. }
1279 | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1280 }
1281 }
1282
1283 pub async fn timeline_frame_visible_for_query(
1284 &self,
1285 frame: &ConsoleFrame,
1286 identity: Option<&str>,
1287 ) -> bool {
1288 let allow_historical_identity = identity == Some(frame.identity.as_str());
1289 let identity_records = self.inner.identity_read_model.current().await;
1290 Box::pin(frame_is_visible(
1291 &self.inner,
1292 frame,
1293 allow_historical_identity,
1294 &identity_records,
1295 ))
1296 .await
1297 .unwrap_or(false)
1298 }
1299
1300 pub async fn send(
1301 &self,
1302 request: ConsoleSendRequest,
1303 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1304 validate_send_request(&request)?;
1305 let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1306 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1307 };
1308 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1309 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1310 };
1311 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1312 return Err(ConsoleSendError::UnknownIdentity(request.identity));
1313 }
1314 if !member_is_addressable(&resolved.member) {
1315 return Err(ConsoleSendError::NotAddressable(request.identity));
1316 }
1317 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1318 return Err(ConsoleSendError::Retired(request.identity));
1319 }
1320
1321 let content = content_input_from_value(&request.content)?;
1322 let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1323 assert_member_accepts_images(
1324 &resolved.handle,
1325 resolved.entry.runtime.session_service(),
1326 &resolved.runtime_identity,
1327 &content,
1328 )
1329 .await
1330 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1331
1332 let dedupe_key = send_dedupe_key(
1333 &resolved.entry.runtime_key,
1334 &request.identity,
1335 &request.origin,
1336 &request.idempotency_key,
1337 );
1338 let handling_mode_value = request
1339 .handling_mode
1340 .as_deref()
1341 .unwrap_or("queue")
1342 .to_string();
1343 let request_fingerprint =
1344 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1345 if let Some(existing) = self
1346 .inner
1347 .store
1348 .frame_by_dedupe_key(&dedupe_key)
1349 .await
1350 .map_err(ConsoleSendError::Log)?
1351 {
1352 let same_request = existing.source.source_cursor.as_deref()
1353 == Some(request_fingerprint.as_str())
1354 || existing.source.source_cursor.is_none()
1355 && existing.payload.get("origin").and_then(Value::as_str)
1356 == Some(request.origin.as_str())
1357 && existing.payload.get("content") == Some(&request.content)
1358 && existing
1359 .payload
1360 .get("handling_mode")
1361 .and_then(Value::as_str)
1362 == Some(handling_mode_value.as_str());
1363 if !same_request {
1364 return Err(ConsoleSendError::IdempotencyConflict(
1365 request.idempotency_key,
1366 ));
1367 }
1368 return Ok(accepted_from_frame(&existing));
1369 }
1370
1371 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1372 resolved
1373 .entry
1374 .console_events
1375 .reserve_interaction_value(
1376 &resolved.runtime_identity,
1377 Some(resolved.runtime_identity.as_str()),
1378 &interaction_id,
1379 &request.origin,
1380 request.content.clone(),
1381 )
1382 .await
1383 .map_err(ConsoleSendError::State)?;
1384 let session_id = resolved
1385 .handle
1386 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1387 resolved.runtime_identity.as_str(),
1388 ))
1389 .await
1390 .map(|sid| sid.to_string());
1391 let mut new_frame = NewConsoleFrame {
1392 id: None,
1393 dedupe_key,
1394 timestamp_ms: current_time_ms(),
1395 runtime_key: resolved.entry.runtime_key.clone(),
1396 identity: request.identity.clone(),
1397 conversation_id: Some(request.identity.clone()),
1398 session_id: session_id.clone(),
1399 kind: "user_input".to_string(),
1400 status: ConsoleFrameStatus::Accepted,
1401 payload: json!({
1402 "content": request.content,
1403 "origin": request.origin,
1404 "idempotency_key": request.idempotency_key,
1405 "handling_mode": handling_mode_value,
1406 }),
1407 source: ConsoleFrameSource {
1408 kind: ConsoleFrameSourceKind::Send,
1409 source_cursor: Some(request_fingerprint),
1410 },
1411 source_event_id: None,
1412 interaction_id: Some(interaction_id.clone()),
1413 turn_id: None,
1414 run_id: None,
1415 parent_frame_id: None,
1416 caused_by_frame_id: None,
1417 };
1418 if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1419 new_frame.payload = redacted;
1420 new_frame.status = ConsoleFrameStatus::Redacted;
1421 }
1422
1423 let _ = SendState::Requested
1424 .apply(SendTransition::PersistAccepted)
1425 .map_err(ConsoleSendError::State)?;
1426 let outcome = self
1427 .inner
1428 .store
1429 .append_if_absent(new_frame)
1430 .await
1431 .map_err(ConsoleSendError::Log)?;
1432 if outcome.disposition == AppendDisposition::Existing {
1433 return Ok(accepted_from_frame(&outcome.frame));
1434 }
1435 let _ = self
1436 .inner
1437 .event_tx
1438 .send(ConsoleTimelineEvent::ConsoleFrame {
1439 frame: outcome.frame.clone(),
1440 });
1441 let accepted = accepted_from_frame(&outcome.frame);
1442
1443 let (dispatching, _effects) = SendState::AcceptedPersisted
1444 .apply(SendTransition::StartDispatch)
1445 .map_err(ConsoleSendError::State)?;
1446 update_frame_status_and_emit(
1447 &self.inner,
1448 &outcome.frame.id,
1449 ConsoleFrameStatus::Dispatching,
1450 )
1451 .await
1452 .map_err(ConsoleSendError::Log)?;
1453
1454 spawn_console_send_dispatch(
1455 self.inner.clone(),
1456 resolved,
1457 content,
1458 handling_mode,
1459 dispatching,
1460 outcome.frame,
1461 interaction_id,
1462 );
1463 Ok(accepted)
1464 }
1465
1466 pub async fn reserve_identity_first_interaction(
1467 &self,
1468 request: ConsoleSendRequest,
1469 session_id: Option<&str>,
1470 ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1471 validate_send_request(&request)?;
1472 let _content = content_input_from_value(&request.content)?;
1473 let runtime_key =
1474 Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1475 let handling_mode_value = request
1476 .handling_mode
1477 .as_deref()
1478 .unwrap_or("queue")
1479 .to_string();
1480 let dedupe_key = send_dedupe_key(
1481 &runtime_key,
1482 &request.identity,
1483 &request.origin,
1484 &request.idempotency_key,
1485 );
1486 let request_fingerprint =
1487 send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1488 if let Some(existing) = self
1489 .inner
1490 .store
1491 .frame_by_dedupe_key(&dedupe_key)
1492 .await
1493 .map_err(ConsoleSendError::Log)?
1494 {
1495 let same_request = existing.source.source_cursor.as_deref()
1496 == Some(request_fingerprint.as_str())
1497 || existing.source.source_cursor.is_none()
1498 && existing.payload.get("origin").and_then(Value::as_str)
1499 == Some(request.origin.as_str())
1500 && existing.payload.get("content") == Some(&request.content)
1501 && existing
1502 .payload
1503 .get("handling_mode")
1504 .and_then(Value::as_str)
1505 == Some(handling_mode_value.as_str());
1506 if !same_request {
1507 return Err(ConsoleSendError::IdempotencyConflict(
1508 request.idempotency_key,
1509 ));
1510 }
1511 return Ok(accepted_from_frame(&existing));
1512 }
1513
1514 let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1515 let new_frame = NewConsoleFrame {
1516 id: None,
1517 dedupe_key,
1518 timestamp_ms: current_time_ms(),
1519 runtime_key,
1520 identity: request.identity.clone(),
1521 conversation_id: Some(request.identity.clone()),
1522 session_id: session_id.map(ToString::to_string),
1523 kind: "user_input".to_string(),
1524 status: ConsoleFrameStatus::Accepted,
1525 payload: json!({
1526 "content": request.content,
1527 "origin": request.origin,
1528 "idempotency_key": request.idempotency_key,
1529 "handling_mode": handling_mode_value,
1530 }),
1531 source: ConsoleFrameSource {
1532 kind: ConsoleFrameSourceKind::Send,
1533 source_cursor: Some(request_fingerprint),
1534 },
1535 source_event_id: None,
1536 interaction_id: Some(interaction_id),
1537 turn_id: None,
1538 run_id: None,
1539 parent_frame_id: None,
1540 caused_by_frame_id: None,
1541 };
1542 let outcome = self
1543 .inner
1544 .store
1545 .append_if_absent(new_frame)
1546 .await
1547 .map_err(ConsoleSendError::Log)?;
1548 let _ = self
1549 .inner
1550 .event_tx
1551 .send(ConsoleTimelineEvent::ConsoleFrame {
1552 frame: outcome.frame.clone(),
1553 });
1554 Ok(accepted_from_frame(&outcome.frame))
1555 }
1556
1557 async fn runtime_key_for_identity_first_send(
1558 &self,
1559 identity: &str,
1560 ) -> Result<String, ConsoleSendError> {
1561 let entries = self
1562 .inner
1563 .runtimes
1564 .read()
1565 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1566 .clone();
1567 if entries.is_empty() {
1568 return Ok("identity-first".to_string());
1569 }
1570
1571 let mut identity_runtime_keys = Vec::new();
1572 let mut hidden_durable_match = false;
1573 let mut durable_matches = Vec::new();
1574 for entry in entries.values() {
1575 let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1576 continue;
1577 };
1578 identity_runtime_keys.push(entry.runtime_key.clone());
1579 if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1580 continue;
1581 }
1582 for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1583 {
1584 let Ok(parsed_identity) =
1585 crate::identity_first::AgentIdentity::parse(&runtime_identity)
1586 else {
1587 continue;
1588 };
1589 if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1590 let record = identity_record_for_status(entry, &status);
1591 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1592 hidden_durable_match = true;
1593 continue;
1594 }
1595 durable_matches.push((
1596 entry.clone(),
1597 identity_runtime.clone(),
1598 parsed_identity,
1599 record,
1600 ));
1601 }
1602 }
1603 }
1604 if durable_matches.len() > 1 {
1605 let candidates = durable_matches
1606 .iter()
1607 .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1608 .collect::<Vec<_>>()
1609 .join(", ");
1610 return Err(ConsoleSendError::InvalidRequest(format!(
1611 "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1612 )));
1613 }
1614 if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1615 durable_matches.into_iter().next()
1616 {
1617 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1618 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1619 &entry,
1620 identity,
1621 &record,
1622 &live_records,
1623 ))
1624 .await;
1625 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1626 &entry,
1627 &durable_live_records,
1628 ))
1629 .await;
1630 if let Some(ambiguous_error) =
1631 ambiguous_live_alias_error(identity, &visible_durable_live_records)
1632 {
1633 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1634 }
1635 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1636 &entry,
1637 identity_runtime.as_ref(),
1638 &parsed_identity,
1639 identity,
1640 &record,
1641 &durable_live_records,
1642 )
1643 .await
1644 .map_err(ConsoleSendError::InvalidRequest)?
1645 {
1646 record = rebound_record;
1647 }
1648 if let Some(stale_error) =
1649 stale_durable_record_error(identity, &record, &durable_live_records)
1650 {
1651 return Err(ConsoleSendError::InvalidRequest(stale_error));
1652 }
1653 return Ok(entry.runtime_key);
1654 }
1655 if entries.values().any(|entry| {
1656 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1657 }) {
1658 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1659 }
1660 if hidden_durable_match {
1661 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1662 }
1663
1664 match identity_runtime_keys.as_slice() {
1665 [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1666 [runtime_key] => Ok(runtime_key.clone()),
1667 _ => Err(ConsoleSendError::InvalidRequest(format!(
1668 "identity-first send for '{identity}' did not match exactly one registered runtime"
1669 ))),
1670 }
1671 }
1672
1673 pub async fn mark_interaction_delivery_failed(
1674 &self,
1675 input_frame_id: &str,
1676 ) -> Result<(), ConsoleSendError> {
1677 update_frame_status_and_emit(
1678 &self.inner,
1679 input_frame_id,
1680 ConsoleFrameStatus::DeliveryFailed,
1681 )
1682 .await
1683 .map_err(ConsoleSendError::Log)?;
1684 Ok(())
1685 }
1686
1687 pub async fn mark_interaction_delivered(
1688 &self,
1689 input_frame_id: &str,
1690 ) -> Result<(), ConsoleSendError> {
1691 update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1692 .await
1693 .map_err(ConsoleSendError::Log)?;
1694 Ok(())
1695 }
1696
1697 pub async fn mark_steer_interaction_delivered(
1698 &self,
1699 input_frame_id: &str,
1700 interaction_id: &str,
1701 ) -> Result<(), ConsoleSendError> {
1702 let Some(updated) = update_frame_status_and_emit(
1703 &self.inner,
1704 input_frame_id,
1705 ConsoleFrameStatus::Delivered,
1706 )
1707 .await
1708 .map_err(ConsoleSendError::Log)?
1709 else {
1710 return Ok(());
1711 };
1712 append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1713 .await
1714 .map_err(ConsoleSendError::Log)?;
1715 Ok(())
1716 }
1717
1718 pub async fn binary_blob_store_for_identity(
1719 &self,
1720 identity: &str,
1721 ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1722 if identity.trim().is_empty() {
1723 return Err(ConsoleSendError::InvalidRequest(
1724 "identity must be non-empty".to_string(),
1725 ));
1726 }
1727 let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1728 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1729 };
1730 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1731 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1732 };
1733 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1734 return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1735 }
1736 if !member_is_addressable(&resolved.member) {
1737 return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1738 }
1739 if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1740 return Err(ConsoleSendError::Retired(identity.to_string()));
1741 }
1742 Ok(resolved.entry.runtime.binary_blob_store())
1743 }
1744
1745 pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1746 self.inner
1747 .runtimes
1748 .read()
1749 .map(|entries| {
1750 entries
1751 .values()
1752 .filter_map(|entry| entry.runtime.binary_blob_store())
1753 .collect()
1754 })
1755 .unwrap_or_default()
1756 }
1757
1758 async fn resolve_send_member(
1759 &self,
1760 identity: &str,
1761 ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1762 let matches = Box::pin(self.resolve_visible_members(identity)).await;
1763 if matches.len() > 1 {
1764 let candidates = matches
1765 .iter()
1766 .map(|resolved| resolved.runtime_identity.clone())
1767 .collect::<Vec<_>>()
1768 .join(", ");
1769 return Err(ConsoleSendError::AmbiguousIdentity {
1770 identity: identity.to_string(),
1771 candidates,
1772 });
1773 }
1774 let Some(resolved) = matches.into_iter().next() else {
1775 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1776 return Err(ConsoleSendError::InvalidRequest(stale_error));
1777 }
1778 return Ok(None);
1779 };
1780 if let Some(record) = identity_record_for_resolved_member(&resolved).await
1781 && let Some(stale_error) =
1782 reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1783 {
1784 return Err(ConsoleSendError::InvalidRequest(stale_error));
1785 }
1786 if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1787 return Err(ConsoleSendError::InvalidRequest(stale_error));
1788 }
1789 Ok(Some(resolved))
1790 }
1791
1792 async fn durable_send_binding_error(
1793 &self,
1794 identity: &str,
1795 ) -> Result<Option<String>, ConsoleSendError> {
1796 let entries = self
1797 .inner
1798 .runtimes
1799 .read()
1800 .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1801 .clone();
1802 if entries.values().any(|entry| {
1803 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1804 }) {
1805 return Ok(None);
1806 }
1807 let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1808 let mut durable_matches = Vec::new();
1809 for entry in entries.values() {
1810 let Some(identity_runtime) = entry.identity_runtime.clone() else {
1811 continue;
1812 };
1813 for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1814 let Ok(parsed_identity) =
1815 crate::identity_first::AgentIdentity::parse(&raw_identity)
1816 else {
1817 continue;
1818 };
1819 let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1820 continue;
1821 };
1822 let record = identity_record_for_status(entry, &status);
1823 if !Box::pin(console_identity_record_visible(entry, &record)).await {
1824 continue;
1825 }
1826 durable_matches.push((
1827 entry.clone(),
1828 identity_runtime.clone(),
1829 parsed_identity,
1830 record,
1831 ));
1832 }
1833 }
1834 if durable_matches.len() > 1 {
1835 let candidates = durable_matches
1836 .iter()
1837 .map(|(_, _, _, record)| record.runtime_member_id.clone())
1838 .collect::<Vec<_>>()
1839 .join(", ");
1840 return Err(ConsoleSendError::AmbiguousIdentity {
1841 identity: identity.to_string(),
1842 candidates,
1843 });
1844 }
1845 let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1846 durable_matches.into_iter().next()
1847 else {
1848 return Ok(None);
1849 };
1850 let durable_live_records = Box::pin(self.live_records_for_durable_record(
1851 &entry,
1852 identity,
1853 &durable_record,
1854 &live_records,
1855 ))
1856 .await;
1857 let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1858 &entry,
1859 &durable_live_records,
1860 ))
1861 .await;
1862 if let Some(ambiguous_error) =
1863 ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1864 {
1865 return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1866 }
1867 if let Some(rebound_record) = self_heal_durable_session_mismatch(
1868 &entry,
1869 identity_runtime.as_ref(),
1870 &parsed_identity,
1871 identity,
1872 &durable_record,
1873 &durable_live_records,
1874 )
1875 .await
1876 .map_err(ConsoleSendError::InvalidRequest)?
1877 {
1878 durable_record = rebound_record;
1879 }
1880 Ok(stale_durable_record_error(
1881 identity,
1882 &durable_record,
1883 &durable_live_records,
1884 ))
1885 }
1886
1887 async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1888 let entries = self
1889 .inner
1890 .runtimes
1891 .read()
1892 .ok()
1893 .map(|entries| entries.clone())
1894 .unwrap_or_default();
1895 let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1896 let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1897 for entry in entries.values() {
1898 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1899 if raw_identities.is_empty() {
1900 continue;
1901 }
1902 let mids = raw_identities
1903 .iter()
1904 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
1905 .collect::<Vec<_>>();
1906 for resolved in Box::pin(member_sources_for_entry(entry)).await {
1907 let session_id = resolved
1908 .handle
1909 .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1910 .await
1911 .map(|sid| sid.to_string())
1912 .unwrap_or_default();
1913 if mids.contains(&resolved.member.agent_identity) {
1914 exact_matches.push((session_id, resolved));
1915 } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1916 {
1917 label_matches.push((session_id, resolved));
1918 }
1919 }
1920 }
1921 let mut matches = exact_matches;
1922 matches.extend(label_matches);
1923 let mut seen_members = BTreeSet::new();
1924 matches.retain(|(session_id, resolved)| {
1925 seen_members.insert((
1926 resolved.entry.runtime_key.clone(),
1927 resolved.source_mob_id.clone(),
1928 session_id.clone(),
1929 resolved.runtime_identity.clone(),
1930 ))
1931 });
1932 matches.sort_by(|left, right| right.0.cmp(&left.0));
1933 matches.into_iter().map(|(_, resolved)| resolved).collect()
1934 }
1935
1936 async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1937 let mut visible = Vec::new();
1938 for resolved in Box::pin(self.resolve_members(identity)).await {
1939 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1940 continue;
1941 };
1942 if Box::pin(resolved_member_visible(&resolved, &record)).await {
1943 visible.push(resolved);
1944 }
1945 }
1946 visible
1947 }
1948
1949 async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1950 let mut records = Vec::new();
1951 for resolved in Box::pin(self.resolve_members(identity)).await {
1952 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1953 continue;
1954 };
1955 if Box::pin(resolved_member_visible(&resolved, &record)).await {
1956 records.push(record);
1957 }
1958 }
1959 records
1960 }
1961
1962 async fn live_records_for_durable_record(
1963 &self,
1964 entry: &RuntimeEntry,
1965 requested_identity: &str,
1966 durable: &ConsoleIdentityRecord,
1967 requested_live_records: &[ConsoleIdentityRecord],
1968 ) -> Vec<ConsoleIdentityRecord> {
1969 let mut records = if durable.identity == requested_identity {
1970 requested_live_records.to_vec()
1971 } else {
1972 Box::pin(self.live_records_for_identity(&durable.identity)).await
1973 };
1974 for runtime_record in Box::pin(live_records_for_runtime_member(
1975 entry,
1976 &durable.runtime_member_id,
1977 ))
1978 .await
1979 {
1980 if records.iter().any(|record| {
1981 record.runtime_key == runtime_record.runtime_key
1982 && record.runtime_member_id == runtime_record.runtime_member_id
1983 && record.session_id == runtime_record.session_id
1984 && record.identity == runtime_record.identity
1985 }) {
1986 continue;
1987 }
1988 records.push(runtime_record);
1989 }
1990 records
1991 }
1992}
1993
1994async fn console_identity_record_visible(
1995 entry: &RuntimeEntry,
1996 record: &ConsoleIdentityRecord,
1997) -> bool {
1998 if !entry.visibility_policy.identity_visible(record) {
1999 return false;
2000 }
2001 let mut bound_live_member_seen = false;
2002 let mut wrong_live_projection_seen = false;
2003 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2004 if resolved.runtime_identity != record.runtime_member_id {
2005 continue;
2006 }
2007 bound_live_member_seen = true;
2008 let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2009 continue;
2010 };
2011 if live_record.identity != record.identity {
2012 wrong_live_projection_seen = true;
2013 continue;
2014 }
2015 if raw_resolved_member_visible(&resolved, &live_record) {
2016 return true;
2017 }
2018 }
2019 if wrong_live_projection_seen {
2020 return true;
2021 }
2022 !bound_live_member_seen
2023}
2024
2025async fn live_record_shadowed_by_hidden_durable_binding(
2026 resolved: &ResolvedConsoleMember,
2027 record: &ConsoleIdentityRecord,
2028) -> bool {
2029 let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2030 return false;
2031 };
2032 let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2033 return false;
2034 };
2035 let Ok(status) = identity_runtime.status(&identity).await else {
2036 return false;
2037 };
2038 let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2039 return false;
2040 };
2041 if bound_runtime_id.as_str() == resolved.runtime_identity {
2042 return false;
2043 }
2044 for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2045 if candidate.runtime_identity != bound_runtime_id.as_str() {
2046 continue;
2047 }
2048 let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2049 continue;
2050 };
2051 if bound_record.identity == record.identity
2052 && !raw_resolved_member_visible(&candidate, &bound_record)
2053 {
2054 return true;
2055 }
2056 }
2057 false
2058}
2059
2060async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2061 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2062 .unwrap_or_else(|| frame.identity.clone());
2063 let frame_session_id = frame.session_id.as_deref();
2064 let mut saw_hidden = false;
2065 let mut saw_visible = false;
2066 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2067 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2068 continue;
2069 };
2070 let session_matches =
2071 frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2072 let runtime_matches = record.runtime_member_id == runtime_member_id;
2073 let identity_matches = record.identity == frame.identity;
2074 if !session_matches && !runtime_matches && !identity_matches {
2075 continue;
2076 }
2077 if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2078 return true;
2079 }
2080 if Box::pin(resolved_member_visible(&resolved, &record)).await {
2081 saw_visible = true;
2082 } else {
2083 saw_hidden = true;
2084 }
2085 }
2086 saw_hidden && !saw_visible
2087}
2088
2089async fn live_records_for_runtime_member(
2090 entry: &RuntimeEntry,
2091 runtime_member_id: &str,
2092) -> Vec<ConsoleIdentityRecord> {
2093 let mut records = Vec::new();
2094 for resolved in Box::pin(member_sources_for_entry(entry)).await {
2095 if resolved.runtime_identity != runtime_member_id {
2096 continue;
2097 }
2098 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2099 continue;
2100 };
2101 records.push(record);
2102 }
2103 records
2104}
2105
2106async fn visible_live_records_for_entry(
2107 entry: &RuntimeEntry,
2108 records: &[ConsoleIdentityRecord],
2109) -> Vec<ConsoleIdentityRecord> {
2110 let mut visible = Vec::new();
2111 for record in records {
2112 if record.runtime_key != entry.runtime_key
2113 || Box::pin(console_identity_record_visible(entry, record)).await
2114 {
2115 visible.push(record.clone());
2116 }
2117 }
2118 visible
2119}
2120
2121async fn reconcile_stale_live_record_binding(
2122 entry: &RuntimeEntry,
2123 requested_identity: &str,
2124 live_record: &ConsoleIdentityRecord,
2125) -> Option<String> {
2126 let identity_runtime = entry.identity_runtime.as_ref()?;
2127 for status in identity_runtime.statuses().await {
2128 let matches_runtime = status
2129 .agent_runtime_id
2130 .as_ref()
2131 .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2132 if !matches_runtime {
2133 continue;
2134 }
2135 let durable_record = identity_record_for_status(entry, &status);
2136 if let Some(session_mismatch) =
2137 stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2138 {
2139 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2140 continue;
2141 };
2142 let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2143 return Some(format!(
2144 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2145 ));
2146 };
2147 return match identity_runtime
2148 .rebind_session_after_live_respawn(&status.identity, session_id)
2149 .await
2150 {
2151 Ok(_) => {
2152 tracing::warn!(
2153 requested_identity,
2154 identity = %durable_record.identity,
2155 runtime_member_id = %durable_record.runtime_member_id,
2156 old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2157 new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2158 "self-healed stale durable identity alias session mismatch"
2159 );
2160 None
2161 }
2162 Err(err) => Some(format!(
2163 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2164 durable_record.identity,
2165 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2166 )),
2167 };
2168 }
2169 if let Some(stale_error) = stale_durable_record_error(
2170 requested_identity,
2171 &durable_record,
2172 std::slice::from_ref(live_record),
2173 ) {
2174 return Some(stale_error);
2175 }
2176 }
2177 None
2178}
2179
2180async fn self_heal_durable_session_mismatch(
2181 entry: &RuntimeEntry,
2182 identity_runtime: &crate::identity_first::IdentityRuntime,
2183 parsed_identity: &crate::identity_first::AgentIdentity,
2184 requested_identity: &str,
2185 durable: &ConsoleIdentityRecord,
2186 live_records: &[ConsoleIdentityRecord],
2187) -> Result<Option<ConsoleIdentityRecord>, String> {
2188 let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2189 return Ok(None);
2190 };
2191 let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2192 return Ok(None);
2193 };
2194 let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2195 format!(
2196 "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2197 )
2198 })?;
2199 identity_runtime
2200 .rebind_session_after_live_respawn(parsed_identity, session_id)
2201 .await
2202 .map_err(|err| {
2203 format!(
2204 "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2205 durable.identity, live_session_id
2206 )
2207 })?;
2208 tracing::warn!(
2209 requested_identity,
2210 identity = %durable.identity,
2211 runtime_member_id = %durable.runtime_member_id,
2212 old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2213 new_session_id = live_session_id,
2214 "self-healed stale durable identity alias session mismatch"
2215 );
2216 let status = identity_runtime
2217 .status(parsed_identity)
2218 .await
2219 .map_err(|err| {
2220 format!(
2221 "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2222 durable.identity
2223 )
2224 })?;
2225 Ok(Some(identity_record_for_status(entry, &status)))
2226}
2227
2228fn stale_durable_session_mismatch<'a>(
2229 durable: &ConsoleIdentityRecord,
2230 live_records: &'a [ConsoleIdentityRecord],
2231) -> Option<&'a ConsoleIdentityRecord> {
2232 live_records.iter().find(|record| {
2233 record.identity == durable.identity
2234 && record.runtime_member_id == durable.runtime_member_id
2235 && durable.session_id.is_some()
2236 && record.session_id.is_some()
2237 && durable.session_id != record.session_id
2238 })
2239}
2240
2241fn stale_durable_record_error(
2242 requested_identity: &str,
2243 durable: &ConsoleIdentityRecord,
2244 live_records: &[ConsoleIdentityRecord],
2245) -> Option<String> {
2246 let matching_live = live_records
2247 .iter()
2248 .filter(|record| record.identity == durable.identity)
2249 .collect::<Vec<_>>();
2250 if let Some(wrong_projection) = live_records.iter().find(|record| {
2251 record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2252 }) {
2253 return Some(format!(
2254 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2255 durable.identity, durable.runtime_member_id, wrong_projection.identity
2256 ));
2257 }
2258 if matching_live.is_empty() {
2259 return None;
2260 }
2261 if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2262 return Some(format!(
2263 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2264 durable.identity,
2265 durable.runtime_member_id,
2266 durable.session_id.as_deref().unwrap_or("<none>"),
2267 session_mismatch.session_id.as_deref().unwrap_or("<none>")
2268 ));
2269 }
2270 if matching_live
2271 .iter()
2272 .any(|record| record.runtime_member_id == durable.runtime_member_id)
2273 {
2274 return None;
2275 }
2276 let live_candidates = matching_live
2277 .iter()
2278 .map(|record| record.runtime_member_id.as_str())
2279 .collect::<Vec<_>>()
2280 .join(", ");
2281 Some(format!(
2282 "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2283 durable.identity, durable.runtime_member_id, live_candidates
2284 ))
2285}
2286
2287fn ambiguous_live_alias_error(
2288 requested_identity: &str,
2289 live_records: &[ConsoleIdentityRecord],
2290) -> Option<String> {
2291 if live_records.len() <= 1 {
2292 return None;
2293 }
2294 let candidates = live_records
2295 .iter()
2296 .map(|record| {
2297 format!(
2298 "{}@{}",
2299 record.runtime_member_id,
2300 record
2301 .labels
2302 .get("source_mob_id")
2303 .map(String::as_str)
2304 .unwrap_or(record.runtime_key.as_str())
2305 )
2306 })
2307 .collect::<Vec<_>>()
2308 .join(", ");
2309 Some(format!(
2310 "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2311 ))
2312}
2313
2314fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2315 crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2318}
2319
2320async fn retire_stale_console_members_for_identity(
2321 handle: &MobHandle,
2322 durable_identities: &[String],
2323 keep_runtime_member_id: Option<&str>,
2324) -> Result<(), String> {
2325 let stale_members = handle
2326 .list_members_including_retiring()
2327 .await
2328 .into_iter()
2329 .filter(|member| {
2330 (durable_identities.iter().any(|durable_identity| {
2331 member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2332 }) || member.labels.get("agent_identity").is_some_and(|identity| {
2333 durable_identities
2334 .iter()
2335 .any(|durable_identity| identity == durable_identity)
2336 })) && keep_runtime_member_id
2337 .map(|keep| {
2338 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2340 != keep
2341 })
2342 .unwrap_or(true)
2343 })
2344 .map(|member| member.agent_identity)
2345 .collect::<Vec<_>>();
2346 for member_id in stale_members {
2347 match handle.retire(member_id).await {
2348 Ok(()) => {}
2349 Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2350 Err(err) => return Err(err.to_string()),
2351 }
2352 }
2353 Ok(())
2354}
2355
2356async fn retire_stale_console_members_for_runtime_entry(
2357 entry: &RuntimeEntry,
2358 durable_identities: &[String],
2359) -> Result<(), String> {
2360 let primary_handle = entry.runtime.handle();
2361 let primary_mob_id = primary_handle.mob_id().to_string();
2362 let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2363 if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2364 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2365 .await
2366 .unwrap_or_default()
2367 {
2368 if mob_id.as_str() != primary_mob_id {
2369 handles.push((mob_id.to_string(), handle));
2370 }
2371 }
2372 }
2373 let mut seen_handles = BTreeSet::new();
2374 for (mob_id, handle) in handles {
2375 if !seen_handles.insert(mob_id) {
2376 continue;
2377 }
2378 retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2379 }
2380 Ok(())
2381}
2382
2383fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2384 is_recoverable_lifecycle_cleanup_error(error)
2385}
2386
2387fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2388 if frames.is_empty() {
2389 return true;
2390 }
2391 let latest_session_history_timestamp_ms = frames
2392 .iter()
2393 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2394 .map(|frame| frame.timestamp_ms)
2395 .max();
2396 if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2397 return current_time_ms().saturating_sub(latest_timestamp_ms)
2398 >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2399 }
2400 frames.iter().any(|frame| {
2401 matches!(
2402 frame.kind.as_str(),
2403 "turn_started"
2404 | "run_started"
2405 | "reasoning_delta"
2406 | "reasoning_complete"
2407 | "tool_call_requested"
2408 | "tool_call"
2409 | "tool_execution_started"
2410 | "tool_execution_completed"
2411 | "tool_result_received"
2412 )
2413 })
2414}
2415
2416fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2417 match frame.kind.as_str() {
2418 "user_input" | "run_started" => true,
2419 "interaction_started" => frame
2420 .payload
2421 .get("content")
2422 .or_else(|| frame.payload.get("prompt"))
2423 .is_some(),
2424 _ => false,
2425 }
2426}
2427
2428fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2429 let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2430 for record in records {
2431 by_identity
2432 .entry(identity_record_dedupe_key(&record))
2433 .and_modify(|current| {
2434 if identity_record_prefer(&record, current) {
2435 *current = record.clone();
2436 }
2437 })
2438 .or_insert(record);
2439 }
2440 by_identity.into_values().collect()
2441}
2442
2443fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2444 format!(
2445 "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2446 record.identity,
2447 record.runtime_key,
2448 record
2449 .labels
2450 .get("source_mob_id")
2451 .map(String::as_str)
2452 .unwrap_or(""),
2453 record.runtime_member_id,
2454 record.session_id.as_deref().unwrap_or("")
2455 )
2456}
2457
2458fn identity_record_prefer(
2459 candidate: &ConsoleIdentityRecord,
2460 current: &ConsoleIdentityRecord,
2461) -> bool {
2462 let candidate_is_live_label_projection =
2463 candidate
2464 .labels
2465 .get("agent_identity")
2466 .is_some_and(|identity| {
2467 identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2468 || candidate.runtime_member_id != candidate.identity
2469 });
2470 let current_is_live_label_projection =
2471 current
2472 .labels
2473 .get("agent_identity")
2474 .is_some_and(|identity| {
2475 identity == &strip_namespace(¤t.identity, "").unwrap_or_default()
2476 || current.runtime_member_id != current.identity
2477 });
2478 if candidate_is_live_label_projection != current_is_live_label_projection {
2479 return candidate_is_live_label_projection;
2480 }
2481 let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2482 let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2483 if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2484 return candidate_has_distinct_runtime_binding;
2485 }
2486 let candidate_live = candidate.addressable && candidate.health != "retired";
2487 let current_live = current.addressable && current.health != "retired";
2488 if candidate_live != current_live {
2489 return candidate_live;
2490 }
2491 candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2492}
2493
2494async fn collect_identity_records(
2495 inner: &Arc<AggregatorInner>,
2496 mode: IdentityCollectionMode,
2497) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2498 let entries = inner
2499 .runtimes
2500 .read()
2501 .map_err(|_| runtime_registry_lock_error())?
2502 .clone();
2503 let mut identities = Vec::new();
2504 for entry in entries.values() {
2505 if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2506 let topology_peers =
2507 identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2508 for status in identity_runtime.statuses().await {
2509 let mut record = identity_record_for_status(entry, &status);
2510 record.topology_peers = topology_peers
2511 .get(&record.identity)
2512 .cloned()
2513 .unwrap_or_default();
2514 if Box::pin(console_identity_record_visible(entry, &record)).await {
2515 identities.push(record);
2516 }
2517 }
2518 if mode == IdentityCollectionMode::CachedOnly {
2519 continue;
2520 }
2521 }
2522 let members = if entry.identity_runtime.is_some() {
2523 match Box::pin(tokio::time::timeout(
2524 IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2525 member_sources_for_entry(entry),
2526 ))
2527 .await
2528 {
2529 Ok(members) => members,
2530 Err(_) => {
2531 tracing::warn!(
2532 runtime_key = %entry.runtime_key,
2533 "identity-first live member refresh timed out; keeping cached identity read model"
2534 );
2535 Vec::new()
2536 }
2537 }
2538 } else {
2539 Box::pin(member_sources_for_entry(entry)).await
2540 };
2541 for resolved in members {
2542 if let Some(record) = identity_record_for_resolved_member(&resolved).await
2543 && Box::pin(resolved_member_visible(&resolved, &record)).await
2544 {
2545 identities.push(record);
2546 }
2547 }
2548 }
2549 Ok(dedupe_identity_records(identities))
2550}
2551
2552async fn identity_runtime_topology_peers(
2553 entry: &RuntimeEntry,
2554 identity_runtime: &crate::identity_first::IdentityRuntime,
2555) -> BTreeMap<String, Vec<String>> {
2556 let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2557 for edge in identity_runtime.desired_peer_edges().await {
2558 let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2559 let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2560 peers.entry(a.clone()).or_default().insert(b.clone());
2561 peers.entry(b).or_default().insert(a);
2562 }
2563 peers
2564 .into_iter()
2565 .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2566 .collect()
2567}
2568
2569fn spawn_identity_backfills_for_records(
2570 inner: Arc<AggregatorInner>,
2571 records: &[ConsoleIdentityRecord],
2572) {
2573 if !inner.options.session_history_backfill_enabled {
2574 return;
2575 }
2576 let entries = match inner.runtimes.read() {
2577 Ok(entries) => entries.clone(),
2578 Err(_) => return,
2579 };
2580 for record in records {
2581 if record.health == "dormant" || record.health == "uninitialized" {
2582 continue;
2583 }
2584 let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2585 continue;
2586 };
2587 let Some(session_id) = record.session_id.clone() else {
2588 continue;
2589 };
2590 spawn_session_history_backfill_target(
2591 inner.clone(),
2592 SessionBackfillTarget {
2593 entry,
2594 record: record.clone(),
2595 session_id,
2596 },
2597 false,
2598 );
2599 }
2600}
2601
2602async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2603 let mut resolved = Vec::new();
2604 let primary_handle = entry.runtime.handle();
2605 let primary_mob_id = primary_handle.mob_id().to_string();
2606 for member in primary_handle.list_members_observation_snapshot().await {
2607 resolved.push(ResolvedConsoleMember {
2608 entry: entry.clone(),
2609 handle: primary_handle.clone(),
2610 runtime_identity: crate::member_comms_id::runtime_alias_str(
2613 member.agent_identity.as_str(),
2614 )
2615 .into_owned(),
2616 source_mob_id: primary_mob_id.clone(),
2617 member,
2618 });
2619 }
2620
2621 let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2622 return resolved;
2623 };
2624 if !entry.visibility_policy.include_implicit_delegate_members() {
2625 return resolved;
2626 }
2627 for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2628 .await
2629 .unwrap_or_default()
2630 {
2631 if mob_id.as_str() == primary_mob_id {
2632 continue;
2633 }
2634 for member in handle.list_members_observation_snapshot().await {
2635 resolved.push(ResolvedConsoleMember {
2636 entry: entry.clone(),
2637 handle: handle.clone(),
2638 runtime_identity: crate::member_comms_id::runtime_alias_str(
2641 member.agent_identity.as_str(),
2642 )
2643 .into_owned(),
2644 source_mob_id: mob_id.to_string(),
2645 member,
2646 });
2647 }
2648 }
2649 resolved
2650}
2651
2652async fn dispatch_message_to_resolved_member(
2653 resolved: &ResolvedConsoleMember,
2654 content: ContentInput,
2655 handling_mode: meerkat_core::types::HandlingMode,
2656) -> Result<String, String> {
2657 let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2658 match send_message_on_mob_with_mode(
2659 &resolved.handle,
2660 &resolved.runtime_identity,
2661 content.clone(),
2662 handling_mode,
2663 )
2664 .await
2665 {
2666 Ok(session_id) => Ok(session_id),
2667 Err(err) if is_not_externally_addressable(&err) => {
2668 let member = resolved
2669 .handle
2670 .member(&mid)
2671 .await
2672 .map_err(|err| err.to_string())?;
2673 let _receipt = member
2674 .internal_turn(content)
2675 .await
2676 .map_err(|err| err.to_string())?;
2677 resolved
2678 .handle
2679 .resolve_bridge_session_id_observation(&mid)
2680 .await
2681 .map(|sid| sid.to_string())
2682 .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2683 }
2684 Err(err) => Err(err.to_string()),
2685 }
2686}
2687
2688fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2689 matches!(
2690 err,
2691 MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2692 )
2693}
2694
2695fn spawn_console_send_dispatch(
2696 inner: Arc<AggregatorInner>,
2697 resolved: ResolvedConsoleMember,
2698 content: ContentInput,
2699 handling_mode: meerkat_core::types::HandlingMode,
2700 dispatching: SendState,
2701 user_frame: ConsoleFrame,
2702 interaction_id: String,
2703) {
2704 tokio::spawn(async move {
2705 match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2706 Ok(_) => {
2707 let _ = dispatching.apply(SendTransition::MarkDelivered);
2708 if let Err(err) = update_frame_status_and_emit(
2709 &inner,
2710 &user_frame.id,
2711 ConsoleFrameStatus::Delivered,
2712 )
2713 .await
2714 {
2715 tracing::warn!(
2716 frame_id = %user_frame.id,
2717 error = %err,
2718 "failed to update console send delivery status"
2719 );
2720 }
2721 if handling_mode == HandlingMode::Steer
2722 && let Err(err) =
2723 append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2724 {
2725 tracing::warn!(
2726 frame_id = %user_frame.id,
2727 interaction_id = %interaction_id,
2728 error = %err,
2729 "failed to append console steer terminal frame"
2730 );
2731 }
2732 }
2733 Err(err) => {
2734 let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2735 if let Err(update_err) = update_frame_status_and_emit(
2736 &inner,
2737 &user_frame.id,
2738 ConsoleFrameStatus::DeliveryFailed,
2739 )
2740 .await
2741 {
2742 tracing::warn!(
2743 frame_id = %user_frame.id,
2744 error = %update_err,
2745 "failed to update console send failure status"
2746 );
2747 }
2748 let failure_frame = NewConsoleFrame {
2749 id: None,
2750 dedupe_key: format!("delivery-failed:{}", user_frame.id),
2751 timestamp_ms: current_time_ms(),
2752 runtime_key: user_frame.runtime_key,
2753 identity: user_frame.identity,
2754 conversation_id: user_frame.conversation_id,
2755 session_id: user_frame.session_id,
2756 kind: "message_delivery_failed".to_string(),
2757 status: ConsoleFrameStatus::DeliveryFailed,
2758 payload: json!({ "reason": err }),
2759 source: ConsoleFrameSource {
2760 kind: ConsoleFrameSourceKind::Synthetic,
2761 source_cursor: None,
2762 },
2763 source_event_id: None,
2764 interaction_id: Some(interaction_id),
2765 turn_id: None,
2766 run_id: None,
2767 parent_frame_id: Some(user_frame.id.clone()),
2768 caused_by_frame_id: Some(user_frame.id),
2769 };
2770 if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2771 tracing::warn!(
2772 error = %append_err,
2773 "failed to append console send failure frame"
2774 );
2775 }
2776 }
2777 }
2778 });
2779}
2780
2781async fn append_steer_delivery_terminal(
2782 inner: &AggregatorInner,
2783 user_frame: &ConsoleFrame,
2784 interaction_id: &str,
2785) -> ConsoleLogResult<AppendOutcome> {
2786 append_and_emit(
2787 inner,
2788 NewConsoleFrame {
2789 id: None,
2790 dedupe_key: format!("steer-delivered:{}", user_frame.id),
2791 timestamp_ms: current_time_ms(),
2792 runtime_key: user_frame.runtime_key.clone(),
2793 identity: user_frame.identity.clone(),
2794 conversation_id: user_frame.conversation_id.clone(),
2795 session_id: user_frame.session_id.clone(),
2796 kind: "interaction_complete".to_string(),
2797 status: ConsoleFrameStatus::Completed,
2798 payload: json!({
2799 "reason": "steer_delivered",
2800 "handling_mode": "steer",
2801 }),
2802 source: ConsoleFrameSource {
2803 kind: ConsoleFrameSourceKind::Synthetic,
2804 source_cursor: None,
2805 },
2806 source_event_id: None,
2807 interaction_id: Some(interaction_id.to_string()),
2808 turn_id: None,
2809 run_id: None,
2810 parent_frame_id: Some(user_frame.id.clone()),
2811 caused_by_frame_id: Some(user_frame.id.clone()),
2812 },
2813 )
2814 .await
2815}
2816
2817#[derive(Debug)]
2818pub enum ConsoleSendError {
2819 UnknownIdentity(String),
2820 AmbiguousIdentity {
2821 identity: String,
2822 candidates: String,
2823 },
2824 NotAddressable(String),
2825 Retired(String),
2826 InvalidContent(String),
2827 InvalidHandlingMode(String),
2828 InvalidRequest(String),
2829 IdempotencyConflict(String),
2830 State(&'static str),
2831 Dispatch(String),
2832 Log(ConsoleLogError),
2833}
2834
2835impl std::fmt::Display for ConsoleSendError {
2836 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2837 match self {
2838 Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2839 Self::AmbiguousIdentity {
2840 identity,
2841 candidates,
2842 } => write!(
2843 f,
2844 "ambiguous live identity alias {identity}: candidates [{candidates}]"
2845 ),
2846 Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2847 Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2848 Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2849 Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2850 Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2851 Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2852 Self::State(message) => write!(f, "console send state error: {message}"),
2853 Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2854 Self::Log(err) => write!(f, "console log error: {err}"),
2855 }
2856 }
2857}
2858
2859impl std::error::Error for ConsoleSendError {}
2860
2861async fn backfill_session_history(
2862 inner: Arc<AggregatorInner>,
2863 runtime_key: String,
2864 force_refresh: bool,
2865) -> ConsoleLogResult<()> {
2866 if !inner.options.session_history_backfill_enabled {
2867 return Ok(());
2868 }
2869 let Some(entry) = inner
2870 .runtimes
2871 .read()
2872 .ok()
2873 .and_then(|entries| entries.get(&runtime_key).cloned())
2874 else {
2875 return Ok(());
2876 };
2877 let members = Box::pin(member_sources_for_entry(&entry)).await;
2878 let mut targets = Vec::new();
2879 for resolved in members {
2880 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2881 continue;
2882 };
2883 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
2884 continue;
2885 }
2886 let Some(session_id) = record.session_id.clone() else {
2887 continue;
2888 };
2889 targets.push(SessionBackfillTarget {
2890 entry: entry.clone(),
2891 record,
2892 session_id,
2893 });
2894 }
2895 backfill_session_history_targets(inner, targets, force_refresh).await
2896}
2897
2898#[derive(Clone)]
2899struct SessionBackfillTarget {
2900 entry: RuntimeEntry,
2901 record: ConsoleIdentityRecord,
2902 session_id: String,
2903}
2904
2905async fn backfill_session_history_targets(
2906 inner: Arc<AggregatorInner>,
2907 targets: Vec<SessionBackfillTarget>,
2908 force_refresh: bool,
2909) -> ConsoleLogResult<()> {
2910 let mut tasks = tokio::task::JoinSet::new();
2911 for target in targets {
2912 tasks.spawn(backfill_one_session_history(
2913 inner.clone(),
2914 target,
2915 force_refresh,
2916 ));
2917 }
2918 let mut first_error = None;
2919 while let Some(result) = tasks.join_next().await {
2920 match result {
2921 Ok(Ok(())) => {}
2922 Ok(Err(err)) => {
2923 if first_error.is_none() {
2924 first_error = Some(err);
2925 }
2926 }
2927 Err(err) => {
2928 if first_error.is_none() {
2929 first_error = Some(Box::new(std::io::Error::other(format!(
2930 "session backfill task failed: {err}"
2931 ))) as ConsoleLogError);
2932 }
2933 }
2934 }
2935 }
2936 if let Some(err) = first_error {
2937 Err(err)
2938 } else {
2939 Ok(())
2940 }
2941}
2942
2943async fn backfill_one_session_history(
2944 inner: Arc<AggregatorInner>,
2945 target: SessionBackfillTarget,
2946 force_refresh: bool,
2947) -> ConsoleLogResult<()> {
2948 let _permit = inner
2949 .session_backfill_permits
2950 .clone()
2951 .acquire_owned()
2952 .await
2953 .map_err(|err| -> ConsoleLogError {
2954 Box::new(std::io::Error::other(format!(
2955 "session backfill limiter closed: {err}"
2956 )))
2957 })?;
2958 let SessionBackfillTarget {
2959 entry,
2960 record,
2961 session_id,
2962 } = target;
2963 let watermark_runtime_key =
2964 session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2965 let watermark = inner
2966 .store
2967 .source_watermark(
2968 &watermark_runtime_key,
2969 ConsoleFrameSourceKind::SessionHistory,
2970 )
2971 .await?;
2972 let now_ms = current_time_ms();
2973 let mut offset = watermark
2974 .as_deref()
2975 .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2976 .unwrap_or(0);
2977 if !force_refresh
2978 && watermark.as_deref().is_some_and(|watermark| {
2979 session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2980 })
2981 {
2982 return Ok(());
2983 }
2984 loop {
2985 let page = match entry
2986 .runtime
2987 .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2988 .await
2989 {
2990 Ok(page) => page,
2991 Err(err) => {
2992 record_session_history_backfill_failure(
2993 &inner,
2994 &watermark_runtime_key,
2995 &entry.runtime_key,
2996 &record.identity,
2997 &session_id,
2998 offset,
2999 err.to_string(),
3000 )
3001 .await?;
3002 break;
3003 }
3004 };
3005 let page_value = match serde_json::to_value(page) {
3006 Ok(value) => value,
3007 Err(err) => {
3008 record_session_history_backfill_failure(
3009 &inner,
3010 &watermark_runtime_key,
3011 &entry.runtime_key,
3012 &record.identity,
3013 &session_id,
3014 offset,
3015 err.to_string(),
3016 )
3017 .await?;
3018 break;
3019 }
3020 };
3021 let base_offset = page_value
3022 .get("offset")
3023 .and_then(Value::as_u64)
3024 .unwrap_or(offset as u64) as usize;
3025 let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3026 record_session_history_backfill_failure(
3027 &inner,
3028 &watermark_runtime_key,
3029 &entry.runtime_key,
3030 &record.identity,
3031 &session_id,
3032 offset,
3033 "session history page missing messages".to_string(),
3034 )
3035 .await?;
3036 break;
3037 };
3038 if messages.is_empty() {
3039 if offset > 0 {
3040 record_session_history_watermark(
3041 &inner,
3042 &watermark_runtime_key,
3043 &session_id,
3044 offset,
3045 )
3046 .await?;
3047 }
3048 break;
3049 }
3050 for (idx, message) in messages.iter().enumerate() {
3051 let absolute_offset = base_offset + idx;
3052 let frames = frames_from_session_history_message_with_namespace(
3053 &entry.runtime_key,
3054 &record.identity,
3055 &entry.identity_namespace,
3056 &session_id,
3057 absolute_offset,
3058 message.clone(),
3059 );
3060 for mut frame in frames {
3061 if history_frame_has_existing_counterpart(&inner, &frame).await? {
3062 continue;
3063 }
3064 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3065 frame.payload = redacted;
3066 frame.status = ConsoleFrameStatus::Redacted;
3067 }
3068 append_and_emit(&inner, frame).await?;
3069 }
3070 }
3071 let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3072 tracing::warn!(
3079 session_id = %session_id,
3080 requested_offset = offset,
3081 page_base_offset = base_offset,
3082 page_len = messages.len(),
3083 "session history backfill made no forward progress; stopping to avoid a livelock"
3084 );
3085 break;
3086 };
3087 offset = next_offset;
3088 record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3089 .await?;
3090 let has_more = page_value
3091 .get("has_more")
3092 .and_then(Value::as_bool)
3093 .unwrap_or(false);
3094 if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3095 break;
3096 }
3097 }
3098 Ok(())
3099}
3100
3101fn advance_backfill_offset(
3106 requested_offset: usize,
3107 base_offset: usize,
3108 page_len: usize,
3109) -> Option<usize> {
3110 let next = base_offset.saturating_add(page_len);
3111 (next > requested_offset).then_some(next)
3112}
3113
3114async fn record_session_history_watermark(
3115 inner: &AggregatorInner,
3116 watermark_runtime_key: &str,
3117 session_id: &str,
3118 offset: usize,
3119) -> ConsoleLogResult<()> {
3120 inner
3121 .store
3122 .record_source_watermark(
3123 watermark_runtime_key,
3124 ConsoleFrameSourceKind::SessionHistory,
3125 &format_session_history_watermark(session_id, offset, current_time_ms()),
3126 )
3127 .await
3128}
3129
3130fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3131 if !inner.options.session_history_backfill_enabled {
3132 return;
3133 }
3134 tokio::spawn(async move {
3135 {
3136 let mut active = inner.active_session_backfills.lock().await;
3137 if !active.insert(runtime_key.clone()) {
3138 return;
3139 }
3140 }
3141 let result = Box::pin(backfill_session_history(
3142 inner.clone(),
3143 runtime_key.clone(),
3144 false,
3145 ))
3146 .await;
3147 let mut active = inner.active_session_backfills.lock().await;
3148 active.remove(&runtime_key);
3149 drop(active);
3150 if let Err(err) = result {
3151 tracing::warn!(
3152 runtime_key = %runtime_key,
3153 error = %err,
3154 "console session-history backfill failed"
3155 );
3156 }
3157 });
3158}
3159
3160fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3161 if !inner.options.session_history_backfill_enabled {
3162 return;
3163 }
3164 tokio::spawn(async move {
3165 let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3166 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3167 loop {
3168 interval.tick().await;
3169 let runtime_still_registered = inner
3170 .runtimes
3171 .read()
3172 .ok()
3173 .is_some_and(|entries| entries.contains_key(&runtime_key));
3174 if !runtime_still_registered {
3175 break;
3176 }
3177 spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3178 }
3179 });
3180}
3181
3182fn spawn_session_history_backfill_target(
3183 inner: Arc<AggregatorInner>,
3184 target: SessionBackfillTarget,
3185 force_refresh: bool,
3186) {
3187 if !inner.options.session_history_backfill_enabled {
3188 return;
3189 }
3190 tokio::spawn(async move {
3191 let active_key = targeted_session_history_active_key(&target, force_refresh);
3192 let result =
3193 run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3194 if let Err(err) = result {
3195 tracing::warn!(
3196 active_key = %active_key,
3197 error = %err,
3198 "console targeted session-history backfill failed"
3199 );
3200 }
3201 });
3202}
3203
3204async fn run_targeted_session_history_backfill(
3205 inner: Arc<AggregatorInner>,
3206 target: SessionBackfillTarget,
3207 force_refresh: bool,
3208) -> ConsoleLogResult<()> {
3209 let active_key = targeted_session_history_active_key(&target, force_refresh);
3210 {
3211 let mut active = inner.active_session_backfills.lock().await;
3212 if !active.insert(active_key.clone()) {
3213 return Ok(());
3214 }
3215 }
3216 let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3217 let mut active = inner.active_session_backfills.lock().await;
3218 active.remove(&active_key);
3219 result
3220}
3221
3222fn targeted_session_history_active_key(
3223 target: &SessionBackfillTarget,
3224 force_refresh: bool,
3225) -> String {
3226 let mode = if force_refresh { "force" } else { "refresh" };
3227 format!(
3228 "{}:session-history:{}:{mode}",
3229 target.entry.runtime_key, target.session_id
3230 )
3231}
3232
3233fn spawn_session_history_backfill_for_identity(
3234 inner: Arc<AggregatorInner>,
3235 identity: String,
3236 force_refresh: bool,
3237) {
3238 if !inner.options.session_history_backfill_enabled {
3239 return;
3240 }
3241 tokio::spawn(async move {
3242 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3243 spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3244 }
3245 });
3246}
3247
3248fn spawn_opportunistic_session_history_backfill_for_identity(
3249 inner: Arc<AggregatorInner>,
3250 identity: String,
3251) {
3252 if !inner.options.session_history_backfill_enabled {
3253 return;
3254 }
3255 tokio::spawn(async move {
3256 for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3257 let active_key = format!(
3258 "{}:session-history:{}",
3259 target.entry.runtime_key, target.session_id
3260 );
3261 {
3262 let mut seen = inner.opportunistic_session_backfills.lock().await;
3263 if !seen.insert(active_key) {
3264 continue;
3265 }
3266 }
3267 spawn_session_history_backfill_target(inner.clone(), target, false);
3268 }
3269 });
3270}
3271
3272#[cfg(test)]
3273async fn session_backfill_target_for_identity(
3274 inner: &AggregatorInner,
3275 identity: &str,
3276) -> Option<SessionBackfillTarget> {
3277 session_backfill_targets_for_identity(inner, identity)
3278 .await
3279 .into_iter()
3280 .next()
3281}
3282
3283async fn session_backfill_targets_for_identity(
3284 inner: &AggregatorInner,
3285 identity: &str,
3286) -> Vec<SessionBackfillTarget> {
3287 let entries = inner
3288 .runtimes
3289 .read()
3290 .ok()
3291 .map(|entries| entries.clone())
3292 .unwrap_or_default();
3293 let mut targets = Vec::new();
3294 for entry in entries.values() {
3295 let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3296 if raw_identities.is_empty() {
3297 continue;
3298 }
3299 let mids = raw_identities
3300 .iter()
3301 .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3302 .collect::<Vec<_>>();
3303 for resolved in Box::pin(member_sources_for_entry(entry))
3304 .await
3305 .into_iter()
3306 .filter(|candidate| {
3307 resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3308 })
3309 {
3310 let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3311 continue;
3312 };
3313 if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3314 continue;
3315 }
3316 let Some(session_id) = record.session_id.clone() else {
3317 continue;
3318 };
3319 targets.push(SessionBackfillTarget {
3320 entry: entry.clone(),
3321 record,
3322 session_id,
3323 });
3324 }
3325 }
3326 targets
3327}
3328
3329fn resolved_member_matches_raw_identities(
3330 resolved: &ResolvedConsoleMember,
3331 raw_identities: &[String],
3332 mids: &[AgentIdentity],
3333) -> bool {
3334 mids.contains(&resolved.member.agent_identity)
3335 || resolved
3336 .member
3337 .labels
3338 .get("agent_identity")
3339 .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3340}
3341
3342async fn recover_lagged_source_events(
3343 inner: Arc<AggregatorInner>,
3344 runtime_key: &str,
3345 console_events: &ConsoleEventStore,
3346) -> ConsoleLogResult<()> {
3347 let watermark = inner
3348 .store
3349 .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3350 .await?;
3351 match console_events.replay_all(watermark.as_deref()).await {
3352 Ok(events) => {
3353 for envelope in events {
3354 project_console_event(inner.clone(), runtime_key, envelope).await?;
3355 }
3356 }
3357 Err(err) => {
3358 append_source_gap(
3359 &inner,
3360 runtime_key,
3361 format!(
3362 "{}:{}:{}",
3363 err.error, err.stream, err.requested_last_event_id
3364 ),
3365 )
3366 .await?;
3367 }
3368 }
3369 Ok(())
3370}
3371
3372async fn append_source_gap(
3373 inner: &AggregatorInner,
3374 runtime_key: &str,
3375 reason: String,
3376) -> ConsoleLogResult<()> {
3377 append_and_emit(
3378 inner,
3379 NewConsoleFrame {
3380 id: None,
3381 dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3382 timestamp_ms: current_time_ms(),
3383 runtime_key: runtime_key.to_string(),
3384 identity: "__console__".to_string(),
3385 conversation_id: None,
3386 session_id: None,
3387 kind: "replay_unavailable".to_string(),
3388 status: ConsoleFrameStatus::DeliveryFailed,
3389 payload: json!({
3390 "reason": reason,
3391 "source_kind": "console_event",
3392 }),
3393 source: ConsoleFrameSource {
3394 kind: ConsoleFrameSourceKind::Synthetic,
3395 source_cursor: None,
3396 },
3397 source_event_id: None,
3398 interaction_id: None,
3399 turn_id: None,
3400 run_id: None,
3401 parent_frame_id: None,
3402 caused_by_frame_id: None,
3403 },
3404 )
3405 .await?;
3406 let _ = inner
3407 .event_tx
3408 .send(ConsoleTimelineEvent::ReplayUnavailable {
3409 requested_cursor: format!("source-gap:{runtime_key}"),
3410 latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3411 });
3412 Ok(())
3413}
3414
3415async fn record_session_history_backfill_failure(
3420 inner: &AggregatorInner,
3421 watermark_runtime_key: &str,
3422 runtime_key: &str,
3423 identity: &str,
3424 session_id: &str,
3425 offset: usize,
3426 reason: String,
3427) -> ConsoleLogResult<()> {
3428 record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3429 append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3430}
3431
3432fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3435 format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3436}
3437
3438async fn append_backfill_gap(
3439 inner: &AggregatorInner,
3440 runtime_key: &str,
3441 identity: &str,
3442 session_id: &str,
3443 reason: String,
3444) -> ConsoleLogResult<()> {
3445 append_and_emit(
3446 inner,
3447 NewConsoleFrame {
3448 id: None,
3449 dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3454 timestamp_ms: current_time_ms(),
3455 runtime_key: runtime_key.to_string(),
3456 identity: identity.to_string(),
3457 conversation_id: Some(identity.to_string()),
3458 session_id: Some(session_id.to_string()),
3459 kind: "replay_unavailable".to_string(),
3460 status: ConsoleFrameStatus::DeliveryFailed,
3461 payload: json!({
3462 "reason": reason,
3463 "source_kind": "session_history",
3464 }),
3465 source: ConsoleFrameSource {
3466 kind: ConsoleFrameSourceKind::Synthetic,
3467 source_cursor: None,
3468 },
3469 source_event_id: None,
3470 interaction_id: None,
3471 turn_id: None,
3472 run_id: None,
3473 parent_frame_id: None,
3474 caused_by_frame_id: None,
3475 },
3476 )
3477 .await?;
3478 Ok(())
3479}
3480
3481async fn project_console_event(
3482 inner: Arc<AggregatorInner>,
3483 runtime_key: &str,
3484 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3485) -> ConsoleLogResult<()> {
3486 let Some(entry) = inner
3487 .runtimes
3488 .read()
3489 .ok()
3490 .and_then(|entries| entries.get(runtime_key).cloned())
3491 else {
3492 return Ok(());
3493 };
3494 let mut frame = frame_from_console_event(&entry, envelope);
3495 if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3496 frame.payload = redacted;
3497 frame.status = ConsoleFrameStatus::Redacted;
3498 }
3499 let source_cursor = frame
3500 .source_event_id
3501 .clone()
3502 .unwrap_or_else(|| frame.dedupe_key.clone());
3503 let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3504 Some(frame.identity.clone())
3505 } else {
3506 None
3507 };
3508 let opportunistic_refresh_identity = if refresh_identity.is_none()
3509 && console_event_should_start_session_history_backfill(&frame)
3510 {
3511 Some(frame.identity.clone())
3512 } else {
3513 None
3514 };
3515 append_and_emit(&inner, frame).await?;
3516 inner
3517 .store
3518 .record_source_watermark(
3519 &entry.runtime_key,
3520 ConsoleFrameSourceKind::ConsoleEvent,
3521 &source_cursor,
3522 )
3523 .await?;
3524 if let Some(identity) = refresh_identity {
3525 spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3526 } else if let Some(identity) = opportunistic_refresh_identity {
3527 spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3528 }
3529 Ok(())
3530}
3531
3532fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3533 matches!(
3534 frame.kind.as_str(),
3535 "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3536 ) || frame.session_id.is_some()
3537}
3538
3539fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3540 if frame.identity == SYSTEM_EVENT_IDENTITY {
3541 return false;
3542 }
3543 matches!(
3544 frame.kind.as_str(),
3545 "turn_started"
3546 | "run_started"
3547 | "reasoning_complete"
3548 | "tool_call_requested"
3549 | "tool_call"
3550 | "tool_execution_started"
3551 | "text_delta"
3552 | "system_notice"
3553 )
3554}
3555
3556async fn append_and_emit(
3557 inner: &AggregatorInner,
3558 frame: NewConsoleFrame,
3559) -> ConsoleLogResult<AppendOutcome> {
3560 let outcome = inner.store.append_if_absent(frame).await?;
3561 if outcome.disposition == AppendDisposition::Inserted {
3562 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3563 frame: outcome.frame.clone(),
3564 });
3565 }
3566 Ok(outcome)
3567}
3568
3569async fn update_frame_status_and_emit(
3570 inner: &AggregatorInner,
3571 frame_id: &str,
3572 status: ConsoleFrameStatus,
3573) -> ConsoleLogResult<Option<ConsoleFrame>> {
3574 let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3575 return Ok(None);
3576 };
3577 let update_marker = NewConsoleFrame {
3578 id: None,
3579 dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3580 timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3581 runtime_key: updated.runtime_key.clone(),
3582 identity: updated.identity.clone(),
3583 conversation_id: updated.conversation_id.clone(),
3584 session_id: updated.session_id.clone(),
3585 kind: "frame_updated".to_string(),
3586 status: updated.status,
3587 payload: json!({ "frame": updated.clone() }),
3588 source: ConsoleFrameSource {
3589 kind: ConsoleFrameSourceKind::Synthetic,
3590 source_cursor: None,
3591 },
3592 source_event_id: None,
3593 interaction_id: updated.interaction_id.clone(),
3594 turn_id: updated.turn_id.clone(),
3595 run_id: updated.run_id.clone(),
3596 parent_frame_id: Some(updated.id.clone()),
3597 caused_by_frame_id: Some(updated.id.clone()),
3598 };
3599 let outcome = inner.store.append_if_absent(update_marker).await?;
3600 if outcome.disposition == AppendDisposition::Inserted {
3601 let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3602 frame: outcome.frame,
3603 });
3604 }
3605 Ok(Some(updated))
3606}
3607
3608fn frame_from_console_event(
3609 entry: &RuntimeEntry,
3610 envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3611) -> NewConsoleFrame {
3612 let event_id = envelope.event_id;
3613 let interaction_id = envelope.interaction_id;
3614 let event_type = envelope.event_type;
3615 let payload = envelope.data;
3616 let turn_id = payload
3617 .get("turn_id")
3618 .and_then(Value::as_str)
3619 .map(ToString::to_string);
3620 let run_id = payload
3621 .get("run_id")
3622 .and_then(Value::as_str)
3623 .map(ToString::to_string);
3624 let status = match event_type.as_str() {
3625 "interaction_started" => ConsoleFrameStatus::Accepted,
3626 "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3627 "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3628 _ => ConsoleFrameStatus::Delivered,
3629 };
3630 let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3631 let dedupe_key = reasoning_payload_text(&event_type, &payload)
3632 .map(|reasoning_text| {
3633 let turn_scope = interaction_id
3636 .as_deref()
3637 .or(turn_id.as_deref())
3638 .or(run_id.as_deref())
3639 .unwrap_or(event_id.as_str());
3640 format!(
3641 "console-reasoning:{}:{}:{}",
3642 entry.runtime_key,
3643 turn_scope,
3644 hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3645 )
3646 })
3647 .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3648 NewConsoleFrame {
3649 id: Some(event_id.clone()),
3650 dedupe_key,
3651 timestamp_ms: envelope.timestamp_ms,
3652 runtime_key: entry.runtime_key.clone(),
3653 identity: identity.clone(),
3654 conversation_id: Some(identity),
3655 session_id: payload
3656 .get("session_id")
3657 .and_then(Value::as_str)
3658 .map(ToString::to_string),
3659 kind: event_type,
3660 status,
3661 payload,
3662 source: ConsoleFrameSource {
3663 kind: ConsoleFrameSourceKind::ConsoleEvent,
3664 source_cursor: None,
3665 },
3666 source_event_id: Some(event_id),
3667 interaction_id,
3668 turn_id,
3669 run_id,
3670 parent_frame_id: None,
3671 caused_by_frame_id: None,
3672 }
3673}
3674
3675#[cfg(test)]
3676fn frame_from_session_history_message(
3677 runtime_key: &str,
3678 identity: &str,
3679 session_id: &str,
3680 offset: usize,
3681 message: Value,
3682) -> Option<NewConsoleFrame> {
3683 frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3684 .into_iter()
3685 .next()
3686}
3687
3688#[cfg(test)]
3689fn frames_from_session_history_message(
3690 runtime_key: &str,
3691 identity: &str,
3692 session_id: &str,
3693 offset: usize,
3694 message: Value,
3695) -> Vec<NewConsoleFrame> {
3696 frames_from_session_history_message_with_namespace(
3697 runtime_key,
3698 identity,
3699 "",
3700 session_id,
3701 offset,
3702 message,
3703 )
3704}
3705
3706#[allow(clippy::too_many_arguments)]
3714fn session_history_assistant_image_frames(
3715 runtime_key: &str,
3716 identity: &str,
3717 session_id: &str,
3718 offset: usize,
3719 result_idx: usize,
3720 timestamp_ms: u64,
3721 tool_use_id: &str,
3722 result_text: &str,
3723) -> Vec<NewConsoleFrame> {
3724 let Ok(image_result) = serde_json::from_str::<
3725 meerkat_core::image_generation::ImageGenerationToolResult,
3726 >(result_text) else {
3727 return Vec::new();
3728 };
3729 image_result
3730 .images
3731 .iter()
3732 .enumerate()
3733 .map(|(image_idx, image)| NewConsoleFrame {
3734 id: None,
3735 dedupe_key: format!(
3738 "console-assistant-image:{runtime_key}:{}",
3739 image.blob_ref.blob_id
3740 ),
3741 timestamp_ms,
3742 runtime_key: runtime_key.to_string(),
3743 identity: identity.to_string(),
3744 conversation_id: Some(identity.to_string()),
3745 session_id: Some(session_id.to_string()),
3746 kind: "assistant_image".to_string(),
3747 status: ConsoleFrameStatus::Completed,
3748 payload: json!({
3749 "source_event_type": "session_history",
3750 "tool_call_id": tool_use_id,
3751 "image_id": image.image_id.0.to_string(),
3752 "blob_id": image.blob_ref.blob_id,
3753 "media_type": image.media_type.as_str(),
3754 "width": image.width,
3755 "height": image.height,
3756 "revised_prompt": image_result.revised_prompt.clone(),
3757 "type": "session_history",
3758 }),
3759 source: ConsoleFrameSource {
3760 kind: ConsoleFrameSourceKind::SessionHistory,
3761 source_cursor: Some(format!(
3762 "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3763 )),
3764 },
3765 source_event_id: None,
3766 interaction_id: None,
3767 turn_id: None,
3768 run_id: None,
3769 parent_frame_id: None,
3770 caused_by_frame_id: None,
3771 })
3772 .collect()
3773}
3774
3775fn frames_from_session_history_message_with_namespace(
3776 runtime_key: &str,
3777 identity: &str,
3778 identity_namespace: &str,
3779 session_id: &str,
3780 offset: usize,
3781 message: Value,
3782) -> Vec<NewConsoleFrame> {
3783 let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3784 let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3785 return Vec::new();
3786 };
3787 if let Message::ToolResults {
3788 results,
3789 created_at,
3790 } = parsed
3791 {
3792 return results
3793 .into_iter()
3794 .enumerate()
3795 .flat_map(|(idx, result)| {
3796 let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3797 let result_text = result.text_content();
3798 let tool_use_id = result.tool_use_id.clone();
3799 let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3800 let mut frames = vec![NewConsoleFrame {
3801 id: None,
3802 dedupe_key: format!(
3803 "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3804 ),
3805 timestamp_ms,
3806 runtime_key: runtime_key.to_string(),
3807 identity: identity.to_string(),
3808 conversation_id: Some(identity.to_string()),
3809 session_id: Some(session_id.to_string()),
3810 kind: "tool_execution_completed".to_string(),
3811 status: ConsoleFrameStatus::Completed,
3812 payload: json!({
3813 "id": tool_use_id,
3814 "tool_call_id": tool_use_id,
3815 "result": result_text,
3816 "content": content,
3817 "is_error": result.is_error,
3818 "source_event_type": "session_history",
3819 "type": "session_history",
3820 }),
3821 source: ConsoleFrameSource {
3822 kind: ConsoleFrameSourceKind::SessionHistory,
3823 source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3824 },
3825 source_event_id: None,
3826 interaction_id: None,
3827 turn_id: None,
3828 run_id: None,
3829 parent_frame_id: None,
3830 caused_by_frame_id: None,
3831 }];
3832 frames.extend(session_history_assistant_image_frames(
3840 runtime_key,
3841 identity,
3842 session_id,
3843 offset,
3844 idx,
3845 timestamp_ms,
3846 &tool_use_id,
3847 &result_text,
3848 ));
3849 frames
3850 })
3851 .collect();
3852 }
3853 let (kind, timestamp_ms, payload) = match &parsed {
3854 Message::User(user) => {
3855 if session_history_user_message_is_scaffold(&message) {
3856 return Vec::new();
3857 }
3858 (
3859 "user_input",
3860 user.created_at.timestamp_millis().max(0) as u64,
3861 json!({
3862 "content": user.content,
3863 "message": message,
3864 }),
3865 )
3866 }
3867 Message::BlockAssistant(assistant) => {
3870 let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3871 (
3872 "interaction_complete",
3873 assistant.created_at.timestamp_millis().max(0) as u64,
3874 json!({
3875 "result": text,
3876 "text": text,
3877 "message": message,
3878 "source_event_type": "session_history",
3879 "type": "session_history",
3880 }),
3881 )
3882 }
3883 Message::SystemNotice(notice) => (
3884 "system_notice",
3885 notice.created_at.timestamp_millis().max(0) as u64,
3886 json!({
3887 "message": message,
3888 "kind": notice.kind,
3889 "render_class": notice.kind.render_class(),
3890 "body": notice.body,
3891 "blocks": notice.blocks,
3892 "source_event_type": "session_history",
3893 "type": "session_history",
3894 }),
3895 ),
3896 Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3897 };
3898 let mut frames = vec![NewConsoleFrame {
3899 id: None,
3900 dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3901 timestamp_ms,
3902 runtime_key: runtime_key.to_string(),
3903 identity: identity.to_string(),
3904 conversation_id: Some(identity.to_string()),
3905 session_id: Some(session_id.to_string()),
3906 kind: kind.to_string(),
3907 status: ConsoleFrameStatus::Completed,
3908 payload,
3909 source: ConsoleFrameSource {
3910 kind: ConsoleFrameSourceKind::SessionHistory,
3911 source_cursor: Some(format!("{session_id}:{offset}")),
3912 },
3913 source_event_id: None,
3914 interaction_id: None,
3915 turn_id: None,
3916 run_id: None,
3917 parent_frame_id: None,
3918 caused_by_frame_id: None,
3919 }];
3920 if let Message::BlockAssistant(assistant) = &parsed {
3921 frames.extend(spawn_initial_message_frames_from_assistant(
3922 runtime_key,
3923 identity,
3924 identity_namespace,
3925 session_id,
3926 offset,
3927 assistant,
3928 &payload_hash,
3929 ));
3930 }
3931 frames
3932}
3933
3934fn spawn_initial_message_frames_from_assistant(
3935 runtime_key: &str,
3936 parent_identity: &str,
3937 identity_namespace: &str,
3938 session_id: &str,
3939 offset: usize,
3940 assistant: &meerkat_core::types::BlockAssistantMessage,
3941 payload_hash: &str,
3942) -> Vec<NewConsoleFrame> {
3943 let mut frames = Vec::new();
3944 for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3945 if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3946 continue;
3947 }
3948 let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3949 continue;
3950 };
3951 for (spawn_idx, (target_identity, initial_message)) in
3952 spawn_initial_messages_from_tool_args(&args)
3953 .into_iter()
3954 .enumerate()
3955 {
3956 let target_identity = apply_namespace(&target_identity, identity_namespace);
3957 let message_content = match &initial_message {
3958 Value::String(text) => json!([
3959 {
3960 "type": "text",
3961 "text": text,
3962 }
3963 ]),
3964 other => other.clone(),
3965 };
3966 let message = match &initial_message {
3967 Value::String(text) => json!({
3968 "role": "user",
3969 "content": text,
3970 "created_at": assistant.created_at,
3971 }),
3972 other => json!({
3973 "role": "user",
3974 "content": other,
3975 "created_at": assistant.created_at,
3976 }),
3977 };
3978 frames.push(NewConsoleFrame {
3979 id: None,
3980 dedupe_key: format!(
3981 "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3982 ),
3983 timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3984 runtime_key: runtime_key.to_string(),
3985 identity: target_identity.clone(),
3986 conversation_id: Some(target_identity),
3987 session_id: None,
3988 kind: "user_input".to_string(),
3989 status: ConsoleFrameStatus::Completed,
3990 payload: json!({
3991 "content": message_content,
3992 "message": message,
3993 "source_event_type": "session_history_spawn_initial_message",
3994 "type": "session_history",
3995 "tool_call_id": tool.id,
3996 "parent_identity": parent_identity,
3997 "via_tool": tool.name,
3998 }),
3999 source: ConsoleFrameSource {
4000 kind: ConsoleFrameSourceKind::SessionHistory,
4001 source_cursor: Some(format!(
4002 "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4003 )),
4004 },
4005 source_event_id: None,
4006 interaction_id: None,
4007 turn_id: None,
4008 run_id: None,
4009 parent_frame_id: None,
4010 caused_by_frame_id: None,
4011 });
4012 }
4013 }
4014 frames
4015}
4016
4017fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4018 let mut messages = Vec::new();
4019 if let Some(message) = spawn_initial_message_from_record(args) {
4020 messages.push(message);
4021 }
4022 if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4023 for spec in specs {
4024 if let Some(message) = spawn_initial_message_from_record(spec) {
4025 messages.push(message);
4026 }
4027 }
4028 }
4029 messages
4030}
4031
4032fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4033 let target_identity = record
4034 .get("member_id")
4035 .or_else(|| record.get("agent_identity"))
4036 .and_then(Value::as_str)?
4037 .trim();
4038 if target_identity.is_empty() {
4039 return None;
4040 }
4041 let initial_message = record
4042 .get("initial_message")
4043 .or_else(|| record.get("task"))?
4044 .clone();
4045 Some((target_identity.to_string(), initial_message))
4046}
4047
4048fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4049 message
4050 .get("content")
4051 .is_some_and(scaffold_message_content_is_noise)
4052}
4053
4054fn scaffold_message_content_is_noise(value: &Value) -> bool {
4055 match value {
4056 Value::String(text) => scaffold_message_text_is_noise(text),
4057 Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4058 Value::Object(map) => ["text", "content", "message"]
4059 .iter()
4060 .filter_map(|key| map.get(*key))
4061 .any(scaffold_message_content_is_noise),
4062 _ => false,
4063 }
4064}
4065
4066fn scaffold_message_text_is_noise(text: &str) -> bool {
4067 let trimmed = text.trim_start();
4068 trimmed.starts_with("[PEER UPDATE]")
4069 || trimmed
4070 .to_ascii_lowercase()
4071 .starts_with("you have been spawned")
4072}
4073
4074fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4075 let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4076 rest.split(':').next()?.parse().ok()
4077}
4078
4079fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4080 format!("{session_id}:{offset}:{checked_at_ms}")
4081}
4082
4083fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4084 let Some(checked_at_ms) = watermark
4085 .rsplit_once(':')
4086 .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4087 else {
4088 return false;
4089 };
4090 let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4091 let ttl_ms = if offset > 0 {
4092 SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4093 } else {
4094 SESSION_HISTORY_REFRESH_TTL_MS
4095 };
4096 now_ms.saturating_sub(checked_at_ms) < ttl_ms
4097}
4098
4099async fn history_frame_has_existing_counterpart(
4100 inner: &AggregatorInner,
4101 frame: &NewConsoleFrame,
4102) -> ConsoleLogResult<bool> {
4103 let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4104 let Some(fingerprint) = fingerprint else {
4105 return Ok(false);
4106 };
4107 let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4108 let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4109 let mut after = None;
4110 loop {
4111 let page = inner
4112 .store
4113 .query_frames(ConsoleTimelineQuery {
4114 identity: Some(frame.identity.clone()),
4115 conversation_id: frame.conversation_id.clone(),
4116 after,
4117 limit: 1_000,
4118 })
4119 .await?;
4120 for existing in &page.frames {
4121 let same_session = existing.session_id == frame.session_id
4122 || existing.session_id.is_none()
4123 || frame.session_id.is_none();
4124 if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4125 continue;
4126 }
4127 if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4128 == Some(&fingerprint)
4129 {
4130 return Ok(true);
4131 }
4132 if assistant_terminal
4133 && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4134 {
4135 let turn_key = existing
4136 .interaction_id
4137 .as_deref()
4138 .or(existing.turn_id.as_deref())
4139 .or(existing.run_id.as_deref())
4140 .unwrap_or("session");
4141 let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4142 aggregated.push_str(delta);
4143 if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4144 return Ok(true);
4145 }
4146 }
4147 }
4148 if page.frames.is_empty() || page.next_cursor.is_none() {
4149 return Ok(false);
4150 }
4151 after = page.next_cursor;
4152 }
4153}
4154
4155fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4156 format!("{runtime_key}:session-history:{session_id}")
4157}
4158
4159fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4160 match kind {
4161 "user_input" | "interaction_started" => payload
4162 .get("content")
4163 .map(stable_value_fingerprint)
4164 .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4165 "tool_execution_completed" => tool_result_fingerprint(payload),
4166 "text_delta" => {
4167 text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4168 }
4169 kind if is_reasoning_event_kind(kind) => {
4170 reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4171 }
4172 "text_complete" | "interaction_complete" | "run_completed" => {
4173 assistant_terminal_fingerprint(kind, payload)
4174 }
4175 _ => None,
4176 }
4177}
4178
4179fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4180 let id = payload
4181 .get("tool_call_id")
4182 .or_else(|| payload.get("id"))
4183 .and_then(Value::as_str)
4184 .unwrap_or("");
4185 let result = payload
4186 .get("result")
4187 .or_else(|| payload.get("content"))
4188 .map(stable_value_fingerprint)?;
4189 Some(format!("{id}:{result}"))
4190}
4191
4192fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4193 match kind {
4194 "text_complete" | "interaction_complete" | "run_completed" => payload
4195 .get("text")
4196 .or_else(|| payload.get("result"))
4197 .or_else(|| payload.get("content"))
4198 .map(stable_value_fingerprint),
4199 _ => None,
4200 }
4201}
4202
4203fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4204 if kind != "text_delta" {
4205 return None;
4206 }
4207 payload
4208 .get("delta")
4209 .or_else(|| payload.get("text"))
4210 .or_else(|| payload.get("content"))
4211 .and_then(Value::as_str)
4212 .or_else(|| payload.as_str())
4213}
4214
4215fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4216 if !is_reasoning_event_kind(kind) {
4217 return None;
4218 }
4219 if kind == "reasoning_delta" {
4220 return payload
4221 .get("delta")
4222 .or_else(|| payload.get("text"))
4223 .or_else(|| payload.get("content"))
4224 .or_else(|| payload.get("summary"))
4225 .or_else(|| payload.get("reasoning"))
4226 .and_then(Value::as_str)
4227 .or_else(|| payload.as_str());
4228 }
4229 payload
4230 .get("summary")
4231 .or_else(|| payload.get("text"))
4232 .or_else(|| payload.get("content"))
4233 .or_else(|| payload.get("delta"))
4234 .or_else(|| payload.get("reasoning"))
4235 .and_then(Value::as_str)
4236 .or_else(|| payload.as_str())
4237}
4238
4239fn is_reasoning_event_kind(kind: &str) -> bool {
4240 matches!(
4241 kind,
4242 "reasoning_delta"
4243 | "reasoning_complete"
4244 | "reasoning_summary"
4245 | "reasoning_summary_delta"
4246 | "reasoning_summary_complete"
4247 )
4248}
4249
4250fn stable_value_fingerprint(value: &Value) -> String {
4251 if let Some(text) = content_value_text(value) {
4252 return normalize_transcript_fingerprint_text(&text);
4253 }
4254 match value {
4255 Value::String(text) => normalize_transcript_fingerprint_text(text),
4256 other => serde_json::to_string(other).unwrap_or_default(),
4257 }
4258}
4259
4260fn content_value_text(value: &Value) -> Option<String> {
4261 match value {
4262 Value::String(text) => Some(text.clone()),
4263 Value::Array(items) => {
4264 let text = items
4265 .iter()
4266 .filter_map(content_value_text)
4267 .collect::<String>();
4268 (!text.is_empty()).then_some(text)
4269 }
4270 Value::Object(map) => ["text", "content", "message", "blocks"]
4271 .iter()
4272 .filter_map(|key| map.get(*key))
4273 .find_map(content_value_text),
4274 _ => None,
4275 }
4276}
4277
4278fn normalize_transcript_fingerprint_text(text: &str) -> String {
4279 let trimmed = text.trim();
4280 trimmed
4281 .strip_prefix("[EVENT via rpc] ")
4282 .unwrap_or(trimmed)
4283 .trim()
4284 .to_string()
4285}
4286
4287#[derive(Debug, Clone, Copy)]
4288enum CachedIdentityVisibility {
4289 Visible,
4290 Hidden,
4291 Missing,
4292}
4293
4294async fn frame_is_visible(
4295 inner: &AggregatorInner,
4296 frame: &ConsoleFrame,
4297 allow_historical_identity: bool,
4298 identity_records: &[ConsoleIdentityRecord],
4299) -> ConsoleLogResult<bool> {
4300 let mut identity_visibility_cache = HashMap::new();
4301 Box::pin(frame_is_visible_cached(
4302 inner,
4303 frame,
4304 allow_historical_identity,
4305 &mut identity_visibility_cache,
4306 identity_records,
4307 ))
4308 .await
4309}
4310
4311async fn frame_is_visible_cached(
4312 inner: &AggregatorInner,
4313 frame: &ConsoleFrame,
4314 allow_historical_identity: bool,
4315 identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4316 identity_records: &[ConsoleIdentityRecord],
4317) -> ConsoleLogResult<bool> {
4318 let entry = {
4319 let entries = inner
4320 .runtimes
4321 .read()
4322 .map_err(|_| runtime_registry_lock_error())?;
4323 if entries.is_empty() {
4324 return Ok(true);
4325 }
4326 let Some(entry) = entries.get(&frame.runtime_key) else {
4327 return Ok(false);
4328 };
4329 entry.clone()
4330 };
4331 if frame.identity != "__console__" {
4332 let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4333 let identity_visibility =
4334 if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4335 cached
4336 } else {
4337 let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4338 .unwrap_or_else(|| frame.identity.clone());
4339 let visibility = match identity_records.iter().find(|record| {
4340 record.runtime_key == frame.runtime_key
4341 && (record.identity == frame.identity
4342 || record.runtime_member_id == runtime_member_id)
4343 }) {
4344 Some(record) => {
4345 if Box::pin(console_identity_record_visible(&entry, record)).await {
4346 CachedIdentityVisibility::Visible
4347 } else {
4348 CachedIdentityVisibility::Hidden
4349 }
4350 }
4351 None => CachedIdentityVisibility::Missing,
4352 };
4353 identity_visibility_cache.insert(cache_key, visibility);
4354 visibility
4355 };
4356 match identity_visibility {
4357 CachedIdentityVisibility::Visible => {
4358 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4359 return Ok(false);
4360 }
4361 }
4362 CachedIdentityVisibility::Hidden => return Ok(false),
4363 CachedIdentityVisibility::Missing => {
4364 if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4365 return Ok(false);
4366 }
4367 return Ok(
4368 allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4369 );
4370 }
4371 }
4372 }
4373 Ok(entry.visibility_policy.frame_visible(frame))
4374}
4375
4376async fn identity_record_for_member(
4377 entry: &RuntimeEntry,
4378 handle: &MobHandle,
4379 member: &MobMemberListEntry,
4380) -> Option<ConsoleIdentityRecord> {
4381 let runtime_member_id =
4384 crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4385 let durable_identity = member
4386 .labels
4387 .get("agent_identity")
4388 .filter(|value| !value.trim().is_empty())
4389 .map_or(runtime_member_id.as_str(), String::as_str);
4390 let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4391 let addressable = member_is_addressable(member);
4392 let visibility = match member.status {
4393 meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4394 ConsoleVisibility::RetiredReadable
4395 }
4396 meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4399 ConsoleVisibility::Unreachable
4400 }
4401 _ if addressable => ConsoleVisibility::Addressable,
4402 _ => ConsoleVisibility::Hidden,
4403 };
4404 let session_id = handle
4405 .resolve_bridge_session_id_observation(&member.agent_identity)
4406 .await
4407 .map(|sid| sid.to_string());
4408 let mut labels = member.labels.clone();
4409 labels
4410 .entry("role".to_string())
4411 .or_insert_with(|| member.role.to_string());
4412 crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4418 if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4419 crate::console_spawn::merge_registered_labels(&mut labels, ®istered);
4420 }
4421 let display_name = labels
4422 .get("display_name")
4423 .cloned()
4424 .unwrap_or_else(|| runtime_member_id.clone());
4425 Some(ConsoleIdentityRecord {
4426 identity,
4427 display_name,
4428 runtime_key: entry.runtime_key.clone(),
4429 runtime_member_id,
4430 session_id,
4431 visibility,
4432 addressable,
4433 health: if addressable {
4434 "ready"
4435 } else {
4436 "hidden_by_policy"
4437 }
4438 .to_string(),
4439 topology_peers: member
4440 .wired_to
4441 .iter()
4442 .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4443 .collect(),
4444 labels,
4445 })
4446}
4447
4448async fn identity_record_for_resolved_member(
4449 resolved: &ResolvedConsoleMember,
4450) -> Option<ConsoleIdentityRecord> {
4451 let mut record =
4452 identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4453 if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4454 record
4455 .labels
4456 .entry("source_mob_id".to_string())
4457 .or_insert_with(|| resolved.source_mob_id.clone());
4458 }
4459 Some(record)
4460}
4461
4462fn identity_record_for_status(
4463 entry: &RuntimeEntry,
4464 status: &crate::identity_first::IdentityStatus,
4465) -> ConsoleIdentityRecord {
4466 let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4467 let runtime_member_id = status
4468 .agent_runtime_id
4469 .as_ref()
4470 .map(crate::identity_first::AgentRuntimeId::as_str)
4471 .unwrap_or_else(|| status.identity.as_str())
4472 .to_string();
4473 let addressable = status.addressability
4474 == crate::identity_first::AgentAddressability::Addressable
4475 && matches!(
4476 status.state,
4477 crate::identity_first::IdentityLifecycleState::Active
4478 | crate::identity_first::IdentityLifecycleState::Dormant
4479 | crate::identity_first::IdentityLifecycleState::Uninitialized
4480 );
4481 let visibility = match status.state {
4482 crate::identity_first::IdentityLifecycleState::Retiring => {
4483 ConsoleVisibility::RetiredReadable
4484 }
4485 crate::identity_first::IdentityLifecycleState::Broken
4486 | crate::identity_first::IdentityLifecycleState::Suspended => {
4487 ConsoleVisibility::Unreachable
4488 }
4489 _ if addressable => ConsoleVisibility::Addressable,
4490 _ => ConsoleVisibility::Hidden,
4491 };
4492 let health = match status.state {
4493 crate::identity_first::IdentityLifecycleState::Active => "ready",
4494 crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4495 crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4496 crate::identity_first::IdentityLifecycleState::Broken => "broken",
4497 crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4498 crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4499 }
4500 .to_string();
4501 let display_name = status
4502 .display_name
4503 .as_ref()
4504 .map(crate::identity_first::DisplayName::as_str)
4505 .unwrap_or_else(|| status.identity.as_str())
4506 .to_string();
4507 ConsoleIdentityRecord {
4508 identity,
4509 display_name,
4510 runtime_key: entry.runtime_key.clone(),
4511 runtime_member_id,
4512 session_id: status.session_id.as_ref().map(ToString::to_string),
4513 visibility,
4514 addressable,
4515 health,
4516 topology_peers: Vec::new(),
4517 labels: status.labels.clone(),
4518 }
4519}
4520
4521pub(crate) fn is_implicit_delegate_member(
4522 role: &str,
4523 labels: &std::collections::BTreeMap<String, String>,
4524) -> bool {
4525 role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4526}
4527
4528fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4529 member
4530 .labels
4531 .get("addressable")
4532 .map(|value| !value.eq_ignore_ascii_case("false"))
4533 .unwrap_or(true)
4534}
4535
4536fn apply_namespace(identity: &str, namespace: &str) -> String {
4537 let namespace = namespace.trim().trim_matches('/');
4538 if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4539 identity.to_string()
4540 } else {
4541 format!("{namespace}/{identity}")
4542 }
4543}
4544
4545fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4546 let namespace = namespace.trim().trim_matches('/');
4547 if namespace.is_empty() {
4548 return Some(identity.to_string());
4549 }
4550 identity
4551 .strip_prefix(namespace)
4552 .and_then(|rest| rest.strip_prefix('/'))
4553 .map(ToString::to_string)
4554}
4555
4556fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4557 let namespace = namespace.trim().trim_matches('/');
4558 if namespace.is_empty() {
4559 return vec![identity.to_string()];
4560 }
4561 let mut candidates = Vec::new();
4562 if let Some(stripped) = strip_namespace(identity, namespace) {
4563 candidates.push(stripped);
4564 candidates.push(identity.to_string());
4565 }
4566 candidates.sort();
4567 candidates.dedup();
4568 candidates
4569}
4570
4571fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4572 namespace_match_candidates(identity, namespace)
4573 .iter()
4574 .any(|candidate| candidate.starts_with("rt:"))
4575}
4576
4577fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4578 entries.iter().any(|entry| {
4579 requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4580 })
4581}
4582
4583fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4584 if request.identity.trim().is_empty() {
4585 return Err(ConsoleSendError::InvalidRequest(
4586 "identity must be non-empty".to_string(),
4587 ));
4588 }
4589 if request.origin.trim().is_empty() {
4590 return Err(ConsoleSendError::InvalidRequest(
4591 "origin must be non-empty".to_string(),
4592 ));
4593 }
4594 if request.idempotency_key.trim().is_empty() {
4595 return Err(ConsoleSendError::InvalidRequest(
4596 "idempotency_key must be non-empty".to_string(),
4597 ));
4598 }
4599 Ok(())
4600}
4601
4602fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4603 let content: ContentInput = serde_json::from_value(value.clone())
4604 .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4605 match &content {
4606 ContentInput::Text(text) if text.trim().is_empty() => Err(
4607 ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4608 ),
4609 ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4610 "content blocks must be non-empty".to_string(),
4611 )),
4612 _ => Ok(content),
4613 }
4614}
4615
4616fn parse_handling_mode(
4617 value: Option<&str>,
4618) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4619 match value.unwrap_or("queue") {
4620 "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4621 "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4622 other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4623 }
4624}
4625
4626fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4627 ConsoleInteractionAccepted {
4628 interaction_id: frame
4629 .interaction_id
4630 .clone()
4631 .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4632 identity: frame.identity.clone(),
4633 conversation_id: frame.conversation_id.clone(),
4634 session_id: frame.session_id.clone(),
4635 input_frame_id: frame.id.clone(),
4636 cursor: frame.cursor.clone(),
4637 status: frame.status,
4638 }
4639}
4640
4641fn send_dedupe_key(
4642 runtime_key: &str,
4643 identity: &str,
4644 origin: &str,
4645 idempotency_key: &str,
4646) -> String {
4647 format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4648}
4649
4650fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4651 let content_json = serde_json::to_string(content).unwrap_or_default();
4652 hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4653}
4654
4655fn hash_short(value: &str) -> String {
4656 let mut hasher = Sha256::new();
4657 hasher.update(value.as_bytes());
4658 let digest = hasher.finalize();
4659 to_hex(&digest[..8])
4660}
4661
4662fn to_hex(bytes: &[u8]) -> String {
4663 const HEX: &[u8; 16] = b"0123456789abcdef";
4664 let mut out = String::with_capacity(bytes.len() * 2);
4665 for byte in bytes {
4666 out.push(HEX[(byte >> 4) as usize] as char);
4667 out.push(HEX[(byte & 0x0f) as usize] as char);
4668 }
4669 out
4670}
4671
4672fn current_time_ms() -> u64 {
4673 match SystemTime::now().duration_since(UNIX_EPOCH) {
4674 Ok(duration) => duration.as_millis() as u64,
4675 Err(_) => 0,
4676 }
4677}
4678
4679fn runtime_registry_lock_error() -> ConsoleLogError {
4680 Box::new(std::io::Error::other(
4681 "console runtime registry lock poisoned",
4682 ))
4683}
4684
4685#[cfg(test)]
4686#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4687mod tests {
4688 #[test]
4693 fn backfill_gap_dedupe_key_is_stable_per_session() {
4694 use super::backfill_gap_dedupe_key;
4695 let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4696 let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4697 assert_eq!(a, b, "same target must reuse one gap frame");
4698 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4699 assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4700 assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4703 }
4704
4705 use std::sync::Arc;
4706 use std::sync::atomic::{AtomicUsize, Ordering};
4707 use std::time::Duration;
4708 use std::time::Instant;
4709
4710 use futures::StreamExt;
4711 use meerkat::{AgentFactory, Config, build_ephemeral_service};
4712 use meerkat_client::types::LlmStream;
4713 use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4714 use meerkat_core::{
4715 AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4716 RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4717 SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4718 SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4719 StreamError,
4720 };
4721 use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4722 use serde_json::json;
4723
4724 use super::*;
4725 use crate::mob_handle_runtime::MobBootstrapSpec;
4726
4727 #[derive(Debug)]
4728 struct HideHiddenNoiseFrames;
4729
4730 impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4731 fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4732 frame.kind != "hidden_noise"
4733 }
4734 }
4735
4736 #[derive(Debug)]
4737 struct HideRuntimeMemberIdentity(&'static str);
4738
4739 impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4740 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4741 record.runtime_member_id != self.0
4742 }
4743 }
4744
4745 #[derive(Debug)]
4746 struct HideRuntimeMemberOnly(&'static str);
4747
4748 impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4749 fn member_visible(&self, member: &ConsoleMember) -> bool {
4750 member.agent_identity != self.0
4751 }
4752 }
4753
4754 #[derive(Debug)]
4755 struct HideConsoleIdentity(&'static str);
4756
4757 impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4758 fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4759 record.identity != self.0
4760 }
4761 }
4762
4763 struct CountingConsoleLogStore {
4764 inner: InMemoryConsoleLogStore,
4765 source_watermark_calls: AtomicUsize,
4766 record_watermark_calls: AtomicUsize,
4767 }
4768
4769 impl CountingConsoleLogStore {
4770 fn new() -> Self {
4771 Self {
4772 inner: InMemoryConsoleLogStore::new(),
4773 source_watermark_calls: AtomicUsize::new(0),
4774 record_watermark_calls: AtomicUsize::new(0),
4775 }
4776 }
4777
4778 fn source_watermark_calls(&self) -> usize {
4779 self.source_watermark_calls.load(Ordering::SeqCst)
4780 }
4781 }
4782
4783 struct SlowTestClient {
4784 delay: Duration,
4785 }
4786
4787 #[async_trait::async_trait]
4788 impl LlmClient for SlowTestClient {
4789 fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4790 Ok(messages.to_vec())
4791 }
4792
4793 fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4794 let delay = self.delay;
4795 let delayed_text = futures::stream::once(async move {
4796 tokio::time::sleep(delay).await;
4797 Ok(LlmEvent::TextDelta {
4798 delta: "slow ok".to_string(),
4799 meta: None,
4800 })
4801 });
4802 let done = futures::stream::once(async {
4803 Ok(LlmEvent::Done {
4804 outcome: LlmDoneOutcome::Success {
4805 stop_reason: StopReason::EndTurn,
4806 },
4807 })
4808 });
4809 Box::pin(delayed_text.chain(done))
4810 }
4811
4812 fn provider(&self) -> meerkat_core::Provider {
4813 meerkat_core::Provider::Other
4814 }
4815
4816 async fn health_check(&self) -> Result<(), LlmError> {
4817 Ok(())
4818 }
4819 }
4820
4821 #[derive(Clone)]
4822 struct DelayedHistorySessionService {
4823 inner: Arc<dyn MobSessionService>,
4824 delay: Duration,
4825 read_calls: Arc<AtomicUsize>,
4826 active_reads: Arc<AtomicUsize>,
4827 max_active_reads: Arc<AtomicUsize>,
4828 }
4829
4830 impl DelayedHistorySessionService {
4831 fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4832 Self {
4833 inner,
4834 delay,
4835 read_calls: Arc::new(AtomicUsize::new(0)),
4836 active_reads: Arc::new(AtomicUsize::new(0)),
4837 max_active_reads: Arc::new(AtomicUsize::new(0)),
4838 }
4839 }
4840
4841 fn read_calls(&self) -> usize {
4842 self.read_calls.load(Ordering::SeqCst)
4843 }
4844
4845 fn max_active_reads(&self) -> usize {
4846 self.max_active_reads.load(Ordering::SeqCst)
4847 }
4848 }
4849
4850 #[async_trait::async_trait]
4851 impl SessionService for DelayedHistorySessionService {
4852 async fn create_session(
4853 &self,
4854 req: meerkat_core::CreateSessionRequest,
4855 ) -> Result<RunResult, SessionError> {
4856 self.inner.create_session(req).await
4857 }
4858
4859 async fn start_turn(
4860 &self,
4861 id: &SessionId,
4862 req: StartTurnRequest,
4863 ) -> Result<RunResult, SessionError> {
4864 self.inner.start_turn(id, req).await
4865 }
4866
4867 async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4868 self.inner.interrupt(id).await
4869 }
4870
4871 async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4872 self.inner.read(id).await
4873 }
4874
4875 async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4876 self.inner.list(query).await
4877 }
4878
4879 async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4880 self.inner.archive(id).await
4881 }
4882
4883 async fn subscribe_session_events(
4884 &self,
4885 id: &SessionId,
4886 ) -> Result<EventStream, StreamError> {
4887 SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4888 }
4889 }
4890
4891 #[async_trait::async_trait]
4892 impl SessionServiceCommsExt for DelayedHistorySessionService {
4893 async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4894 self.inner.comms_runtime(session_id).await
4895 }
4896 }
4897
4898 #[async_trait::async_trait]
4899 impl SessionServiceControlExt for DelayedHistorySessionService {
4900 async fn append_system_context(
4901 &self,
4902 id: &SessionId,
4903 req: AppendSystemContextRequest,
4904 ) -> Result<AppendSystemContextResult, SessionControlError> {
4905 self.inner.append_system_context(id, req).await
4906 }
4907 }
4908
4909 #[async_trait::async_trait]
4910 impl SessionServiceHistoryExt for DelayedHistorySessionService {
4911 async fn read_history(
4912 &self,
4913 id: &SessionId,
4914 query: SessionHistoryQuery,
4915 ) -> Result<SessionHistoryPage, SessionError> {
4916 self.read_calls.fetch_add(1, Ordering::SeqCst);
4917 let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4918 self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4919 tokio::time::sleep(self.delay).await;
4920 let result = self.inner.read_history(id, query).await;
4921 self.active_reads.fetch_sub(1, Ordering::SeqCst);
4922 result
4923 }
4924 }
4925
4926 #[async_trait::async_trait]
4927 impl MobSessionService for DelayedHistorySessionService {
4928 fn supports_persistent_sessions(&self) -> bool {
4929 self.inner.supports_persistent_sessions()
4930 }
4931
4932 fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4933 self.inner.runtime_adapter()
4934 }
4935
4936 async fn session_belongs_to_mob(
4937 &self,
4938 session_id: &SessionId,
4939 mob_id: &meerkat_mob::MobId,
4940 ) -> bool {
4941 self.inner.session_belongs_to_mob(session_id, mob_id).await
4942 }
4943
4944 async fn cancel_all_checkpointers(&self) {
4945 self.inner.cancel_all_checkpointers().await;
4946 }
4947
4948 async fn rearm_all_checkpointers(&self) {
4949 self.inner.rearm_all_checkpointers().await;
4950 }
4951 }
4952
4953 #[async_trait::async_trait]
4954 impl ConsoleLogStore for CountingConsoleLogStore {
4955 async fn append_if_absent(
4956 &self,
4957 frame: NewConsoleFrame,
4958 ) -> ConsoleLogResult<AppendOutcome> {
4959 self.inner.append_if_absent(frame).await
4960 }
4961
4962 async fn update_frame_status(
4963 &self,
4964 frame_id: &str,
4965 status: ConsoleFrameStatus,
4966 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4967 self.inner.update_frame_status(frame_id, status).await
4968 }
4969
4970 async fn query_frames(
4971 &self,
4972 query: ConsoleTimelineQuery,
4973 ) -> ConsoleLogResult<ConsoleTimelinePage> {
4974 self.inner.query_frames(query).await
4975 }
4976
4977 async fn query_windowed_frames(
4978 &self,
4979 query: ConsoleTimelineWindowQuery,
4980 ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4981 self.inner.query_windowed_frames(query).await
4982 }
4983
4984 async fn frame_by_dedupe_key(
4985 &self,
4986 dedupe_key: &str,
4987 ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4988 self.inner.frame_by_dedupe_key(dedupe_key).await
4989 }
4990
4991 async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4992 self.inner.latest_cursor().await
4993 }
4994
4995 async fn clear_frames(&self) -> ConsoleLogResult<()> {
4996 self.inner.clear_frames().await
4997 }
4998
4999 async fn record_source_watermark(
5000 &self,
5001 runtime_key: &str,
5002 source_kind: ConsoleFrameSourceKind,
5003 source_cursor: &str,
5004 ) -> ConsoleLogResult<()> {
5005 self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5006 self.inner
5007 .record_source_watermark(runtime_key, source_kind, source_cursor)
5008 .await
5009 }
5010
5011 async fn source_watermark(
5012 &self,
5013 runtime_key: &str,
5014 source_kind: ConsoleFrameSourceKind,
5015 ) -> ConsoleLogResult<Option<String>> {
5016 self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5017 self.inner.source_watermark(runtime_key, source_kind).await
5018 }
5019 }
5020
5021 async fn build_single_member_runtime() -> UnifiedRuntime {
5022 build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5023 }
5024
5025 async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5026 let definition = MobDefinition::from_toml(
5027 r#"
5028[mob]
5029id = "console-aggregator-perf-test"
5030
5031[profiles.worker]
5032model = "gpt-5.5"
5033external_addressable = true
5034
5035[profiles.worker.tools]
5036comms = true
5037"#,
5038 )
5039 .expect("definition parses");
5040 let runtime = UnifiedRuntime::builder()
5041 .definition(definition)
5042 .default_llm_client(client)
5043 .build()
5044 .await
5045 .expect("runtime builds");
5046 runtime
5047 .spawn(SpawnMemberSpec::from_wire(
5048 "worker".to_string(),
5049 "agent-a".to_string(),
5050 Some("You are agent-a.".into()),
5051 None,
5052 None,
5053 ))
5054 .await
5055 .expect("member spawns");
5056 runtime
5057 }
5058
5059 async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5060 let definition = MobDefinition::from_toml(&format!(
5061 r#"
5062[mob]
5063id = "{mob_id}"
5064
5065[profiles.worker]
5066model = "gpt-5.5"
5067external_addressable = true
5068
5069[profiles.worker.tools]
5070comms = true
5071"#
5072 ))
5073 .expect("definition parses");
5074 UnifiedRuntime::builder()
5075 .definition(definition)
5076 .default_llm_client(Arc::new(TestClient::default()))
5077 .build()
5078 .await
5079 .expect("runtime builds")
5080 }
5081
5082 async fn build_stress_runtime(
5083 member_count: usize,
5084 history_delay: Duration,
5085 ) -> (
5086 tempfile::TempDir,
5087 Arc<UnifiedRuntime>,
5088 DelayedHistorySessionService,
5089 ) {
5090 let temp_dir = tempfile::tempdir().expect("temp dir");
5091 let session_path = temp_dir.path().join("sessions");
5092 std::fs::create_dir_all(&session_path).expect("session path");
5093 let factory = AgentFactory::new(&session_path).comms(true);
5094 let base_service = Arc::new(build_ephemeral_service(
5095 factory,
5096 Config::default(),
5097 member_count + 8,
5098 ));
5099 let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5100 let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5101 let definition = MobDefinition::from_toml(
5102 r#"
5103[mob]
5104id = "console-aggregator-stress-test"
5105
5106[profiles.worker]
5107model = "gpt-5.5"
5108external_addressable = true
5109
5110[profiles.worker.tools]
5111comms = true
5112"#,
5113 )
5114 .expect("definition parses");
5115 let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5116 .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5117 allow_ephemeral_sessions: true,
5118 notify_orchestrator_on_resume: true,
5119 default_llm_client: Some(Arc::new(TestClient::default())),
5120 });
5121 let runtime = Arc::new(
5122 UnifiedRuntime::bootstrap(
5123 spec,
5124 crate::types::MobKitConfig {
5125 modules: Vec::new(),
5126 discovery: crate::types::DiscoverySpec {
5127 namespace: "stress".to_string(),
5128 modules: Vec::new(),
5129 },
5130 pre_spawn: Vec::new(),
5131 },
5132 Duration::from_secs(5),
5133 )
5134 .await
5135 .expect("runtime boots"),
5136 );
5137 for idx in 0..member_count {
5138 runtime
5139 .spawn(SpawnMemberSpec::from_wire(
5140 "worker".to_string(),
5141 format!("agent-{idx}"),
5142 Some(format!("You are agent-{idx}.").into()),
5143 None,
5144 None,
5145 ))
5146 .await
5147 .expect("member spawns");
5148 }
5149 (temp_dir, runtime, delayed_service)
5150 }
5151
5152 fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5153 RuntimeEntry {
5154 runtime_key: runtime_key.to_string(),
5155 identity_namespace: "test".to_string(),
5156 runtime: runtime.mob_runtime().clone(),
5157 identity_runtime: runtime.identity_runtime().cloned(),
5158 console_events: runtime.console_events(),
5159 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5160 }
5161 }
5162
5163 fn console_envelope_for_test(
5164 event_id: &str,
5165 interaction_id: Option<&str>,
5166 event_type: &str,
5167 data: Value,
5168 ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5169 crate::console_contracts::ConsoleIdentityEventEnvelope {
5170 event_id: event_id.to_string(),
5171 interaction_id: interaction_id.map(ToString::to_string),
5172 identity: "agent-a".to_string(),
5173 event_type: event_type.to_string(),
5174 timestamp_ms: 1_000,
5175 data,
5176 }
5177 }
5178
5179 fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5180 ConsoleIdentityRecord {
5181 identity: identity.to_string(),
5182 display_name: identity.to_string(),
5183 runtime_key: "runtime-cache".to_string(),
5184 runtime_member_id: identity.to_string(),
5185 session_id: Some(format!("session-{identity}")),
5186 visibility: ConsoleVisibility::Addressable,
5187 addressable: true,
5188 health: "ready".to_string(),
5189 topology_peers: Vec::new(),
5190 labels: BTreeMap::new(),
5191 }
5192 }
5193
5194 #[test]
5195 fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5196 let mut durable = identity_record_for_test("test/review:singleton");
5197 durable.runtime_member_id = "rt:review:singleton:1".to_string();
5198 durable.session_id = Some("session-stale".to_string());
5199 durable
5200 .labels
5201 .insert("agent_identity".to_string(), "review:singleton".to_string());
5202
5203 let mut live = identity_record_for_test("test/review:singleton");
5204 live.runtime_member_id = "rt:review:singleton:0".to_string();
5205 live.session_id = Some("session-live".to_string());
5206 live.labels
5207 .insert("agent_identity".to_string(), "review:singleton".to_string());
5208
5209 let records = dedupe_identity_records(vec![durable, live]);
5210
5211 assert_eq!(
5212 records.len(),
5213 2,
5214 "list projection must expose stale durable/live alias split-brain instead of picking one record"
5215 );
5216 assert!(
5217 records
5218 .iter()
5219 .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5220 );
5221 assert!(
5222 records
5223 .iter()
5224 .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5225 );
5226 }
5227
5228 #[test]
5229 fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5230 let mut first = identity_record_for_test("test/review:singleton");
5231 first.runtime_member_id = "rt:review:singleton:0".to_string();
5232 first
5233 .labels
5234 .insert("agent_identity".to_string(), "review:singleton".to_string());
5235
5236 let mut second = identity_record_for_test("test/review:singleton");
5237 second.runtime_member_id = "rt:review:singleton:1".to_string();
5238 second
5239 .labels
5240 .insert("agent_identity".to_string(), "review:singleton".to_string());
5241
5242 let records = dedupe_identity_records(vec![first, second]);
5243
5244 assert_eq!(
5245 records.len(),
5246 2,
5247 "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5248 );
5249 }
5250
5251 #[test]
5252 fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5253 let mut first = identity_record_for_test("test/review:singleton");
5254 first.runtime_member_id = "rt:review:singleton:0".to_string();
5255 first.session_id = None;
5256 first
5257 .labels
5258 .insert("agent_identity".to_string(), "review:singleton".to_string());
5259 first
5260 .labels
5261 .insert("source_mob_id".to_string(), "mob-a".to_string());
5262
5263 let mut second = first.clone();
5264 second
5265 .labels
5266 .insert("source_mob_id".to_string(), "mob-b".to_string());
5267
5268 let records = dedupe_identity_records(vec![first, second]);
5269
5270 assert_eq!(
5271 records.len(),
5272 2,
5273 "same member/session aliases from different source mobs must remain visible"
5274 );
5275 }
5276
5277 #[test]
5278 fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5279 let durable = identity_record_for_test("test/review:singleton");
5280 let live = durable.clone();
5281
5282 let records = dedupe_identity_records(vec![durable, live]);
5283
5284 assert_eq!(
5285 records.len(),
5286 1,
5287 "healthy durable and primary live projections should not duplicate the roster"
5288 );
5289 }
5290
5291 #[test]
5292 fn stale_durable_record_error_detects_session_split_brain() {
5293 let mut durable = identity_record_for_test("test/review:singleton");
5294 durable.runtime_member_id = "rt:review:singleton:0".to_string();
5295 durable.session_id = Some("session-durable".to_string());
5296
5297 let mut live = durable.clone();
5298 live.session_id = Some("session-live".to_string());
5299
5300 let err = stale_durable_record_error("review:singleton", &durable, &[live])
5301 .expect("session mismatch must be stale split-brain");
5302
5303 assert!(
5304 err.contains("stale durable identity alias"),
5305 "unexpected error: {err}"
5306 );
5307 assert!(
5308 err.contains("session-durable") && err.contains("session-live"),
5309 "error should name both sessions: {err}"
5310 );
5311 }
5312
5313 async fn identity_runtime_for_test(
5314 identities: &[&str],
5315 ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5316 {
5317 let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5318 crate::identity_first::IdentityRuntimeConfig {
5319 continuity_store: Arc::new(
5320 crate::identity_first::LocalContinuityStore::in_memory()?
5321 ),
5322 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5323 runtime_instance_id: "console-aggregator-identity-test".to_string(),
5324 has_runtime_store: true,
5325 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5326 bridge: None,
5327 default_timeout: None,
5328 },
5329 ));
5330 for identity in identities {
5331 let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5332 let record = crate::identity_first::ContinuityRecord {
5333 identity: identity.clone(),
5334 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5335 "rt:{}:0",
5336 identity.as_str()
5337 ))?,
5338 session_id: SessionId::new(),
5339 generation: crate::identity_first::ContinuityGeneration::new(0),
5340 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5341 };
5342 runtime
5343 .register(
5344 crate::identity_first::DurableAgentSpec {
5345 identity: identity.clone(),
5346 profile: meerkat_mob::ProfileName::from("worker"),
5347 addressability: crate::identity_first::AgentAddressability::Addressable,
5348 display_name: None,
5349 labels: BTreeMap::new(),
5350 context: None,
5351 additional_instructions: Vec::new(),
5352 initial_message: None,
5353 runtime_mode_override: None,
5354 backend: None,
5355 binding: None,
5356 },
5357 crate::identity_first::IdentityLifecycleState::Active,
5358 Some(record),
5359 Some(crate::identity_first::LeaseGrant {
5360 identity,
5361 fencing_token: crate::identity_first::FencingToken::new(1),
5362 ttl: Duration::from_mins(5),
5363 }),
5364 )
5365 .await;
5366 }
5367 Ok(runtime)
5368 }
5369
5370 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5371 async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5372 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5373 let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5374 let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5375 let aggregator = MobKitConsoleAggregator::in_memory();
5376 aggregator.register_runtime_handles_with_policy(
5377 "runtime-a",
5378 "",
5379 runtime.mob_runtime().clone(),
5380 Some(identity_runtime),
5381 runtime.console_events(),
5382 Arc::new(AllowAllConsoleVisibilityPolicy),
5383 );
5384
5385 assert!(
5386 aggregator
5387 .inspect_identity("rt:review:singleton:0")
5388 .await?
5389 .is_none(),
5390 "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5391 );
5392 assert!(
5393 !aggregator.retire_identity("rt:review:singleton:0").await?,
5394 "runtime-member-id shaped requests must not retire a durable rt:* identity"
5395 );
5396 let reserve_err = aggregator
5397 .reserve_identity_first_interaction(
5398 ConsoleSendRequest {
5399 identity: "rt:review:singleton:0".to_string(),
5400 content: json!("hello"),
5401 origin: "console:test".to_string(),
5402 idempotency_key: "rt-shadow-reserve".to_string(),
5403 handling_mode: Some("queue".to_string()),
5404 },
5405 None,
5406 )
5407 .await
5408 .expect_err(
5409 "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5410 );
5411 assert!(
5412 reserve_err.to_string().contains("unknown identity"),
5413 "unexpected reserve error: {reserve_err}"
5414 );
5415 let send_err = aggregator
5416 .send(ConsoleSendRequest {
5417 identity: "rt:review:singleton:0".to_string(),
5418 content: json!("hello"),
5419 origin: "console:test".to_string(),
5420 idempotency_key: "rt-shadow-send".to_string(),
5421 handling_mode: Some("queue".to_string()),
5422 })
5423 .await
5424 .expect_err(
5425 "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5426 );
5427 assert!(
5428 send_err.to_string().contains("unknown identity"),
5429 "unexpected send error: {send_err}"
5430 );
5431 let blob_err = match aggregator
5432 .binary_blob_store_for_identity("rt:review:singleton:0")
5433 .await
5434 {
5435 Ok(_) => panic!(
5436 "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5437 ),
5438 Err(err) => err,
5439 };
5440 assert!(
5441 blob_err.to_string().contains("unknown identity"),
5442 "unexpected blob error: {blob_err}"
5443 );
5444
5445 let _ = runtime.mob_handle().stop().await;
5446 Ok(())
5447 }
5448
5449 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5450 async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5451 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5452 let runtime = build_empty_runtime("identity-label-test").await;
5453 let mut labels = BTreeMap::new();
5454 labels.insert(
5455 "agent_identity".to_string(),
5456 "channel:C0SMOKEOB3".to_string(),
5457 );
5458 labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5459 runtime
5460 .spawn(
5461 SpawnMemberSpec::from_wire(
5462 "worker".to_string(),
5463 "rt:channel:C0SMOKEOB3:0".to_string(),
5464 Some("You are C0SMOKEOB3.".into()),
5465 None,
5466 None,
5467 )
5468 .with_labels(labels),
5469 )
5470 .await
5471 .expect("member spawns");
5472
5473 let entry = RuntimeEntry {
5474 runtime_key: "runtime-a".to_string(),
5475 identity_namespace: String::new(),
5476 runtime: runtime.mob_runtime().clone(),
5477 identity_runtime: runtime.identity_runtime().cloned(),
5478 console_events: runtime.console_events(),
5479 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5480 };
5481 let aggregator = MobKitConsoleAggregator::in_memory();
5482 aggregator
5483 .inner
5484 .runtimes
5485 .write()
5486 .expect("runtime registry")
5487 .insert("runtime-a".to_string(), entry);
5488
5489 let records = aggregator.list_identities_fresh().await?;
5490 let record = records
5491 .iter()
5492 .find(|record| record.identity == "channel:C0SMOKEOB3")
5493 .expect("durable identity is exposed");
5494 assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5495
5496 let inspection = aggregator
5497 .inspect_identity("channel:C0SMOKEOB3")
5498 .await?
5499 .expect("durable identity resolves back to runtime member");
5500 assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5501 assert_eq!(
5502 inspection.identity.runtime_member_id,
5503 "rt:channel:C0SMOKEOB3:0"
5504 );
5505
5506 let _ = runtime.mob_handle().stop().await;
5507 Ok(())
5508 }
5509
5510 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5511 async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
5512 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5513 let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
5514 let mut labels = BTreeMap::new();
5515 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5516 runtime
5517 .spawn(
5518 SpawnMemberSpec::from_wire(
5519 "worker".to_string(),
5520 "rt:review:singleton:0".to_string(),
5521 Some("You are the Review Agent.".into()),
5522 None,
5523 None,
5524 )
5525 .with_labels(labels),
5526 )
5527 .await?;
5528 let hidden_session_id = runtime
5529 .mob_handle()
5530 .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
5531 "rt:review:singleton:0",
5532 ))
5533 .await
5534 .map(|sid| sid.to_string());
5535 let mut duplicate_labels = BTreeMap::new();
5536 duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5537 runtime
5538 .spawn(
5539 SpawnMemberSpec::from_wire(
5540 "worker".to_string(),
5541 "rt:review:singleton:1".to_string(),
5542 Some("You are a visible duplicate Review Agent.".into()),
5543 None,
5544 None,
5545 )
5546 .with_labels(duplicate_labels),
5547 )
5548 .await?;
5549
5550 let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5551 let aggregator = MobKitConsoleAggregator::in_memory();
5552 aggregator.register_runtime_handles_with_policy(
5553 "runtime-a",
5554 "",
5555 runtime.mob_runtime().clone(),
5556 Some(identity_runtime),
5557 runtime.console_events(),
5558 Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5559 );
5560
5561 assert!(
5562 aggregator
5563 .inspect_identity("review:singleton")
5564 .await?
5565 .is_none(),
5566 "member-hidden live alias must not inspect through aggregator"
5567 );
5568 let records = aggregator.list_identities_fresh().await?;
5569 assert!(
5570 records
5571 .iter()
5572 .all(|record| record.identity != "review:singleton"),
5573 "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5574 );
5575 let appended = aggregator
5576 .store()
5577 .append_if_absent(NewConsoleFrame {
5578 id: None,
5579 dedupe_key: "member-hidden-frame".to_string(),
5580 timestamp_ms: 1,
5581 runtime_key: "runtime-a".to_string(),
5582 identity: "review:singleton".to_string(),
5583 conversation_id: Some("review:singleton".to_string()),
5584 session_id: hidden_session_id,
5585 kind: "text_delta".to_string(),
5586 status: ConsoleFrameStatus::Delivered,
5587 payload: json!({ "delta": "hidden" }),
5588 source: ConsoleFrameSource {
5589 kind: ConsoleFrameSourceKind::ConsoleEvent,
5590 source_cursor: None,
5591 },
5592 source_event_id: Some("member-hidden-frame".to_string()),
5593 interaction_id: None,
5594 turn_id: None,
5595 run_id: None,
5596 parent_frame_id: None,
5597 caused_by_frame_id: None,
5598 })
5599 .await?;
5600 assert!(
5601 !aggregator
5602 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5603 frame: appended.frame.clone()
5604 })
5605 .await,
5606 "member-hidden live alias must not pass timeline event visibility"
5607 );
5608 let identity_only = aggregator
5609 .store()
5610 .append_if_absent(NewConsoleFrame {
5611 id: None,
5612 dedupe_key: "member-hidden-identity-only-frame".to_string(),
5613 timestamp_ms: 2,
5614 runtime_key: "runtime-a".to_string(),
5615 identity: "review:singleton".to_string(),
5616 conversation_id: Some("review:singleton".to_string()),
5617 session_id: None,
5618 kind: "text_delta".to_string(),
5619 status: ConsoleFrameStatus::Delivered,
5620 payload: json!({ "delta": "identity-only-hidden" }),
5621 source: ConsoleFrameSource {
5622 kind: ConsoleFrameSourceKind::ConsoleEvent,
5623 source_cursor: None,
5624 },
5625 source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5626 interaction_id: None,
5627 turn_id: None,
5628 run_id: None,
5629 parent_frame_id: None,
5630 caused_by_frame_id: None,
5631 })
5632 .await?;
5633 assert!(
5634 !aggregator
5635 .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5636 frame: identity_only.frame.clone()
5637 })
5638 .await,
5639 "member-hidden durable alias must not leak identity-only timeline frames"
5640 );
5641 let page = aggregator
5642 .query_timeline(ConsoleTimelineQuery {
5643 identity: Some("review:singleton".to_string()),
5644 limit: 10,
5645 ..ConsoleTimelineQuery::default()
5646 })
5647 .await?;
5648 assert!(
5649 page.frames.is_empty(),
5650 "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5651 );
5652 assert!(
5653 !aggregator.retire_identity("review:singleton").await?,
5654 "member-hidden live alias must not retire through aggregator"
5655 );
5656 let reserve_err = aggregator
5657 .reserve_identity_first_interaction(
5658 ConsoleSendRequest {
5659 identity: "review:singleton".to_string(),
5660 content: json!("hello"),
5661 origin: "console:test".to_string(),
5662 idempotency_key: "member-hidden-reserve".to_string(),
5663 handling_mode: Some("queue".to_string()),
5664 },
5665 None,
5666 )
5667 .await
5668 .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5669 assert!(
5670 reserve_err.to_string().contains("unknown identity"),
5671 "unexpected reserve error: {reserve_err}"
5672 );
5673 let send_err = aggregator
5674 .send(ConsoleSendRequest {
5675 identity: "review:singleton".to_string(),
5676 content: json!("hello"),
5677 origin: "console:test".to_string(),
5678 idempotency_key: "member-hidden-send".to_string(),
5679 handling_mode: Some("queue".to_string()),
5680 })
5681 .await
5682 .expect_err("member-hidden live alias must not receive sends");
5683 assert!(
5684 send_err.to_string().contains("unknown identity"),
5685 "unexpected send error: {send_err}"
5686 );
5687 let blob_result = aggregator
5688 .binary_blob_store_for_identity("review:singleton")
5689 .await;
5690 let Err(blob_err) = blob_result else {
5691 panic!("member-hidden live alias must not expose blob store");
5692 };
5693 assert!(
5694 blob_err.to_string().contains("unknown identity"),
5695 "unexpected blob error: {blob_err}"
5696 );
5697
5698 let _ = runtime.mob_handle().stop().await;
5699 Ok(())
5700 }
5701
5702 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5703 async fn explicit_session_backfill_target_resolves_durable_identity_label()
5704 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5705 let runtime = build_empty_runtime("identity-label-backfill-test").await;
5706 let mut labels = BTreeMap::new();
5707 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5708 runtime
5709 .spawn(
5710 SpawnMemberSpec::from_wire(
5711 "worker".to_string(),
5712 "rt:review:singleton:0".to_string(),
5713 Some("You are the Review Agent.".into()),
5714 None,
5715 None,
5716 )
5717 .with_labels(labels),
5718 )
5719 .await
5720 .expect("member spawns");
5721
5722 let entry = RuntimeEntry {
5723 runtime_key: "runtime-a".to_string(),
5724 identity_namespace: String::new(),
5725 runtime: runtime.mob_runtime().clone(),
5726 identity_runtime: runtime.identity_runtime().cloned(),
5727 console_events: runtime.console_events(),
5728 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5729 };
5730 let aggregator = MobKitConsoleAggregator::in_memory();
5731 aggregator
5732 .inner
5733 .runtimes
5734 .write()
5735 .expect("runtime registry")
5736 .insert("runtime-a".to_string(), entry);
5737
5738 let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5739 .await
5740 .expect("durable label should resolve targeted session backfill");
5741 assert_eq!(target.record.identity, "review:singleton");
5742 assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5743
5744 let _ = runtime.mob_handle().stop().await;
5745 Ok(())
5746 }
5747
5748 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5749 async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5750 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5751 let runtime = build_empty_runtime("identity-stale-durable-test").await;
5752 let mut labels = BTreeMap::new();
5753 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5754 runtime
5755 .spawn(
5756 SpawnMemberSpec::from_wire(
5757 "worker".to_string(),
5758 "rt:review:singleton:0".to_string(),
5759 Some("You are the live Review Agent.".into()),
5760 None,
5761 None,
5762 )
5763 .with_labels(labels),
5764 )
5765 .await
5766 .expect("member spawns");
5767
5768 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5769 crate::identity_first::IdentityRuntimeConfig {
5770 continuity_store: Arc::new(
5771 crate::identity_first::LocalContinuityStore::in_memory()?
5772 ),
5773 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5774 runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5775 has_runtime_store: true,
5776 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5777 bridge: None,
5778 default_timeout: None,
5779 },
5780 ));
5781 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5782 let record = crate::identity_first::ContinuityRecord {
5783 identity: identity.clone(),
5784 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5785 "rt:review:singleton:1",
5786 )?,
5787 session_id: SessionId::new(),
5788 generation: crate::identity_first::ContinuityGeneration::new(1),
5789 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5790 };
5791 identity_runtime
5792 .register(
5793 crate::identity_first::DurableAgentSpec {
5794 identity: identity.clone(),
5795 profile: meerkat_mob::ProfileName::from("worker"),
5796 addressability: crate::identity_first::AgentAddressability::Addressable,
5797 display_name: None,
5798 labels: BTreeMap::new(),
5799 context: None,
5800 additional_instructions: Vec::new(),
5801 initial_message: None,
5802 runtime_mode_override: None,
5803 backend: None,
5804 binding: None,
5805 },
5806 crate::identity_first::IdentityLifecycleState::Active,
5807 Some(record),
5808 Some(crate::identity_first::LeaseGrant {
5809 identity,
5810 fencing_token: crate::identity_first::FencingToken::new(7),
5811 ttl: Duration::from_mins(5),
5812 }),
5813 )
5814 .await;
5815
5816 let aggregator = MobKitConsoleAggregator::in_memory();
5817 aggregator
5818 .inner
5819 .runtimes
5820 .write()
5821 .expect("runtime registry")
5822 .insert(
5823 "runtime-a".to_string(),
5824 RuntimeEntry {
5825 runtime_key: "runtime-a".to_string(),
5826 identity_namespace: String::new(),
5827 runtime: runtime.mob_runtime().clone(),
5828 identity_runtime: Some(identity_runtime),
5829 console_events: runtime.console_events(),
5830 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5831 },
5832 );
5833
5834 let err = aggregator
5835 .inspect_identity("review:singleton")
5836 .await
5837 .expect_err("stale durable binding must not inspect as healthy");
5838 assert!(
5839 err.to_string().contains("stale durable identity alias"),
5840 "unexpected error: {err}"
5841 );
5842 let retire_err = aggregator
5843 .retire_identity("review:singleton")
5844 .await
5845 .expect_err("stale durable binding must not retire as healthy");
5846 assert!(
5847 retire_err
5848 .to_string()
5849 .contains("stale durable identity alias"),
5850 "unexpected retire error: {retire_err}"
5851 );
5852 let send_err = aggregator
5853 .send(ConsoleSendRequest {
5854 identity: "review:singleton".to_string(),
5855 content: json!("hello"),
5856 origin: "console:test".to_string(),
5857 idempotency_key: "stale-durable-send".to_string(),
5858 handling_mode: Some("queue".to_string()),
5859 })
5860 .await
5861 .expect_err("stale durable binding must not send to the wrong live member");
5862 assert!(
5863 send_err
5864 .to_string()
5865 .contains("stale durable identity alias"),
5866 "unexpected send error: {send_err}"
5867 );
5868 let reserve_err = aggregator
5869 .reserve_identity_first_interaction(
5870 ConsoleSendRequest {
5871 identity: "review:singleton".to_string(),
5872 content: json!("hello"),
5873 origin: "console:test".to_string(),
5874 idempotency_key: "stale-durable-reserve".to_string(),
5875 handling_mode: Some("queue".to_string()),
5876 },
5877 None,
5878 )
5879 .await
5880 .expect_err("identity-first reserve must not bypass stale durable binding");
5881 assert!(
5882 reserve_err
5883 .to_string()
5884 .contains("stale durable identity alias"),
5885 "unexpected reserve error: {reserve_err}"
5886 );
5887 let blob_err = match aggregator
5888 .binary_blob_store_for_identity("review:singleton")
5889 .await
5890 {
5891 Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5892 Err(err) => err,
5893 };
5894 assert!(
5895 blob_err
5896 .to_string()
5897 .contains("stale durable identity alias"),
5898 "unexpected blob error: {blob_err}"
5899 );
5900 let records = aggregator.list_identities_fresh().await?;
5901 assert!(
5902 records
5903 .iter()
5904 .any(|record| record.identity == "review:singleton"
5905 && record.runtime_member_id == "rt:review:singleton:0"),
5906 "live alias should still be listed; records: {records:#?}"
5907 );
5908 assert!(
5909 records
5910 .iter()
5911 .any(|record| record.identity == "review:singleton"
5912 && record.runtime_member_id == "rt:review:singleton:1"),
5913 "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5914 );
5915
5916 let _ = runtime.mob_handle().stop().await;
5917 Ok(())
5918 }
5919
5920 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5921 async fn identity_first_alias_self_heals_session_only_split_brain()
5922 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5923 let runtime = build_empty_runtime("identity-session-rebind-test").await;
5924 let mut labels = BTreeMap::new();
5925 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5926 runtime
5927 .spawn(
5928 SpawnMemberSpec::from_wire(
5929 "worker".to_string(),
5930 "rt:review:singleton:0".to_string(),
5931 Some("You are the live Review Agent.".into()),
5932 None,
5933 None,
5934 )
5935 .with_labels(labels),
5936 )
5937 .await
5938 .expect("member spawns");
5939 let live_session_id = runtime
5940 .mob_handle()
5941 .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
5942 "rt:review:singleton:0",
5943 ))
5944 .await
5945 .expect("live member has bridge session");
5946
5947 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5948 crate::identity_first::IdentityRuntimeConfig {
5949 continuity_store: Arc::new(
5950 crate::identity_first::LocalContinuityStore::in_memory()?
5951 ),
5952 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5953 runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
5954 has_runtime_store: true,
5955 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5956 bridge: None,
5957 default_timeout: None,
5958 },
5959 ));
5960 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5961 let stale_session_id = SessionId::new();
5962 assert_ne!(stale_session_id, live_session_id);
5963 let record = crate::identity_first::ContinuityRecord {
5964 identity: identity.clone(),
5965 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5966 "rt:review:singleton:0",
5967 )?,
5968 session_id: stale_session_id,
5969 generation: crate::identity_first::ContinuityGeneration::new(1),
5970 checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
5971 };
5972 identity_runtime
5973 .register(
5974 crate::identity_first::DurableAgentSpec {
5975 identity: identity.clone(),
5976 profile: meerkat_mob::ProfileName::from("worker"),
5977 addressability: crate::identity_first::AgentAddressability::Addressable,
5978 display_name: None,
5979 labels: BTreeMap::new(),
5980 context: None,
5981 additional_instructions: Vec::new(),
5982 initial_message: None,
5983 runtime_mode_override: None,
5984 backend: None,
5985 binding: None,
5986 },
5987 crate::identity_first::IdentityLifecycleState::Active,
5988 Some(record),
5989 Some(crate::identity_first::LeaseGrant {
5990 identity: identity.clone(),
5991 fencing_token: crate::identity_first::FencingToken::new(7),
5992 ttl: Duration::from_mins(5),
5993 }),
5994 )
5995 .await;
5996
5997 let aggregator = MobKitConsoleAggregator::in_memory();
5998 aggregator
5999 .inner
6000 .runtimes
6001 .write()
6002 .expect("runtime registry")
6003 .insert(
6004 "runtime-a".to_string(),
6005 RuntimeEntry {
6006 runtime_key: "runtime-a".to_string(),
6007 identity_namespace: String::new(),
6008 runtime: runtime.mob_runtime().clone(),
6009 identity_runtime: Some(identity_runtime.clone()),
6010 console_events: runtime.console_events(),
6011 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6012 },
6013 );
6014
6015 aggregator
6016 .reserve_identity_first_interaction(
6017 ConsoleSendRequest {
6018 identity: "review:singleton".to_string(),
6019 content: json!("hello"),
6020 origin: "console:test".to_string(),
6021 idempotency_key: "session-rebind-reserve".to_string(),
6022 handling_mode: Some("queue".to_string()),
6023 },
6024 None,
6025 )
6026 .await
6027 .expect("session-only split-brain should self-heal before reserve");
6028
6029 let rebound_status = identity_runtime.status(&identity).await?;
6030 assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6031
6032 let inspection = aggregator
6033 .inspect_identity("review:singleton")
6034 .await?
6035 .expect("rebound identity should inspect");
6036 let live_session_string = live_session_id.to_string();
6037 assert_eq!(
6038 inspection.identity.session_id.as_deref(),
6039 Some(live_session_string.as_str())
6040 );
6041
6042 let _ = runtime.mob_handle().stop().await;
6043 Ok(())
6044 }
6045
6046 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6047 async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6048 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6049 let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6050 let mut labels = BTreeMap::new();
6051 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6052 runtime
6053 .spawn(
6054 SpawnMemberSpec::from_wire(
6055 "worker".to_string(),
6056 "rt:review:singleton:0".to_string(),
6057 Some("You are the mislabeled Review Agent.".into()),
6058 None,
6059 None,
6060 )
6061 .with_labels(labels),
6062 )
6063 .await
6064 .expect("member spawns");
6065
6066 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6067 crate::identity_first::IdentityRuntimeConfig {
6068 continuity_store: Arc::new(
6069 crate::identity_first::LocalContinuityStore::in_memory()?
6070 ),
6071 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6072 runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6073 has_runtime_store: true,
6074 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6075 bridge: None,
6076 default_timeout: None,
6077 },
6078 ));
6079 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6080 let record = crate::identity_first::ContinuityRecord {
6081 identity: identity.clone(),
6082 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6083 "rt:review:singleton:0",
6084 )?,
6085 session_id: SessionId::new(),
6086 generation: crate::identity_first::ContinuityGeneration::new(1),
6087 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6088 };
6089 identity_runtime
6090 .register(
6091 crate::identity_first::DurableAgentSpec {
6092 identity: identity.clone(),
6093 profile: meerkat_mob::ProfileName::from("worker"),
6094 addressability: crate::identity_first::AgentAddressability::Addressable,
6095 display_name: None,
6096 labels: BTreeMap::new(),
6097 context: None,
6098 additional_instructions: Vec::new(),
6099 initial_message: None,
6100 runtime_mode_override: None,
6101 backend: None,
6102 binding: None,
6103 },
6104 crate::identity_first::IdentityLifecycleState::Active,
6105 Some(record),
6106 Some(crate::identity_first::LeaseGrant {
6107 identity,
6108 fencing_token: crate::identity_first::FencingToken::new(7),
6109 ttl: Duration::from_mins(5),
6110 }),
6111 )
6112 .await;
6113
6114 let aggregator = MobKitConsoleAggregator::in_memory();
6115 aggregator
6116 .inner
6117 .runtimes
6118 .write()
6119 .expect("runtime registry")
6120 .insert(
6121 "runtime-a".to_string(),
6122 RuntimeEntry {
6123 runtime_key: "runtime-a".to_string(),
6124 identity_namespace: String::new(),
6125 runtime: runtime.mob_runtime().clone(),
6126 identity_runtime: Some(identity_runtime),
6127 console_events: runtime.console_events(),
6128 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6129 },
6130 );
6131
6132 let err = aggregator
6133 .inspect_identity("review:singleton")
6134 .await
6135 .expect_err("wrong projected live identity must not inspect as healthy");
6136 assert!(
6137 err.to_string()
6138 .contains("projects identity other:singleton"),
6139 "unexpected error: {err}"
6140 );
6141 let retire_err = aggregator
6142 .retire_identity("review:singleton")
6143 .await
6144 .expect_err("wrong projected live identity must not retire as healthy");
6145 assert!(
6146 retire_err
6147 .to_string()
6148 .contains("projects identity other:singleton"),
6149 "unexpected retire error: {retire_err}"
6150 );
6151 let send_err = aggregator
6152 .send(ConsoleSendRequest {
6153 identity: "review:singleton".to_string(),
6154 content: json!("hello"),
6155 origin: "console:test".to_string(),
6156 idempotency_key: "wrong-projection-send".to_string(),
6157 handling_mode: Some("queue".to_string()),
6158 })
6159 .await
6160 .expect_err("wrong projected live identity must not send as unknown");
6161 assert!(
6162 send_err
6163 .to_string()
6164 .contains("projects identity other:singleton"),
6165 "unexpected send error: {send_err}"
6166 );
6167 let reserve_err = aggregator
6168 .reserve_identity_first_interaction(
6169 ConsoleSendRequest {
6170 identity: "review:singleton".to_string(),
6171 content: json!("hello"),
6172 origin: "console:test".to_string(),
6173 idempotency_key: "wrong-projection-reserve".to_string(),
6174 handling_mode: Some("queue".to_string()),
6175 },
6176 None,
6177 )
6178 .await
6179 .expect_err("identity-first reserve must not bypass wrong live projection");
6180 assert!(
6181 reserve_err
6182 .to_string()
6183 .contains("projects identity other:singleton"),
6184 "unexpected reserve error: {reserve_err}"
6185 );
6186 let blob_err = match aggregator
6187 .binary_blob_store_for_identity("review:singleton")
6188 .await
6189 {
6190 Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6191 Err(err) => err,
6192 };
6193 assert!(
6194 blob_err
6195 .to_string()
6196 .contains("projects identity other:singleton"),
6197 "unexpected blob error: {blob_err}"
6198 );
6199 let projected_err = aggregator
6200 .inspect_identity("other:singleton")
6201 .await
6202 .expect_err("wrong projected alias must not inspect through live-only fallback");
6203 assert!(
6204 projected_err
6205 .to_string()
6206 .contains("identity runtime binding for review:singleton"),
6207 "unexpected projected-alias error: {projected_err}"
6208 );
6209 let projected_retire_err = aggregator
6210 .retire_identity("other:singleton")
6211 .await
6212 .expect_err("wrong projected alias must not retire through live-only fallback");
6213 assert!(
6214 projected_retire_err
6215 .to_string()
6216 .contains("identity runtime binding for review:singleton"),
6217 "unexpected projected-alias retire error: {projected_retire_err}"
6218 );
6219
6220 let _ = runtime.mob_handle().stop().await;
6221 Ok(())
6222 }
6223
6224 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6225 async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6226 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6227 let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6228 let mut labels = BTreeMap::new();
6229 labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6230 runtime
6231 .spawn(
6232 SpawnMemberSpec::from_wire(
6233 "worker".to_string(),
6234 "rt:review:singleton:0".to_string(),
6235 Some("You are a hidden wrong projection.".into()),
6236 None,
6237 None,
6238 )
6239 .with_labels(labels),
6240 )
6241 .await
6242 .expect("member spawns");
6243
6244 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6245 crate::identity_first::IdentityRuntimeConfig {
6246 continuity_store: Arc::new(
6247 crate::identity_first::LocalContinuityStore::in_memory()?
6248 ),
6249 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6250 runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6251 has_runtime_store: true,
6252 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6253 bridge: None,
6254 default_timeout: None,
6255 },
6256 ));
6257 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6258 identity_runtime
6259 .register(
6260 crate::identity_first::DurableAgentSpec {
6261 identity: identity.clone(),
6262 profile: meerkat_mob::ProfileName::from("worker"),
6263 addressability: crate::identity_first::AgentAddressability::Addressable,
6264 display_name: None,
6265 labels: BTreeMap::new(),
6266 context: None,
6267 additional_instructions: Vec::new(),
6268 initial_message: None,
6269 runtime_mode_override: None,
6270 backend: None,
6271 binding: None,
6272 },
6273 crate::identity_first::IdentityLifecycleState::Active,
6274 Some(crate::identity_first::ContinuityRecord {
6275 identity: identity.clone(),
6276 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6277 "rt:review:singleton:0",
6278 )?,
6279 session_id: SessionId::new(),
6280 generation: crate::identity_first::ContinuityGeneration::new(1),
6281 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6282 }),
6283 Some(crate::identity_first::LeaseGrant {
6284 identity,
6285 fencing_token: crate::identity_first::FencingToken::new(7),
6286 ttl: Duration::from_mins(5),
6287 }),
6288 )
6289 .await;
6290
6291 let aggregator = MobKitConsoleAggregator::in_memory();
6292 aggregator
6293 .inner
6294 .runtimes
6295 .write()
6296 .expect("runtime registry")
6297 .insert(
6298 "runtime-a".to_string(),
6299 RuntimeEntry {
6300 runtime_key: "runtime-a".to_string(),
6301 identity_namespace: String::new(),
6302 runtime: runtime.mob_runtime().clone(),
6303 identity_runtime: Some(identity_runtime),
6304 console_events: runtime.console_events(),
6305 visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6306 },
6307 );
6308
6309 let err = aggregator
6310 .inspect_identity("review:singleton")
6311 .await
6312 .expect_err("hidden wrong projection must still fail durable binding validation");
6313 assert!(
6314 err.to_string()
6315 .contains("projects identity other:singleton"),
6316 "unexpected hidden wrong-projection error: {err}"
6317 );
6318 let reserve_err = aggregator
6319 .reserve_identity_first_interaction(
6320 ConsoleSendRequest {
6321 identity: "review:singleton".to_string(),
6322 content: json!("hello"),
6323 origin: "console:test".to_string(),
6324 idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6325 handling_mode: Some("queue".to_string()),
6326 },
6327 None,
6328 )
6329 .await
6330 .expect_err("reserve must not hide wrong projection from durable validation");
6331 assert!(
6332 reserve_err
6333 .to_string()
6334 .contains("projects identity other:singleton"),
6335 "unexpected reserve error: {reserve_err}"
6336 );
6337
6338 let _ = runtime.mob_handle().stop().await;
6339 Ok(())
6340 }
6341
6342 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6343 async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6344 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6345 let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6346 let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6347 let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6348 crate::identity_first::IdentityRuntimeConfig {
6349 continuity_store: Arc::new(
6350 crate::identity_first::LocalContinuityStore::in_memory()?
6351 ),
6352 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6353 runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6354 has_runtime_store: true,
6355 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6356 bridge: None,
6357 default_timeout: None,
6358 },
6359 ));
6360 let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6361 crate::identity_first::IdentityRuntimeConfig {
6362 continuity_store: Arc::new(
6363 crate::identity_first::LocalContinuityStore::in_memory()?
6364 ),
6365 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6366 runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6367 has_runtime_store: true,
6368 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6369 bridge: None,
6370 default_timeout: None,
6371 },
6372 ));
6373 for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6374 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6375 identity_runtime
6376 .register(
6377 crate::identity_first::DurableAgentSpec {
6378 identity: identity.clone(),
6379 profile: meerkat_mob::ProfileName::from("worker"),
6380 addressability: crate::identity_first::AgentAddressability::Addressable,
6381 display_name: None,
6382 labels: BTreeMap::new(),
6383 context: None,
6384 additional_instructions: Vec::new(),
6385 initial_message: None,
6386 runtime_mode_override: None,
6387 backend: None,
6388 binding: None,
6389 },
6390 crate::identity_first::IdentityLifecycleState::Active,
6391 Some(crate::identity_first::ContinuityRecord {
6392 identity: identity.clone(),
6393 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6394 "rt:review:singleton:0",
6395 )?,
6396 session_id: SessionId::new(),
6397 generation: crate::identity_first::ContinuityGeneration::new(1),
6398 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6399 }),
6400 Some(crate::identity_first::LeaseGrant {
6401 identity,
6402 fencing_token: crate::identity_first::FencingToken::new(7),
6403 ttl: Duration::from_mins(5),
6404 }),
6405 )
6406 .await;
6407 }
6408
6409 let aggregator = MobKitConsoleAggregator::in_memory();
6410 {
6411 let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6412 runtimes.insert(
6413 "a-hidden".to_string(),
6414 RuntimeEntry {
6415 runtime_key: "a-hidden".to_string(),
6416 identity_namespace: String::new(),
6417 runtime: hidden_runtime.mob_runtime().clone(),
6418 identity_runtime: Some(hidden_identity_runtime),
6419 console_events: hidden_runtime.console_events(),
6420 visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6421 },
6422 );
6423 runtimes.insert(
6424 "b-visible".to_string(),
6425 RuntimeEntry {
6426 runtime_key: "b-visible".to_string(),
6427 identity_namespace: String::new(),
6428 runtime: visible_runtime.mob_runtime().clone(),
6429 identity_runtime: Some(visible_identity_runtime),
6430 console_events: visible_runtime.console_events(),
6431 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6432 },
6433 );
6434 }
6435
6436 let inspection = aggregator
6437 .inspect_identity("rt:review:singleton:0")
6438 .await?
6439 .expect("visible runtime-id match should be returned");
6440 assert_eq!(inspection.identity.runtime_key, "b-visible");
6441
6442 let _ = hidden_runtime.mob_handle().stop().await;
6443 let _ = visible_runtime.mob_handle().stop().await;
6444 Ok(())
6445 }
6446
6447 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6448 async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6449 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6450 let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6451 let mut primary_labels = BTreeMap::new();
6452 primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6453 runtime
6454 .spawn(
6455 SpawnMemberSpec::from_wire(
6456 "worker".to_string(),
6457 "rt:review:singleton:0".to_string(),
6458 Some("You are the primary Review Agent.".into()),
6459 None,
6460 None,
6461 )
6462 .with_labels(primary_labels),
6463 )
6464 .await
6465 .expect("primary member spawns");
6466
6467 let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6468 let mut delegate_labels = BTreeMap::new();
6469 delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6470 delegate_runtime
6471 .spawn(
6472 SpawnMemberSpec::from_wire(
6473 "worker".to_string(),
6474 "rt:review:singleton:0".to_string(),
6475 Some("You are the wrong projected Review Agent.".into()),
6476 None,
6477 None,
6478 )
6479 .with_labels(delegate_labels),
6480 )
6481 .await
6482 .expect("delegate member spawns");
6483 runtime
6484 .mob_runtime()
6485 .agent_mob_mcp_state()
6486 .expect("runtime should expose mob MCP state")
6487 .mob_insert_handle(
6488 meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6489 delegate_runtime.mob_handle().clone(),
6490 )
6491 .await;
6492
6493 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6494 crate::identity_first::IdentityRuntimeConfig {
6495 continuity_store: Arc::new(
6496 crate::identity_first::LocalContinuityStore::in_memory()?
6497 ),
6498 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6499 runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
6500 has_runtime_store: true,
6501 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6502 bridge: None,
6503 default_timeout: None,
6504 },
6505 ));
6506 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6507 let record = crate::identity_first::ContinuityRecord {
6508 identity: identity.clone(),
6509 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6510 "rt:review:singleton:0",
6511 )?,
6512 session_id: SessionId::new(),
6513 generation: crate::identity_first::ContinuityGeneration::new(1),
6514 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6515 };
6516 identity_runtime
6517 .register(
6518 crate::identity_first::DurableAgentSpec {
6519 identity: identity.clone(),
6520 profile: meerkat_mob::ProfileName::from("worker"),
6521 addressability: crate::identity_first::AgentAddressability::Addressable,
6522 display_name: None,
6523 labels: BTreeMap::new(),
6524 context: None,
6525 additional_instructions: Vec::new(),
6526 initial_message: None,
6527 runtime_mode_override: None,
6528 backend: None,
6529 binding: None,
6530 },
6531 crate::identity_first::IdentityLifecycleState::Active,
6532 Some(record),
6533 Some(crate::identity_first::LeaseGrant {
6534 identity,
6535 fencing_token: crate::identity_first::FencingToken::new(7),
6536 ttl: Duration::from_mins(5),
6537 }),
6538 )
6539 .await;
6540
6541 let aggregator = MobKitConsoleAggregator::in_memory();
6542 aggregator
6543 .inner
6544 .runtimes
6545 .write()
6546 .expect("runtime registry")
6547 .insert(
6548 "runtime-a".to_string(),
6549 RuntimeEntry {
6550 runtime_key: "runtime-a".to_string(),
6551 identity_namespace: String::new(),
6552 runtime: runtime.mob_runtime().clone(),
6553 identity_runtime: Some(identity_runtime),
6554 console_events: runtime.console_events(),
6555 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6556 },
6557 );
6558
6559 let err = aggregator
6560 .inspect_identity("review:singleton")
6561 .await
6562 .expect_err("cross-source wrong projection must not be hidden by primary source");
6563 assert!(
6564 err.to_string()
6565 .contains("projects identity other:singleton")
6566 || err.to_string().contains("ambiguous live identity alias"),
6567 "unexpected error: {err}"
6568 );
6569
6570 let _ = delegate_runtime.mob_handle().stop().await;
6571 let _ = runtime.mob_handle().stop().await;
6572 Ok(())
6573 }
6574
6575 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6576 async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
6577 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6578 let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
6579 let mut labels = BTreeMap::new();
6580 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6581 runtime
6582 .spawn(
6583 SpawnMemberSpec::from_wire(
6584 "worker".to_string(),
6585 "rt:review:singleton:0".to_string(),
6586 Some("You are the first Review Agent.".into()),
6587 None,
6588 None,
6589 )
6590 .with_labels(labels.clone()),
6591 )
6592 .await
6593 .expect("first member spawns");
6594 runtime
6595 .spawn(
6596 SpawnMemberSpec::from_wire(
6597 "worker".to_string(),
6598 "rt:review:singleton:1".to_string(),
6599 Some("You are the second Review Agent.".into()),
6600 None,
6601 None,
6602 )
6603 .with_labels(labels),
6604 )
6605 .await
6606 .expect("second member spawns");
6607
6608 let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6609 crate::identity_first::IdentityRuntimeConfig {
6610 continuity_store: Arc::new(
6611 crate::identity_first::LocalContinuityStore::in_memory()?
6612 ),
6613 lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6614 runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
6615 has_runtime_store: true,
6616 durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6617 bridge: None,
6618 default_timeout: None,
6619 },
6620 ));
6621 let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6622 identity_runtime
6623 .register(
6624 crate::identity_first::DurableAgentSpec {
6625 identity: identity.clone(),
6626 profile: meerkat_mob::ProfileName::from("worker"),
6627 addressability: crate::identity_first::AgentAddressability::Addressable,
6628 display_name: None,
6629 labels: BTreeMap::new(),
6630 context: None,
6631 additional_instructions: Vec::new(),
6632 initial_message: None,
6633 runtime_mode_override: None,
6634 backend: None,
6635 binding: None,
6636 },
6637 crate::identity_first::IdentityLifecycleState::Active,
6638 Some(crate::identity_first::ContinuityRecord {
6639 identity: identity.clone(),
6640 agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6641 "rt:review:singleton:0",
6642 )?,
6643 session_id: SessionId::new(),
6644 generation: crate::identity_first::ContinuityGeneration::new(1),
6645 checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6646 }),
6647 Some(crate::identity_first::LeaseGrant {
6648 identity,
6649 fencing_token: crate::identity_first::FencingToken::new(8),
6650 ttl: Duration::from_mins(5),
6651 }),
6652 )
6653 .await;
6654
6655 let aggregator = MobKitConsoleAggregator::in_memory();
6656 aggregator
6657 .inner
6658 .runtimes
6659 .write()
6660 .expect("runtime registry")
6661 .insert(
6662 "runtime-a".to_string(),
6663 RuntimeEntry {
6664 runtime_key: "runtime-a".to_string(),
6665 identity_namespace: String::new(),
6666 runtime: runtime.mob_runtime().clone(),
6667 identity_runtime: Some(identity_runtime),
6668 console_events: runtime.console_events(),
6669 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6670 },
6671 );
6672
6673 let err = aggregator
6674 .inspect_identity("review:singleton")
6675 .await
6676 .expect_err("duplicate live alias must not inspect via durable fast path");
6677 assert!(
6678 err.to_string().contains("ambiguous live identity alias"),
6679 "unexpected error: {err}"
6680 );
6681 let retire_err = aggregator
6682 .retire_identity("review:singleton")
6683 .await
6684 .expect_err("duplicate live alias must not retire via durable fast path");
6685 assert!(
6686 retire_err
6687 .to_string()
6688 .contains("ambiguous live identity alias"),
6689 "unexpected retire error: {retire_err}"
6690 );
6691 let runtime_id_err = aggregator
6692 .inspect_identity("rt:review:singleton:0")
6693 .await
6694 .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6695 assert!(
6696 runtime_id_err
6697 .to_string()
6698 .contains("ambiguous live identity alias"),
6699 "unexpected runtime-id inspect error: {runtime_id_err}"
6700 );
6701 let runtime_id_retire_err = aggregator
6702 .retire_identity("rt:review:singleton:0")
6703 .await
6704 .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6705 assert!(
6706 runtime_id_retire_err
6707 .to_string()
6708 .contains("ambiguous live identity alias")
6709 || runtime_id_retire_err
6710 .to_string()
6711 .contains("stale durable identity alias"),
6712 "unexpected runtime-id retire error: {runtime_id_retire_err}"
6713 );
6714
6715 let _ = runtime.mob_handle().stop().await;
6716 Ok(())
6717 }
6718
6719 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6720 async fn inspect_identity_ignores_hidden_live_alias_candidates()
6721 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6722 let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6723 let mut labels = BTreeMap::new();
6724 labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6725 runtime
6726 .spawn(
6727 SpawnMemberSpec::from_wire(
6728 "worker".to_string(),
6729 "rt:review:singleton:0".to_string(),
6730 Some("You are the visible Review Agent.".into()),
6731 None,
6732 None,
6733 )
6734 .with_labels(labels.clone()),
6735 )
6736 .await
6737 .expect("visible member spawns");
6738 runtime
6739 .spawn(
6740 SpawnMemberSpec::from_wire(
6741 "worker".to_string(),
6742 "rt:review:singleton:1".to_string(),
6743 Some("You are hidden maintenance noise.".into()),
6744 None,
6745 None,
6746 )
6747 .with_labels(labels),
6748 )
6749 .await
6750 .expect("hidden member spawns");
6751
6752 let aggregator = MobKitConsoleAggregator::in_memory();
6753 aggregator.register_runtime_handles_with_policy(
6754 "runtime-a",
6755 "",
6756 runtime.mob_runtime().clone(),
6757 None,
6758 runtime.console_events(),
6759 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6760 );
6761
6762 let inspection = aggregator
6763 .inspect_identity("review:singleton")
6764 .await?
6765 .expect("visible live alias should inspect without hidden ambiguity");
6766 assert_eq!(inspection.identity.identity, "review:singleton");
6767 assert_eq!(
6768 inspection.identity.runtime_member_id,
6769 "rt:review:singleton:0"
6770 );
6771
6772 let retired = aggregator.retire_identity("review:singleton").await?;
6773 assert!(
6774 retired,
6775 "visible live alias should retire without hidden ambiguity"
6776 );
6777
6778 let _ = runtime.mob_handle().stop().await;
6779 Ok(())
6780 }
6781
6782 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6783 async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6784 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6785 let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6786 let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6787 let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6788 let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6789
6790 let aggregator = MobKitConsoleAggregator::in_memory();
6791 aggregator.register_runtime_handles_with_policy(
6792 "a-hidden",
6793 "",
6794 hidden_runtime.mob_runtime().clone(),
6795 Some(hidden_identity_runtime),
6796 hidden_runtime.console_events(),
6797 Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6798 );
6799 aggregator.register_runtime_handles_with_policy(
6800 "b-visible",
6801 "",
6802 visible_runtime.mob_runtime().clone(),
6803 Some(visible_identity_runtime),
6804 visible_runtime.console_events(),
6805 Arc::new(AllowAllConsoleVisibilityPolicy),
6806 );
6807
6808 let accepted = aggregator
6809 .reserve_identity_first_interaction(
6810 ConsoleSendRequest {
6811 identity: "review:singleton".to_string(),
6812 content: json!("hello"),
6813 origin: "console:test".to_string(),
6814 idempotency_key: "visible-after-hidden-reserve".to_string(),
6815 handling_mode: Some("queue".to_string()),
6816 },
6817 None,
6818 )
6819 .await?;
6820 let page = aggregator
6821 .query_timeline(ConsoleTimelineQuery {
6822 identity: Some("review:singleton".to_string()),
6823 ..ConsoleTimelineQuery::default()
6824 })
6825 .await?;
6826 let frame = page
6827 .frames
6828 .iter()
6829 .find(|frame| frame.id == accepted.input_frame_id)
6830 .ok_or("accepted frame missing from timeline")?;
6831 assert_eq!(frame.runtime_key, "b-visible");
6832
6833 let _ = hidden_runtime.mob_handle().stop().await;
6834 let _ = visible_runtime.mob_handle().stop().await;
6835 Ok(())
6836 }
6837
6838 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6839 async fn identity_first_list_identities_projects_cached_topology_peers()
6840 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6841 let runtime = build_empty_runtime("identity-topology-cache-test").await;
6842 let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6843 identity_runtime
6844 .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6845 crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6846 crate::identity_first::AgentIdentity::parse("agent:beta")?,
6847 )?])
6848 .await;
6849
6850 let aggregator = MobKitConsoleAggregator::in_memory();
6851 aggregator.register_runtime_handles_with_policy(
6852 "identity-first",
6853 "",
6854 runtime.mob_runtime().clone(),
6855 Some(identity_runtime),
6856 runtime.console_events(),
6857 Arc::new(AllowAllConsoleVisibilityPolicy),
6858 );
6859
6860 let identities = aggregator.list_identities().await?;
6861 let alpha = identities
6862 .iter()
6863 .find(|record| record.identity == "agent:alpha")
6864 .ok_or("agent:alpha identity missing")?;
6865 assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6866 let inspection = aggregator
6867 .inspect_identity("agent:alpha")
6868 .await?
6869 .ok_or("agent:alpha inspection missing")?;
6870 assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6871
6872 let _ = runtime.mob_handle().stop().await;
6873 Ok(())
6874 }
6875
6876 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6877 async fn identity_first_list_identities_includes_member_only_spawned_workers()
6878 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6879 let runtime = build_empty_runtime("identity-member-only-test").await;
6880 let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6881 let aggregator = MobKitConsoleAggregator::in_memory();
6882 aggregator.register_runtime_handles_with_policy(
6883 "identity-first",
6884 "",
6885 runtime.mob_runtime().clone(),
6886 Some(identity_runtime),
6887 runtime.console_events(),
6888 Arc::new(AllowAllConsoleVisibilityPolicy),
6889 );
6890
6891 runtime
6892 .spawn(SpawnMemberSpec::from_wire(
6893 "worker".to_string(),
6894 "agent:beta".to_string(),
6895 Some("You are beta.".into()),
6896 None,
6897 None,
6898 ))
6899 .await?;
6900
6901 let identities = aggregator.list_identities().await?;
6902 assert!(
6903 identities
6904 .iter()
6905 .any(|record| record.identity == "agent:beta"),
6906 "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6907 );
6908
6909 let _ = runtime.mob_handle().stop().await;
6910 Ok(())
6911 }
6912
6913 #[tokio::test]
6914 async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6915 let aggregator = MobKitConsoleAggregator::in_memory();
6916 let record = identity_record_for_test("agent-cached");
6917 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6918 aggregator
6919 .inner
6920 .identity_read_model
6921 .primed
6922 .store(true, Ordering::Release);
6923 let _guard = aggregator
6924 .inner
6925 .identity_read_model
6926 .refresh_lock
6927 .clone()
6928 .lock_owned()
6929 .await;
6930
6931 let identities =
6932 tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6933 .await
6934 .expect("hot identity list should not wait for refresh lock")
6935 .expect("identity list succeeds");
6936
6937 assert_eq!(identities, vec![record]);
6938 }
6939
6940 #[tokio::test]
6941 async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6942 let aggregator = MobKitConsoleAggregator::in_memory();
6943 let guard = aggregator
6944 .inner
6945 .identity_read_model
6946 .refresh_lock
6947 .clone()
6948 .lock_owned()
6949 .await;
6950 let waiter = tokio::spawn({
6951 let aggregator = aggregator.clone();
6952 async move { aggregator.list_identities().await }
6953 });
6954
6955 tokio::time::sleep(Duration::from_millis(20)).await;
6956 assert!(
6957 !waiter.is_finished(),
6958 "cold identity list should wait for the in-flight refresh to finish"
6959 );
6960
6961 let record = identity_record_for_test("agent-primed");
6962 *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6963 aggregator
6964 .inner
6965 .identity_read_model
6966 .primed
6967 .store(true, Ordering::Release);
6968 drop(guard);
6969
6970 let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6971 .await
6972 .expect("cold identity list waiter should resume")
6973 .expect("waiter joins")
6974 .expect("identity list succeeds");
6975 assert_eq!(identities, vec![record]);
6976 }
6977
6978 #[tokio::test]
6979 async fn query_timeline_reads_from_aggregate_store() {
6980 let aggregator = MobKitConsoleAggregator::in_memory();
6981 let frame = NewConsoleFrame {
6982 id: None,
6983 dedupe_key: "event-1".to_string(),
6984 timestamp_ms: 1,
6985 runtime_key: "runtime-a".to_string(),
6986 identity: "agent-a".to_string(),
6987 conversation_id: Some("agent-a".to_string()),
6988 session_id: None,
6989 kind: "text_delta".to_string(),
6990 status: ConsoleFrameStatus::Delivered,
6991 payload: json!({ "delta": "hello" }),
6992 source: ConsoleFrameSource {
6993 kind: ConsoleFrameSourceKind::ConsoleEvent,
6994 source_cursor: None,
6995 },
6996 source_event_id: Some("event-1".to_string()),
6997 interaction_id: None,
6998 turn_id: None,
6999 run_id: None,
7000 parent_frame_id: None,
7001 caused_by_frame_id: None,
7002 };
7003 aggregator
7004 .store()
7005 .append_if_absent(frame)
7006 .await
7007 .expect("append frame");
7008
7009 let page = aggregator
7010 .query_timeline(ConsoleTimelineQuery {
7011 identity: Some("agent-a".to_string()),
7012 limit: 10,
7013 ..ConsoleTimelineQuery::default()
7014 })
7015 .await
7016 .expect("query timeline");
7017 assert_eq!(page.frames.len(), 1);
7018 assert_eq!(page.frames[0].kind, "text_delta");
7019 }
7020
7021 #[tokio::test]
7022 async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7023 let aggregator = MobKitConsoleAggregator::in_memory();
7024 let runtime = build_single_member_runtime().await;
7025 let entry = runtime_entry_for_test("runtime-a", &runtime);
7026 for event_type in ["reasoning_delta", "reasoning_complete"] {
7027 let frame = frame_from_console_event(
7028 &entry,
7029 console_envelope_for_test(
7030 event_type,
7031 Some("interaction-a"),
7032 event_type,
7033 json!({
7034 "delta": "I should inspect the file first.",
7035 "text": "I should inspect the file first.",
7036 "turn_id": "turn-a",
7037 }),
7038 ),
7039 );
7040 aggregator
7041 .store()
7042 .append_if_absent(frame)
7043 .await
7044 .expect("append reasoning frame");
7045 }
7046
7047 let page = aggregator
7048 .query_timeline(ConsoleTimelineQuery {
7049 identity: Some("test/agent-a".to_string()),
7050 limit: 10,
7051 ..ConsoleTimelineQuery::default()
7052 })
7053 .await
7054 .expect("query timeline");
7055
7056 assert_eq!(
7057 page.frames.len(),
7058 1,
7059 "identical reasoning re-emissions in one turn should collapse"
7060 );
7061 assert_eq!(page.frames[0].kind, "reasoning_delta");
7062 assert_eq!(
7063 page.frames[0].dedupe_key,
7064 format!(
7065 "console-reasoning:runtime-a:interaction-a:{}",
7066 hash_short("I should inspect the file first.")
7067 )
7068 );
7069 }
7070
7071 #[tokio::test]
7072 async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7073 let aggregator = MobKitConsoleAggregator::in_memory();
7074 let runtime = build_single_member_runtime().await;
7075 let entry = runtime_entry_for_test("runtime-a", &runtime);
7076 for (event_id, text) in [
7077 ("reasoning-1", "I should inspect the file first."),
7078 ("reasoning-2", "Now I should run the focused test."),
7079 ] {
7080 let frame = frame_from_console_event(
7081 &entry,
7082 console_envelope_for_test(
7083 event_id,
7084 Some("interaction-a"),
7085 "reasoning_complete",
7086 json!({
7087 "text": text,
7088 "turn_id": "turn-a",
7089 }),
7090 ),
7091 );
7092 aggregator
7093 .store()
7094 .append_if_absent(frame)
7095 .await
7096 .expect("append reasoning frame");
7097 }
7098
7099 let page = aggregator
7100 .query_timeline(ConsoleTimelineQuery {
7101 identity: Some("test/agent-a".to_string()),
7102 limit: 10,
7103 ..ConsoleTimelineQuery::default()
7104 })
7105 .await
7106 .expect("query timeline");
7107
7108 assert_eq!(
7109 page.frames.len(),
7110 2,
7111 "distinct same-turn reasoning segments should not collapse"
7112 );
7113 assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7114 }
7115
7116 #[tokio::test]
7117 async fn identity_recent_anchor_respects_query_limit() {
7118 let aggregator = MobKitConsoleAggregator::in_memory();
7119 aggregator
7120 .store()
7121 .append_if_absent(NewConsoleFrame {
7122 id: None,
7123 dedupe_key: "anchored-user-input".to_string(),
7124 timestamp_ms: 1,
7125 runtime_key: "runtime-a".to_string(),
7126 identity: "agent-a".to_string(),
7127 conversation_id: Some("agent-a".to_string()),
7128 session_id: None,
7129 kind: "user_input".to_string(),
7130 status: ConsoleFrameStatus::Delivered,
7131 payload: json!({ "content": "anchor me" }),
7132 source: ConsoleFrameSource {
7133 kind: ConsoleFrameSourceKind::Synthetic,
7134 source_cursor: None,
7135 },
7136 source_event_id: Some("anchored-user-input".to_string()),
7137 interaction_id: Some("turn-a".to_string()),
7138 turn_id: None,
7139 run_id: None,
7140 parent_frame_id: None,
7141 caused_by_frame_id: None,
7142 })
7143 .await
7144 .expect("append anchor");
7145 for idx in 2..=40 {
7146 aggregator
7147 .store()
7148 .append_if_absent(NewConsoleFrame {
7149 id: None,
7150 dedupe_key: format!("noisy-tail-{idx}"),
7151 timestamp_ms: idx,
7152 runtime_key: "runtime-a".to_string(),
7153 identity: "agent-a".to_string(),
7154 conversation_id: Some("agent-a".to_string()),
7155 session_id: None,
7156 kind: "reasoning_delta".to_string(),
7157 status: ConsoleFrameStatus::Delivered,
7158 payload: json!({ "delta": idx }),
7159 source: ConsoleFrameSource {
7160 kind: ConsoleFrameSourceKind::ConsoleEvent,
7161 source_cursor: None,
7162 },
7163 source_event_id: Some(format!("noisy-tail-{idx}")),
7164 interaction_id: Some("turn-a".to_string()),
7165 turn_id: None,
7166 run_id: None,
7167 parent_frame_id: None,
7168 caused_by_frame_id: None,
7169 })
7170 .await
7171 .expect("append noisy tail");
7172 }
7173
7174 let page = aggregator
7175 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7176 identity: Some("agent-a".to_string()),
7177 mode: ConsoleTimelineMode::Recent,
7178 limit: 5,
7179 ..ConsoleTimelineWindowQuery::default()
7180 })
7181 .await
7182 .expect("query recent identity timeline");
7183
7184 assert_eq!(
7185 page.frames.len(),
7186 5,
7187 "identity anchor merge must not exceed the requested limit"
7188 );
7189 assert!(
7190 page.frames
7191 .iter()
7192 .any(|frame| frame.dedupe_key == "anchored-user-input"),
7193 "the bounded result should still retain the useful turn anchor: {:#?}",
7194 page.frames
7195 );
7196 assert_eq!(
7197 page.frames.last().and_then(|frame| frame.cursor.seq()),
7198 Some(40)
7199 );
7200 }
7201
7202 #[tokio::test]
7203 async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7204 let aggregator = MobKitConsoleAggregator::in_memory();
7205 let runtime = build_single_member_runtime().await;
7206 let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7207 entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7208 aggregator
7209 .inner
7210 .runtimes
7211 .write()
7212 .expect("runtime registry")
7213 .insert("runtime-a".to_string(), entry);
7214
7215 for idx in 0..1_500 {
7216 aggregator
7217 .store()
7218 .append_if_absent(NewConsoleFrame {
7219 id: None,
7220 dedupe_key: format!("hidden-gap-{idx}"),
7221 timestamp_ms: idx,
7222 runtime_key: "runtime-a".to_string(),
7223 identity: "agent-a".to_string(),
7224 conversation_id: Some("agent-a".to_string()),
7225 session_id: None,
7226 kind: "hidden_noise".to_string(),
7227 status: ConsoleFrameStatus::Delivered,
7228 payload: json!({ "delta": idx }),
7229 source: ConsoleFrameSource {
7230 kind: ConsoleFrameSourceKind::ConsoleEvent,
7231 source_cursor: None,
7232 },
7233 source_event_id: Some(format!("hidden-gap-{idx}")),
7234 interaction_id: None,
7235 turn_id: None,
7236 run_id: None,
7237 parent_frame_id: None,
7238 caused_by_frame_id: None,
7239 })
7240 .await
7241 .expect("append hidden frame");
7242 }
7243 aggregator
7244 .store()
7245 .append_if_absent(NewConsoleFrame {
7246 id: None,
7247 dedupe_key: "visible-after-gap".to_string(),
7248 timestamp_ms: 1_501,
7249 runtime_key: "runtime-a".to_string(),
7250 identity: "agent-a".to_string(),
7251 conversation_id: Some("agent-a".to_string()),
7252 session_id: None,
7253 kind: "interaction_complete".to_string(),
7254 status: ConsoleFrameStatus::Delivered,
7255 payload: json!({ "text": "visible after hidden gap" }),
7256 source: ConsoleFrameSource {
7257 kind: ConsoleFrameSourceKind::ConsoleEvent,
7258 source_cursor: None,
7259 },
7260 source_event_id: Some("visible-after-gap".to_string()),
7261 interaction_id: Some("turn-visible".to_string()),
7262 turn_id: None,
7263 run_id: None,
7264 parent_frame_id: None,
7265 caused_by_frame_id: None,
7266 })
7267 .await
7268 .expect("append visible frame");
7269
7270 let page = aggregator
7271 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7272 identity: Some("agent-a".to_string()),
7273 mode: ConsoleTimelineMode::Since,
7274 limit: 1,
7275 ..ConsoleTimelineWindowQuery::default()
7276 })
7277 .await
7278 .expect("query timeline");
7279 assert_eq!(page.frames.len(), 1);
7280 assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
7281 assert_eq!(
7282 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7283 Some(1_501)
7284 );
7285 let _ = runtime.mob_handle().stop().await;
7286 }
7287
7288 #[tokio::test]
7289 async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
7290 let aggregator = MobKitConsoleAggregator::in_memory();
7291 for idx in 1..=300 {
7292 aggregator
7293 .store()
7294 .append_if_absent(NewConsoleFrame {
7295 id: None,
7296 dedupe_key: format!("visible-{idx}"),
7297 timestamp_ms: idx,
7298 runtime_key: "runtime-a".to_string(),
7299 identity: "agent-a".to_string(),
7300 conversation_id: Some("agent-a".to_string()),
7301 session_id: None,
7302 kind: "interaction_complete".to_string(),
7303 status: ConsoleFrameStatus::Delivered,
7304 payload: json!({ "text": format!("visible {idx}") }),
7305 source: ConsoleFrameSource {
7306 kind: ConsoleFrameSourceKind::ConsoleEvent,
7307 source_cursor: None,
7308 },
7309 source_event_id: Some(format!("visible-{idx}")),
7310 interaction_id: Some(format!("turn-{idx}")),
7311 turn_id: None,
7312 run_id: None,
7313 parent_frame_id: None,
7314 caused_by_frame_id: None,
7315 })
7316 .await
7317 .expect("append visible frame");
7318 }
7319
7320 let first = aggregator
7321 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7322 identity: Some("agent-a".to_string()),
7323 mode: ConsoleTimelineMode::Since,
7324 limit: 10,
7325 ..ConsoleTimelineWindowQuery::default()
7326 })
7327 .await
7328 .expect("query first page");
7329 assert_eq!(first.frames.len(), 10);
7330 assert_eq!(
7331 first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7332 Some(10)
7333 );
7334
7335 let second = aggregator
7336 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7337 identity: Some("agent-a".to_string()),
7338 mode: ConsoleTimelineMode::Since,
7339 after: first.next_cursor,
7340 limit: 10,
7341 ..ConsoleTimelineWindowQuery::default()
7342 })
7343 .await
7344 .expect("query second page");
7345 assert_eq!(second.frames.len(), 10);
7346 assert_eq!(second.frames[0].dedupe_key, "visible-11");
7347 assert_eq!(
7348 second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7349 Some(20)
7350 );
7351 }
7352
7353 #[tokio::test]
7354 async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
7355 let store = Arc::new(CountingConsoleLogStore::new());
7356 let aggregator = MobKitConsoleAggregator::new(store.clone());
7357 let runtime = build_single_member_runtime().await;
7358 aggregator
7359 .inner
7360 .runtimes
7361 .write()
7362 .expect("runtime registry")
7363 .insert(
7364 "runtime-a".to_string(),
7365 runtime_entry_for_test("runtime-a", &runtime),
7366 );
7367 let inserted = store
7368 .append_if_absent(NewConsoleFrame {
7369 id: None,
7370 dedupe_key: "event-1".to_string(),
7371 timestamp_ms: 1,
7372 runtime_key: "runtime-a".to_string(),
7373 identity: "agent-a".to_string(),
7374 conversation_id: Some("agent-a".to_string()),
7375 session_id: None,
7376 kind: "text_delta".to_string(),
7377 status: ConsoleFrameStatus::Delivered,
7378 payload: json!({ "delta": "hello" }),
7379 source: ConsoleFrameSource {
7380 kind: ConsoleFrameSourceKind::ConsoleEvent,
7381 source_cursor: None,
7382 },
7383 source_event_id: Some("event-1".to_string()),
7384 interaction_id: None,
7385 turn_id: None,
7386 run_id: None,
7387 parent_frame_id: None,
7388 caused_by_frame_id: None,
7389 })
7390 .await
7391 .expect("append frame");
7392
7393 let page = aggregator
7394 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7395 identity: Some("agent-a".to_string()),
7396 mode: ConsoleTimelineMode::Since,
7397 after: Some(inserted.frame.cursor),
7398 limit: 10,
7399 ..ConsoleTimelineWindowQuery::default()
7400 })
7401 .await
7402 .expect("query continuation");
7403
7404 assert!(page.frames.is_empty());
7405 assert_eq!(
7406 store.source_watermark_calls(),
7407 0,
7408 "empty since continuation must not synchronously force session-history backfill"
7409 );
7410 let _ = runtime.mob_handle().stop().await;
7411 }
7412
7413 #[test]
7414 fn legacy_timeline_struct_literals_remain_source_compatible() {
7415 let query = ConsoleTimelineQuery {
7416 identity: Some("agent-a".to_string()),
7417 conversation_id: None,
7418 after: Some(ConsoleCursor::from("console:1")),
7419 limit: 10,
7420 };
7421 let page = ConsoleTimelinePage {
7422 frames: Vec::new(),
7423 next_cursor: query.after.clone(),
7424 };
7425
7426 assert_eq!(query.limit, 10);
7427 assert_eq!(
7428 page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7429 Some(1)
7430 );
7431 }
7432
7433 #[tokio::test]
7434 async fn query_timeline_is_store_local_for_registered_runtimes() {
7435 let store = Arc::new(CountingConsoleLogStore::new());
7436 let aggregator = MobKitConsoleAggregator::new(store.clone());
7437 let runtime = build_single_member_runtime().await;
7438 aggregator
7439 .inner
7440 .runtimes
7441 .write()
7442 .expect("runtime registry")
7443 .insert(
7444 "runtime-a".to_string(),
7445 runtime_entry_for_test("runtime-a", &runtime),
7446 );
7447 store
7448 .append_if_absent(NewConsoleFrame {
7449 id: None,
7450 dedupe_key: "event-1".to_string(),
7451 timestamp_ms: 1,
7452 runtime_key: "runtime-a".to_string(),
7453 identity: "agent-a".to_string(),
7454 conversation_id: Some("agent-a".to_string()),
7455 session_id: None,
7456 kind: "text_delta".to_string(),
7457 status: ConsoleFrameStatus::Delivered,
7458 payload: json!({ "delta": "hello" }),
7459 source: ConsoleFrameSource {
7460 kind: ConsoleFrameSourceKind::ConsoleEvent,
7461 source_cursor: None,
7462 },
7463 source_event_id: Some("event-1".to_string()),
7464 interaction_id: None,
7465 turn_id: None,
7466 run_id: None,
7467 parent_frame_id: None,
7468 caused_by_frame_id: None,
7469 })
7470 .await
7471 .expect("append frame");
7472
7473 let page = tokio::time::timeout(
7474 Duration::from_millis(250),
7475 aggregator.query_timeline(ConsoleTimelineQuery {
7476 identity: Some("agent-a".to_string()),
7477 limit: 10,
7478 ..ConsoleTimelineQuery::default()
7479 }),
7480 )
7481 .await
7482 .expect("timeline query should not wait for session history")
7483 .expect("timeline query succeeds");
7484
7485 assert_eq!(page.frames.len(), 1);
7486 assert_eq!(
7487 store.source_watermark_calls(),
7488 0,
7489 "query_timeline must not synchronously touch session-history watermarks"
7490 );
7491 let _ = runtime.mob_handle().stop().await;
7492 }
7493
7494 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7495 async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
7496 let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
7497 delay: Duration::from_secs(2),
7498 }))
7499 .await;
7500 let aggregator = MobKitConsoleAggregator::in_memory();
7501 aggregator
7502 .inner
7503 .runtimes
7504 .write()
7505 .expect("runtime registry")
7506 .insert(
7507 "runtime-a".to_string(),
7508 runtime_entry_for_test("runtime-a", &runtime),
7509 );
7510
7511 let start = Instant::now();
7512 let accepted = tokio::time::timeout(
7513 Duration::from_millis(300),
7514 aggregator.send(ConsoleSendRequest {
7515 identity: "test/agent-a".to_string(),
7516 content: json!("hello slow agent"),
7517 origin: "console:test".to_string(),
7518 idempotency_key: "nonblocking-send".to_string(),
7519 handling_mode: Some("queue".to_string()),
7520 }),
7521 )
7522 .await
7523 .expect("console send should return once the input is accepted")
7524 .expect("send succeeds");
7525
7526 assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
7527 assert!(
7528 start.elapsed() < Duration::from_secs(1),
7529 "console send should not wait for the delayed LLM turn"
7530 );
7531
7532 wait_for_session_history_text(
7533 &aggregator,
7534 "test/agent-a",
7535 "slow ok",
7536 Duration::from_secs(5),
7537 )
7538 .await
7539 .expect("background dispatch should still complete and project history");
7540 let _ = runtime.mob_handle().stop().await;
7541 }
7542
7543 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7544 async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
7545 {
7546 let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
7547 let aggregator = MobKitConsoleAggregator::in_memory();
7548 aggregator.register_runtime(ConsoleRuntimeRegistration {
7549 runtime_key: "runtime-late".to_string(),
7550 runtime: runtime.clone(),
7551 identity_namespace: "late".to_string(),
7552 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7553 });
7554
7555 runtime
7556 .spawn(SpawnMemberSpec::from_wire(
7557 "worker".to_string(),
7558 "agent-late".to_string(),
7559 Some("You are agent-late.".into()),
7560 None,
7561 None,
7562 ))
7563 .await
7564 .expect("late member spawns");
7565 let session_id = send_message_on_mob_with_mode(
7566 &runtime.mob_handle(),
7567 "agent-late",
7568 ContentInput::Text("hello after registration".to_string()),
7569 meerkat_core::types::HandlingMode::Queue,
7570 )
7571 .await
7572 .expect("direct member send succeeds");
7573 wait_for_identity_record(
7574 &aggregator,
7575 "late/agent-late",
7576 Some(session_id.as_str()),
7577 Duration::from_secs(5),
7578 )
7579 .await?;
7580
7581 wait_for_session_history_text(
7582 &aggregator,
7583 "late/agent-late",
7584 "You are agent-late.",
7585 Duration::from_secs(5),
7586 )
7587 .await?;
7588 let _ = runtime.mob_handle().stop().await;
7589 Ok(())
7590 }
7591
7592 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7593 async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
7594 -> Result<(), String> {
7595 let runtime = build_single_member_runtime().await;
7596 let entry = runtime_entry_for_test("runtime-a", &runtime);
7597 let resolved = member_sources_for_entry(&entry)
7598 .await
7599 .into_iter()
7600 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7601 .expect("agent-a member exists");
7602 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7603 .await
7604 .expect("identity record exists");
7605 let session_id = record.session_id.expect("agent-a has a session");
7606 wait_for_runtime_session_history_text(
7607 &runtime,
7608 &session_id,
7609 "You are agent-a.",
7610 Duration::from_secs(5),
7611 )
7612 .await?;
7613
7614 let aggregator = MobKitConsoleAggregator::in_memory();
7615 aggregator
7616 .inner
7617 .runtimes
7618 .write()
7619 .expect("runtime registry")
7620 .insert("runtime-a".to_string(), entry);
7621 let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
7622 aggregator
7623 .store()
7624 .record_source_watermark(
7625 &watermark_key,
7626 ConsoleFrameSourceKind::SessionHistory,
7627 &format_session_history_watermark(&session_id, 0, current_time_ms()),
7628 )
7629 .await
7630 .expect("record fresh empty watermark");
7631
7632 let page = tokio::time::timeout(
7636 Duration::from_mins(1),
7637 aggregator.query_timeline(ConsoleTimelineQuery {
7638 identity: Some("test/agent-a".to_string()),
7639 limit: 20,
7640 ..ConsoleTimelineQuery::default()
7641 }),
7642 )
7643 .await
7644 .expect("explicit identity query should not stall")
7645 .expect("query succeeds");
7646
7647 if !page.frames.is_empty() {
7648 assert!(
7649 page.frames.iter().any(|frame| {
7650 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7651 && session_history_content_text(frame).as_deref()
7652 == Some("You are agent-a.")
7653 }),
7654 "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
7655 page.frames
7656 );
7657 }
7658 wait_for_session_history_text(
7659 &aggregator,
7660 "test/agent-a",
7661 "You are agent-a.",
7662 Duration::from_secs(5),
7663 )
7664 .await?;
7665 let _ = runtime.mob_handle().stop().await;
7666 Ok(())
7667 }
7668
7669 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7670 async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
7671 {
7672 let store = Arc::new(CountingConsoleLogStore::new());
7673 let aggregator = MobKitConsoleAggregator::new(store.clone());
7674 let runtime = build_single_member_runtime().await;
7675 let entry = runtime_entry_for_test("runtime-a", &runtime);
7676 let resolved = member_sources_for_entry(&entry)
7677 .await
7678 .into_iter()
7679 .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7680 .expect("agent-a member exists");
7681 let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7682 .await
7683 .expect("identity record exists");
7684 let session_id = record.session_id.expect("agent-a has a session");
7685 wait_for_runtime_session_history_text(
7686 &runtime,
7687 &session_id,
7688 "You are agent-a.",
7689 Duration::from_secs(5),
7690 )
7691 .await?;
7692 aggregator
7693 .inner
7694 .runtimes
7695 .write()
7696 .expect("runtime registry")
7697 .insert("runtime-a".to_string(), entry);
7698
7699 store
7700 .append_if_absent(NewConsoleFrame {
7701 id: None,
7702 dedupe_key: "stale-session-history-agent-a".to_string(),
7703 timestamp_ms: 10,
7704 runtime_key: "runtime-a".to_string(),
7705 identity: "test/agent-a".to_string(),
7706 conversation_id: Some("test/agent-a".to_string()),
7707 session_id: Some(session_id.clone()),
7708 kind: "user_input".to_string(),
7709 status: ConsoleFrameStatus::Delivered,
7710 payload: json!({
7711 "text": "stale projected history",
7712 "type": "user_input",
7713 }),
7714 source: ConsoleFrameSource {
7715 kind: ConsoleFrameSourceKind::SessionHistory,
7716 source_cursor: Some("stale-session-history-agent-a".to_string()),
7717 },
7718 source_event_id: Some("stale-session-history-agent-a".to_string()),
7719 interaction_id: None,
7720 turn_id: None,
7721 run_id: None,
7722 parent_frame_id: None,
7723 caused_by_frame_id: None,
7724 })
7725 .await
7726 .expect("append stale history frame");
7727
7728 let fresh_prompt = "fresh prompt after stale history";
7729 let sent_session_id = send_message_on_mob_with_mode(
7730 &runtime.mob_handle(),
7731 "agent-a",
7732 ContentInput::Text(fresh_prompt.to_string()),
7733 meerkat_core::types::HandlingMode::Queue,
7734 )
7735 .await
7736 .expect("direct member send succeeds");
7737 assert_eq!(sent_session_id, session_id);
7738 wait_for_runtime_session_history_text(
7739 &runtime,
7740 &session_id,
7741 fresh_prompt,
7742 Duration::from_secs(5),
7743 )
7744 .await?;
7745
7746 let page = aggregator
7747 .query_timeline(ConsoleTimelineQuery {
7748 identity: Some("test/agent-a".to_string()),
7749 limit: 50,
7750 ..ConsoleTimelineQuery::default()
7751 })
7752 .await
7753 .expect("query child timeline");
7754
7755 assert!(
7756 page.frames.iter().any(|frame| {
7757 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7758 && session_history_content_text(frame).as_deref()
7759 == Some("stale projected history")
7760 }),
7761 "explicit identity query should return existing history before async refresh; frames: {:#?}",
7762 page.frames
7763 );
7764 wait_for_session_history_text(
7765 &aggregator,
7766 "test/agent-a",
7767 fresh_prompt,
7768 Duration::from_secs(5),
7769 )
7770 .await?;
7771 assert!(
7772 store.source_watermark_calls() > 0,
7773 "stale existing session history should force a targeted source refresh"
7774 );
7775 let _ = runtime.mob_handle().stop().await;
7776 Ok(())
7777 }
7778
7779 async fn wait_for_session_history_text(
7780 aggregator: &MobKitConsoleAggregator,
7781 identity: &str,
7782 expected: &str,
7783 timeout: Duration,
7784 ) -> Result<(), String> {
7785 let deadline = Instant::now() + timeout;
7786 let mut observed = Vec::new();
7787 while Instant::now() < deadline {
7788 let _ = aggregator.list_identities().await;
7789 let page = aggregator
7790 .query_timeline(ConsoleTimelineQuery {
7791 identity: Some(identity.to_string()),
7792 limit: 20,
7793 ..ConsoleTimelineQuery::default()
7794 })
7795 .await
7796 .expect("query timeline");
7797 observed = page.frames;
7798 if observed.iter().any(|frame| {
7799 frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7800 && session_history_content_text(frame).as_deref() == Some(expected)
7801 }) {
7802 return Ok(());
7803 }
7804 tokio::time::sleep(Duration::from_millis(25)).await;
7805 }
7806
7807 Err(format!(
7808 "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7809 ))
7810 }
7811
7812 async fn wait_for_session_backfill_target(
7813 aggregator: &MobKitConsoleAggregator,
7814 identity: &str,
7815 timeout: Duration,
7816 ) -> Result<String, String> {
7817 let deadline = Instant::now() + timeout;
7818 while Instant::now() < deadline {
7819 if let Some(target) =
7820 session_backfill_target_for_identity(&aggregator.inner, identity).await
7821 {
7822 return Ok(target.session_id);
7823 }
7824 tokio::time::sleep(Duration::from_millis(25)).await;
7825 }
7826 Err(format!(
7827 "session backfill target for {identity:?} was not resolvable"
7828 ))
7829 }
7830
7831 async fn wait_for_runtime_session_history_text(
7832 runtime: &UnifiedRuntime,
7833 session_id: &str,
7834 expected: &str,
7835 timeout: Duration,
7836 ) -> Result<(), String> {
7837 let deadline = Instant::now() + timeout;
7838 let mut observed = Vec::new();
7839 while Instant::now() < deadline {
7840 let page = runtime
7841 .mob_runtime()
7842 .read_session_history(session_id, 0, Some(20))
7843 .await
7844 .map_err(|err| err.to_string())?;
7845 observed = page.messages;
7846 if observed.iter().enumerate().any(|(idx, message)| {
7847 let Some(message) = serde_json::to_value(message).ok() else {
7848 return false;
7849 };
7850 frames_from_session_history_message(
7851 "runtime-a",
7852 "test/agent-a",
7853 session_id,
7854 idx,
7855 message,
7856 )
7857 .iter()
7858 .any(|frame| {
7859 matches!(
7860 frame.kind.as_str(),
7861 "user_input" | "system_notice" | "interaction_complete"
7862 ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7863 })
7864 }) {
7865 return Ok(());
7866 }
7867 tokio::time::sleep(Duration::from_millis(25)).await;
7868 }
7869
7870 Err(format!(
7871 "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7872 ))
7873 }
7874
7875 async fn wait_for_identity_record(
7876 aggregator: &MobKitConsoleAggregator,
7877 identity: &str,
7878 session_id: Option<&str>,
7879 timeout: Duration,
7880 ) -> Result<(), String> {
7881 let deadline = Instant::now() + timeout;
7882 let mut observed = Vec::new();
7883 while Instant::now() < deadline {
7884 observed = aggregator
7885 .list_identities()
7886 .await
7887 .map_err(|err| err.to_string())?;
7888 if observed.iter().any(|record| {
7889 record.identity == identity && record.session_id.as_deref() == session_id
7890 }) {
7891 return Ok(());
7892 }
7893 tokio::time::sleep(Duration::from_millis(25)).await;
7894 }
7895
7896 Err(format!(
7897 "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7898 ))
7899 }
7900
7901 fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7902 session_history_payload_text(&frame.payload)
7903 }
7904
7905 fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7906 session_history_payload_text(&frame.payload)
7907 }
7908
7909 fn session_history_payload_text(payload: &Value) -> Option<String> {
7910 if let Some(text) = payload.get("text").and_then(Value::as_str) {
7911 return Some(text.to_string());
7912 }
7913 if let Some(text) = payload.get("result").and_then(Value::as_str) {
7914 return Some(text.to_string());
7915 }
7916 if let Some(text) = payload.get("body").and_then(Value::as_str) {
7917 return Some(text.to_string());
7918 }
7919 match payload.get("content")? {
7920 Value::String(text) => Some(text.clone()),
7921 Value::Array(blocks) => Some(
7922 blocks
7923 .iter()
7924 .filter_map(|block| block.get("text").and_then(Value::as_str))
7925 .collect::<Vec<_>>()
7926 .join(""),
7927 ),
7928 _ => None,
7929 }
7930 }
7931
7932 #[tokio::test]
7933 async fn query_timeline_handles_large_store_without_backfill_calls() {
7934 let store = Arc::new(CountingConsoleLogStore::new());
7935 let aggregator = MobKitConsoleAggregator::new(store.clone());
7936 for idx in 0..5_000 {
7937 store
7938 .append_if_absent(NewConsoleFrame {
7939 id: None,
7940 dedupe_key: format!("event-{idx}"),
7941 timestamp_ms: idx,
7942 runtime_key: "runtime-a".to_string(),
7943 identity: "agent-a".to_string(),
7944 conversation_id: Some("agent-a".to_string()),
7945 session_id: Some("session-a".to_string()),
7946 kind: "text_delta".to_string(),
7947 status: ConsoleFrameStatus::Delivered,
7948 payload: json!({ "delta": idx }),
7949 source: ConsoleFrameSource {
7950 kind: ConsoleFrameSourceKind::ConsoleEvent,
7951 source_cursor: None,
7952 },
7953 source_event_id: Some(format!("event-{idx}")),
7954 interaction_id: None,
7955 turn_id: None,
7956 run_id: None,
7957 parent_frame_id: None,
7958 caused_by_frame_id: None,
7959 })
7960 .await
7961 .expect("append frame");
7962 }
7963
7964 let page = aggregator
7965 .query_timeline(ConsoleTimelineQuery {
7966 identity: Some("agent-a".to_string()),
7967 limit: 1_000,
7968 ..ConsoleTimelineQuery::default()
7969 })
7970 .await
7971 .expect("large query");
7972
7973 assert_eq!(page.frames.len(), 1_000);
7974 assert_eq!(store.source_watermark_calls(), 0);
7975 }
7976
7977 #[tokio::test]
7978 async fn query_timeline_rejects_future_cursor_after_store_reset() {
7979 let aggregator = MobKitConsoleAggregator::in_memory();
7980 let err = aggregator
7981 .query_timeline_windowed(ConsoleTimelineWindowQuery {
7982 after: Some(ConsoleCursor::from("console:99")),
7983 limit: 10,
7984 ..ConsoleTimelineWindowQuery::default()
7985 })
7986 .await
7987 .expect_err("future cursor on empty/reset store must be replay-unavailable");
7988
7989 assert!(
7990 err.to_string()
7991 .contains("beyond the current store frontier"),
7992 "unexpected error: {err}"
7993 );
7994 }
7995
7996 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7997 async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7998 let store = Arc::new(CountingConsoleLogStore::new());
7999 let aggregator = MobKitConsoleAggregator::new(store.clone());
8000 let runtime = build_single_member_runtime().await;
8001 aggregator
8002 .inner
8003 .runtimes
8004 .write()
8005 .expect("runtime registry")
8006 .insert(
8007 "runtime-a".to_string(),
8008 runtime_entry_for_test("runtime-a", &runtime),
8009 );
8010 let session_id =
8011 wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8012 .await
8013 .expect("spawned member is resolvable for targeted backfill");
8014 wait_for_runtime_session_history_text(
8015 &runtime,
8016 &session_id,
8017 "You are agent-a.",
8018 Duration::from_secs(5),
8019 )
8020 .await
8021 .expect("spawned member kickoff is readable before live-frame race");
8022 store
8023 .append_if_absent(NewConsoleFrame {
8024 id: None,
8025 dedupe_key: "live-tool-before-history".to_string(),
8026 timestamp_ms: 10,
8027 runtime_key: "runtime-a".to_string(),
8028 identity: "test/agent-a".to_string(),
8029 conversation_id: Some("test/agent-a".to_string()),
8030 session_id: None,
8031 kind: "tool_execution_started".to_string(),
8032 status: ConsoleFrameStatus::Delivered,
8033 payload: json!({
8034 "id": "call-live",
8035 "name": "king_search",
8036 "source_event_type": "tool_execution_started",
8037 "type": "tool_execution_started",
8038 }),
8039 source: ConsoleFrameSource {
8040 kind: ConsoleFrameSourceKind::ConsoleEvent,
8041 source_cursor: Some("live-tool-before-history".to_string()),
8042 },
8043 source_event_id: Some("live-tool-before-history".to_string()),
8044 interaction_id: None,
8045 turn_id: None,
8046 run_id: None,
8047 parent_frame_id: None,
8048 caused_by_frame_id: None,
8049 })
8050 .await
8051 .expect("append live tool frame");
8052
8053 let page = aggregator
8054 .query_timeline(ConsoleTimelineQuery {
8055 identity: Some("test/agent-a".to_string()),
8056 limit: 20,
8057 ..ConsoleTimelineQuery::default()
8058 })
8059 .await
8060 .expect("query child timeline");
8061
8062 assert!(
8063 page.frames
8064 .iter()
8065 .any(|frame| frame.kind == "tool_execution_started"),
8066 "initial explicit child timeline should return existing store frames without waiting for session history"
8067 );
8068
8069 let mut observed_backfill = false;
8070 for _ in 0..80 {
8071 let page = aggregator
8072 .query_timeline(ConsoleTimelineQuery {
8073 identity: Some("test/agent-a".to_string()),
8074 limit: 20,
8075 ..ConsoleTimelineQuery::default()
8076 })
8077 .await
8078 .expect("query child timeline after scheduled backfill");
8079 if page.frames.iter().any(|frame| {
8080 frame.kind == "user_input"
8081 && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8082 && frame.payload.to_string().contains("You are agent-a.")
8083 }) {
8084 observed_backfill = true;
8085 break;
8086 }
8087 tokio::time::sleep(Duration::from_millis(25)).await;
8088 }
8089 assert!(
8090 observed_backfill,
8091 "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8092 );
8093 let _ = runtime.mob_handle().stop().await;
8094 }
8095
8096 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8097 async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8098 let store = Arc::new(CountingConsoleLogStore::new());
8099 let aggregator = MobKitConsoleAggregator::new(store.clone());
8100 let (_temp, runtime, _delayed_service) =
8101 build_stress_runtime(64, Duration::from_millis(0)).await;
8102 aggregator
8103 .inner
8104 .runtimes
8105 .write()
8106 .expect("runtime registry")
8107 .insert(
8108 "runtime-a".to_string(),
8109 runtime_entry_for_test("runtime-a", runtime.as_ref()),
8110 );
8111 for idx in 0..1_000 {
8112 store
8113 .append_if_absent(NewConsoleFrame {
8114 id: None,
8115 dedupe_key: format!("event-{idx}"),
8116 timestamp_ms: idx,
8117 runtime_key: "runtime-a".to_string(),
8118 identity: "test/agent-0".to_string(),
8119 conversation_id: Some("test/agent-0".to_string()),
8120 session_id: Some("session-a".to_string()),
8121 kind: "text_delta".to_string(),
8122 status: ConsoleFrameStatus::Delivered,
8123 payload: json!({ "delta": idx }),
8124 source: ConsoleFrameSource {
8125 kind: ConsoleFrameSourceKind::ConsoleEvent,
8126 source_cursor: None,
8127 },
8128 source_event_id: Some(format!("event-{idx}")),
8129 interaction_id: None,
8130 turn_id: None,
8131 run_id: None,
8132 parent_frame_id: None,
8133 caused_by_frame_id: None,
8134 })
8135 .await
8136 .expect("append frame");
8137 }
8138
8139 let page = tokio::time::timeout(
8146 Duration::from_mins(1),
8147 aggregator.query_timeline(ConsoleTimelineQuery {
8148 identity: Some("test/agent-0".to_string()),
8149 limit: 1_000,
8150 ..ConsoleTimelineQuery::default()
8151 }),
8152 )
8153 .await
8154 .expect("identity timeline query should not hang or rediscover the roster per frame")
8155 .expect("timeline query succeeds");
8156
8157 assert_eq!(page.frames.len(), 1_000);
8158 assert_eq!(store.source_watermark_calls(), 0);
8159 let _ = runtime.mob_handle().stop().await;
8160 }
8161
8162 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8163 async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8164 const MEMBER_COUNT: usize = 32;
8165 let (_temp, runtime, delayed_service) =
8166 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8167 let aggregator = MobKitConsoleAggregator::in_memory();
8168 aggregator
8169 .inner
8170 .runtimes
8171 .write()
8172 .expect("runtime registry")
8173 .insert(
8174 "runtime-stress".to_string(),
8175 runtime_entry_for_test("runtime-stress", &runtime),
8176 );
8177
8178 let started = Instant::now();
8179 aggregator
8180 .refresh_session_history()
8181 .await
8182 .expect("stress refresh");
8183 let elapsed = started.elapsed();
8184
8185 assert!(
8186 delayed_service.read_calls() >= MEMBER_COUNT,
8187 "expected at least one history read per member, saw {}",
8188 delayed_service.read_calls()
8189 );
8190 assert!(
8191 delayed_service.max_active_reads() > 1,
8192 "session history backfill should fan out instead of reading members serially"
8193 );
8194 assert!(
8195 delayed_service.max_active_reads()
8196 <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8197 "session history backfill should respect the default concurrency limit"
8198 );
8199 assert!(
8200 elapsed < Duration::from_millis(600),
8201 "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8202 MEMBER_COUNT * 40
8203 );
8204 let _ = runtime.mob_handle().stop().await;
8205 }
8206
8207 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8208 async fn session_history_backfill_respects_configured_concurrency_limit() {
8209 const MEMBER_COUNT: usize = 16;
8210 let (_temp, runtime, delayed_service) =
8211 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8212 let aggregator =
8213 MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8214 max_concurrent_session_backfills: 4,
8215 ..ConsoleAggregatorOptions::default()
8216 });
8217 aggregator
8218 .inner
8219 .runtimes
8220 .write()
8221 .expect("runtime registry")
8222 .insert(
8223 "runtime-stress".to_string(),
8224 runtime_entry_for_test("runtime-stress", &runtime),
8225 );
8226
8227 aggregator
8228 .refresh_session_history()
8229 .await
8230 .expect("stress refresh");
8231
8232 assert!(
8233 delayed_service.max_active_reads() <= 4,
8234 "configured concurrency limit should bound session history reads, saw {}",
8235 delayed_service.max_active_reads()
8236 );
8237 let _ = runtime.mob_handle().stop().await;
8238 }
8239
8240 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8241 async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8242 const MEMBER_COUNT: usize = 24;
8243 const LIVE_EVENT_COUNT: usize = 2_048;
8244 let (_temp, runtime, _delayed_service) =
8245 build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
8246 let aggregator = MobKitConsoleAggregator::in_memory();
8247 aggregator.register_runtime(ConsoleRuntimeRegistration {
8248 runtime_key: "runtime-burst".to_string(),
8249 runtime: runtime.clone(),
8250 identity_namespace: "stress".to_string(),
8251 visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8252 });
8253 let console_events = runtime.console_events();
8254 for idx in 0..LIVE_EVENT_COUNT {
8255 console_events
8256 .append(
8257 "agent-0",
8258 Some("burst-turn".to_string()),
8259 "text_delta",
8260 json!({ "delta": format!("frame-{idx}") }),
8261 )
8262 .await;
8263 }
8264
8265 let deadline = Instant::now() + Duration::from_secs(30);
8270 let mut observed = 0;
8271 while Instant::now() < deadline {
8272 observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
8273 if observed >= LIVE_EVENT_COUNT {
8274 break;
8275 }
8276 tokio::time::sleep(Duration::from_millis(25)).await;
8277 }
8278
8279 assert_eq!(
8280 observed, LIVE_EVENT_COUNT,
8281 "live pump should not drop frames while slow background backfill is running"
8282 );
8283 let _ = runtime.mob_handle().stop().await;
8284 }
8285
8286 async fn count_console_event_frames(
8287 aggregator: &MobKitConsoleAggregator,
8288 identity: &str,
8289 ) -> usize {
8290 let mut after = None;
8291 let mut count = 0;
8292 loop {
8293 let page = aggregator
8294 .query_timeline(ConsoleTimelineQuery {
8295 identity: Some(identity.to_string()),
8296 after,
8297 limit: 1_000,
8298 ..ConsoleTimelineQuery::default()
8299 })
8300 .await
8301 .expect("query burst timeline");
8302 if page.frames.is_empty() {
8303 break;
8304 }
8305 count += page
8306 .frames
8307 .iter()
8308 .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
8309 .count();
8310 after = page.next_cursor;
8311 if after.is_none() {
8312 break;
8313 }
8314 }
8315 count
8316 }
8317
8318 #[tokio::test]
8319 async fn status_updates_get_replayable_aggregate_cursors() {
8320 let aggregator = MobKitConsoleAggregator::in_memory();
8321 let frame = NewConsoleFrame {
8322 id: None,
8323 dedupe_key: "send-1".to_string(),
8324 timestamp_ms: 1,
8325 runtime_key: "runtime-a".to_string(),
8326 identity: "agent-a".to_string(),
8327 conversation_id: Some("agent-a".to_string()),
8328 session_id: Some("session-1".to_string()),
8329 kind: "user_input".to_string(),
8330 status: ConsoleFrameStatus::Accepted,
8331 payload: json!({ "content": "hello" }),
8332 source: ConsoleFrameSource {
8333 kind: ConsoleFrameSourceKind::Send,
8334 source_cursor: None,
8335 },
8336 source_event_id: None,
8337 interaction_id: Some("interaction-1".to_string()),
8338 turn_id: None,
8339 run_id: None,
8340 parent_frame_id: None,
8341 caused_by_frame_id: None,
8342 };
8343 let inserted = aggregator
8344 .store()
8345 .append_if_absent(frame)
8346 .await
8347 .expect("append frame");
8348
8349 update_frame_status_and_emit(
8350 &aggregator.inner,
8351 &inserted.frame.id,
8352 ConsoleFrameStatus::Delivered,
8353 )
8354 .await
8355 .expect("update status");
8356
8357 let page = aggregator
8358 .query_timeline(ConsoleTimelineQuery {
8359 identity: Some("agent-a".to_string()),
8360 after: Some(inserted.frame.cursor.clone()),
8361 limit: 10,
8362 ..ConsoleTimelineQuery::default()
8363 })
8364 .await
8365 .expect("query timeline");
8366 assert_eq!(page.frames.len(), 1);
8367 assert_eq!(page.frames[0].kind, "frame_updated");
8368 assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
8369 assert_eq!(
8370 page.frames[0]
8371 .payload
8372 .get("frame")
8373 .and_then(|frame| frame.get("status"))
8374 .and_then(Value::as_str),
8375 Some("delivered")
8376 );
8377 }
8378
8379 #[tokio::test]
8380 async fn steer_delivery_appends_terminal_control_frame() {
8381 let aggregator = MobKitConsoleAggregator::in_memory();
8382 let frame = NewConsoleFrame {
8383 id: None,
8384 dedupe_key: "send-steer-1".to_string(),
8385 timestamp_ms: 1,
8386 runtime_key: "runtime-a".to_string(),
8387 identity: "agent-a".to_string(),
8388 conversation_id: Some("agent-a".to_string()),
8389 session_id: Some("session-1".to_string()),
8390 kind: "user_input".to_string(),
8391 status: ConsoleFrameStatus::Delivered,
8392 payload: json!({
8393 "content": "operator steer",
8394 "handling_mode": "steer",
8395 }),
8396 source: ConsoleFrameSource {
8397 kind: ConsoleFrameSourceKind::Send,
8398 source_cursor: None,
8399 },
8400 source_event_id: None,
8401 interaction_id: Some("interaction-steer-1".to_string()),
8402 turn_id: None,
8403 run_id: None,
8404 parent_frame_id: None,
8405 caused_by_frame_id: None,
8406 };
8407 let inserted = aggregator
8408 .store()
8409 .append_if_absent(frame)
8410 .await
8411 .expect("append steer input");
8412
8413 append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
8414 .await
8415 .expect("append steer terminal");
8416
8417 let page = aggregator
8418 .query_timeline(ConsoleTimelineQuery {
8419 identity: Some("agent-a".to_string()),
8420 after: Some(inserted.frame.cursor.clone()),
8421 limit: 10,
8422 ..ConsoleTimelineQuery::default()
8423 })
8424 .await
8425 .expect("query timeline");
8426
8427 assert_eq!(page.frames.len(), 1);
8428 let terminal = &page.frames[0];
8429 assert_eq!(terminal.kind, "interaction_complete");
8430 assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
8431 assert_eq!(
8432 terminal.interaction_id.as_deref(),
8433 Some("interaction-steer-1")
8434 );
8435 assert_eq!(
8436 terminal.parent_frame_id.as_deref(),
8437 Some(inserted.frame.id.as_str())
8438 );
8439 assert_eq!(
8440 terminal.payload.get("reason").and_then(Value::as_str),
8441 Some("steer_delivered")
8442 );
8443 }
8444
8445 #[test]
8446 fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
8447 let live = transcript_fingerprint(
8448 "reasoning_complete",
8449 &json!({ "text": "I should inspect the file first." }),
8450 );
8451 let replay = transcript_fingerprint(
8452 "reasoning_complete",
8453 &json!({ "summary": "I should inspect the file first." }),
8454 );
8455 let delta = transcript_fingerprint(
8456 "reasoning_delta",
8457 &json!({ "delta": "I should inspect the file first." }),
8458 );
8459
8460 assert_eq!(live.as_deref(), Some("I should inspect the file first."));
8461 assert_eq!(live, replay);
8462 assert_eq!(live, delta);
8463 }
8464
8465 #[tokio::test]
8466 async fn history_counterpart_scan_is_not_capped_to_one_page() {
8467 let aggregator = MobKitConsoleAggregator::in_memory();
8468 for idx in 0..1_005 {
8469 aggregator
8470 .store()
8471 .append_if_absent(NewConsoleFrame {
8472 id: None,
8473 dedupe_key: format!("filler-{idx}"),
8474 timestamp_ms: idx,
8475 runtime_key: "runtime-a".to_string(),
8476 identity: "agent-a".to_string(),
8477 conversation_id: Some("agent-a".to_string()),
8478 session_id: Some("session-a".to_string()),
8479 kind: "text_delta".to_string(),
8480 status: ConsoleFrameStatus::Completed,
8481 payload: json!({ "delta": idx }),
8482 source: ConsoleFrameSource {
8483 kind: ConsoleFrameSourceKind::ConsoleEvent,
8484 source_cursor: None,
8485 },
8486 source_event_id: Some(format!("filler-{idx}")),
8487 interaction_id: None,
8488 turn_id: None,
8489 run_id: None,
8490 parent_frame_id: None,
8491 caused_by_frame_id: None,
8492 })
8493 .await
8494 .expect("append filler");
8495 }
8496 aggregator
8497 .store()
8498 .append_if_absent(NewConsoleFrame {
8499 id: None,
8500 dedupe_key: "live-user-input".to_string(),
8501 timestamp_ms: 2_000,
8502 runtime_key: "runtime-a".to_string(),
8503 identity: "agent-a".to_string(),
8504 conversation_id: Some("agent-a".to_string()),
8505 session_id: Some("session-a".to_string()),
8506 kind: "user_input".to_string(),
8507 status: ConsoleFrameStatus::Delivered,
8508 payload: json!({ "content": "already here" }),
8509 source: ConsoleFrameSource {
8510 kind: ConsoleFrameSourceKind::ConsoleEvent,
8511 source_cursor: None,
8512 },
8513 source_event_id: Some("live-user-input".to_string()),
8514 interaction_id: None,
8515 turn_id: None,
8516 run_id: None,
8517 parent_frame_id: None,
8518 caused_by_frame_id: None,
8519 })
8520 .await
8521 .expect("append live input");
8522
8523 let history = NewConsoleFrame {
8524 id: None,
8525 dedupe_key: "history-user-input".to_string(),
8526 timestamp_ms: 3_000,
8527 runtime_key: "runtime-a".to_string(),
8528 identity: "agent-a".to_string(),
8529 conversation_id: Some("agent-a".to_string()),
8530 session_id: Some("session-a".to_string()),
8531 kind: "user_input".to_string(),
8532 status: ConsoleFrameStatus::Completed,
8533 payload: json!({ "content": "already here" }),
8534 source: ConsoleFrameSource {
8535 kind: ConsoleFrameSourceKind::SessionHistory,
8536 source_cursor: Some("session-a:1006".to_string()),
8537 },
8538 source_event_id: None,
8539 interaction_id: None,
8540 turn_id: None,
8541 run_id: None,
8542 parent_frame_id: None,
8543 caused_by_frame_id: None,
8544 };
8545
8546 assert!(
8547 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8548 .await
8549 .expect("counterpart scan")
8550 );
8551 }
8552
8553 #[tokio::test]
8554 async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
8555 let aggregator = MobKitConsoleAggregator::in_memory();
8556 aggregator
8557 .store()
8558 .append_if_absent(NewConsoleFrame {
8559 id: None,
8560 dedupe_key: "live-user-input".to_string(),
8561 timestamp_ms: 2_000,
8562 runtime_key: "runtime-a".to_string(),
8563 identity: "agent-a".to_string(),
8564 conversation_id: Some("agent-a".to_string()),
8565 session_id: Some("session-a".to_string()),
8566 kind: "user_input".to_string(),
8567 status: ConsoleFrameStatus::Delivered,
8568 payload: json!({ "content": "hello from operator" }),
8569 source: ConsoleFrameSource {
8570 kind: ConsoleFrameSourceKind::ConsoleEvent,
8571 source_cursor: None,
8572 },
8573 source_event_id: Some("live-user-input".to_string()),
8574 interaction_id: None,
8575 turn_id: None,
8576 run_id: None,
8577 parent_frame_id: None,
8578 caused_by_frame_id: None,
8579 })
8580 .await
8581 .expect("append live input");
8582
8583 let history = NewConsoleFrame {
8584 id: None,
8585 dedupe_key: "history-user-input".to_string(),
8586 timestamp_ms: 3_000,
8587 runtime_key: "runtime-a".to_string(),
8588 identity: "agent-a".to_string(),
8589 conversation_id: Some("agent-a".to_string()),
8590 session_id: Some("session-a".to_string()),
8591 kind: "user_input".to_string(),
8592 status: ConsoleFrameStatus::Completed,
8593 payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
8594 source: ConsoleFrameSource {
8595 kind: ConsoleFrameSourceKind::SessionHistory,
8596 source_cursor: Some("session-a:2".to_string()),
8597 },
8598 source_event_id: None,
8599 interaction_id: None,
8600 turn_id: None,
8601 run_id: None,
8602 parent_frame_id: None,
8603 caused_by_frame_id: None,
8604 };
8605
8606 assert!(
8607 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8608 .await
8609 .expect("counterpart scan")
8610 );
8611 }
8612
8613 #[tokio::test]
8614 async fn history_counterpart_scan_matches_content_block_user_prompts() {
8615 let aggregator = MobKitConsoleAggregator::in_memory();
8616 aggregator
8617 .store()
8618 .append_if_absent(NewConsoleFrame {
8619 id: None,
8620 dedupe_key: "live-user-input".to_string(),
8621 timestamp_ms: 2_000,
8622 runtime_key: "runtime-a".to_string(),
8623 identity: "agent-a".to_string(),
8624 conversation_id: Some("agent-a".to_string()),
8625 session_id: Some("session-a".to_string()),
8626 kind: "user_input".to_string(),
8627 status: ConsoleFrameStatus::Delivered,
8628 payload: json!({ "content": "hello from operator" }),
8629 source: ConsoleFrameSource {
8630 kind: ConsoleFrameSourceKind::ConsoleEvent,
8631 source_cursor: None,
8632 },
8633 source_event_id: Some("live-user-input".to_string()),
8634 interaction_id: None,
8635 turn_id: None,
8636 run_id: None,
8637 parent_frame_id: None,
8638 caused_by_frame_id: None,
8639 })
8640 .await
8641 .expect("append live input");
8642
8643 let history = NewConsoleFrame {
8644 id: None,
8645 dedupe_key: "history-user-input".to_string(),
8646 timestamp_ms: 3_000,
8647 runtime_key: "runtime-a".to_string(),
8648 identity: "agent-a".to_string(),
8649 conversation_id: Some("agent-a".to_string()),
8650 session_id: Some("session-a".to_string()),
8651 kind: "user_input".to_string(),
8652 status: ConsoleFrameStatus::Completed,
8653 payload: json!({
8654 "content": [{ "type": "text", "text": "hello from operator" }]
8655 }),
8656 source: ConsoleFrameSource {
8657 kind: ConsoleFrameSourceKind::SessionHistory,
8658 source_cursor: Some("session-a:2".to_string()),
8659 },
8660 source_event_id: None,
8661 interaction_id: None,
8662 turn_id: None,
8663 run_id: None,
8664 parent_frame_id: None,
8665 caused_by_frame_id: None,
8666 };
8667
8668 assert!(
8669 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8670 .await
8671 .expect("counterpart scan")
8672 );
8673 }
8674
8675 #[tokio::test]
8676 async fn history_counterpart_scan_matches_live_tool_results() {
8677 let aggregator = MobKitConsoleAggregator::in_memory();
8678 aggregator
8679 .store()
8680 .append_if_absent(NewConsoleFrame {
8681 id: None,
8682 dedupe_key: "live-tool-result".to_string(),
8683 timestamp_ms: 2_000,
8684 runtime_key: "runtime-a".to_string(),
8685 identity: "agent-a".to_string(),
8686 conversation_id: Some("agent-a".to_string()),
8687 session_id: Some("session-a".to_string()),
8688 kind: "tool_execution_completed".to_string(),
8689 status: ConsoleFrameStatus::Delivered,
8690 payload: json!({
8691 "id": "call-1",
8692 "tool_call_id": "call-1",
8693 "result": "{ \"count\": 70 }"
8694 }),
8695 source: ConsoleFrameSource {
8696 kind: ConsoleFrameSourceKind::ConsoleEvent,
8697 source_cursor: None,
8698 },
8699 source_event_id: Some("live-tool-result".to_string()),
8700 interaction_id: None,
8701 turn_id: None,
8702 run_id: None,
8703 parent_frame_id: None,
8704 caused_by_frame_id: None,
8705 })
8706 .await
8707 .expect("append live tool result");
8708
8709 let history = NewConsoleFrame {
8710 id: None,
8711 dedupe_key: "history-tool-result".to_string(),
8712 timestamp_ms: 3_000,
8713 runtime_key: "runtime-a".to_string(),
8714 identity: "agent-a".to_string(),
8715 conversation_id: Some("agent-a".to_string()),
8716 session_id: Some("session-a".to_string()),
8717 kind: "tool_execution_completed".to_string(),
8718 status: ConsoleFrameStatus::Completed,
8719 payload: json!({
8720 "id": "call-1",
8721 "tool_call_id": "call-1",
8722 "result": "{ \"count\": 70 }",
8723 "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
8724 }),
8725 source: ConsoleFrameSource {
8726 kind: ConsoleFrameSourceKind::SessionHistory,
8727 source_cursor: Some("session-a:3:0".to_string()),
8728 },
8729 source_event_id: None,
8730 interaction_id: None,
8731 turn_id: None,
8732 run_id: None,
8733 parent_frame_id: None,
8734 caused_by_frame_id: None,
8735 };
8736
8737 assert!(
8738 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8739 .await
8740 .expect("counterpart scan")
8741 );
8742 }
8743
8744 #[tokio::test]
8745 async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
8746 let aggregator = MobKitConsoleAggregator::in_memory();
8747 for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
8748 aggregator
8749 .store()
8750 .append_if_absent(NewConsoleFrame {
8751 id: None,
8752 dedupe_key: format!("live-delta-{idx}"),
8753 timestamp_ms: 2_000 + idx as u64,
8754 runtime_key: "runtime-a".to_string(),
8755 identity: "agent-a".to_string(),
8756 conversation_id: Some("agent-a".to_string()),
8757 session_id: Some("session-a".to_string()),
8758 kind: "text_delta".to_string(),
8759 status: ConsoleFrameStatus::Delivered,
8760 payload: json!({ "delta": delta }),
8761 source: ConsoleFrameSource {
8762 kind: ConsoleFrameSourceKind::ConsoleEvent,
8763 source_cursor: None,
8764 },
8765 source_event_id: Some(format!("live-delta-{idx}")),
8766 interaction_id: Some("turn-a".to_string()),
8767 turn_id: None,
8768 run_id: None,
8769 parent_frame_id: None,
8770 caused_by_frame_id: None,
8771 })
8772 .await
8773 .expect("append live delta");
8774 }
8775
8776 let history = NewConsoleFrame {
8777 id: None,
8778 dedupe_key: "history-assistant-complete".to_string(),
8779 timestamp_ms: 3_000,
8780 runtime_key: "runtime-a".to_string(),
8781 identity: "agent-a".to_string(),
8782 conversation_id: Some("agent-a".to_string()),
8783 session_id: Some("session-a".to_string()),
8784 kind: "interaction_complete".to_string(),
8785 status: ConsoleFrameStatus::Completed,
8786 payload: json!({ "result": "Ready and standing by." }),
8787 source: ConsoleFrameSource {
8788 kind: ConsoleFrameSourceKind::SessionHistory,
8789 source_cursor: Some("session-a:3".to_string()),
8790 },
8791 source_event_id: None,
8792 interaction_id: None,
8793 turn_id: None,
8794 run_id: None,
8795 parent_frame_id: None,
8796 caused_by_frame_id: None,
8797 };
8798
8799 assert!(
8800 history_frame_has_existing_counterpart(&aggregator.inner, &history)
8801 .await
8802 .expect("counterpart scan")
8803 );
8804 }
8805
8806 #[test]
8807 fn session_history_watermark_key_is_session_scoped() {
8808 assert_ne!(
8809 session_history_watermark_runtime_key("runtime-a", "session-1"),
8810 session_history_watermark_runtime_key("runtime-a", "session-2")
8811 );
8812 }
8813
8814 #[test]
8815 fn backfill_offset_advances_only_on_strict_forward_progress() {
8816 assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
8818 assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
8820 assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
8822 assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
8824
8825 assert_eq!(advance_backfill_offset(500, 0, 500), None); assert_eq!(advance_backfill_offset(500, 0, 400), None); assert_eq!(advance_backfill_offset(1000, 500, 500), None); assert_eq!(advance_backfill_offset(10, 10, 0), None); assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
8834 }
8835
8836 #[test]
8837 fn session_history_watermarks_are_cursor_and_ttl_aware() {
8838 let legacy = "session:with:colon:42";
8839 let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8840 let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8841
8842 assert_eq!(
8843 parse_session_history_watermark(legacy, "session:with:colon"),
8844 Some(42)
8845 );
8846 assert_eq!(
8847 parse_session_history_watermark(&checked, "session:with:colon"),
8848 Some(43)
8849 );
8850 assert!(session_history_watermark_is_fresh(
8851 &checked,
8852 "session:with:colon",
8853 1_500
8854 ));
8855 assert!(!session_history_watermark_is_fresh(
8856 &checked,
8857 "session:with:colon",
8858 1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8859 ));
8860 assert!(session_history_watermark_is_fresh(
8861 &empty_checked,
8862 "session:with:colon",
8863 1_500
8864 ));
8865 assert!(!session_history_watermark_is_fresh(
8866 &empty_checked,
8867 "session:with:colon",
8868 1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8869 ));
8870 }
8871
8872 #[test]
8873 fn session_history_messages_project_to_renderable_frames() {
8874 let user = frame_from_session_history_message(
8875 "runtime-a",
8876 "agent-a",
8877 "session-a",
8878 0,
8879 json!({
8880 "role": "user",
8881 "content": "hello",
8882 "timestamp_ms": 10
8883 }),
8884 )
8885 .expect("user history frame");
8886 let assistant = frame_from_session_history_message(
8891 "runtime-a",
8892 "agent-a",
8893 "session-a",
8894 1,
8895 json!({
8896 "role": "block_assistant",
8897 "blocks": [
8898 { "block_type": "text", "data": { "text": "hi there" } }
8899 ],
8900 "stop_reason": "end_turn",
8901 "timestamp_ms": 11
8902 }),
8903 )
8904 .expect("assistant history frame");
8905
8906 assert_eq!(user.kind, "user_input");
8907 assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8908 assert_eq!(
8909 user.payload["content"],
8910 json!([{ "type": "text", "text": "hello" }])
8911 );
8912 assert_eq!(assistant.kind, "interaction_complete");
8913 assert_eq!(assistant.payload["text"], json!("hi there"));
8914 assert!(
8915 assistant
8916 .dedupe_key
8917 .starts_with("session-history:runtime-a:session-a:1:")
8918 );
8919 }
8920
8921 #[test]
8922 fn session_history_projection_filters_scaffold_user_messages() {
8923 let spawn_notice = frame_from_session_history_message(
8924 "runtime-a",
8925 "agent-a",
8926 "session-a",
8927 0,
8928 json!({
8929 "role": "user",
8930 "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8931 "timestamp_ms": 10
8932 }),
8933 );
8934 let peer_update = frame_from_session_history_message(
8935 "runtime-a",
8936 "agent-a",
8937 "session-a",
8938 1,
8939 json!({
8940 "role": "user",
8941 "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8942 "timestamp_ms": 11
8943 }),
8944 );
8945 let real_user = frame_from_session_history_message(
8946 "runtime-a",
8947 "agent-a",
8948 "session-a",
8949 2,
8950 json!({
8951 "role": "user",
8952 "content": "Please review the incident notes.",
8953 "timestamp_ms": 12
8954 }),
8955 );
8956
8957 assert!(spawn_notice.is_none());
8958 assert!(peer_update.is_none());
8959 assert!(real_user.is_some());
8960 }
8961
8962 #[test]
8963 fn session_history_projection_skips_non_transcript_messages() {
8964 let skipped = frame_from_session_history_message(
8965 "runtime-a",
8966 "agent-a",
8967 "session-a",
8968 0,
8969 json!({
8970 "content": "internal system prompt"
8971 }),
8972 );
8973 assert!(skipped.is_none());
8974 }
8975
8976 #[test]
8977 fn session_history_projection_extracts_assistant_blocks() {
8978 let frame = frame_from_session_history_message(
8979 "runtime-a",
8980 "agent-a",
8981 "session-a",
8982 0,
8983 json!({
8984 "role": "block_assistant",
8985 "blocks": [
8986 { "block_type": "text", "data": { "text": "hello " } },
8987 { "block_type": "text", "data": { "text": "there" } }
8988 ],
8989 "stop_reason": "end_turn"
8990 }),
8991 )
8992 .expect("assistant block history frame");
8993 assert_eq!(frame.payload["text"], json!("hello there"));
8994 }
8995
8996 #[test]
8997 fn session_history_projection_extracts_nested_text_block_data() {
8998 let frame = frame_from_session_history_message(
8999 "runtime-a",
9000 "agent-a",
9001 "session-a",
9002 0,
9003 json!({
9004 "role": "block_assistant",
9005 "blocks": [
9006 {
9007 "block_type": "text",
9008 "data": { "text": "Ready and standing by." }
9009 }
9010 ],
9011 "stop_reason": "end_turn",
9012 "created_at": "1970-01-01T00:00:00.010Z"
9013 }),
9014 )
9015 .expect("assistant block history frame");
9016
9017 assert_eq!(frame.kind, "interaction_complete");
9018 assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9019 }
9020
9021 #[test]
9022 fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9023 let frame = frame_from_session_history_message(
9024 "runtime-a",
9025 "agent-a",
9026 "session-a",
9027 0,
9028 json!({
9029 "role": "block_assistant",
9030 "blocks": [
9031 {
9032 "block_type": "reasoning",
9033 "data": { "text": "**Planning**\nI should not be rendered." }
9034 },
9035 {
9036 "block_type": "text",
9037 "data": { "text": "Visible answer." }
9038 }
9039 ],
9040 "stop_reason": "end_turn",
9041 "created_at": "1970-01-01T00:00:00.010Z"
9042 }),
9043 )
9044 .expect("assistant block history frame");
9045
9046 assert_eq!(frame.kind, "interaction_complete");
9047 assert_eq!(frame.payload["result"], json!("Visible answer."));
9048 assert_eq!(frame.payload["text"], json!("Visible answer."));
9049 }
9050
9051 #[test]
9052 fn session_history_projection_leaves_reasoning_only_result_empty() {
9053 let frame = frame_from_session_history_message(
9054 "runtime-a",
9055 "agent-a",
9056 "session-a",
9057 0,
9058 json!({
9059 "role": "block_assistant",
9060 "blocks": [
9061 {
9062 "block_type": "reasoning",
9063 "data": { "text": "Private planning text." }
9064 },
9065 {
9066 "block_type": "tool_use",
9067 "data": { "id": "toolu-1", "name": "peers", "args": {} }
9068 }
9069 ],
9070 "stop_reason": "end_turn",
9071 "created_at": "1970-01-01T00:00:00.010Z"
9072 }),
9073 )
9074 .expect("assistant block history frame");
9075
9076 assert_eq!(frame.kind, "interaction_complete");
9077 assert_eq!(frame.payload["result"], json!(""));
9078 assert_eq!(frame.payload["text"], json!(""));
9079 }
9080
9081 #[test]
9082 fn session_history_projection_preserves_tool_results() {
9083 let frames = frames_from_session_history_message(
9084 "runtime-a",
9085 "agent-a",
9086 "session-a",
9087 5,
9088 json!({
9089 "role": "tool_results",
9090 "results": [
9091 {
9092 "tool_use_id": "call-peers",
9093 "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9094 "is_error": false
9095 }
9096 ],
9097 "created_at": "1970-01-01T00:00:00.050Z"
9098 }),
9099 );
9100
9101 assert_eq!(frames.len(), 1);
9102 let frame = &frames[0];
9103 assert_eq!(frame.kind, "tool_execution_completed");
9104 assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9105 assert_eq!(
9106 frame.payload["result"],
9107 json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9108 );
9109 assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9110 assert_eq!(frame.timestamp_ms, 50);
9111 }
9112
9113 #[test]
9114 fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9115 let image_result = json!({
9122 "operation_id": "00000000-0000-0000-0000-0000000000aa",
9123 "terminal": { "terminal": "generated" },
9124 "images": [
9125 {
9126 "image_id": "00000000-0000-0000-0000-000000000051",
9127 "blob_ref": {
9128 "blob_id": "sha256:test-generated-image",
9129 "media_type": "image/png"
9130 },
9131 "media_type": "image/png",
9132 "width": 1024,
9133 "height": 768
9134 }
9135 ],
9136 "provider_text": { "disposition": "not_emitted" },
9137 "revised_prompt": { "disposition": "not_requested" },
9138 "native_metadata": { "provider": "not_emitted" }
9139 });
9140 serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9143 image_result.clone(),
9144 )
9145 .expect("fixture must be a valid ImageGenerationToolResult");
9146 let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9147
9148 let frames = frames_from_session_history_message(
9149 "runtime-a",
9150 "artist:singleton",
9151 "session-a",
9152 6,
9153 json!({
9154 "role": "tool_results",
9155 "results": [
9156 {
9157 "tool_use_id": "call-image",
9158 "content": result_text,
9159 "is_error": false
9160 }
9161 ],
9162 "created_at": "1970-01-01T00:00:00.060Z"
9163 }),
9164 );
9165
9166 assert!(
9168 frames
9169 .iter()
9170 .any(|frame| frame.kind == "tool_execution_completed"),
9171 "tool_execution_completed frame must still be present: {frames:#?}"
9172 );
9173 let image_frame = frames
9175 .iter()
9176 .find(|frame| frame.kind == "assistant_image")
9177 .expect("backfill must derive an assistant_image frame");
9178 assert_eq!(
9179 image_frame.payload["blob_id"],
9180 json!("sha256:test-generated-image")
9181 );
9182 assert_eq!(
9183 image_frame.payload["image_id"],
9184 json!("00000000-0000-0000-0000-000000000051")
9185 );
9186 assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9187 assert_eq!(image_frame.payload["width"], json!(1024));
9188 assert_eq!(image_frame.payload["height"], json!(768));
9189 assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9190 assert_eq!(
9191 image_frame.source.kind,
9192 ConsoleFrameSourceKind::SessionHistory
9193 );
9194 assert_eq!(image_frame.timestamp_ms, 60);
9195 }
9196
9197 #[test]
9198 fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9199 let frames = frames_from_session_history_message(
9200 "runtime-a",
9201 "review:singleton",
9202 "session-a",
9203 7,
9204 json!({
9205 "role": "block_assistant",
9206 "blocks": [
9207 {
9208 "block_type": "tool_use",
9209 "data": {
9210 "id": "call-spawn",
9211 "name": "mob_spawn_member",
9212 "args": {
9213 "member_id": "review-worker-alpha",
9214 "profile": "review-worker",
9215 "initial_message": "Review Initiative Alpha and save the result."
9216 }
9217 }
9218 }
9219 ],
9220 "stop_reason": "tool_use",
9221 "created_at": "1970-01-01T00:00:00.070Z"
9222 }),
9223 );
9224
9225 assert_eq!(frames.len(), 2);
9226 let child = frames
9227 .iter()
9228 .find(|frame| frame.identity == "review-worker-alpha")
9229 .expect("spawned member initial message frame");
9230 assert_eq!(child.kind, "user_input");
9231 assert_eq!(child.timestamp_ms, 70);
9232 assert_eq!(
9233 child.payload["message"]["content"],
9234 "Review Initiative Alpha and save the result."
9235 );
9236 assert_eq!(child.payload["parent_identity"], "review:singleton");
9237 assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9238 assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9239 }
9240
9241 #[test]
9242 fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9243 let frames = frames_from_session_history_message(
9244 "runtime-a",
9245 "review:singleton",
9246 "session-a",
9247 8,
9248 json!({
9249 "role": "block_assistant",
9250 "blocks": [
9251 {
9252 "block_type": "tool_use",
9253 "data": {
9254 "id": "call-spawn-many",
9255 "name": "mob_spawn_member",
9256 "args": {
9257 "specs": [
9258 {
9259 "agent_identity": "review-worker-alpha",
9260 "profile": "review-worker",
9261 "initial_message": "Review Initiative Alpha."
9262 },
9263 {
9264 "agent_identity": "review-worker-beta",
9265 "profile": "review-worker",
9266 "initial_message": "Review Initiative Beta."
9267 }
9268 ]
9269 }
9270 }
9271 }
9272 ],
9273 "stop_reason": "tool_use",
9274 "created_at": "1970-01-01T00:00:00.080Z"
9275 }),
9276 );
9277
9278 assert_eq!(frames.len(), 3);
9279 let child_messages = frames
9280 .iter()
9281 .filter(|frame| frame.kind == "user_input")
9282 .map(|frame| {
9283 (
9284 frame.identity.as_str(),
9285 frame.payload["message"]["content"]
9286 .as_str()
9287 .unwrap_or_default(),
9288 )
9289 })
9290 .collect::<Vec<_>>();
9291 assert_eq!(
9292 child_messages,
9293 vec![
9294 ("review-worker-alpha", "Review Initiative Alpha."),
9295 ("review-worker-beta", "Review Initiative Beta."),
9296 ]
9297 );
9298 }
9299
9300 #[tokio::test]
9301 async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
9302 let aggregator = MobKitConsoleAggregator::in_memory();
9303 let frames = frames_from_session_history_message(
9304 "runtime-a",
9305 "review:singleton",
9306 "session-a",
9307 9,
9308 json!({
9309 "role": "block_assistant",
9310 "blocks": [
9311 {
9312 "block_type": "tool_use",
9313 "data": {
9314 "id": "call-spawn-many",
9315 "name": "mob_spawn_member",
9316 "args": {
9317 "specs": [
9318 {
9319 "agent_identity": "review-worker-alpha",
9320 "profile": "review-worker",
9321 "initial_message": "Review Initiative Alpha."
9322 },
9323 {
9324 "agent_identity": "review-worker-beta",
9325 "profile": "review-worker",
9326 "initial_message": "Review Initiative Beta."
9327 }
9328 ]
9329 }
9330 }
9331 }
9332 ],
9333 "stop_reason": "tool_use",
9334 "created_at": "1970-01-01T00:00:00.090Z"
9335 }),
9336 );
9337 for frame in frames {
9338 aggregator
9339 .store()
9340 .append_if_absent(frame)
9341 .await
9342 .expect("append projected session-history frame");
9343 }
9344
9345 let child_page = aggregator
9346 .query_timeline(ConsoleTimelineQuery {
9347 identity: Some("review-worker-alpha".to_string()),
9348 limit: 20,
9349 ..ConsoleTimelineQuery::default()
9350 })
9351 .await
9352 .expect("query child timeline");
9353 assert_eq!(child_page.frames.len(), 1);
9354 assert_eq!(child_page.frames[0].kind, "user_input");
9355 assert_eq!(
9356 child_page.frames[0].payload["message"]["content"],
9357 "Review Initiative Alpha."
9358 );
9359 assert_eq!(
9360 child_page.frames[0].payload["parent_identity"],
9361 "review:singleton"
9362 );
9363
9364 let parent_page = aggregator
9365 .query_timeline(ConsoleTimelineQuery {
9366 identity: Some("review:singleton".to_string()),
9367 limit: 20,
9368 ..ConsoleTimelineQuery::default()
9369 })
9370 .await
9371 .expect("query parent timeline");
9372 assert!(
9373 parent_page
9374 .frames
9375 .iter()
9376 .all(|frame| frame.identity == "review:singleton"),
9377 "child initial messages must not leak into parent timeline: {:#?}",
9378 parent_page.frames
9379 );
9380 }
9381
9382 #[test]
9383 fn session_history_projection_applies_namespace_to_spawned_child_identity() {
9384 let frames = frames_from_session_history_message_with_namespace(
9385 "runtime-a",
9386 "test/review:singleton",
9387 "test",
9388 "session-a",
9389 10,
9390 json!({
9391 "role": "block_assistant",
9392 "blocks": [
9393 {
9394 "block_type": "tool_use",
9395 "data": {
9396 "id": "call-spawn-many",
9397 "name": "mob_spawn_member",
9398 "args": {
9399 "specs": [
9400 {
9401 "agent_identity": "review-worker-alpha",
9402 "profile": "review-worker",
9403 "initial_message": "Review Initiative Alpha."
9404 },
9405 {
9406 "agent_identity": "test/review-worker-beta",
9407 "profile": "review-worker",
9408 "initial_message": "Review Initiative Beta."
9409 }
9410 ]
9411 }
9412 }
9413 }
9414 ],
9415 "stop_reason": "tool_use",
9416 "created_at": "1970-01-01T00:00:00.100Z"
9417 }),
9418 );
9419
9420 let child_identities = frames
9421 .iter()
9422 .filter(|frame| frame.kind == "user_input")
9423 .map(|frame| frame.identity.as_str())
9424 .collect::<Vec<_>>();
9425 assert_eq!(
9426 child_identities,
9427 vec!["test/review-worker-alpha", "test/review-worker-beta"]
9428 );
9429 }
9430
9431 #[tokio::test]
9432 async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
9433 let aggregator = MobKitConsoleAggregator::in_memory();
9434 aggregator
9435 .inner
9436 .identity_read_model
9437 .replace(vec![
9438 identity_record_for_test("test/review:singleton"),
9439 identity_record_for_test("test/review-worker-alpha"),
9440 ])
9441 .await;
9442 let frames = frames_from_session_history_message_with_namespace(
9443 "runtime-a",
9444 "test/review:singleton",
9445 "test",
9446 "session-a",
9447 11,
9448 json!({
9449 "role": "block_assistant",
9450 "blocks": [
9451 {
9452 "block_type": "tool_use",
9453 "data": {
9454 "id": "call-spawn",
9455 "name": "mob_spawn_member",
9456 "args": {
9457 "agent_identity": "review-worker-alpha",
9458 "profile": "review-worker",
9459 "initial_message": "Review Initiative Alpha."
9460 }
9461 }
9462 }
9463 ],
9464 "stop_reason": "tool_use",
9465 "created_at": "1970-01-01T00:00:00.110Z"
9466 }),
9467 );
9468 for frame in frames {
9469 aggregator
9470 .store()
9471 .append_if_absent(frame)
9472 .await
9473 .expect("append projected namespaced session-history frame");
9474 }
9475
9476 let namespaced_child_page = aggregator
9477 .query_timeline(ConsoleTimelineQuery {
9478 identity: Some("test/review-worker-alpha".to_string()),
9479 limit: 20,
9480 ..ConsoleTimelineQuery::default()
9481 })
9482 .await
9483 .expect("query namespaced child timeline");
9484 assert_eq!(namespaced_child_page.frames.len(), 1);
9485 assert_eq!(
9486 namespaced_child_page.frames[0].identity,
9487 "test/review-worker-alpha"
9488 );
9489 assert_eq!(
9490 namespaced_child_page.frames[0].payload["message"]["content"],
9491 "Review Initiative Alpha."
9492 );
9493
9494 let raw_child_page = aggregator
9495 .query_timeline(ConsoleTimelineQuery {
9496 identity: Some("review-worker-alpha".to_string()),
9497 limit: 20,
9498 ..ConsoleTimelineQuery::default()
9499 })
9500 .await
9501 .expect("query raw child timeline");
9502 assert!(
9503 raw_child_page.frames.is_empty(),
9504 "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
9505 raw_child_page.frames
9506 );
9507 }
9508
9509 #[test]
9510 fn namespace_helpers_preserve_namespace_named_member_identity() {
9511 assert_eq!(apply_namespace("test", "test"), "test/test");
9512 assert_eq!(
9513 strip_namespace("test/test", "test").as_deref(),
9514 Some("test")
9515 );
9516 assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
9517 assert_eq!(
9518 strip_namespace("test/worker", "test").as_deref(),
9519 Some("worker")
9520 );
9521 assert_eq!(strip_namespace("test", "test"), None);
9522 }
9523
9524 #[test]
9525 fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
9526 assert!(!member_id_matches_durable_identity(
9527 "rt:review:singleton:0",
9528 "review:singleton"
9529 ));
9530 assert!(!member_id_matches_durable_identity(
9531 "review:singleton:gen1",
9532 "review:singleton"
9533 ));
9534 assert!(!member_id_matches_durable_identity(
9535 "review:singleton:1",
9536 "review:singleton"
9537 ));
9538 assert!(!member_id_matches_durable_identity(
9539 "rt:review:singleton:qa:0",
9540 "review:singleton"
9541 ));
9542 assert!(!member_id_matches_durable_identity(
9543 "review:singleton:qa",
9544 "review:singleton"
9545 ));
9546 }
9547
9548 #[test]
9549 fn session_history_projection_uses_rfc3339_created_at_timestamp() {
9550 let frame = frame_from_session_history_message(
9551 "runtime-a",
9552 "agent-a",
9553 "session-a",
9554 0,
9555 json!({
9556 "role": "user",
9557 "content": "hello",
9558 "created_at": "2026-05-12T05:00:06.564227Z"
9559 }),
9560 )
9561 .expect("user history frame");
9562
9563 assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
9564 }
9565
9566 async fn wait_for_kickoff_frames(
9567 aggregator: &MobKitConsoleAggregator,
9568 identity: &str,
9569 timeout: Duration,
9570 ) -> Vec<ConsoleFrame> {
9571 let deadline = Instant::now() + timeout;
9572 let mut observed = Vec::new();
9573 while Instant::now() < deadline {
9574 let page = aggregator
9575 .query_timeline(ConsoleTimelineQuery {
9576 identity: Some(identity.to_string()),
9577 limit: 50,
9578 ..ConsoleTimelineQuery::default()
9579 })
9580 .await
9581 .expect("query timeline");
9582 observed = page.frames;
9583 if observed.iter().any(|frame| frame.kind == "user_input") {
9584 break;
9585 }
9586 tokio::time::sleep(Duration::from_millis(25)).await;
9587 }
9588 observed
9589 }
9590
9591 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9592 async fn spawn_registry_primes_access_lineage() {
9593 let runtime = build_empty_runtime("spawn-access-lineage-test").await;
9594 let sink = runtime
9595 .mob_runtime()
9596 .console_spawn_sink()
9597 .expect("unified runtime installs the console spawn sink at bootstrap");
9598
9599 runtime
9600 .spawn(SpawnMemberSpec::from_wire(
9601 "worker".to_string(),
9602 "worker-3".to_string(),
9603 None,
9604 None,
9605 None,
9606 ))
9607 .await
9608 .expect("member spawns");
9609 sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
9610 mob_id: Some("spawn-access-lineage-test".to_string()),
9611 member_id: "worker-3".to_string(),
9612 identity: "worker-3".to_string(),
9613 initial_message: None,
9614 labels: BTreeMap::new(),
9615 spawned_by: Some("ops-lead".to_string()),
9616 via_tool: "mob_spawn_member".to_string(),
9617 })
9618 .await;
9619
9620 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9621 enabled: true,
9622 admins: vec!["root@example.test".to_string()],
9623 groups: BTreeMap::new(),
9624 rules: vec![crate::access::AccessRule {
9625 id: "bob-ops-lead".to_string(),
9626 subjects: vec!["bob@example.test".to_string()],
9627 actions: vec!["agent.view".to_string()],
9628 agents: vec!["ops-lead".to_string()],
9629 ..crate::access::AccessRule::default()
9630 }],
9631 })
9632 .expect("controller");
9633
9634 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9635 .await;
9636
9637 let bob = controller.view_for_subject(Some("bob@example.test"));
9638 assert!(
9639 bob.can_view_agent("worker-3"),
9640 "spawn registry lineage must let bob see the member ops-lead spawned"
9641 );
9642 assert!(
9643 !bob.can_view_agent("unrelated-agent"),
9644 "agents outside the granted lineage stay denied"
9645 );
9646
9647 let _ = runtime.mob_handle().stop().await;
9648 }
9649
9650 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9651 async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
9652 let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
9653
9654 let mut imposter = SpawnMemberSpec::from_wire(
9657 "worker".to_string(),
9658 "imposter".to_string(),
9659 None,
9660 None,
9661 None,
9662 );
9663 imposter.labels = Some(BTreeMap::from([(
9664 "spawned_by".to_string(),
9665 "ops-lead".to_string(),
9666 )]));
9667 runtime.spawn(imposter).await.expect("imposter spawns");
9668
9669 let mut evader = SpawnMemberSpec::from_wire(
9670 "worker".to_string(),
9671 "evader".to_string(),
9672 None,
9673 None,
9674 None,
9675 );
9676 evader.labels = Some(BTreeMap::from([(
9677 "spawned_by".to_string(),
9678 "secret-lead".to_string(),
9679 )]));
9680 runtime.spawn(evader).await.expect("evader spawns");
9681
9682 let aggregator = MobKitConsoleAggregator::in_memory();
9684 aggregator.register_runtime_handles_with_policy(
9685 "runtime-a",
9686 "",
9687 runtime.mob_runtime().clone(),
9688 None,
9689 runtime.console_events(),
9690 Arc::new(AllowAllConsoleVisibilityPolicy),
9691 );
9692 let identities = aggregator
9693 .list_identities_fresh()
9694 .await
9695 .expect("list identities");
9696 let imposter_record = identities
9697 .iter()
9698 .find(|record| record.identity == "imposter")
9699 .expect("imposter listed");
9700 assert!(
9701 !imposter_record.labels.contains_key("spawned_by"),
9702 "identity records must drop roster-claimed lineage when the spawn registry has none"
9703 );
9704
9705 let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9706 enabled: true,
9707 admins: vec!["root@example.test".to_string()],
9708 groups: BTreeMap::new(),
9709 rules: vec![
9710 crate::access::AccessRule {
9711 id: "bob-ops-lead".to_string(),
9712 subjects: vec!["bob@example.test".to_string()],
9713 actions: vec!["agent.view".to_string()],
9714 agents: vec!["ops-lead".to_string()],
9715 ..crate::access::AccessRule::default()
9716 },
9717 crate::access::AccessRule {
9718 id: "carol-view-all".to_string(),
9719 subjects: vec!["carol@example.test".to_string()],
9720 actions: vec!["agent.view".to_string()],
9721 agents: vec!["*".to_string()],
9722 ..crate::access::AccessRule::default()
9723 },
9724 crate::access::AccessRule {
9725 id: "hide-secret-lead".to_string(),
9726 effect: crate::access::AccessEffect::Deny,
9727 actions: vec!["agent.*".to_string()],
9728 agents: vec!["secret-lead".to_string()],
9729 ..crate::access::AccessRule::default()
9730 },
9731 ],
9732 })
9733 .expect("controller");
9734
9735 crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9736 .await;
9737
9738 let bob = controller.view_for_subject(Some("bob@example.test"));
9739 assert!(
9740 !bob.can_view_agent("imposter"),
9741 "a roster-claimed spawned_by must not mint inherited visibility"
9742 );
9743 let carol = controller.view_for_subject(Some("carol@example.test"));
9744 assert!(
9745 carol.can_view_agent("evader"),
9746 "a roster-claimed spawned_by must not let a member hide behind a denied parent"
9747 );
9748
9749 let _ = runtime.mob_handle().stop().await;
9750 }
9751
9752 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9753 async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
9754 let runtime = build_empty_runtime("agent-spawn-console-test").await;
9755
9756 let sink = runtime
9760 .mob_runtime()
9761 .console_spawn_sink()
9762 .expect("unified runtime installs the console spawn sink at bootstrap");
9763
9764 runtime
9768 .spawn(SpawnMemberSpec::from_wire(
9769 "worker".to_string(),
9770 "worker-3".to_string(),
9771 None,
9772 None,
9773 None,
9774 ))
9775 .await
9776 .expect("member spawns");
9777
9778 let seed = crate::console_spawn::ConsoleSpawnSeed {
9779 mob_id: Some("agent-spawn-console-test".to_string()),
9780 member_id: "worker-3".to_string(),
9781 identity: "worker-3".to_string(),
9782 initial_message: Some(json!("Kick off the person hunt")),
9783 labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
9784 spawned_by: Some("ops-lead".to_string()),
9785 via_tool: "mob_spawn_member".to_string(),
9786 };
9787 sink.project_spawned_member(&seed).await;
9788 sink.project_spawned_member(&seed).await;
9790
9791 let aggregator = MobKitConsoleAggregator::in_memory();
9792 aggregator.register_runtime_handles_with_policy(
9793 "runtime-a",
9794 "test",
9795 runtime.mob_runtime().clone(),
9796 None,
9797 runtime.console_events(),
9798 Arc::new(AllowAllConsoleVisibilityPolicy),
9799 );
9800
9801 let frames =
9802 wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
9803 let kickoffs = frames
9804 .iter()
9805 .filter(|frame| frame.kind == "user_input")
9806 .collect::<Vec<_>>();
9807 assert_eq!(
9808 kickoffs.len(),
9809 1,
9810 "member timeline must contain exactly one kickoff frame; got {frames:#?}"
9811 );
9812 assert_eq!(
9813 kickoffs[0].payload["content"][0]["text"],
9814 "Kick off the person hunt"
9815 );
9816
9817 runtime
9820 .project_console_event_from_unified(&crate::types::EventEnvelope {
9821 event_id: "evt-live-worker-3".to_string(),
9822 source: "test".to_string(),
9823 timestamp_ms: current_time_ms(),
9824 event: crate::types::UnifiedEvent::Agent {
9825 agent_id: "worker-3:0".to_string(),
9826 event_type: "text_delta".to_string(),
9827 payload: Some(json!({ "delta": "on it" })),
9828 },
9829 })
9830 .await;
9831
9832 let deadline = Instant::now() + Duration::from_secs(5);
9833 let mut live_frame = None;
9834 while Instant::now() < deadline && live_frame.is_none() {
9835 let page = aggregator
9836 .query_timeline(ConsoleTimelineQuery {
9837 identity: Some("test/worker-3".to_string()),
9838 limit: 50,
9839 ..ConsoleTimelineQuery::default()
9840 })
9841 .await
9842 .expect("query timeline");
9843 live_frame = page
9844 .frames
9845 .into_iter()
9846 .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
9847 if live_frame.is_none() {
9848 tokio::time::sleep(Duration::from_millis(25)).await;
9849 }
9850 }
9851 let live_frame = live_frame.expect("live runtime event lands in the member chat");
9852 assert_eq!(live_frame.identity, "test/worker-3");
9853
9854 let identities = aggregator
9857 .list_identities_fresh()
9858 .await
9859 .expect("list identities");
9860 let record = identities
9861 .iter()
9862 .find(|record| record.identity == "test/worker-3")
9863 .unwrap_or_else(|| {
9864 panic!("spawned member missing from identity list: {identities:#?}")
9865 });
9866 assert_eq!(
9867 record.labels.get("group").map(String::as_str),
9868 Some("workers")
9869 );
9870 assert_eq!(
9871 record.labels.get("spawned_by").map(String::as_str),
9872 Some("ops-lead")
9873 );
9874
9875 let _ = runtime.mob_handle().stop().await;
9876 }
9877}