Skip to main content

meerkat_mobkit/console_aggregator/
mod.rs

1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::MeerkatId;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23    MobRuntime, MobRuntimeError, assert_member_accepts_images, send_message_on_mob_with_mode,
24};
25use crate::runtime::ConsoleMember;
26use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
27
28pub use state::{
29    ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
30    SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
31    SourceIngestionTransition,
32};
33pub use store::{
34    ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
35    SqliteConsoleLogStore,
36};
37pub use types::{
38    AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
39    ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
40    ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
41    ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
42    ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
43};
44
45const TIMELINE_CHANNEL_CAP: usize = 1024;
46const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
47const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
48const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
49const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
50const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
51const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
52const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
53const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
54const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57enum IdentityCollectionMode {
58    CachedOnly,
59    IncludeLiveMembers,
60}
61
62#[derive(Clone)]
63pub struct MobKitConsoleAggregator {
64    inner: Arc<AggregatorInner>,
65}
66
67#[derive(Debug, Clone, Copy)]
68pub struct ConsoleAggregatorOptions {
69    pub session_history_backfill_enabled: bool,
70    pub max_concurrent_session_backfills: usize,
71}
72
73impl Default for ConsoleAggregatorOptions {
74    fn default() -> Self {
75        Self {
76            session_history_backfill_enabled: true,
77            max_concurrent_session_backfills: 16,
78        }
79    }
80}
81
82struct AggregatorInner {
83    store: Arc<dyn ConsoleLogStore>,
84    runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
85    event_tx: broadcast::Sender<ConsoleTimelineEvent>,
86    active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
87    opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88    session_backfill_permits: Arc<Semaphore>,
89    identity_read_model: ConsoleIdentityReadModel,
90    options: ConsoleAggregatorOptions,
91}
92
93#[derive(Clone)]
94struct ConsoleIdentityReadModel {
95    inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
96    refresh_lock: Arc<tokio::sync::Mutex<()>>,
97    primed: Arc<AtomicBool>,
98}
99
100impl Default for ConsoleIdentityReadModel {
101    fn default() -> Self {
102        Self {
103            inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
104            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
105            primed: Arc::new(AtomicBool::new(false)),
106        }
107    }
108}
109
110impl ConsoleIdentityReadModel {
111    async fn snapshot(
112        &self,
113        inner: Arc<AggregatorInner>,
114    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
115        if !self.primed.load(Ordering::Acquire) {
116            self.prime_now(inner).await?;
117        }
118        Ok(self.inner.read().await.clone())
119    }
120
121    async fn current(&self) -> Vec<ConsoleIdentityRecord> {
122        self.inner.read().await.clone()
123    }
124
125    async fn refresh_now_if_idle(
126        &self,
127        inner: Arc<AggregatorInner>,
128        mode: IdentityCollectionMode,
129    ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
130        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
131            return None;
132        };
133        let result = collect_identity_records(&inner, mode).await;
134        drop(guard);
135        match result {
136            Ok(identities) => {
137                self.replace(identities.clone()).await;
138                Some(Ok(identities))
139            }
140            Err(err) => Some(Err(err)),
141        }
142    }
143
144    async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
145        if self.primed.load(Ordering::Acquire) {
146            return Ok(());
147        }
148        let _guard = self.refresh_lock.clone().lock_owned().await;
149        if self.primed.load(Ordering::Acquire) {
150            return Ok(());
151        }
152        let identities =
153            collect_identity_records(&inner, IdentityCollectionMode::CachedOnly).await?;
154        *self.inner.write().await = identities;
155        self.primed.store(true, Ordering::Release);
156        Ok(())
157    }
158
159    fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
160        let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
161            return;
162        };
163        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
164            return;
165        };
166        let read_model = self.clone();
167        runtime_handle.spawn(async move {
168            let _guard = guard;
169            match collect_identity_records(&inner, IdentityCollectionMode::IncludeLiveMembers).await
170            {
171                Ok(identities) => {
172                    *read_model.inner.write().await = identities;
173                    read_model.primed.store(true, Ordering::Release);
174                }
175                Err(err) => {
176                    tracing::warn!(error = %err, "console identity read-model refresh failed");
177                }
178            }
179        });
180    }
181
182    async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
183        *self.inner.write().await = identities;
184        self.primed.store(true, Ordering::Release);
185    }
186}
187
188#[derive(Clone)]
189struct RuntimeEntry {
190    runtime_key: String,
191    identity_namespace: String,
192    runtime: MobRuntime,
193    identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
194    console_events: ConsoleEventStore,
195    visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
196}
197
198#[derive(Clone)]
199struct ResolvedConsoleMember {
200    entry: RuntimeEntry,
201    handle: MobHandle,
202    member: MobMemberListEntry,
203    source_mob_id: String,
204    runtime_identity: String,
205}
206
207fn console_member_for_resolved_member(
208    resolved: &ResolvedConsoleMember,
209    record: &ConsoleIdentityRecord,
210) -> ConsoleMember {
211    ConsoleMember {
212        agent_identity: resolved.member.agent_identity.to_string(),
213        role: resolved.member.role.to_string(),
214        state: format!("{:?}", resolved.member.state),
215        model_capabilities: Default::default(),
216        runtime_mode: Some(resolved.member.runtime_mode.to_string()),
217        session_id: record.session_id.clone(),
218        wired_to: resolved
219            .member
220            .wired_to
221            .iter()
222            .map(ToString::to_string)
223            .collect(),
224        labels: record.labels.clone(),
225    }
226}
227
228async fn resolved_member_visible(
229    resolved: &ResolvedConsoleMember,
230    record: &ConsoleIdentityRecord,
231) -> bool {
232    if !raw_resolved_member_visible(resolved, record) {
233        return false;
234    }
235    !live_record_shadowed_by_hidden_durable_binding(resolved, record).await
236}
237
238fn raw_resolved_member_visible(
239    resolved: &ResolvedConsoleMember,
240    record: &ConsoleIdentityRecord,
241) -> bool {
242    resolved.entry.visibility_policy.identity_visible(record)
243        && resolved
244            .entry
245            .visibility_policy
246            .member_visible(&console_member_for_resolved_member(resolved, record))
247}
248
249pub trait ConsoleVisibilityPolicy: Send + Sync {
250    fn include_implicit_delegate_members(&self) -> bool {
251        true
252    }
253
254    fn member_visible(&self, _member: &ConsoleMember) -> bool {
255        true
256    }
257
258    fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
259        true
260    }
261
262    fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
263        true
264    }
265
266    fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
267        None
268    }
269}
270
271#[derive(Debug, Default)]
272pub struct AllowAllConsoleVisibilityPolicy;
273
274impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
275
276#[derive(Debug, Default)]
277pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
278
279impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
280    fn include_implicit_delegate_members(&self) -> bool {
281        false
282    }
283
284    fn member_visible(&self, member: &ConsoleMember) -> bool {
285        !is_implicit_delegate_member(member.role.as_str(), &member.labels)
286    }
287
288    fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
289        !is_implicit_delegate_member(
290            record
291                .labels
292                .get("role")
293                .map(String::as_str)
294                .unwrap_or_default(),
295            &record.labels,
296        )
297    }
298}
299
300#[derive(Clone)]
301pub struct ConsoleRuntimeRegistration {
302    pub runtime_key: String,
303    pub runtime: Arc<UnifiedRuntime>,
304    pub identity_namespace: String,
305    pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
306}
307
308impl MobKitConsoleAggregator {
309    pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
310        Self::new_with_options(store, ConsoleAggregatorOptions::default())
311    }
312
313    pub fn new_with_options(
314        store: Arc<dyn ConsoleLogStore>,
315        mut options: ConsoleAggregatorOptions,
316    ) -> Self {
317        options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
318        let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
319        Self {
320            inner: Arc::new(AggregatorInner {
321                store,
322                runtimes: RwLock::new(BTreeMap::new()),
323                event_tx,
324                active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
325                opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
326                session_backfill_permits: Arc::new(Semaphore::new(
327                    options.max_concurrent_session_backfills,
328                )),
329                identity_read_model: ConsoleIdentityReadModel::default(),
330                options,
331            }),
332        }
333    }
334
335    pub fn in_memory() -> Self {
336        Self::new(Arc::new(InMemoryConsoleLogStore::new()))
337    }
338
339    pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
340        Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
341    }
342
343    pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
344        self.inner.event_tx.subscribe()
345    }
346
347    pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
348        self.inner.store.clone()
349    }
350
351    pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
352        let identity_runtime = registration.runtime.identity_runtime().cloned();
353        self.register_runtime_handles_with_policy(
354            registration.runtime_key,
355            registration.identity_namespace,
356            registration.runtime.mob_runtime().clone(),
357            identity_runtime,
358            registration.runtime.console_events(),
359            registration.visibility_policy,
360        );
361    }
362
363    pub(crate) fn register_runtime_handles_with_policy(
364        &self,
365        runtime_key: impl Into<String>,
366        identity_namespace: impl Into<String>,
367        runtime: MobRuntime,
368        identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
369        console_events: ConsoleEventStore,
370        visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
371    ) {
372        let runtime_key = runtime_key.into();
373        let identity_namespace = identity_namespace.into();
374        let entry = RuntimeEntry {
375            runtime_key: runtime_key.clone(),
376            identity_namespace,
377            runtime,
378            identity_runtime,
379            console_events: console_events.clone(),
380            visibility_policy,
381        };
382        if let Ok(mut runtimes) = self.inner.runtimes.write() {
383            runtimes.insert(runtime_key.clone(), entry);
384        }
385        self.inner
386            .identity_read_model
387            .refresh_soon(self.inner.clone());
388        let inner = self.inner.clone();
389        let events_for_live = console_events.clone();
390        let events_for_live_recovery = console_events.clone();
391        let runtime_key_for_live = runtime_key.clone();
392        tokio::spawn(async move {
393            let mut rx = events_for_live.subscribe();
394            loop {
395                match rx.recv().await {
396                    Ok(envelope) => {
397                        let _ =
398                            project_console_event(inner.clone(), &runtime_key_for_live, envelope)
399                                .await;
400                    }
401                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
402                        let _ = recover_lagged_source_events(
403                            inner.clone(),
404                            &runtime_key_for_live,
405                            &events_for_live_recovery,
406                        )
407                        .await;
408                    }
409                    Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
410                }
411            }
412        });
413
414        let inner = self.inner.clone();
415        let events_for_replay = console_events;
416        let runtime_key_for_replay = runtime_key;
417        tokio::spawn(async move {
418            let mut ingestion_state = SourceIngestionState::Registered;
419            if let Ok((next, _effects)) =
420                ingestion_state.apply(SourceIngestionTransition::StartBackfill)
421            {
422                ingestion_state = next;
423            }
424            if let Ok(events) = events_for_replay.replay_all(None).await {
425                for envelope in events {
426                    let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
427                        .await;
428                }
429            }
430            spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
431            spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
432            if let Ok((next, _effects)) =
433                ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
434            {
435                ingestion_state = next;
436            }
437            let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
438        });
439    }
440
441    pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
442        if let Some(identities) = self
443            .inner
444            .identity_read_model
445            .refresh_now_if_idle(
446                self.inner.clone(),
447                IdentityCollectionMode::IncludeLiveMembers,
448            )
449            .await
450        {
451            let identities = identities?;
452            spawn_identity_backfills_for_records(self.inner.clone(), &identities);
453            return Ok(identities);
454        }
455        self.inner
456            .identity_read_model
457            .refresh_soon(self.inner.clone());
458        let identities = self
459            .inner
460            .identity_read_model
461            .snapshot(self.inner.clone())
462            .await?;
463        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464        Ok(identities)
465    }
466
467    #[cfg(test)]
468    pub(crate) async fn list_identities_fresh(
469        &self,
470    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
471        let _guard = self
472            .inner
473            .identity_read_model
474            .refresh_lock
475            .clone()
476            .lock_owned()
477            .await;
478        let identities =
479            collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
480                .await?;
481        self.inner
482            .identity_read_model
483            .replace(identities.clone())
484            .await;
485        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
486        Ok(identities)
487    }
488
489    pub async fn inspect_identity(
490        &self,
491        identity: &str,
492    ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
493        let entries = self
494            .inner
495            .runtimes
496            .read()
497            .map_err(|_| runtime_registry_lock_error())?
498            .values()
499            .cloned()
500            .collect::<Vec<_>>();
501        let live_records = self.live_records_for_identity(identity).await;
502        let mut durable_matches = Vec::new();
503        if !requested_identity_has_runtime_member_alias(identity, &entries) {
504            for entry in &entries {
505                let Some(identity_runtime) = entry.identity_runtime.clone() else {
506                    continue;
507                };
508                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
509                {
510                    let Ok(parsed_identity) =
511                        crate::identity_first::AgentIdentity::parse(&raw_identity)
512                    else {
513                        continue;
514                    };
515                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
516                        continue;
517                    };
518                    let mut record = identity_record_for_status(entry, &status);
519                    let topology_peers =
520                        identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
521                    record.topology_peers = topology_peers
522                        .get(&record.identity)
523                        .cloned()
524                        .unwrap_or_default();
525                    if console_identity_record_visible(entry, &record).await {
526                        durable_matches.push((entry.clone(), record));
527                    }
528                }
529            }
530        }
531        if durable_matches.len() > 1 {
532            let candidates = durable_matches
533                .iter()
534                .map(|(_, record)| record.runtime_member_id.clone())
535                .collect::<Vec<_>>()
536                .join(", ");
537            return Err(format!(
538                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
539            )
540            .into());
541        }
542        if let Some((entry, record)) = durable_matches.into_iter().next() {
543            let durable_live_records = self
544                .live_records_for_durable_record(&entry, identity, &record, &live_records)
545                .await;
546            let visible_durable_live_records =
547                visible_live_records_for_entry(&entry, &durable_live_records).await;
548            if let Some(ambiguous_error) =
549                ambiguous_live_alias_error(identity, &visible_durable_live_records)
550            {
551                return Err(ambiguous_error.into());
552            }
553            if let Some(stale_error) =
554                stale_durable_record_error(identity, &record, &durable_live_records)
555            {
556                return Err(stale_error.into());
557            }
558            if let Some(session_id) = record.session_id.clone() {
559                spawn_session_history_backfill_target(
560                    self.inner.clone(),
561                    SessionBackfillTarget {
562                        entry,
563                        record: record.clone(),
564                        session_id,
565                    },
566                    false,
567                );
568            }
569            return Ok(Some(ConsoleIdentityInspection {
570                peers: record.topology_peers.clone(),
571                identity: record,
572            }));
573        }
574        let mut runtime_id_matches = Vec::new();
575        for entry in &entries {
576            let Some(identity_runtime) = entry.identity_runtime.clone() else {
577                continue;
578            };
579            let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
580                continue;
581            };
582            let Some(status) = identity_runtime
583                .statuses()
584                .await
585                .into_iter()
586                .find(|status| {
587                    status
588                        .agent_runtime_id
589                        .as_ref()
590                        .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
591                })
592            else {
593                continue;
594            };
595            let mut record = identity_record_for_status(entry, &status);
596            let topology_peers =
597                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
598            record.topology_peers = topology_peers
599                .get(&record.identity)
600                .cloned()
601                .unwrap_or_default();
602            if console_identity_record_visible(entry, &record).await {
603                runtime_id_matches.push((entry.clone(), record));
604            }
605        }
606        if runtime_id_matches.len() > 1 {
607            let candidates = runtime_id_matches
608                .iter()
609                .map(|(_, record)| record.identity.clone())
610                .collect::<Vec<_>>()
611                .join(", ");
612            return Err(format!(
613                "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
614            )
615            .into());
616        }
617        if let Some((entry, record)) = runtime_id_matches.into_iter().next() {
618            let durable_live_records = self
619                .live_records_for_durable_record(&entry, identity, &record, &live_records)
620                .await;
621            let visible_durable_live_records =
622                visible_live_records_for_entry(&entry, &durable_live_records).await;
623            if let Some(ambiguous_error) =
624                ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
625            {
626                return Err(ambiguous_error.into());
627            }
628            if let Some(stale_error) =
629                stale_durable_record_error(identity, &record, &durable_live_records)
630            {
631                return Err(stale_error.into());
632            }
633            if let Some(session_id) = record.session_id.clone() {
634                spawn_session_history_backfill_target(
635                    self.inner.clone(),
636                    SessionBackfillTarget {
637                        entry: entry.clone(),
638                        record: record.clone(),
639                        session_id,
640                    },
641                    false,
642                );
643            }
644            return Ok(Some(ConsoleIdentityInspection {
645                peers: record.topology_peers.clone(),
646                identity: record,
647            }));
648        }
649
650        let live_matches = self.resolve_visible_members(identity).await;
651        if live_matches.len() > 1 {
652            let candidates = live_matches
653                .iter()
654                .map(|resolved| resolved.runtime_identity.clone())
655                .collect::<Vec<_>>()
656                .join(", ");
657            return Err(format!(
658                "ambiguous live identity alias {identity}: candidates [{candidates}]"
659            )
660            .into());
661        }
662        if let Some(resolved) = live_matches.into_iter().next() {
663            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
664                return Ok(None);
665            };
666            if let Some(stale_error) =
667                stale_live_record_binding_error(&resolved.entry, identity, &record).await
668            {
669                return Err(stale_error.into());
670            }
671            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
672                && let Ok(parsed_identity) =
673                    crate::identity_first::AgentIdentity::parse(&record.identity)
674                && let Ok(status) = identity_runtime.status(&parsed_identity).await
675            {
676                let durable_record = identity_record_for_status(&resolved.entry, &status);
677                let durable_live_records = self
678                    .live_records_for_durable_record(
679                        &resolved.entry,
680                        identity,
681                        &durable_record,
682                        &live_records,
683                    )
684                    .await;
685                let visible_durable_live_records =
686                    visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
687                if let Some(ambiguous_error) =
688                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
689                {
690                    return Err(ambiguous_error.into());
691                }
692                if let Some(stale_error) =
693                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
694                {
695                    return Err(stale_error.into());
696                }
697                if status
698                    .agent_runtime_id
699                    .as_ref()
700                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
701                {
702                    return Ok(None);
703                }
704            }
705            if !resolved_member_visible(&resolved, &record).await {
706                return Ok(None);
707            }
708            if let Some(session_id) = record.session_id.clone() {
709                spawn_session_history_backfill_target(
710                    self.inner.clone(),
711                    SessionBackfillTarget {
712                        entry: resolved.entry.clone(),
713                        record: record.clone(),
714                        session_id,
715                    },
716                    false,
717                );
718            }
719            let peers = resolved
720                .member
721                .wired_to
722                .iter()
723                .map(ToString::to_string)
724                .collect();
725            return Ok(Some(ConsoleIdentityInspection {
726                identity: record,
727                peers,
728            }));
729        }
730
731        Ok(None)
732    }
733
734    pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
735        let entries = self
736            .inner
737            .runtimes
738            .read()
739            .map_err(|_| runtime_registry_lock_error())?
740            .clone();
741        let live_records = self.live_records_for_identity(identity).await;
742        let entries_vec = entries.values().cloned().collect::<Vec<_>>();
743        let mut durable_matches = Vec::new();
744        if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
745            for entry in entries.values() {
746                let Some(identity_runtime) = entry.identity_runtime.clone() else {
747                    continue;
748                };
749                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
750                {
751                    let Ok(parsed_identity) =
752                        crate::identity_first::AgentIdentity::parse(&raw_identity)
753                    else {
754                        continue;
755                    };
756                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
757                        continue;
758                    };
759                    let record = identity_record_for_status(entry, &status);
760                    if !console_identity_record_visible(entry, &record).await {
761                        continue;
762                    }
763                    durable_matches.push((
764                        entry.clone(),
765                        identity_runtime.clone(),
766                        parsed_identity,
767                        record,
768                    ));
769                }
770            }
771        }
772        if durable_matches.len() > 1 {
773            let candidates = durable_matches
774                .iter()
775                .map(|(_, _, identity, _)| identity.as_str().to_string())
776                .collect::<Vec<_>>()
777                .join(", ");
778            return Err(format!(
779                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
780            )
781            .into());
782        }
783        if let Some((entry, identity_runtime, parsed_identity, record)) =
784            durable_matches.into_iter().next()
785        {
786            let durable_live_records = self
787                .live_records_for_durable_record(&entry, identity, &record, &live_records)
788                .await;
789            let visible_durable_live_records =
790                visible_live_records_for_entry(&entry, &durable_live_records).await;
791            if let Some(ambiguous_error) =
792                ambiguous_live_alias_error(identity, &visible_durable_live_records)
793            {
794                return Err(ambiguous_error.into());
795            }
796            if let Some(stale_error) =
797                stale_durable_record_error(identity, &record, &durable_live_records)
798            {
799                return Err(stale_error.into());
800            }
801            identity_runtime
802                .retire(&parsed_identity)
803                .await
804                .map_err(|err| -> ConsoleLogError {
805                    format!("retire failed for {identity}: {err}").into()
806                })?;
807            {
808                let mut durable_identities =
809                    namespace_match_candidates(identity, &entry.identity_namespace);
810                durable_identities.push(parsed_identity.as_str().to_string());
811                durable_identities.sort();
812                durable_identities.dedup();
813                if let Err(err) =
814                    retire_stale_console_members_for_runtime_entry(&entry, &durable_identities)
815                        .await
816                {
817                    tracing::warn!(
818                        identity,
819                        error = %err,
820                        "stale console member cleanup failed after durable identity retire"
821                    );
822                }
823            }
824            return Ok(true);
825        }
826        let matches = self.resolve_visible_members(identity).await;
827        if matches.len() > 1 {
828            let candidates = matches
829                .iter()
830                .map(|resolved| resolved.runtime_identity.clone())
831                .collect::<Vec<_>>()
832                .join(", ");
833            return Err(format!(
834                "ambiguous live identity alias {identity}: candidates [{candidates}]"
835            )
836            .into());
837        }
838        let mut live_retired_any = false;
839        for resolved in matches {
840            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
841                continue;
842            };
843            if !resolved_member_visible(&resolved, &record).await {
844                continue;
845            }
846            if let Some(stale_error) =
847                stale_live_record_binding_error(&resolved.entry, identity, &record).await
848            {
849                return Err(stale_error.into());
850            }
851            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
852                && let Ok(parsed_identity) =
853                    crate::identity_first::AgentIdentity::parse(&record.identity)
854                && let Ok(status) = identity_runtime.status(&parsed_identity).await
855            {
856                let durable_record = identity_record_for_status(&resolved.entry, &status);
857                let durable_live_records = self
858                    .live_records_for_durable_record(
859                        &resolved.entry,
860                        identity,
861                        &durable_record,
862                        &live_records,
863                    )
864                    .await;
865                let visible_durable_live_records =
866                    visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
867                if let Some(ambiguous_error) =
868                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
869                {
870                    return Err(ambiguous_error.into());
871                }
872                if let Some(stale_error) =
873                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
874                {
875                    return Err(stale_error.into());
876                }
877                if status
878                    .agent_runtime_id
879                    .as_ref()
880                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
881                {
882                    continue;
883                }
884            }
885            resolved
886                .handle
887                .retire(MeerkatId::from(resolved.runtime_identity.as_str()))
888                .await
889                .map_err(|err| -> ConsoleLogError {
890                    format!("retire failed for {identity}: {err}").into()
891                })
892                .or_else(|err| {
893                    if lifecycle_archive_cleanup_completed(&err.to_string()) {
894                        Ok(())
895                    } else {
896                        Err(err)
897                    }
898                })?;
899            live_retired_any = true;
900        }
901        Ok(live_retired_any)
902    }
903
904    pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
905        self.inner.store.clear_frames().await
906    }
907
908    pub async fn query_timeline(
909        &self,
910        query: ConsoleTimelineQuery,
911    ) -> ConsoleLogResult<ConsoleTimelinePage> {
912        let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
913        Ok(ConsoleTimelinePage {
914            frames: page.frames,
915            next_cursor: page.next_cursor,
916        })
917    }
918
919    pub async fn query_timeline_windowed(
920        &self,
921        query: ConsoleTimelineWindowQuery,
922    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
923        self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
924            .await?;
925        if query.after.is_none()
926            && query.before.is_none()
927            && let Some(identity) = query.identity.clone()
928        {
929            let mut probe_query = query.clone();
930            probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
931            let page = self.inner.store.query_windowed_frames(probe_query).await?;
932            if explicit_identity_query_needs_session_history_backfill(&page.frames) {
933                spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
934            }
935        }
936        Box::pin(self.query_timeline_visible(query)).await
937    }
938
939    async fn reject_after_cursor_beyond_store_frontier(
940        &self,
941        after: Option<&ConsoleCursor>,
942    ) -> ConsoleLogResult<()> {
943        let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
944            return Ok(());
945        };
946        let latest_seq = self
947            .inner
948            .store
949            .latest_cursor()
950            .await?
951            .and_then(|cursor| cursor.seq())
952            .unwrap_or(0);
953        if after_seq > latest_seq {
954            return Err(std::io::Error::other(
955                "timeline replay cursor is beyond the current store frontier",
956            )
957            .into());
958        }
959        Ok(())
960    }
961
962    async fn query_timeline_visible(
963        &self,
964        query: ConsoleTimelineWindowQuery,
965    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
966        let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
967        let mut scan_query = query.clone();
968        scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
969        let mut visible_frames = Vec::with_capacity(requested_limit);
970        let mut anchor_frames = Vec::new();
971        let mut identity_visibility_cache = HashMap::new();
972        let identity_records = self.inner.identity_read_model.current().await;
973        let mut next_cursor = query.after.clone();
974        let mut since_last_scanned_cursor = query.after.clone();
975        let mut since_last_delivered_cursor = None;
976        let mut since_stopped_at_visible_limit = false;
977        let mut latest_cursor = None;
978        let mut exhausted = false;
979        let mut scanned = 0usize;
980
981        loop {
982            let page = self
983                .inner
984                .store
985                .query_windowed_frames(scan_query.clone())
986                .await?;
987            latest_cursor = latest_cursor.or(page.latest_cursor.clone());
988            if page.frames.is_empty() {
989                exhausted = true;
990                break;
991            }
992            let raw_len = page.frames.len();
993            scanned = scanned.saturating_add(raw_len);
994            match query.mode {
995                ConsoleTimelineMode::Since => {
996                    if let Some(cursor) = page.next_cursor.clone() {
997                        since_last_scanned_cursor = Some(cursor.clone());
998                        scan_query.after = Some(cursor);
999                    }
1000                }
1001                ConsoleTimelineMode::Recent => {
1002                    if let Some(first) = page.frames.first() {
1003                        scan_query.before = Some(first.cursor.clone());
1004                    }
1005                    next_cursor = page.next_cursor.clone().or(next_cursor);
1006                }
1007            }
1008
1009            for frame in page.frames {
1010                let allow_historical_identity =
1011                    query.identity.as_deref() == Some(frame.identity.as_str());
1012                if frame_is_visible_cached(
1013                    &self.inner,
1014                    &frame,
1015                    allow_historical_identity,
1016                    &mut identity_visibility_cache,
1017                    &identity_records,
1018                )
1019                .await
1020                .unwrap_or(false)
1021                {
1022                    if query.mode == ConsoleTimelineMode::Recent
1023                        && query.identity.is_some()
1024                        && is_identity_timeline_anchor_frame(&frame)
1025                        && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1026                    {
1027                        anchor_frames.push(frame.clone());
1028                    }
1029                    match query.mode {
1030                        ConsoleTimelineMode::Since => {
1031                            if visible_frames.len() >= requested_limit {
1032                                since_stopped_at_visible_limit = true;
1033                                break;
1034                            }
1035                            since_last_delivered_cursor = Some(frame.cursor.clone());
1036                            visible_frames.push(frame);
1037                            if visible_frames.len() >= requested_limit {
1038                                since_stopped_at_visible_limit = true;
1039                                exhausted = false;
1040                                break;
1041                            }
1042                        }
1043                        ConsoleTimelineMode::Recent => {
1044                            visible_frames.push(frame);
1045                        }
1046                    }
1047                }
1048            }
1049            let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1050                && query.identity.is_some()
1051                && anchor_frames.is_empty()
1052                && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1053            if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1054                break;
1055            }
1056            if since_stopped_at_visible_limit {
1057                break;
1058            }
1059            if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1060                exhausted = true;
1061                break;
1062            }
1063            if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1064                break;
1065            }
1066        }
1067
1068        if query.mode == ConsoleTimelineMode::Recent {
1069            visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1070            if visible_frames.len() > requested_limit {
1071                visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1072            }
1073            if !anchor_frames.is_empty() {
1074                let anchor_cursors = anchor_frames
1075                    .iter()
1076                    .map(|frame| frame.cursor.clone())
1077                    .collect::<Vec<_>>();
1078                let mut merged = anchor_frames;
1079                for frame in visible_frames {
1080                    if !merged
1081                        .iter()
1082                        .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1083                    {
1084                        merged.push(frame);
1085                    }
1086                }
1087                merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1088                visible_frames = merged;
1089                if visible_frames.len() > requested_limit {
1090                    let mut anchors = Vec::new();
1091                    let mut tail = Vec::new();
1092                    for frame in visible_frames {
1093                        if anchor_cursors.contains(&frame.cursor) {
1094                            anchors.push(frame);
1095                        } else {
1096                            tail.push(frame);
1097                        }
1098                    }
1099                    visible_frames = if anchors.len() >= requested_limit {
1100                        anchors.split_off(anchors.len() - requested_limit)
1101                    } else {
1102                        let keep_tail = requested_limit.saturating_sub(anchors.len());
1103                        if tail.len() > keep_tail {
1104                            tail = tail.split_off(tail.len() - keep_tail);
1105                        }
1106                        anchors.extend(tail);
1107                        anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1108                        anchors
1109                    };
1110                }
1111            }
1112        }
1113
1114        Ok(ConsoleTimelineWindowPage {
1115            frames: visible_frames,
1116            next_cursor: match query.mode {
1117                ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1118                    since_last_delivered_cursor.or(since_last_scanned_cursor)
1119                }
1120                ConsoleTimelineMode::Since => since_last_scanned_cursor,
1121                ConsoleTimelineMode::Recent => next_cursor,
1122            },
1123            latest_cursor,
1124            exhausted,
1125        })
1126    }
1127
1128    pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1129        let runtime_keys = self
1130            .inner
1131            .runtimes
1132            .read()
1133            .map_err(|_| runtime_registry_lock_error())?
1134            .keys()
1135            .cloned()
1136            .collect::<Vec<_>>();
1137        let results =
1138            join_all(runtime_keys.into_iter().map(|runtime_key| {
1139                backfill_session_history(self.inner.clone(), runtime_key, true)
1140            }))
1141            .await;
1142        for result in results {
1143            result?;
1144        }
1145        Ok(())
1146    }
1147
1148    pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1149        self.inner.store.latest_cursor().await
1150    }
1151
1152    pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1153        match event {
1154            ConsoleTimelineEvent::ConsoleFrame { frame }
1155            | ConsoleTimelineEvent::FrameUpdated { frame } => {
1156                let identity_records = self.inner.identity_read_model.current().await;
1157                frame_is_visible(&self.inner, frame, false, &identity_records)
1158                    .await
1159                    .unwrap_or(false)
1160            }
1161            ConsoleTimelineEvent::SnapshotStarted { .. }
1162            | ConsoleTimelineEvent::SnapshotComplete { .. }
1163            | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1164        }
1165    }
1166
1167    pub async fn timeline_frame_visible_for_query(
1168        &self,
1169        frame: &ConsoleFrame,
1170        identity: Option<&str>,
1171    ) -> bool {
1172        let allow_historical_identity = identity == Some(frame.identity.as_str());
1173        let identity_records = self.inner.identity_read_model.current().await;
1174        frame_is_visible(
1175            &self.inner,
1176            frame,
1177            allow_historical_identity,
1178            &identity_records,
1179        )
1180        .await
1181        .unwrap_or(false)
1182    }
1183
1184    pub async fn send(
1185        &self,
1186        request: ConsoleSendRequest,
1187    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1188        validate_send_request(&request)?;
1189        let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1190            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1191        };
1192        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1193            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1194        };
1195        if !resolved_member_visible(&resolved, &record).await {
1196            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1197        }
1198        if !member_is_addressable(&resolved.member) {
1199            return Err(ConsoleSendError::NotAddressable(request.identity));
1200        }
1201        if resolved.member.state == meerkat_mob::MemberState::Retiring {
1202            return Err(ConsoleSendError::Retired(request.identity));
1203        }
1204
1205        let content = content_input_from_value(&request.content)?;
1206        let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1207        assert_member_accepts_images(
1208            &resolved.handle,
1209            resolved.entry.runtime.session_service(),
1210            &resolved.runtime_identity,
1211            &content,
1212        )
1213        .await
1214        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1215
1216        let dedupe_key = send_dedupe_key(
1217            &resolved.entry.runtime_key,
1218            &request.identity,
1219            &request.origin,
1220            &request.idempotency_key,
1221        );
1222        let handling_mode_value = request
1223            .handling_mode
1224            .as_deref()
1225            .unwrap_or("queue")
1226            .to_string();
1227        let request_fingerprint =
1228            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1229        if let Some(existing) = self
1230            .inner
1231            .store
1232            .frame_by_dedupe_key(&dedupe_key)
1233            .await
1234            .map_err(ConsoleSendError::Log)?
1235        {
1236            let same_request = existing.source.source_cursor.as_deref()
1237                == Some(request_fingerprint.as_str())
1238                || existing.source.source_cursor.is_none()
1239                    && existing.payload.get("origin").and_then(Value::as_str)
1240                        == Some(request.origin.as_str())
1241                    && existing.payload.get("content") == Some(&request.content)
1242                    && existing
1243                        .payload
1244                        .get("handling_mode")
1245                        .and_then(Value::as_str)
1246                        == Some(handling_mode_value.as_str());
1247            if !same_request {
1248                return Err(ConsoleSendError::IdempotencyConflict(
1249                    request.idempotency_key,
1250                ));
1251            }
1252            return Ok(accepted_from_frame(&existing));
1253        }
1254
1255        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1256        resolved
1257            .entry
1258            .console_events
1259            .reserve_interaction_value(
1260                &resolved.runtime_identity,
1261                Some(resolved.runtime_identity.as_str()),
1262                &interaction_id,
1263                &request.origin,
1264                request.content.clone(),
1265            )
1266            .await
1267            .map_err(ConsoleSendError::State)?;
1268        let session_id = resolved
1269            .handle
1270            .resolve_bridge_session_id_observation(&MeerkatId::from(
1271                resolved.runtime_identity.as_str(),
1272            ))
1273            .await
1274            .map(|sid| sid.to_string());
1275        let mut new_frame = NewConsoleFrame {
1276            id: None,
1277            dedupe_key,
1278            timestamp_ms: current_time_ms(),
1279            runtime_key: resolved.entry.runtime_key.clone(),
1280            identity: request.identity.clone(),
1281            conversation_id: Some(request.identity.clone()),
1282            session_id: session_id.clone(),
1283            kind: "user_input".to_string(),
1284            status: ConsoleFrameStatus::Accepted,
1285            payload: json!({
1286                "content": request.content,
1287                "origin": request.origin,
1288                "idempotency_key": request.idempotency_key,
1289                "handling_mode": handling_mode_value,
1290            }),
1291            source: ConsoleFrameSource {
1292                kind: ConsoleFrameSourceKind::Send,
1293                source_cursor: Some(request_fingerprint),
1294            },
1295            source_event_id: None,
1296            interaction_id: Some(interaction_id.clone()),
1297            turn_id: None,
1298            run_id: None,
1299            parent_frame_id: None,
1300            caused_by_frame_id: None,
1301        };
1302        if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1303            new_frame.payload = redacted;
1304            new_frame.status = ConsoleFrameStatus::Redacted;
1305        }
1306
1307        let _ = SendState::Requested
1308            .apply(SendTransition::PersistAccepted)
1309            .map_err(ConsoleSendError::State)?;
1310        let outcome = self
1311            .inner
1312            .store
1313            .append_if_absent(new_frame)
1314            .await
1315            .map_err(ConsoleSendError::Log)?;
1316        if outcome.disposition == AppendDisposition::Existing {
1317            return Ok(accepted_from_frame(&outcome.frame));
1318        }
1319        let _ = self
1320            .inner
1321            .event_tx
1322            .send(ConsoleTimelineEvent::ConsoleFrame {
1323                frame: outcome.frame.clone(),
1324            });
1325        let accepted = accepted_from_frame(&outcome.frame);
1326
1327        let (dispatching, _effects) = SendState::AcceptedPersisted
1328            .apply(SendTransition::StartDispatch)
1329            .map_err(ConsoleSendError::State)?;
1330        update_frame_status_and_emit(
1331            &self.inner,
1332            &outcome.frame.id,
1333            ConsoleFrameStatus::Dispatching,
1334        )
1335        .await
1336        .map_err(ConsoleSendError::Log)?;
1337
1338        spawn_console_send_dispatch(
1339            self.inner.clone(),
1340            resolved,
1341            content,
1342            handling_mode,
1343            dispatching,
1344            outcome.frame,
1345            interaction_id,
1346        );
1347        Ok(accepted)
1348    }
1349
1350    pub async fn reserve_identity_first_interaction(
1351        &self,
1352        request: ConsoleSendRequest,
1353        session_id: Option<&str>,
1354    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1355        validate_send_request(&request)?;
1356        let _content = content_input_from_value(&request.content)?;
1357        let runtime_key =
1358            Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1359        let handling_mode_value = request
1360            .handling_mode
1361            .as_deref()
1362            .unwrap_or("queue")
1363            .to_string();
1364        let dedupe_key = send_dedupe_key(
1365            &runtime_key,
1366            &request.identity,
1367            &request.origin,
1368            &request.idempotency_key,
1369        );
1370        let request_fingerprint =
1371            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1372        if let Some(existing) = self
1373            .inner
1374            .store
1375            .frame_by_dedupe_key(&dedupe_key)
1376            .await
1377            .map_err(ConsoleSendError::Log)?
1378        {
1379            let same_request = existing.source.source_cursor.as_deref()
1380                == Some(request_fingerprint.as_str())
1381                || existing.source.source_cursor.is_none()
1382                    && existing.payload.get("origin").and_then(Value::as_str)
1383                        == Some(request.origin.as_str())
1384                    && existing.payload.get("content") == Some(&request.content)
1385                    && existing
1386                        .payload
1387                        .get("handling_mode")
1388                        .and_then(Value::as_str)
1389                        == Some(handling_mode_value.as_str());
1390            if !same_request {
1391                return Err(ConsoleSendError::IdempotencyConflict(
1392                    request.idempotency_key,
1393                ));
1394            }
1395            return Ok(accepted_from_frame(&existing));
1396        }
1397
1398        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1399        let new_frame = NewConsoleFrame {
1400            id: None,
1401            dedupe_key,
1402            timestamp_ms: current_time_ms(),
1403            runtime_key,
1404            identity: request.identity.clone(),
1405            conversation_id: Some(request.identity.clone()),
1406            session_id: session_id.map(ToString::to_string),
1407            kind: "user_input".to_string(),
1408            status: ConsoleFrameStatus::Accepted,
1409            payload: json!({
1410                "content": request.content,
1411                "origin": request.origin,
1412                "idempotency_key": request.idempotency_key,
1413                "handling_mode": handling_mode_value,
1414            }),
1415            source: ConsoleFrameSource {
1416                kind: ConsoleFrameSourceKind::Send,
1417                source_cursor: Some(request_fingerprint),
1418            },
1419            source_event_id: None,
1420            interaction_id: Some(interaction_id),
1421            turn_id: None,
1422            run_id: None,
1423            parent_frame_id: None,
1424            caused_by_frame_id: None,
1425        };
1426        let outcome = self
1427            .inner
1428            .store
1429            .append_if_absent(new_frame)
1430            .await
1431            .map_err(ConsoleSendError::Log)?;
1432        let _ = self
1433            .inner
1434            .event_tx
1435            .send(ConsoleTimelineEvent::ConsoleFrame {
1436                frame: outcome.frame.clone(),
1437            });
1438        Ok(accepted_from_frame(&outcome.frame))
1439    }
1440
1441    async fn runtime_key_for_identity_first_send(
1442        &self,
1443        identity: &str,
1444    ) -> Result<String, ConsoleSendError> {
1445        let entries = self
1446            .inner
1447            .runtimes
1448            .read()
1449            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1450            .clone();
1451        if entries.is_empty() {
1452            return Ok("identity-first".to_string());
1453        }
1454
1455        let mut identity_runtime_keys = Vec::new();
1456        let mut hidden_durable_match = false;
1457        let mut durable_matches = Vec::new();
1458        for entry in entries.values() {
1459            let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1460                continue;
1461            };
1462            identity_runtime_keys.push(entry.runtime_key.clone());
1463            if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1464                continue;
1465            }
1466            for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1467            {
1468                let Ok(parsed_identity) =
1469                    crate::identity_first::AgentIdentity::parse(&runtime_identity)
1470                else {
1471                    continue;
1472                };
1473                if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1474                    let record = identity_record_for_status(entry, &status);
1475                    if !console_identity_record_visible(entry, &record).await {
1476                        hidden_durable_match = true;
1477                        continue;
1478                    }
1479                    durable_matches.push((entry.clone(), record));
1480                }
1481            }
1482        }
1483        if durable_matches.len() > 1 {
1484            let candidates = durable_matches
1485                .iter()
1486                .map(|(entry, record)| format!("{}@{}", record.identity, entry.runtime_key))
1487                .collect::<Vec<_>>()
1488                .join(", ");
1489            return Err(ConsoleSendError::InvalidRequest(format!(
1490                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1491            )));
1492        }
1493        if let Some((entry, record)) = durable_matches.into_iter().next() {
1494            let live_records = self.live_records_for_identity(identity).await;
1495            let durable_live_records = self
1496                .live_records_for_durable_record(&entry, identity, &record, &live_records)
1497                .await;
1498            let visible_durable_live_records =
1499                visible_live_records_for_entry(&entry, &durable_live_records).await;
1500            if let Some(ambiguous_error) =
1501                ambiguous_live_alias_error(identity, &visible_durable_live_records)
1502            {
1503                return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1504            }
1505            if let Some(stale_error) =
1506                stale_durable_record_error(identity, &record, &durable_live_records)
1507            {
1508                return Err(ConsoleSendError::InvalidRequest(stale_error));
1509            }
1510            return Ok(entry.runtime_key);
1511        }
1512        if entries.values().any(|entry| {
1513            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1514        }) {
1515            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1516        }
1517        if hidden_durable_match {
1518            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1519        }
1520
1521        match identity_runtime_keys.as_slice() {
1522            [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1523            [runtime_key] => Ok(runtime_key.clone()),
1524            _ => Err(ConsoleSendError::InvalidRequest(format!(
1525                "identity-first send for '{identity}' did not match exactly one registered runtime"
1526            ))),
1527        }
1528    }
1529
1530    pub async fn mark_interaction_delivery_failed(
1531        &self,
1532        input_frame_id: &str,
1533    ) -> Result<(), ConsoleSendError> {
1534        update_frame_status_and_emit(
1535            &self.inner,
1536            input_frame_id,
1537            ConsoleFrameStatus::DeliveryFailed,
1538        )
1539        .await
1540        .map_err(ConsoleSendError::Log)?;
1541        Ok(())
1542    }
1543
1544    pub async fn mark_interaction_delivered(
1545        &self,
1546        input_frame_id: &str,
1547    ) -> Result<(), ConsoleSendError> {
1548        update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1549            .await
1550            .map_err(ConsoleSendError::Log)?;
1551        Ok(())
1552    }
1553
1554    pub async fn mark_steer_interaction_delivered(
1555        &self,
1556        input_frame_id: &str,
1557        interaction_id: &str,
1558    ) -> Result<(), ConsoleSendError> {
1559        let Some(updated) = update_frame_status_and_emit(
1560            &self.inner,
1561            input_frame_id,
1562            ConsoleFrameStatus::Delivered,
1563        )
1564        .await
1565        .map_err(ConsoleSendError::Log)?
1566        else {
1567            return Ok(());
1568        };
1569        append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1570            .await
1571            .map_err(ConsoleSendError::Log)?;
1572        Ok(())
1573    }
1574
1575    pub async fn binary_blob_store_for_identity(
1576        &self,
1577        identity: &str,
1578    ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1579        if identity.trim().is_empty() {
1580            return Err(ConsoleSendError::InvalidRequest(
1581                "identity must be non-empty".to_string(),
1582            ));
1583        }
1584        let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1585            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1586        };
1587        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1588            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1589        };
1590        if !resolved_member_visible(&resolved, &record).await {
1591            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1592        }
1593        if !member_is_addressable(&resolved.member) {
1594            return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1595        }
1596        if resolved.member.state == meerkat_mob::MemberState::Retiring {
1597            return Err(ConsoleSendError::Retired(identity.to_string()));
1598        }
1599        Ok(resolved.entry.runtime.binary_blob_store())
1600    }
1601
1602    pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1603        self.inner
1604            .runtimes
1605            .read()
1606            .map(|entries| {
1607                entries
1608                    .values()
1609                    .filter_map(|entry| entry.runtime.binary_blob_store())
1610                    .collect()
1611            })
1612            .unwrap_or_default()
1613    }
1614
1615    async fn resolve_send_member(
1616        &self,
1617        identity: &str,
1618    ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1619        let matches = self.resolve_visible_members(identity).await;
1620        if matches.len() > 1 {
1621            let candidates = matches
1622                .iter()
1623                .map(|resolved| resolved.runtime_identity.clone())
1624                .collect::<Vec<_>>()
1625                .join(", ");
1626            return Err(ConsoleSendError::AmbiguousIdentity {
1627                identity: identity.to_string(),
1628                candidates,
1629            });
1630        }
1631        let Some(resolved) = matches.into_iter().next() else {
1632            if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1633                return Err(ConsoleSendError::InvalidRequest(stale_error));
1634            }
1635            return Ok(None);
1636        };
1637        if let Some(record) = identity_record_for_resolved_member(&resolved).await
1638            && let Some(stale_error) =
1639                stale_live_record_binding_error(&resolved.entry, identity, &record).await
1640        {
1641            return Err(ConsoleSendError::InvalidRequest(stale_error));
1642        }
1643        if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1644            return Err(ConsoleSendError::InvalidRequest(stale_error));
1645        }
1646        Ok(Some(resolved))
1647    }
1648
1649    async fn durable_send_binding_error(
1650        &self,
1651        identity: &str,
1652    ) -> Result<Option<String>, ConsoleSendError> {
1653        let entries = self
1654            .inner
1655            .runtimes
1656            .read()
1657            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1658            .clone();
1659        if entries.values().any(|entry| {
1660            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1661        }) {
1662            return Ok(None);
1663        }
1664        let live_records = self.live_records_for_identity(identity).await;
1665        let mut durable_matches = Vec::new();
1666        for entry in entries.values() {
1667            let Some(identity_runtime) = entry.identity_runtime.clone() else {
1668                continue;
1669            };
1670            for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1671                let Ok(parsed_identity) =
1672                    crate::identity_first::AgentIdentity::parse(&raw_identity)
1673                else {
1674                    continue;
1675                };
1676                let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1677                    continue;
1678                };
1679                let record = identity_record_for_status(entry, &status);
1680                if !console_identity_record_visible(entry, &record).await {
1681                    continue;
1682                }
1683                durable_matches.push((entry.clone(), record));
1684            }
1685        }
1686        if durable_matches.len() > 1 {
1687            let candidates = durable_matches
1688                .iter()
1689                .map(|(_, record)| record.runtime_member_id.clone())
1690                .collect::<Vec<_>>()
1691                .join(", ");
1692            return Err(ConsoleSendError::AmbiguousIdentity {
1693                identity: identity.to_string(),
1694                candidates,
1695            });
1696        }
1697        let Some((entry, durable_record)) = durable_matches.into_iter().next() else {
1698            return Ok(None);
1699        };
1700        let durable_live_records = self
1701            .live_records_for_durable_record(&entry, identity, &durable_record, &live_records)
1702            .await;
1703        let visible_durable_live_records =
1704            visible_live_records_for_entry(&entry, &durable_live_records).await;
1705        if let Some(ambiguous_error) =
1706            ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1707        {
1708            return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1709        }
1710        Ok(stale_durable_record_error(
1711            identity,
1712            &durable_record,
1713            &durable_live_records,
1714        ))
1715    }
1716
1717    async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1718        let entries = self
1719            .inner
1720            .runtimes
1721            .read()
1722            .ok()
1723            .map(|entries| entries.clone())
1724            .unwrap_or_default();
1725        let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1726        let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1727        for entry in entries.values() {
1728            let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1729            if raw_identities.is_empty() {
1730                continue;
1731            }
1732            let mids = raw_identities
1733                .iter()
1734                .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
1735                .collect::<Vec<_>>();
1736            for resolved in member_sources_for_entry(entry).await {
1737                let session_id = resolved
1738                    .handle
1739                    .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1740                    .await
1741                    .map(|sid| sid.to_string())
1742                    .unwrap_or_default();
1743                if mids.contains(&resolved.member.agent_identity) {
1744                    exact_matches.push((session_id, resolved));
1745                } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1746                {
1747                    label_matches.push((session_id, resolved));
1748                }
1749            }
1750        }
1751        let mut matches = exact_matches;
1752        matches.extend(label_matches);
1753        let mut seen_members = BTreeSet::new();
1754        matches.retain(|(session_id, resolved)| {
1755            seen_members.insert((
1756                resolved.entry.runtime_key.clone(),
1757                resolved.source_mob_id.clone(),
1758                session_id.clone(),
1759                resolved.runtime_identity.clone(),
1760            ))
1761        });
1762        matches.sort_by(|left, right| right.0.cmp(&left.0));
1763        matches.into_iter().map(|(_, resolved)| resolved).collect()
1764    }
1765
1766    async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1767        let mut visible = Vec::new();
1768        for resolved in self.resolve_members(identity).await {
1769            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1770                continue;
1771            };
1772            if resolved_member_visible(&resolved, &record).await {
1773                visible.push(resolved);
1774            }
1775        }
1776        visible
1777    }
1778
1779    async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1780        let mut records = Vec::new();
1781        for resolved in self.resolve_members(identity).await {
1782            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1783                continue;
1784            };
1785            if resolved_member_visible(&resolved, &record).await {
1786                records.push(record);
1787            }
1788        }
1789        records
1790    }
1791
1792    async fn live_records_for_durable_record(
1793        &self,
1794        entry: &RuntimeEntry,
1795        requested_identity: &str,
1796        durable: &ConsoleIdentityRecord,
1797        requested_live_records: &[ConsoleIdentityRecord],
1798    ) -> Vec<ConsoleIdentityRecord> {
1799        let mut records = if durable.identity == requested_identity {
1800            requested_live_records.to_vec()
1801        } else {
1802            self.live_records_for_identity(&durable.identity).await
1803        };
1804        for runtime_record in
1805            live_records_for_runtime_member(entry, &durable.runtime_member_id).await
1806        {
1807            if records.iter().any(|record| {
1808                record.runtime_key == runtime_record.runtime_key
1809                    && record.runtime_member_id == runtime_record.runtime_member_id
1810                    && record.session_id == runtime_record.session_id
1811                    && record.identity == runtime_record.identity
1812            }) {
1813                continue;
1814            }
1815            records.push(runtime_record);
1816        }
1817        records
1818    }
1819}
1820
1821async fn console_identity_record_visible(
1822    entry: &RuntimeEntry,
1823    record: &ConsoleIdentityRecord,
1824) -> bool {
1825    if !entry.visibility_policy.identity_visible(record) {
1826        return false;
1827    }
1828    let mut bound_live_member_seen = false;
1829    let mut wrong_live_projection_seen = false;
1830    for resolved in member_sources_for_entry(entry).await {
1831        if resolved.runtime_identity != record.runtime_member_id {
1832            continue;
1833        }
1834        bound_live_member_seen = true;
1835        let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
1836            continue;
1837        };
1838        if live_record.identity != record.identity {
1839            wrong_live_projection_seen = true;
1840            continue;
1841        }
1842        if raw_resolved_member_visible(&resolved, &live_record) {
1843            return true;
1844        }
1845    }
1846    if wrong_live_projection_seen {
1847        return true;
1848    }
1849    !bound_live_member_seen
1850}
1851
1852async fn live_record_shadowed_by_hidden_durable_binding(
1853    resolved: &ResolvedConsoleMember,
1854    record: &ConsoleIdentityRecord,
1855) -> bool {
1856    let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
1857        return false;
1858    };
1859    let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
1860        return false;
1861    };
1862    let Ok(status) = identity_runtime.status(&identity).await else {
1863        return false;
1864    };
1865    let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
1866        return false;
1867    };
1868    if bound_runtime_id.as_str() == resolved.runtime_identity {
1869        return false;
1870    }
1871    for candidate in member_sources_for_entry(&resolved.entry).await {
1872        if candidate.runtime_identity != bound_runtime_id.as_str() {
1873            continue;
1874        }
1875        let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
1876            continue;
1877        };
1878        if bound_record.identity == record.identity
1879            && !raw_resolved_member_visible(&candidate, &bound_record)
1880        {
1881            return true;
1882        }
1883    }
1884    false
1885}
1886
1887async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
1888    let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
1889        .unwrap_or_else(|| frame.identity.clone());
1890    let frame_session_id = frame.session_id.as_deref();
1891    let mut saw_hidden = false;
1892    let mut saw_visible = false;
1893    for resolved in member_sources_for_entry(entry).await {
1894        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1895            continue;
1896        };
1897        let session_matches =
1898            frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
1899        let runtime_matches = record.runtime_member_id == runtime_member_id;
1900        let identity_matches = record.identity == frame.identity;
1901        if !session_matches && !runtime_matches && !identity_matches {
1902            continue;
1903        }
1904        if session_matches && !raw_resolved_member_visible(&resolved, &record) {
1905            return true;
1906        }
1907        if resolved_member_visible(&resolved, &record).await {
1908            saw_visible = true;
1909        } else {
1910            saw_hidden = true;
1911        }
1912    }
1913    saw_hidden && !saw_visible
1914}
1915
1916async fn live_records_for_runtime_member(
1917    entry: &RuntimeEntry,
1918    runtime_member_id: &str,
1919) -> Vec<ConsoleIdentityRecord> {
1920    let mut records = Vec::new();
1921    for resolved in member_sources_for_entry(entry).await {
1922        if resolved.runtime_identity != runtime_member_id {
1923            continue;
1924        }
1925        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1926            continue;
1927        };
1928        records.push(record);
1929    }
1930    records
1931}
1932
1933async fn visible_live_records_for_entry(
1934    entry: &RuntimeEntry,
1935    records: &[ConsoleIdentityRecord],
1936) -> Vec<ConsoleIdentityRecord> {
1937    let mut visible = Vec::new();
1938    for record in records {
1939        if record.runtime_key != entry.runtime_key
1940            || console_identity_record_visible(entry, record).await
1941        {
1942            visible.push(record.clone());
1943        }
1944    }
1945    visible
1946}
1947
1948async fn stale_live_record_binding_error(
1949    entry: &RuntimeEntry,
1950    requested_identity: &str,
1951    live_record: &ConsoleIdentityRecord,
1952) -> Option<String> {
1953    let identity_runtime = entry.identity_runtime.as_ref()?;
1954    for status in identity_runtime.statuses().await {
1955        let matches_runtime = status
1956            .agent_runtime_id
1957            .as_ref()
1958            .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
1959        if !matches_runtime {
1960            continue;
1961        }
1962        let durable_record = identity_record_for_status(entry, &status);
1963        if let Some(stale_error) = stale_durable_record_error(
1964            requested_identity,
1965            &durable_record,
1966            std::slice::from_ref(live_record),
1967        ) {
1968            return Some(stale_error);
1969        }
1970    }
1971    None
1972}
1973
1974fn stale_durable_record_error(
1975    requested_identity: &str,
1976    durable: &ConsoleIdentityRecord,
1977    live_records: &[ConsoleIdentityRecord],
1978) -> Option<String> {
1979    let matching_live = live_records
1980        .iter()
1981        .filter(|record| record.identity == durable.identity)
1982        .collect::<Vec<_>>();
1983    if let Some(wrong_projection) = live_records.iter().find(|record| {
1984        record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
1985    }) {
1986        return Some(format!(
1987            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
1988            durable.identity, durable.runtime_member_id, wrong_projection.identity
1989        ));
1990    }
1991    if matching_live.is_empty() {
1992        return None;
1993    }
1994    if let Some(session_mismatch) = matching_live.iter().find(|record| {
1995        record.runtime_member_id == durable.runtime_member_id
1996            && durable.session_id.is_some()
1997            && record.session_id.is_some()
1998            && durable.session_id != record.session_id
1999    }) {
2000        return Some(format!(
2001            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2002            durable.identity,
2003            durable.runtime_member_id,
2004            durable.session_id.as_deref().unwrap_or("<none>"),
2005            session_mismatch.session_id.as_deref().unwrap_or("<none>")
2006        ));
2007    }
2008    if matching_live
2009        .iter()
2010        .any(|record| record.runtime_member_id == durable.runtime_member_id)
2011    {
2012        return None;
2013    }
2014    let live_candidates = matching_live
2015        .iter()
2016        .map(|record| record.runtime_member_id.as_str())
2017        .collect::<Vec<_>>()
2018        .join(", ");
2019    Some(format!(
2020        "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2021        durable.identity, durable.runtime_member_id, live_candidates
2022    ))
2023}
2024
2025fn ambiguous_live_alias_error(
2026    requested_identity: &str,
2027    live_records: &[ConsoleIdentityRecord],
2028) -> Option<String> {
2029    if live_records.len() <= 1 {
2030        return None;
2031    }
2032    let candidates = live_records
2033        .iter()
2034        .map(|record| {
2035            format!(
2036                "{}@{}",
2037                record.runtime_member_id,
2038                record
2039                    .labels
2040                    .get("source_mob_id")
2041                    .map(String::as_str)
2042                    .unwrap_or(record.runtime_key.as_str())
2043            )
2044        })
2045        .collect::<Vec<_>>()
2046        .join(", ");
2047    Some(format!(
2048        "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2049    ))
2050}
2051
2052fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2053    member_id == durable_identity
2054}
2055
2056async fn retire_stale_console_members_for_identity(
2057    handle: &MobHandle,
2058    durable_identities: &[String],
2059    keep_runtime_member_id: Option<&str>,
2060) -> Result<(), String> {
2061    let stale_members = handle
2062        .list_members_including_retiring()
2063        .await
2064        .into_iter()
2065        .filter(|member| {
2066            (durable_identities.iter().any(|durable_identity| {
2067                member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2068            }) || member.labels.get("agent_identity").is_some_and(|identity| {
2069                durable_identities
2070                    .iter()
2071                    .any(|durable_identity| identity == durable_identity)
2072            })) && keep_runtime_member_id
2073                .map(|keep| member.agent_identity.as_str() != keep)
2074                .unwrap_or(true)
2075        })
2076        .map(|member| member.agent_identity)
2077        .collect::<Vec<_>>();
2078    for member_id in stale_members {
2079        match handle.retire(member_id).await {
2080            Ok(()) => {}
2081            Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2082            Err(err) => return Err(err.to_string()),
2083        }
2084    }
2085    Ok(())
2086}
2087
2088async fn retire_stale_console_members_for_runtime_entry(
2089    entry: &RuntimeEntry,
2090    durable_identities: &[String],
2091) -> Result<(), String> {
2092    let primary_handle = entry.runtime.handle();
2093    let primary_mob_id = primary_handle.mob_id().to_string();
2094    let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2095    if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2096        for (mob_id, handle) in state.mob_handles_snapshot().await {
2097            if mob_id.as_str() != primary_mob_id {
2098                handles.push((mob_id.to_string(), handle));
2099            }
2100        }
2101    }
2102    let mut seen_handles = BTreeSet::new();
2103    for (mob_id, handle) in handles {
2104        if !seen_handles.insert(mob_id) {
2105            continue;
2106        }
2107        retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2108    }
2109    Ok(())
2110}
2111
2112fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2113    error.contains("disposal completed but ArchiveSession failed")
2114        && error.contains("cancel-before-retire failed")
2115        && error.contains("Runtime not ready: running")
2116}
2117
2118fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2119    if frames.is_empty() {
2120        return true;
2121    }
2122    let latest_session_history_timestamp_ms = frames
2123        .iter()
2124        .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2125        .map(|frame| frame.timestamp_ms)
2126        .max();
2127    if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2128        return current_time_ms().saturating_sub(latest_timestamp_ms)
2129            >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2130    }
2131    frames.iter().any(|frame| {
2132        matches!(
2133            frame.kind.as_str(),
2134            "turn_started"
2135                | "run_started"
2136                | "reasoning_delta"
2137                | "reasoning_complete"
2138                | "tool_call_requested"
2139                | "tool_call"
2140                | "tool_execution_started"
2141                | "tool_execution_completed"
2142                | "tool_result_received"
2143        )
2144    })
2145}
2146
2147fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2148    match frame.kind.as_str() {
2149        "user_input" | "run_started" => true,
2150        "interaction_started" => frame
2151            .payload
2152            .get("content")
2153            .or_else(|| frame.payload.get("prompt"))
2154            .is_some(),
2155        _ => false,
2156    }
2157}
2158
2159fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2160    let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2161    for record in records {
2162        by_identity
2163            .entry(identity_record_dedupe_key(&record))
2164            .and_modify(|current| {
2165                if identity_record_prefer(&record, current) {
2166                    *current = record.clone();
2167                }
2168            })
2169            .or_insert(record);
2170    }
2171    by_identity.into_values().collect()
2172}
2173
2174fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2175    format!(
2176        "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2177        record.identity,
2178        record.runtime_key,
2179        record
2180            .labels
2181            .get("source_mob_id")
2182            .map(String::as_str)
2183            .unwrap_or(""),
2184        record.runtime_member_id,
2185        record.session_id.as_deref().unwrap_or("")
2186    )
2187}
2188
2189fn identity_record_prefer(
2190    candidate: &ConsoleIdentityRecord,
2191    current: &ConsoleIdentityRecord,
2192) -> bool {
2193    let candidate_is_live_label_projection =
2194        candidate
2195            .labels
2196            .get("agent_identity")
2197            .is_some_and(|identity| {
2198                identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2199                    || candidate.runtime_member_id != candidate.identity
2200            });
2201    let current_is_live_label_projection =
2202        current
2203            .labels
2204            .get("agent_identity")
2205            .is_some_and(|identity| {
2206                identity == &strip_namespace(&current.identity, "").unwrap_or_default()
2207                    || current.runtime_member_id != current.identity
2208            });
2209    if candidate_is_live_label_projection != current_is_live_label_projection {
2210        return candidate_is_live_label_projection;
2211    }
2212    let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2213    let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2214    if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2215        return candidate_has_distinct_runtime_binding;
2216    }
2217    let candidate_live = candidate.addressable && candidate.health != "retired";
2218    let current_live = current.addressable && current.health != "retired";
2219    if candidate_live != current_live {
2220        return candidate_live;
2221    }
2222    candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2223}
2224
2225async fn collect_identity_records(
2226    inner: &Arc<AggregatorInner>,
2227    mode: IdentityCollectionMode,
2228) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2229    let entries = inner
2230        .runtimes
2231        .read()
2232        .map_err(|_| runtime_registry_lock_error())?
2233        .clone();
2234    let mut identities = Vec::new();
2235    for entry in entries.values() {
2236        if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2237            let topology_peers =
2238                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2239            for status in identity_runtime.statuses().await {
2240                let mut record = identity_record_for_status(entry, &status);
2241                record.topology_peers = topology_peers
2242                    .get(&record.identity)
2243                    .cloned()
2244                    .unwrap_or_default();
2245                if console_identity_record_visible(entry, &record).await {
2246                    identities.push(record);
2247                }
2248            }
2249            if mode == IdentityCollectionMode::CachedOnly {
2250                continue;
2251            }
2252        }
2253        let members = if entry.identity_runtime.is_some() {
2254            match tokio::time::timeout(
2255                IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2256                member_sources_for_entry(entry),
2257            )
2258            .await
2259            {
2260                Ok(members) => members,
2261                Err(_) => {
2262                    tracing::warn!(
2263                        runtime_key = %entry.runtime_key,
2264                        "identity-first live member refresh timed out; keeping cached identity read model"
2265                    );
2266                    Vec::new()
2267                }
2268            }
2269        } else {
2270            member_sources_for_entry(entry).await
2271        };
2272        for resolved in members {
2273            if let Some(record) = identity_record_for_resolved_member(&resolved).await
2274                && resolved_member_visible(&resolved, &record).await
2275            {
2276                identities.push(record);
2277            }
2278        }
2279    }
2280    Ok(dedupe_identity_records(identities))
2281}
2282
2283async fn identity_runtime_topology_peers(
2284    entry: &RuntimeEntry,
2285    identity_runtime: &crate::identity_first::IdentityRuntime,
2286) -> BTreeMap<String, Vec<String>> {
2287    let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2288    for edge in identity_runtime.desired_peer_edges().await {
2289        let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2290        let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2291        peers.entry(a.clone()).or_default().insert(b.clone());
2292        peers.entry(b).or_default().insert(a);
2293    }
2294    peers
2295        .into_iter()
2296        .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2297        .collect()
2298}
2299
2300fn spawn_identity_backfills_for_records(
2301    inner: Arc<AggregatorInner>,
2302    records: &[ConsoleIdentityRecord],
2303) {
2304    if !inner.options.session_history_backfill_enabled {
2305        return;
2306    }
2307    let entries = match inner.runtimes.read() {
2308        Ok(entries) => entries.clone(),
2309        Err(_) => return,
2310    };
2311    for record in records {
2312        if record.health == "dormant" || record.health == "uninitialized" {
2313            continue;
2314        }
2315        let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2316            continue;
2317        };
2318        let Some(session_id) = record.session_id.clone() else {
2319            continue;
2320        };
2321        spawn_session_history_backfill_target(
2322            inner.clone(),
2323            SessionBackfillTarget {
2324                entry,
2325                record: record.clone(),
2326                session_id,
2327            },
2328            false,
2329        );
2330    }
2331}
2332
2333async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2334    let mut resolved = Vec::new();
2335    let primary_handle = entry.runtime.handle();
2336    let primary_mob_id = primary_handle.mob_id().to_string();
2337    for member in primary_handle.list_members_observation_snapshot().await {
2338        resolved.push(ResolvedConsoleMember {
2339            entry: entry.clone(),
2340            handle: primary_handle.clone(),
2341            runtime_identity: member.agent_identity.to_string(),
2342            source_mob_id: primary_mob_id.clone(),
2343            member,
2344        });
2345    }
2346
2347    let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2348        return resolved;
2349    };
2350    if !entry.visibility_policy.include_implicit_delegate_members() {
2351        return resolved;
2352    }
2353    for (mob_id, handle) in state.mob_handles_snapshot().await {
2354        if mob_id.as_str() == primary_mob_id {
2355            continue;
2356        }
2357        for member in handle.list_members_observation_snapshot().await {
2358            resolved.push(ResolvedConsoleMember {
2359                entry: entry.clone(),
2360                handle: handle.clone(),
2361                runtime_identity: member.agent_identity.to_string(),
2362                source_mob_id: mob_id.to_string(),
2363                member,
2364            });
2365        }
2366    }
2367    resolved
2368}
2369
2370async fn dispatch_message_to_resolved_member(
2371    resolved: &ResolvedConsoleMember,
2372    content: ContentInput,
2373    handling_mode: meerkat_core::types::HandlingMode,
2374) -> Result<String, String> {
2375    let mid = MeerkatId::from(resolved.runtime_identity.as_str());
2376    match send_message_on_mob_with_mode(
2377        &resolved.handle,
2378        &resolved.runtime_identity,
2379        content.clone(),
2380        handling_mode,
2381    )
2382    .await
2383    {
2384        Ok(session_id) => Ok(session_id),
2385        Err(err) if is_not_externally_addressable(&err) => {
2386            let member = resolved
2387                .handle
2388                .member(&mid)
2389                .await
2390                .map_err(|err| err.to_string())?;
2391            let _receipt = member
2392                .internal_turn(content)
2393                .await
2394                .map_err(|err| err.to_string())?;
2395            resolved
2396                .handle
2397                .resolve_bridge_session_id_observation(&mid)
2398                .await
2399                .map(|sid| sid.to_string())
2400                .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2401        }
2402        Err(err) => Err(err.to_string()),
2403    }
2404}
2405
2406fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2407    matches!(
2408        err,
2409        MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2410    )
2411}
2412
2413fn spawn_console_send_dispatch(
2414    inner: Arc<AggregatorInner>,
2415    resolved: ResolvedConsoleMember,
2416    content: ContentInput,
2417    handling_mode: meerkat_core::types::HandlingMode,
2418    dispatching: SendState,
2419    user_frame: ConsoleFrame,
2420    interaction_id: String,
2421) {
2422    tokio::spawn(async move {
2423        match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2424            Ok(_) => {
2425                let _ = dispatching.apply(SendTransition::MarkDelivered);
2426                if let Err(err) = update_frame_status_and_emit(
2427                    &inner,
2428                    &user_frame.id,
2429                    ConsoleFrameStatus::Delivered,
2430                )
2431                .await
2432                {
2433                    tracing::warn!(
2434                        frame_id = %user_frame.id,
2435                        error = %err,
2436                        "failed to update console send delivery status"
2437                    );
2438                }
2439                if handling_mode == HandlingMode::Steer
2440                    && let Err(err) =
2441                        append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2442                {
2443                    tracing::warn!(
2444                        frame_id = %user_frame.id,
2445                        interaction_id = %interaction_id,
2446                        error = %err,
2447                        "failed to append console steer terminal frame"
2448                    );
2449                }
2450            }
2451            Err(err) => {
2452                let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2453                if let Err(update_err) = update_frame_status_and_emit(
2454                    &inner,
2455                    &user_frame.id,
2456                    ConsoleFrameStatus::DeliveryFailed,
2457                )
2458                .await
2459                {
2460                    tracing::warn!(
2461                        frame_id = %user_frame.id,
2462                        error = %update_err,
2463                        "failed to update console send failure status"
2464                    );
2465                }
2466                let failure_frame = NewConsoleFrame {
2467                    id: None,
2468                    dedupe_key: format!("delivery-failed:{}", user_frame.id),
2469                    timestamp_ms: current_time_ms(),
2470                    runtime_key: user_frame.runtime_key,
2471                    identity: user_frame.identity,
2472                    conversation_id: user_frame.conversation_id,
2473                    session_id: user_frame.session_id,
2474                    kind: "message_delivery_failed".to_string(),
2475                    status: ConsoleFrameStatus::DeliveryFailed,
2476                    payload: json!({ "reason": err }),
2477                    source: ConsoleFrameSource {
2478                        kind: ConsoleFrameSourceKind::Synthetic,
2479                        source_cursor: None,
2480                    },
2481                    source_event_id: None,
2482                    interaction_id: Some(interaction_id),
2483                    turn_id: None,
2484                    run_id: None,
2485                    parent_frame_id: Some(user_frame.id.clone()),
2486                    caused_by_frame_id: Some(user_frame.id),
2487                };
2488                if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2489                    tracing::warn!(
2490                        error = %append_err,
2491                        "failed to append console send failure frame"
2492                    );
2493                }
2494            }
2495        }
2496    });
2497}
2498
2499async fn append_steer_delivery_terminal(
2500    inner: &AggregatorInner,
2501    user_frame: &ConsoleFrame,
2502    interaction_id: &str,
2503) -> ConsoleLogResult<AppendOutcome> {
2504    append_and_emit(
2505        inner,
2506        NewConsoleFrame {
2507            id: None,
2508            dedupe_key: format!("steer-delivered:{}", user_frame.id),
2509            timestamp_ms: current_time_ms(),
2510            runtime_key: user_frame.runtime_key.clone(),
2511            identity: user_frame.identity.clone(),
2512            conversation_id: user_frame.conversation_id.clone(),
2513            session_id: user_frame.session_id.clone(),
2514            kind: "interaction_complete".to_string(),
2515            status: ConsoleFrameStatus::Completed,
2516            payload: json!({
2517                "reason": "steer_delivered",
2518                "handling_mode": "steer",
2519            }),
2520            source: ConsoleFrameSource {
2521                kind: ConsoleFrameSourceKind::Synthetic,
2522                source_cursor: None,
2523            },
2524            source_event_id: None,
2525            interaction_id: Some(interaction_id.to_string()),
2526            turn_id: None,
2527            run_id: None,
2528            parent_frame_id: Some(user_frame.id.clone()),
2529            caused_by_frame_id: Some(user_frame.id.clone()),
2530        },
2531    )
2532    .await
2533}
2534
2535#[derive(Debug)]
2536pub enum ConsoleSendError {
2537    UnknownIdentity(String),
2538    AmbiguousIdentity {
2539        identity: String,
2540        candidates: String,
2541    },
2542    NotAddressable(String),
2543    Retired(String),
2544    InvalidContent(String),
2545    InvalidHandlingMode(String),
2546    InvalidRequest(String),
2547    IdempotencyConflict(String),
2548    State(&'static str),
2549    Dispatch(String),
2550    Log(ConsoleLogError),
2551}
2552
2553impl std::fmt::Display for ConsoleSendError {
2554    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2555        match self {
2556            Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2557            Self::AmbiguousIdentity {
2558                identity,
2559                candidates,
2560            } => write!(
2561                f,
2562                "ambiguous live identity alias {identity}: candidates [{candidates}]"
2563            ),
2564            Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2565            Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2566            Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2567            Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2568            Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2569            Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2570            Self::State(message) => write!(f, "console send state error: {message}"),
2571            Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2572            Self::Log(err) => write!(f, "console log error: {err}"),
2573        }
2574    }
2575}
2576
2577impl std::error::Error for ConsoleSendError {}
2578
2579async fn backfill_session_history(
2580    inner: Arc<AggregatorInner>,
2581    runtime_key: String,
2582    force_refresh: bool,
2583) -> ConsoleLogResult<()> {
2584    if !inner.options.session_history_backfill_enabled {
2585        return Ok(());
2586    }
2587    let Some(entry) = inner
2588        .runtimes
2589        .read()
2590        .ok()
2591        .and_then(|entries| entries.get(&runtime_key).cloned())
2592    else {
2593        return Ok(());
2594    };
2595    let members = member_sources_for_entry(&entry).await;
2596    let mut targets = Vec::new();
2597    for resolved in members {
2598        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2599            continue;
2600        };
2601        if !resolved_member_visible(&resolved, &record).await {
2602            continue;
2603        }
2604        let Some(session_id) = record.session_id.clone() else {
2605            continue;
2606        };
2607        targets.push(SessionBackfillTarget {
2608            entry: entry.clone(),
2609            record,
2610            session_id,
2611        });
2612    }
2613    backfill_session_history_targets(inner, targets, force_refresh).await
2614}
2615
2616#[derive(Clone)]
2617struct SessionBackfillTarget {
2618    entry: RuntimeEntry,
2619    record: ConsoleIdentityRecord,
2620    session_id: String,
2621}
2622
2623async fn backfill_session_history_targets(
2624    inner: Arc<AggregatorInner>,
2625    targets: Vec<SessionBackfillTarget>,
2626    force_refresh: bool,
2627) -> ConsoleLogResult<()> {
2628    let mut tasks = tokio::task::JoinSet::new();
2629    for target in targets {
2630        tasks.spawn(backfill_one_session_history(
2631            inner.clone(),
2632            target,
2633            force_refresh,
2634        ));
2635    }
2636    let mut first_error = None;
2637    while let Some(result) = tasks.join_next().await {
2638        match result {
2639            Ok(Ok(())) => {}
2640            Ok(Err(err)) => {
2641                if first_error.is_none() {
2642                    first_error = Some(err);
2643                }
2644            }
2645            Err(err) => {
2646                if first_error.is_none() {
2647                    first_error = Some(Box::new(std::io::Error::other(format!(
2648                        "session backfill task failed: {err}"
2649                    ))) as ConsoleLogError);
2650                }
2651            }
2652        }
2653    }
2654    if let Some(err) = first_error {
2655        Err(err)
2656    } else {
2657        Ok(())
2658    }
2659}
2660
2661async fn backfill_one_session_history(
2662    inner: Arc<AggregatorInner>,
2663    target: SessionBackfillTarget,
2664    force_refresh: bool,
2665) -> ConsoleLogResult<()> {
2666    let _permit = inner
2667        .session_backfill_permits
2668        .clone()
2669        .acquire_owned()
2670        .await
2671        .map_err(|err| -> ConsoleLogError {
2672            Box::new(std::io::Error::other(format!(
2673                "session backfill limiter closed: {err}"
2674            )))
2675        })?;
2676    let SessionBackfillTarget {
2677        entry,
2678        record,
2679        session_id,
2680    } = target;
2681    let watermark_runtime_key =
2682        session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2683    let watermark = inner
2684        .store
2685        .source_watermark(
2686            &watermark_runtime_key,
2687            ConsoleFrameSourceKind::SessionHistory,
2688        )
2689        .await?;
2690    let now_ms = current_time_ms();
2691    let mut offset = watermark
2692        .as_deref()
2693        .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2694        .unwrap_or(0);
2695    if !force_refresh
2696        && watermark.as_deref().is_some_and(|watermark| {
2697            session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2698        })
2699    {
2700        return Ok(());
2701    }
2702    loop {
2703        let page = match entry
2704            .runtime
2705            .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2706            .await
2707        {
2708            Ok(page) => page,
2709            Err(err) => {
2710                append_backfill_gap(
2711                    &inner,
2712                    &entry.runtime_key,
2713                    &record.identity,
2714                    err.to_string(),
2715                )
2716                .await?;
2717                break;
2718            }
2719        };
2720        let page_value = match serde_json::to_value(page) {
2721            Ok(value) => value,
2722            Err(err) => {
2723                append_backfill_gap(
2724                    &inner,
2725                    &entry.runtime_key,
2726                    &record.identity,
2727                    err.to_string(),
2728                )
2729                .await?;
2730                break;
2731            }
2732        };
2733        let base_offset = page_value
2734            .get("offset")
2735            .and_then(Value::as_u64)
2736            .unwrap_or(offset as u64) as usize;
2737        let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
2738            append_backfill_gap(
2739                &inner,
2740                &entry.runtime_key,
2741                &record.identity,
2742                "session history page missing messages".to_string(),
2743            )
2744            .await?;
2745            break;
2746        };
2747        if messages.is_empty() {
2748            if offset > 0 {
2749                record_session_history_watermark(
2750                    &inner,
2751                    &watermark_runtime_key,
2752                    &session_id,
2753                    offset,
2754                )
2755                .await?;
2756            }
2757            break;
2758        }
2759        for (idx, message) in messages.iter().enumerate() {
2760            let absolute_offset = base_offset + idx;
2761            let frames = frames_from_session_history_message_with_namespace(
2762                &entry.runtime_key,
2763                &record.identity,
2764                &entry.identity_namespace,
2765                &session_id,
2766                absolute_offset,
2767                message.clone(),
2768            );
2769            for mut frame in frames {
2770                if history_frame_has_existing_counterpart(&inner, &frame).await? {
2771                    continue;
2772                }
2773                if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
2774                    frame.payload = redacted;
2775                    frame.status = ConsoleFrameStatus::Redacted;
2776                }
2777                append_and_emit(&inner, frame).await?;
2778            }
2779        }
2780        offset = base_offset + messages.len();
2781        record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
2782            .await?;
2783        let has_more = page_value
2784            .get("has_more")
2785            .and_then(Value::as_bool)
2786            .unwrap_or(false);
2787        if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
2788            break;
2789        }
2790    }
2791    Ok(())
2792}
2793
2794async fn record_session_history_watermark(
2795    inner: &AggregatorInner,
2796    watermark_runtime_key: &str,
2797    session_id: &str,
2798    offset: usize,
2799) -> ConsoleLogResult<()> {
2800    inner
2801        .store
2802        .record_source_watermark(
2803            watermark_runtime_key,
2804            ConsoleFrameSourceKind::SessionHistory,
2805            &format_session_history_watermark(session_id, offset, current_time_ms()),
2806        )
2807        .await
2808}
2809
2810fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
2811    if !inner.options.session_history_backfill_enabled {
2812        return;
2813    }
2814    tokio::spawn(async move {
2815        {
2816            let mut active = inner.active_session_backfills.lock().await;
2817            if !active.insert(runtime_key.clone()) {
2818                return;
2819            }
2820        }
2821        let result = backfill_session_history(inner.clone(), runtime_key.clone(), false).await;
2822        let mut active = inner.active_session_backfills.lock().await;
2823        active.remove(&runtime_key);
2824        drop(active);
2825        if let Err(err) = result {
2826            tracing::warn!(
2827                runtime_key = %runtime_key,
2828                error = %err,
2829                "console session-history backfill failed"
2830            );
2831        }
2832    });
2833}
2834
2835fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
2836    if !inner.options.session_history_backfill_enabled {
2837        return;
2838    }
2839    tokio::spawn(async move {
2840        let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
2841        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2842        loop {
2843            interval.tick().await;
2844            let runtime_still_registered = inner
2845                .runtimes
2846                .read()
2847                .ok()
2848                .is_some_and(|entries| entries.contains_key(&runtime_key));
2849            if !runtime_still_registered {
2850                break;
2851            }
2852            spawn_session_history_backfill(inner.clone(), runtime_key.clone());
2853        }
2854    });
2855}
2856
2857fn spawn_session_history_backfill_target(
2858    inner: Arc<AggregatorInner>,
2859    target: SessionBackfillTarget,
2860    force_refresh: bool,
2861) {
2862    if !inner.options.session_history_backfill_enabled {
2863        return;
2864    }
2865    tokio::spawn(async move {
2866        let active_key = targeted_session_history_active_key(&target, force_refresh);
2867        let result =
2868            run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
2869        if let Err(err) = result {
2870            tracing::warn!(
2871                active_key = %active_key,
2872                error = %err,
2873                "console targeted session-history backfill failed"
2874            );
2875        }
2876    });
2877}
2878
2879async fn run_targeted_session_history_backfill(
2880    inner: Arc<AggregatorInner>,
2881    target: SessionBackfillTarget,
2882    force_refresh: bool,
2883) -> ConsoleLogResult<()> {
2884    let active_key = targeted_session_history_active_key(&target, force_refresh);
2885    {
2886        let mut active = inner.active_session_backfills.lock().await;
2887        if !active.insert(active_key.clone()) {
2888            return Ok(());
2889        }
2890    }
2891    let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
2892    let mut active = inner.active_session_backfills.lock().await;
2893    active.remove(&active_key);
2894    result
2895}
2896
2897fn targeted_session_history_active_key(
2898    target: &SessionBackfillTarget,
2899    force_refresh: bool,
2900) -> String {
2901    let mode = if force_refresh { "force" } else { "refresh" };
2902    format!(
2903        "{}:session-history:{}:{mode}",
2904        target.entry.runtime_key, target.session_id
2905    )
2906}
2907
2908fn spawn_session_history_backfill_for_identity(
2909    inner: Arc<AggregatorInner>,
2910    identity: String,
2911    force_refresh: bool,
2912) {
2913    if !inner.options.session_history_backfill_enabled {
2914        return;
2915    }
2916    tokio::spawn(async move {
2917        for target in session_backfill_targets_for_identity(&inner, &identity).await {
2918            spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
2919        }
2920    });
2921}
2922
2923fn spawn_opportunistic_session_history_backfill_for_identity(
2924    inner: Arc<AggregatorInner>,
2925    identity: String,
2926) {
2927    if !inner.options.session_history_backfill_enabled {
2928        return;
2929    }
2930    tokio::spawn(async move {
2931        for target in session_backfill_targets_for_identity(&inner, &identity).await {
2932            let active_key = format!(
2933                "{}:session-history:{}",
2934                target.entry.runtime_key, target.session_id
2935            );
2936            {
2937                let mut seen = inner.opportunistic_session_backfills.lock().await;
2938                if !seen.insert(active_key) {
2939                    continue;
2940                }
2941            }
2942            spawn_session_history_backfill_target(inner.clone(), target, false);
2943        }
2944    });
2945}
2946
2947#[cfg(test)]
2948async fn session_backfill_target_for_identity(
2949    inner: &AggregatorInner,
2950    identity: &str,
2951) -> Option<SessionBackfillTarget> {
2952    session_backfill_targets_for_identity(inner, identity)
2953        .await
2954        .into_iter()
2955        .next()
2956}
2957
2958async fn session_backfill_targets_for_identity(
2959    inner: &AggregatorInner,
2960    identity: &str,
2961) -> Vec<SessionBackfillTarget> {
2962    let entries = inner
2963        .runtimes
2964        .read()
2965        .ok()
2966        .map(|entries| entries.clone())
2967        .unwrap_or_default();
2968    let mut targets = Vec::new();
2969    for entry in entries.values() {
2970        let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2971        if raw_identities.is_empty() {
2972            continue;
2973        }
2974        let mids = raw_identities
2975            .iter()
2976            .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
2977            .collect::<Vec<_>>();
2978        for resolved in member_sources_for_entry(entry)
2979            .await
2980            .into_iter()
2981            .filter(|candidate| {
2982                resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
2983            })
2984        {
2985            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2986                continue;
2987            };
2988            if !resolved_member_visible(&resolved, &record).await {
2989                continue;
2990            }
2991            let Some(session_id) = record.session_id.clone() else {
2992                continue;
2993            };
2994            targets.push(SessionBackfillTarget {
2995                entry: entry.clone(),
2996                record,
2997                session_id,
2998            });
2999        }
3000    }
3001    targets
3002}
3003
3004fn resolved_member_matches_raw_identities(
3005    resolved: &ResolvedConsoleMember,
3006    raw_identities: &[String],
3007    mids: &[MeerkatId],
3008) -> bool {
3009    mids.contains(&resolved.member.agent_identity)
3010        || resolved
3011            .member
3012            .labels
3013            .get("agent_identity")
3014            .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3015}
3016
3017async fn recover_lagged_source_events(
3018    inner: Arc<AggregatorInner>,
3019    runtime_key: &str,
3020    console_events: &ConsoleEventStore,
3021) -> ConsoleLogResult<()> {
3022    let watermark = inner
3023        .store
3024        .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3025        .await?;
3026    match console_events.replay_all(watermark.as_deref()).await {
3027        Ok(events) => {
3028            for envelope in events {
3029                project_console_event(inner.clone(), runtime_key, envelope).await?;
3030            }
3031        }
3032        Err(err) => {
3033            append_source_gap(
3034                &inner,
3035                runtime_key,
3036                format!(
3037                    "{}:{}:{}",
3038                    err.error, err.stream, err.requested_last_event_id
3039                ),
3040            )
3041            .await?;
3042        }
3043    }
3044    Ok(())
3045}
3046
3047async fn append_source_gap(
3048    inner: &AggregatorInner,
3049    runtime_key: &str,
3050    reason: String,
3051) -> ConsoleLogResult<()> {
3052    append_and_emit(
3053        inner,
3054        NewConsoleFrame {
3055            id: None,
3056            dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3057            timestamp_ms: current_time_ms(),
3058            runtime_key: runtime_key.to_string(),
3059            identity: "__console__".to_string(),
3060            conversation_id: None,
3061            session_id: None,
3062            kind: "replay_unavailable".to_string(),
3063            status: ConsoleFrameStatus::DeliveryFailed,
3064            payload: json!({
3065                "reason": reason,
3066                "source_kind": "console_event",
3067            }),
3068            source: ConsoleFrameSource {
3069                kind: ConsoleFrameSourceKind::Synthetic,
3070                source_cursor: None,
3071            },
3072            source_event_id: None,
3073            interaction_id: None,
3074            turn_id: None,
3075            run_id: None,
3076            parent_frame_id: None,
3077            caused_by_frame_id: None,
3078        },
3079    )
3080    .await?;
3081    let _ = inner
3082        .event_tx
3083        .send(ConsoleTimelineEvent::ReplayUnavailable {
3084            requested_cursor: format!("source-gap:{runtime_key}"),
3085            latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3086        });
3087    Ok(())
3088}
3089
3090async fn append_backfill_gap(
3091    inner: &AggregatorInner,
3092    runtime_key: &str,
3093    identity: &str,
3094    reason: String,
3095) -> ConsoleLogResult<()> {
3096    append_and_emit(
3097        inner,
3098        NewConsoleFrame {
3099            id: None,
3100            dedupe_key: format!(
3101                "session-backfill-gap:{runtime_key}:{identity}:{}",
3102                current_time_ms()
3103            ),
3104            timestamp_ms: current_time_ms(),
3105            runtime_key: runtime_key.to_string(),
3106            identity: identity.to_string(),
3107            conversation_id: Some(identity.to_string()),
3108            session_id: None,
3109            kind: "replay_unavailable".to_string(),
3110            status: ConsoleFrameStatus::DeliveryFailed,
3111            payload: json!({
3112                "reason": reason,
3113                "source_kind": "session_history",
3114            }),
3115            source: ConsoleFrameSource {
3116                kind: ConsoleFrameSourceKind::Synthetic,
3117                source_cursor: None,
3118            },
3119            source_event_id: None,
3120            interaction_id: None,
3121            turn_id: None,
3122            run_id: None,
3123            parent_frame_id: None,
3124            caused_by_frame_id: None,
3125        },
3126    )
3127    .await?;
3128    Ok(())
3129}
3130
3131async fn project_console_event(
3132    inner: Arc<AggregatorInner>,
3133    runtime_key: &str,
3134    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3135) -> ConsoleLogResult<()> {
3136    let Some(entry) = inner
3137        .runtimes
3138        .read()
3139        .ok()
3140        .and_then(|entries| entries.get(runtime_key).cloned())
3141    else {
3142        return Ok(());
3143    };
3144    let mut frame = frame_from_console_event(&entry, envelope);
3145    if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3146        frame.payload = redacted;
3147        frame.status = ConsoleFrameStatus::Redacted;
3148    }
3149    let source_cursor = frame
3150        .source_event_id
3151        .clone()
3152        .unwrap_or_else(|| frame.dedupe_key.clone());
3153    let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3154        Some(frame.identity.clone())
3155    } else {
3156        None
3157    };
3158    let opportunistic_refresh_identity = if refresh_identity.is_none()
3159        && console_event_should_start_session_history_backfill(&frame)
3160    {
3161        Some(frame.identity.clone())
3162    } else {
3163        None
3164    };
3165    append_and_emit(&inner, frame).await?;
3166    inner
3167        .store
3168        .record_source_watermark(
3169            &entry.runtime_key,
3170            ConsoleFrameSourceKind::ConsoleEvent,
3171            &source_cursor,
3172        )
3173        .await?;
3174    if let Some(identity) = refresh_identity {
3175        spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3176    } else if let Some(identity) = opportunistic_refresh_identity {
3177        spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3178    }
3179    Ok(())
3180}
3181
3182fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3183    matches!(
3184        frame.kind.as_str(),
3185        "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3186    ) || frame.session_id.is_some()
3187}
3188
3189fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3190    if frame.identity == SYSTEM_EVENT_IDENTITY {
3191        return false;
3192    }
3193    matches!(
3194        frame.kind.as_str(),
3195        "turn_started"
3196            | "run_started"
3197            | "reasoning_complete"
3198            | "tool_call_requested"
3199            | "tool_call"
3200            | "tool_execution_started"
3201            | "text_delta"
3202            | "system_notice"
3203    )
3204}
3205
3206async fn append_and_emit(
3207    inner: &AggregatorInner,
3208    frame: NewConsoleFrame,
3209) -> ConsoleLogResult<AppendOutcome> {
3210    let outcome = inner.store.append_if_absent(frame).await?;
3211    if outcome.disposition == AppendDisposition::Inserted {
3212        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3213            frame: outcome.frame.clone(),
3214        });
3215    }
3216    Ok(outcome)
3217}
3218
3219async fn update_frame_status_and_emit(
3220    inner: &AggregatorInner,
3221    frame_id: &str,
3222    status: ConsoleFrameStatus,
3223) -> ConsoleLogResult<Option<ConsoleFrame>> {
3224    let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3225        return Ok(None);
3226    };
3227    let update_marker = NewConsoleFrame {
3228        id: None,
3229        dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3230        timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3231        runtime_key: updated.runtime_key.clone(),
3232        identity: updated.identity.clone(),
3233        conversation_id: updated.conversation_id.clone(),
3234        session_id: updated.session_id.clone(),
3235        kind: "frame_updated".to_string(),
3236        status: updated.status,
3237        payload: json!({ "frame": updated.clone() }),
3238        source: ConsoleFrameSource {
3239            kind: ConsoleFrameSourceKind::Synthetic,
3240            source_cursor: None,
3241        },
3242        source_event_id: None,
3243        interaction_id: updated.interaction_id.clone(),
3244        turn_id: updated.turn_id.clone(),
3245        run_id: updated.run_id.clone(),
3246        parent_frame_id: Some(updated.id.clone()),
3247        caused_by_frame_id: Some(updated.id.clone()),
3248    };
3249    let outcome = inner.store.append_if_absent(update_marker).await?;
3250    if outcome.disposition == AppendDisposition::Inserted {
3251        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3252            frame: outcome.frame,
3253        });
3254    }
3255    Ok(Some(updated))
3256}
3257
3258fn frame_from_console_event(
3259    entry: &RuntimeEntry,
3260    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3261) -> NewConsoleFrame {
3262    let turn_id = envelope
3263        .data
3264        .get("turn_id")
3265        .and_then(Value::as_str)
3266        .map(ToString::to_string);
3267    let run_id = envelope
3268        .data
3269        .get("run_id")
3270        .and_then(Value::as_str)
3271        .map(ToString::to_string);
3272    let status = match envelope.event_type.as_str() {
3273        "interaction_started" => ConsoleFrameStatus::Accepted,
3274        "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3275        "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3276        _ => ConsoleFrameStatus::Delivered,
3277    };
3278    let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3279    NewConsoleFrame {
3280        id: Some(envelope.event_id.clone()),
3281        dedupe_key: format!("console-event:{}:{}", entry.runtime_key, envelope.event_id),
3282        timestamp_ms: envelope.timestamp_ms,
3283        runtime_key: entry.runtime_key.clone(),
3284        identity: identity.clone(),
3285        conversation_id: Some(identity),
3286        session_id: envelope
3287            .data
3288            .get("session_id")
3289            .and_then(Value::as_str)
3290            .map(ToString::to_string),
3291        kind: envelope.event_type,
3292        status,
3293        payload: envelope.data,
3294        source: ConsoleFrameSource {
3295            kind: ConsoleFrameSourceKind::ConsoleEvent,
3296            source_cursor: None,
3297        },
3298        source_event_id: Some(envelope.event_id),
3299        interaction_id: envelope.interaction_id,
3300        turn_id,
3301        run_id,
3302        parent_frame_id: None,
3303        caused_by_frame_id: None,
3304    }
3305}
3306
3307#[cfg(test)]
3308fn frame_from_session_history_message(
3309    runtime_key: &str,
3310    identity: &str,
3311    session_id: &str,
3312    offset: usize,
3313    message: Value,
3314) -> Option<NewConsoleFrame> {
3315    frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3316        .into_iter()
3317        .next()
3318}
3319
3320#[cfg(test)]
3321fn frames_from_session_history_message(
3322    runtime_key: &str,
3323    identity: &str,
3324    session_id: &str,
3325    offset: usize,
3326    message: Value,
3327) -> Vec<NewConsoleFrame> {
3328    frames_from_session_history_message_with_namespace(
3329        runtime_key,
3330        identity,
3331        "",
3332        session_id,
3333        offset,
3334        message,
3335    )
3336}
3337
3338fn frames_from_session_history_message_with_namespace(
3339    runtime_key: &str,
3340    identity: &str,
3341    identity_namespace: &str,
3342    session_id: &str,
3343    offset: usize,
3344    message: Value,
3345) -> Vec<NewConsoleFrame> {
3346    let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3347    let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3348        return Vec::new();
3349    };
3350    if let Message::ToolResults {
3351        results,
3352        created_at,
3353    } = parsed
3354    {
3355        return results
3356            .into_iter()
3357            .enumerate()
3358            .map(|(idx, result)| {
3359                let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3360                let result_text = result.text_content();
3361                let tool_use_id = result.tool_use_id.clone();
3362                NewConsoleFrame {
3363                    id: None,
3364                    dedupe_key: format!(
3365                        "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3366                    ),
3367                    timestamp_ms: created_at.timestamp_millis().max(0) as u64,
3368                    runtime_key: runtime_key.to_string(),
3369                    identity: identity.to_string(),
3370                    conversation_id: Some(identity.to_string()),
3371                    session_id: Some(session_id.to_string()),
3372                    kind: "tool_execution_completed".to_string(),
3373                    status: ConsoleFrameStatus::Completed,
3374                    payload: json!({
3375                        "id": tool_use_id,
3376                        "tool_call_id": tool_use_id,
3377                        "result": result_text,
3378                        "content": content,
3379                        "is_error": result.is_error,
3380                        "source_event_type": "session_history",
3381                        "type": "session_history",
3382                    }),
3383                    source: ConsoleFrameSource {
3384                        kind: ConsoleFrameSourceKind::SessionHistory,
3385                        source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3386                    },
3387                    source_event_id: None,
3388                    interaction_id: None,
3389                    turn_id: None,
3390                    run_id: None,
3391                    parent_frame_id: None,
3392                    caused_by_frame_id: None,
3393                }
3394            })
3395            .collect();
3396    }
3397    let (kind, timestamp_ms, payload) = match &parsed {
3398        Message::User(user) => {
3399            if session_history_user_message_is_scaffold(&message) {
3400                return Vec::new();
3401            }
3402            (
3403                "user_input",
3404                user.created_at.timestamp_millis().max(0) as u64,
3405                json!({
3406                    "content": user.content,
3407                    "message": message,
3408                }),
3409            )
3410        }
3411        Message::Assistant(assistant) => (
3412            "interaction_complete",
3413            assistant.created_at.timestamp_millis().max(0) as u64,
3414            json!({
3415                "result": assistant.content,
3416                "text": assistant.content,
3417                "message": message,
3418                "source_event_type": "session_history",
3419                "type": "session_history",
3420            }),
3421        ),
3422        Message::BlockAssistant(assistant) => {
3423            let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3424            (
3425                "interaction_complete",
3426                assistant.created_at.timestamp_millis().max(0) as u64,
3427                json!({
3428                    "result": text,
3429                    "text": text,
3430                    "message": message,
3431                    "source_event_type": "session_history",
3432                    "type": "session_history",
3433                }),
3434            )
3435        }
3436        Message::SystemNotice(notice) => (
3437            "system_notice",
3438            notice.created_at.timestamp_millis().max(0) as u64,
3439            json!({
3440                "message": message,
3441                "kind": notice.kind,
3442                "render_class": notice.kind.render_class(),
3443                "body": notice.body,
3444                "blocks": notice.blocks,
3445                "source_event_type": "session_history",
3446                "type": "session_history",
3447            }),
3448        ),
3449        Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3450    };
3451    let mut frames = vec![NewConsoleFrame {
3452        id: None,
3453        dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3454        timestamp_ms,
3455        runtime_key: runtime_key.to_string(),
3456        identity: identity.to_string(),
3457        conversation_id: Some(identity.to_string()),
3458        session_id: Some(session_id.to_string()),
3459        kind: kind.to_string(),
3460        status: ConsoleFrameStatus::Completed,
3461        payload,
3462        source: ConsoleFrameSource {
3463            kind: ConsoleFrameSourceKind::SessionHistory,
3464            source_cursor: Some(format!("{session_id}:{offset}")),
3465        },
3466        source_event_id: None,
3467        interaction_id: None,
3468        turn_id: None,
3469        run_id: None,
3470        parent_frame_id: None,
3471        caused_by_frame_id: None,
3472    }];
3473    if let Message::BlockAssistant(assistant) = &parsed {
3474        frames.extend(spawn_initial_message_frames_from_assistant(
3475            runtime_key,
3476            identity,
3477            identity_namespace,
3478            session_id,
3479            offset,
3480            assistant,
3481            &payload_hash,
3482        ));
3483    }
3484    frames
3485}
3486
3487fn spawn_initial_message_frames_from_assistant(
3488    runtime_key: &str,
3489    parent_identity: &str,
3490    identity_namespace: &str,
3491    session_id: &str,
3492    offset: usize,
3493    assistant: &meerkat_core::types::BlockAssistantMessage,
3494    payload_hash: &str,
3495) -> Vec<NewConsoleFrame> {
3496    let mut frames = Vec::new();
3497    for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3498        if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3499            continue;
3500        }
3501        let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3502            continue;
3503        };
3504        for (spawn_idx, (target_identity, initial_message)) in
3505            spawn_initial_messages_from_tool_args(&args)
3506                .into_iter()
3507                .enumerate()
3508        {
3509            let target_identity = apply_namespace(&target_identity, identity_namespace);
3510            let message_content = match &initial_message {
3511                Value::String(text) => json!([
3512                    {
3513                        "type": "text",
3514                        "text": text,
3515                    }
3516                ]),
3517                other => other.clone(),
3518            };
3519            let message = match &initial_message {
3520                Value::String(text) => json!({
3521                    "role": "user",
3522                    "content": text,
3523                    "created_at": assistant.created_at,
3524                }),
3525                other => json!({
3526                    "role": "user",
3527                    "content": other,
3528                    "created_at": assistant.created_at,
3529                }),
3530            };
3531            frames.push(NewConsoleFrame {
3532                id: None,
3533                dedupe_key: format!(
3534                    "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3535                ),
3536                timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3537                runtime_key: runtime_key.to_string(),
3538                identity: target_identity.clone(),
3539                conversation_id: Some(target_identity),
3540                session_id: None,
3541                kind: "user_input".to_string(),
3542                status: ConsoleFrameStatus::Completed,
3543                payload: json!({
3544                    "content": message_content,
3545                    "message": message,
3546                    "source_event_type": "session_history_spawn_initial_message",
3547                    "type": "session_history",
3548                    "tool_call_id": tool.id,
3549                    "parent_identity": parent_identity,
3550                    "via_tool": tool.name,
3551                }),
3552                source: ConsoleFrameSource {
3553                    kind: ConsoleFrameSourceKind::SessionHistory,
3554                    source_cursor: Some(format!(
3555                        "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
3556                    )),
3557                },
3558                source_event_id: None,
3559                interaction_id: None,
3560                turn_id: None,
3561                run_id: None,
3562                parent_frame_id: None,
3563                caused_by_frame_id: None,
3564            });
3565        }
3566    }
3567    frames
3568}
3569
3570fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
3571    let mut messages = Vec::new();
3572    if let Some(message) = spawn_initial_message_from_record(args) {
3573        messages.push(message);
3574    }
3575    if let Some(specs) = args.get("specs").and_then(Value::as_array) {
3576        for spec in specs {
3577            if let Some(message) = spawn_initial_message_from_record(spec) {
3578                messages.push(message);
3579            }
3580        }
3581    }
3582    messages
3583}
3584
3585fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
3586    let target_identity = record
3587        .get("member_id")
3588        .or_else(|| record.get("agent_identity"))
3589        .and_then(Value::as_str)?
3590        .trim();
3591    if target_identity.is_empty() {
3592        return None;
3593    }
3594    let initial_message = record
3595        .get("initial_message")
3596        .or_else(|| record.get("task"))?
3597        .clone();
3598    Some((target_identity.to_string(), initial_message))
3599}
3600
3601fn session_history_user_message_is_scaffold(message: &Value) -> bool {
3602    message
3603        .get("content")
3604        .is_some_and(scaffold_message_content_is_noise)
3605}
3606
3607fn scaffold_message_content_is_noise(value: &Value) -> bool {
3608    match value {
3609        Value::String(text) => scaffold_message_text_is_noise(text),
3610        Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
3611        Value::Object(map) => ["text", "content", "message"]
3612            .iter()
3613            .filter_map(|key| map.get(*key))
3614            .any(scaffold_message_content_is_noise),
3615        _ => false,
3616    }
3617}
3618
3619fn scaffold_message_text_is_noise(text: &str) -> bool {
3620    let trimmed = text.trim_start();
3621    trimmed.starts_with("[PEER UPDATE]")
3622        || trimmed
3623            .to_ascii_lowercase()
3624            .starts_with("you have been spawned")
3625}
3626
3627fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
3628    let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
3629    rest.split(':').next()?.parse().ok()
3630}
3631
3632fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
3633    format!("{session_id}:{offset}:{checked_at_ms}")
3634}
3635
3636fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
3637    let Some(checked_at_ms) = watermark
3638        .rsplit_once(':')
3639        .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
3640    else {
3641        return false;
3642    };
3643    let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
3644    let ttl_ms = if offset > 0 {
3645        SESSION_HISTORY_GROWING_REFRESH_TTL_MS
3646    } else {
3647        SESSION_HISTORY_REFRESH_TTL_MS
3648    };
3649    now_ms.saturating_sub(checked_at_ms) < ttl_ms
3650}
3651
3652async fn history_frame_has_existing_counterpart(
3653    inner: &AggregatorInner,
3654    frame: &NewConsoleFrame,
3655) -> ConsoleLogResult<bool> {
3656    let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
3657    let Some(fingerprint) = fingerprint else {
3658        return Ok(false);
3659    };
3660    let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
3661    let mut delta_text_by_turn = BTreeMap::<String, String>::new();
3662    let mut after = None;
3663    loop {
3664        let page = inner
3665            .store
3666            .query_frames(ConsoleTimelineQuery {
3667                identity: Some(frame.identity.clone()),
3668                conversation_id: frame.conversation_id.clone(),
3669                after,
3670                limit: 1_000,
3671            })
3672            .await?;
3673        for existing in &page.frames {
3674            let same_session = existing.session_id == frame.session_id
3675                || existing.session_id.is_none()
3676                || frame.session_id.is_none();
3677            if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
3678                continue;
3679            }
3680            if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
3681                == Some(&fingerprint)
3682            {
3683                return Ok(true);
3684            }
3685            if assistant_terminal
3686                && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
3687            {
3688                let turn_key = existing
3689                    .interaction_id
3690                    .as_deref()
3691                    .or(existing.turn_id.as_deref())
3692                    .or(existing.run_id.as_deref())
3693                    .unwrap_or("session");
3694                let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
3695                aggregated.push_str(delta);
3696                if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
3697                    return Ok(true);
3698                }
3699            }
3700        }
3701        if page.frames.is_empty() || page.next_cursor.is_none() {
3702            return Ok(false);
3703        }
3704        after = page.next_cursor;
3705    }
3706}
3707
3708fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
3709    format!("{runtime_key}:session-history:{session_id}")
3710}
3711
3712fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3713    match kind {
3714        "user_input" | "interaction_started" => payload
3715            .get("content")
3716            .map(stable_value_fingerprint)
3717            .or_else(|| payload.get("message").map(stable_value_fingerprint)),
3718        "tool_execution_completed" => tool_result_fingerprint(payload),
3719        "text_delta" => {
3720            text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
3721        }
3722        "text_complete" | "interaction_complete" | "run_completed" => {
3723            assistant_terminal_fingerprint(kind, payload)
3724        }
3725        _ => None,
3726    }
3727}
3728
3729fn tool_result_fingerprint(payload: &Value) -> Option<String> {
3730    let id = payload
3731        .get("tool_call_id")
3732        .or_else(|| payload.get("id"))
3733        .and_then(Value::as_str)
3734        .unwrap_or("");
3735    let result = payload
3736        .get("result")
3737        .or_else(|| payload.get("content"))
3738        .map(stable_value_fingerprint)?;
3739    Some(format!("{id}:{result}"))
3740}
3741
3742fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3743    match kind {
3744        "text_complete" | "interaction_complete" | "run_completed" => payload
3745            .get("text")
3746            .or_else(|| payload.get("result"))
3747            .or_else(|| payload.get("content"))
3748            .map(stable_value_fingerprint),
3749        _ => None,
3750    }
3751}
3752
3753fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
3754    if kind != "text_delta" {
3755        return None;
3756    }
3757    payload
3758        .get("delta")
3759        .or_else(|| payload.get("text"))
3760        .or_else(|| payload.get("content"))
3761        .and_then(Value::as_str)
3762        .or_else(|| payload.as_str())
3763}
3764
3765fn stable_value_fingerprint(value: &Value) -> String {
3766    if let Some(text) = content_value_text(value) {
3767        return normalize_transcript_fingerprint_text(&text);
3768    }
3769    match value {
3770        Value::String(text) => normalize_transcript_fingerprint_text(text),
3771        other => serde_json::to_string(other).unwrap_or_default(),
3772    }
3773}
3774
3775fn content_value_text(value: &Value) -> Option<String> {
3776    match value {
3777        Value::String(text) => Some(text.clone()),
3778        Value::Array(items) => {
3779            let text = items
3780                .iter()
3781                .filter_map(content_value_text)
3782                .collect::<String>();
3783            (!text.is_empty()).then_some(text)
3784        }
3785        Value::Object(map) => ["text", "content", "message", "blocks"]
3786            .iter()
3787            .filter_map(|key| map.get(*key))
3788            .find_map(content_value_text),
3789        _ => None,
3790    }
3791}
3792
3793fn normalize_transcript_fingerprint_text(text: &str) -> String {
3794    let trimmed = text.trim();
3795    trimmed
3796        .strip_prefix("[EVENT via rpc] ")
3797        .unwrap_or(trimmed)
3798        .trim()
3799        .to_string()
3800}
3801
3802#[derive(Debug, Clone, Copy)]
3803enum CachedIdentityVisibility {
3804    Visible,
3805    Hidden,
3806    Missing,
3807}
3808
3809async fn frame_is_visible(
3810    inner: &AggregatorInner,
3811    frame: &ConsoleFrame,
3812    allow_historical_identity: bool,
3813    identity_records: &[ConsoleIdentityRecord],
3814) -> ConsoleLogResult<bool> {
3815    let mut identity_visibility_cache = HashMap::new();
3816    frame_is_visible_cached(
3817        inner,
3818        frame,
3819        allow_historical_identity,
3820        &mut identity_visibility_cache,
3821        identity_records,
3822    )
3823    .await
3824}
3825
3826async fn frame_is_visible_cached(
3827    inner: &AggregatorInner,
3828    frame: &ConsoleFrame,
3829    allow_historical_identity: bool,
3830    identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
3831    identity_records: &[ConsoleIdentityRecord],
3832) -> ConsoleLogResult<bool> {
3833    let entry = {
3834        let entries = inner
3835            .runtimes
3836            .read()
3837            .map_err(|_| runtime_registry_lock_error())?;
3838        if entries.is_empty() {
3839            return Ok(true);
3840        }
3841        let Some(entry) = entries.get(&frame.runtime_key) else {
3842            return Ok(false);
3843        };
3844        entry.clone()
3845    };
3846    if frame.identity != "__console__" {
3847        let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
3848        let identity_visibility =
3849            if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
3850                cached
3851            } else {
3852                let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
3853                    .unwrap_or_else(|| frame.identity.clone());
3854                let visibility = match identity_records.iter().find(|record| {
3855                    record.runtime_key == frame.runtime_key
3856                        && (record.identity == frame.identity
3857                            || record.runtime_member_id == runtime_member_id)
3858                }) {
3859                    Some(record) => {
3860                        if console_identity_record_visible(&entry, record).await {
3861                            CachedIdentityVisibility::Visible
3862                        } else {
3863                            CachedIdentityVisibility::Hidden
3864                        }
3865                    }
3866                    None => CachedIdentityVisibility::Missing,
3867                };
3868                identity_visibility_cache.insert(cache_key, visibility);
3869                visibility
3870            };
3871        match identity_visibility {
3872            CachedIdentityVisibility::Visible => {
3873                if frame_matches_hidden_member(&entry, frame).await {
3874                    return Ok(false);
3875                }
3876            }
3877            CachedIdentityVisibility::Hidden => return Ok(false),
3878            CachedIdentityVisibility::Missing => {
3879                if frame_matches_hidden_member(&entry, frame).await {
3880                    return Ok(false);
3881                }
3882                return Ok(
3883                    allow_historical_identity && entry.visibility_policy.frame_visible(frame)
3884                );
3885            }
3886        }
3887    }
3888    Ok(entry.visibility_policy.frame_visible(frame))
3889}
3890
3891async fn identity_record_for_member(
3892    entry: &RuntimeEntry,
3893    handle: &MobHandle,
3894    member: &MobMemberListEntry,
3895) -> Option<ConsoleIdentityRecord> {
3896    let runtime_member_id = member.agent_identity.to_string();
3897    let durable_identity = member
3898        .labels
3899        .get("agent_identity")
3900        .filter(|value| !value.trim().is_empty())
3901        .map_or(runtime_member_id.as_str(), String::as_str);
3902    let identity = apply_namespace(durable_identity, &entry.identity_namespace);
3903    let addressable = member_is_addressable(member);
3904    let visibility = if member.state == meerkat_mob::MemberState::Retiring {
3905        ConsoleVisibility::RetiredReadable
3906    } else if addressable {
3907        ConsoleVisibility::Addressable
3908    } else {
3909        ConsoleVisibility::Hidden
3910    };
3911    let session_id = handle
3912        .resolve_bridge_session_id_observation(&member.agent_identity)
3913        .await
3914        .map(|sid| sid.to_string());
3915    let display_name = member
3916        .labels
3917        .get("display_name")
3918        .cloned()
3919        .unwrap_or_else(|| runtime_member_id.clone());
3920    let mut labels = member.labels.clone();
3921    labels
3922        .entry("role".to_string())
3923        .or_insert_with(|| member.role.to_string());
3924    Some(ConsoleIdentityRecord {
3925        identity,
3926        display_name,
3927        runtime_key: entry.runtime_key.clone(),
3928        runtime_member_id,
3929        session_id,
3930        visibility,
3931        addressable,
3932        health: if addressable {
3933            "ready"
3934        } else {
3935            "hidden_by_policy"
3936        }
3937        .to_string(),
3938        topology_peers: member.wired_to.iter().map(ToString::to_string).collect(),
3939        labels,
3940    })
3941}
3942
3943async fn identity_record_for_resolved_member(
3944    resolved: &ResolvedConsoleMember,
3945) -> Option<ConsoleIdentityRecord> {
3946    let mut record =
3947        identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
3948    if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
3949        record
3950            .labels
3951            .entry("source_mob_id".to_string())
3952            .or_insert_with(|| resolved.source_mob_id.clone());
3953    }
3954    Some(record)
3955}
3956
3957fn identity_record_for_status(
3958    entry: &RuntimeEntry,
3959    status: &crate::identity_first::IdentityStatus,
3960) -> ConsoleIdentityRecord {
3961    let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
3962    let runtime_member_id = status
3963        .agent_runtime_id
3964        .as_ref()
3965        .map(crate::identity_first::AgentRuntimeId::as_str)
3966        .unwrap_or_else(|| status.identity.as_str())
3967        .to_string();
3968    let addressable = status.addressability
3969        == crate::identity_first::AgentAddressability::Addressable
3970        && matches!(
3971            status.state,
3972            crate::identity_first::IdentityLifecycleState::Active
3973                | crate::identity_first::IdentityLifecycleState::Dormant
3974                | crate::identity_first::IdentityLifecycleState::Uninitialized
3975        );
3976    let visibility = match status.state {
3977        crate::identity_first::IdentityLifecycleState::Retiring => {
3978            ConsoleVisibility::RetiredReadable
3979        }
3980        crate::identity_first::IdentityLifecycleState::Broken
3981        | crate::identity_first::IdentityLifecycleState::Suspended => {
3982            ConsoleVisibility::Unreachable
3983        }
3984        _ if addressable => ConsoleVisibility::Addressable,
3985        _ => ConsoleVisibility::Hidden,
3986    };
3987    let health = match status.state {
3988        crate::identity_first::IdentityLifecycleState::Active => "ready",
3989        crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
3990        crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
3991        crate::identity_first::IdentityLifecycleState::Broken => "broken",
3992        crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
3993        crate::identity_first::IdentityLifecycleState::Retiring => "retired",
3994    }
3995    .to_string();
3996    let display_name = status
3997        .display_name
3998        .as_ref()
3999        .map(crate::identity_first::DisplayName::as_str)
4000        .unwrap_or_else(|| status.identity.as_str())
4001        .to_string();
4002    ConsoleIdentityRecord {
4003        identity,
4004        display_name,
4005        runtime_key: entry.runtime_key.clone(),
4006        runtime_member_id,
4007        session_id: status.session_id.as_ref().map(ToString::to_string),
4008        visibility,
4009        addressable,
4010        health,
4011        topology_peers: Vec::new(),
4012        labels: status.labels.clone(),
4013    }
4014}
4015
4016pub(crate) fn is_implicit_delegate_member(
4017    role: &str,
4018    labels: &std::collections::BTreeMap<String, String>,
4019) -> bool {
4020    role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4021}
4022
4023fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4024    member
4025        .labels
4026        .get("addressable")
4027        .map(|value| !value.eq_ignore_ascii_case("false"))
4028        .unwrap_or(true)
4029}
4030
4031fn apply_namespace(identity: &str, namespace: &str) -> String {
4032    let namespace = namespace.trim().trim_matches('/');
4033    if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4034        identity.to_string()
4035    } else {
4036        format!("{namespace}/{identity}")
4037    }
4038}
4039
4040fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4041    let namespace = namespace.trim().trim_matches('/');
4042    if namespace.is_empty() {
4043        return Some(identity.to_string());
4044    }
4045    identity
4046        .strip_prefix(namespace)
4047        .and_then(|rest| rest.strip_prefix('/'))
4048        .map(ToString::to_string)
4049}
4050
4051fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4052    let namespace = namespace.trim().trim_matches('/');
4053    if namespace.is_empty() {
4054        return vec![identity.to_string()];
4055    }
4056    let mut candidates = Vec::new();
4057    if let Some(stripped) = strip_namespace(identity, namespace) {
4058        candidates.push(stripped);
4059        candidates.push(identity.to_string());
4060    }
4061    candidates.sort();
4062    candidates.dedup();
4063    candidates
4064}
4065
4066fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4067    namespace_match_candidates(identity, namespace)
4068        .iter()
4069        .any(|candidate| candidate.starts_with("rt:"))
4070}
4071
4072fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4073    entries.iter().any(|entry| {
4074        requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4075    })
4076}
4077
4078fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4079    if request.identity.trim().is_empty() {
4080        return Err(ConsoleSendError::InvalidRequest(
4081            "identity must be non-empty".to_string(),
4082        ));
4083    }
4084    if request.origin.trim().is_empty() {
4085        return Err(ConsoleSendError::InvalidRequest(
4086            "origin must be non-empty".to_string(),
4087        ));
4088    }
4089    if request.idempotency_key.trim().is_empty() {
4090        return Err(ConsoleSendError::InvalidRequest(
4091            "idempotency_key must be non-empty".to_string(),
4092        ));
4093    }
4094    Ok(())
4095}
4096
4097fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4098    let content: ContentInput = serde_json::from_value(value.clone())
4099        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4100    match &content {
4101        ContentInput::Text(text) if text.trim().is_empty() => Err(
4102            ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4103        ),
4104        ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4105            "content blocks must be non-empty".to_string(),
4106        )),
4107        _ => Ok(content),
4108    }
4109}
4110
4111fn parse_handling_mode(
4112    value: Option<&str>,
4113) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4114    match value.unwrap_or("queue") {
4115        "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4116        "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4117        other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4118    }
4119}
4120
4121fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4122    ConsoleInteractionAccepted {
4123        interaction_id: frame
4124            .interaction_id
4125            .clone()
4126            .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4127        identity: frame.identity.clone(),
4128        conversation_id: frame.conversation_id.clone(),
4129        session_id: frame.session_id.clone(),
4130        input_frame_id: frame.id.clone(),
4131        cursor: frame.cursor.clone(),
4132        status: frame.status,
4133    }
4134}
4135
4136fn send_dedupe_key(
4137    runtime_key: &str,
4138    identity: &str,
4139    origin: &str,
4140    idempotency_key: &str,
4141) -> String {
4142    format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4143}
4144
4145fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4146    let content_json = serde_json::to_string(content).unwrap_or_default();
4147    hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4148}
4149
4150fn hash_short(value: &str) -> String {
4151    let mut hasher = Sha256::new();
4152    hasher.update(value.as_bytes());
4153    let digest = hasher.finalize();
4154    to_hex(&digest[..8])
4155}
4156
4157fn to_hex(bytes: &[u8]) -> String {
4158    const HEX: &[u8; 16] = b"0123456789abcdef";
4159    let mut out = String::with_capacity(bytes.len() * 2);
4160    for byte in bytes {
4161        out.push(HEX[(byte >> 4) as usize] as char);
4162        out.push(HEX[(byte & 0x0f) as usize] as char);
4163    }
4164    out
4165}
4166
4167fn current_time_ms() -> u64 {
4168    match SystemTime::now().duration_since(UNIX_EPOCH) {
4169        Ok(duration) => duration.as_millis() as u64,
4170        Err(_) => 0,
4171    }
4172}
4173
4174fn runtime_registry_lock_error() -> ConsoleLogError {
4175    Box::new(std::io::Error::other(
4176        "console runtime registry lock poisoned",
4177    ))
4178}
4179
4180#[cfg(test)]
4181#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4182mod tests {
4183    use std::sync::Arc;
4184    use std::sync::atomic::{AtomicUsize, Ordering};
4185    use std::time::Duration;
4186    use std::time::Instant;
4187
4188    use futures::StreamExt;
4189    use meerkat::{AgentFactory, Config, build_ephemeral_service};
4190    use meerkat_client::types::LlmStream;
4191    use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4192    use meerkat_core::{
4193        AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4194        RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4195        SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4196        SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4197        StreamError,
4198    };
4199    use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4200    use serde_json::json;
4201
4202    use super::*;
4203    use crate::mob_handle_runtime::MobBootstrapSpec;
4204
4205    #[derive(Debug)]
4206    struct HideHiddenNoiseFrames;
4207
4208    impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4209        fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4210            frame.kind != "hidden_noise"
4211        }
4212    }
4213
4214    #[derive(Debug)]
4215    struct HideRuntimeMemberIdentity(&'static str);
4216
4217    impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4218        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4219            record.runtime_member_id != self.0
4220        }
4221    }
4222
4223    #[derive(Debug)]
4224    struct HideRuntimeMemberOnly(&'static str);
4225
4226    impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4227        fn member_visible(&self, member: &ConsoleMember) -> bool {
4228            member.agent_identity != self.0
4229        }
4230    }
4231
4232    #[derive(Debug)]
4233    struct HideConsoleIdentity(&'static str);
4234
4235    impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4236        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4237            record.identity != self.0
4238        }
4239    }
4240
4241    struct CountingConsoleLogStore {
4242        inner: InMemoryConsoleLogStore,
4243        source_watermark_calls: AtomicUsize,
4244        record_watermark_calls: AtomicUsize,
4245    }
4246
4247    impl CountingConsoleLogStore {
4248        fn new() -> Self {
4249            Self {
4250                inner: InMemoryConsoleLogStore::new(),
4251                source_watermark_calls: AtomicUsize::new(0),
4252                record_watermark_calls: AtomicUsize::new(0),
4253            }
4254        }
4255
4256        fn source_watermark_calls(&self) -> usize {
4257            self.source_watermark_calls.load(Ordering::SeqCst)
4258        }
4259    }
4260
4261    struct SlowTestClient {
4262        delay: Duration,
4263    }
4264
4265    #[async_trait::async_trait]
4266    impl LlmClient for SlowTestClient {
4267        fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4268            Ok(messages.to_vec())
4269        }
4270
4271        fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4272            let delay = self.delay;
4273            let delayed_text = futures::stream::once(async move {
4274                tokio::time::sleep(delay).await;
4275                Ok(LlmEvent::TextDelta {
4276                    delta: "slow ok".to_string(),
4277                    meta: None,
4278                })
4279            });
4280            let done = futures::stream::once(async {
4281                Ok(LlmEvent::Done {
4282                    outcome: LlmDoneOutcome::Success {
4283                        stop_reason: StopReason::EndTurn,
4284                    },
4285                })
4286            });
4287            Box::pin(delayed_text.chain(done))
4288        }
4289
4290        fn provider(&self) -> &'static str {
4291            "slow-test"
4292        }
4293
4294        async fn health_check(&self) -> Result<(), LlmError> {
4295            Ok(())
4296        }
4297    }
4298
4299    #[derive(Clone)]
4300    struct DelayedHistorySessionService {
4301        inner: Arc<dyn MobSessionService>,
4302        delay: Duration,
4303        read_calls: Arc<AtomicUsize>,
4304        active_reads: Arc<AtomicUsize>,
4305        max_active_reads: Arc<AtomicUsize>,
4306    }
4307
4308    impl DelayedHistorySessionService {
4309        fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4310            Self {
4311                inner,
4312                delay,
4313                read_calls: Arc::new(AtomicUsize::new(0)),
4314                active_reads: Arc::new(AtomicUsize::new(0)),
4315                max_active_reads: Arc::new(AtomicUsize::new(0)),
4316            }
4317        }
4318
4319        fn read_calls(&self) -> usize {
4320            self.read_calls.load(Ordering::SeqCst)
4321        }
4322
4323        fn max_active_reads(&self) -> usize {
4324            self.max_active_reads.load(Ordering::SeqCst)
4325        }
4326    }
4327
4328    #[async_trait::async_trait]
4329    impl SessionService for DelayedHistorySessionService {
4330        async fn create_session(
4331            &self,
4332            req: meerkat_core::CreateSessionRequest,
4333        ) -> Result<RunResult, SessionError> {
4334            self.inner.create_session(req).await
4335        }
4336
4337        async fn start_turn(
4338            &self,
4339            id: &SessionId,
4340            req: StartTurnRequest,
4341        ) -> Result<RunResult, SessionError> {
4342            self.inner.start_turn(id, req).await
4343        }
4344
4345        async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4346            self.inner.interrupt(id).await
4347        }
4348
4349        async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4350            self.inner.read(id).await
4351        }
4352
4353        async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4354            self.inner.list(query).await
4355        }
4356
4357        async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4358            self.inner.archive(id).await
4359        }
4360
4361        async fn subscribe_session_events(
4362            &self,
4363            id: &SessionId,
4364        ) -> Result<EventStream, StreamError> {
4365            SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4366        }
4367    }
4368
4369    #[async_trait::async_trait]
4370    impl SessionServiceCommsExt for DelayedHistorySessionService {
4371        async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4372            self.inner.comms_runtime(session_id).await
4373        }
4374    }
4375
4376    #[async_trait::async_trait]
4377    impl SessionServiceControlExt for DelayedHistorySessionService {
4378        async fn append_system_context(
4379            &self,
4380            id: &SessionId,
4381            req: AppendSystemContextRequest,
4382        ) -> Result<AppendSystemContextResult, SessionControlError> {
4383            self.inner.append_system_context(id, req).await
4384        }
4385    }
4386
4387    #[async_trait::async_trait]
4388    impl SessionServiceHistoryExt for DelayedHistorySessionService {
4389        async fn read_history(
4390            &self,
4391            id: &SessionId,
4392            query: SessionHistoryQuery,
4393        ) -> Result<SessionHistoryPage, SessionError> {
4394            self.read_calls.fetch_add(1, Ordering::SeqCst);
4395            let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4396            self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4397            tokio::time::sleep(self.delay).await;
4398            let result = self.inner.read_history(id, query).await;
4399            self.active_reads.fetch_sub(1, Ordering::SeqCst);
4400            result
4401        }
4402    }
4403
4404    #[async_trait::async_trait]
4405    impl MobSessionService for DelayedHistorySessionService {
4406        fn supports_persistent_sessions(&self) -> bool {
4407            self.inner.supports_persistent_sessions()
4408        }
4409
4410        fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4411            self.inner.runtime_adapter()
4412        }
4413
4414        async fn session_belongs_to_mob(
4415            &self,
4416            session_id: &SessionId,
4417            mob_id: &meerkat_mob::MobId,
4418        ) -> bool {
4419            self.inner.session_belongs_to_mob(session_id, mob_id).await
4420        }
4421
4422        async fn cancel_all_checkpointers(&self) {
4423            self.inner.cancel_all_checkpointers().await;
4424        }
4425
4426        async fn rearm_all_checkpointers(&self) {
4427            self.inner.rearm_all_checkpointers().await;
4428        }
4429    }
4430
4431    #[async_trait::async_trait]
4432    impl ConsoleLogStore for CountingConsoleLogStore {
4433        async fn append_if_absent(
4434            &self,
4435            frame: NewConsoleFrame,
4436        ) -> ConsoleLogResult<AppendOutcome> {
4437            self.inner.append_if_absent(frame).await
4438        }
4439
4440        async fn update_frame_status(
4441            &self,
4442            frame_id: &str,
4443            status: ConsoleFrameStatus,
4444        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4445            self.inner.update_frame_status(frame_id, status).await
4446        }
4447
4448        async fn query_frames(
4449            &self,
4450            query: ConsoleTimelineQuery,
4451        ) -> ConsoleLogResult<ConsoleTimelinePage> {
4452            self.inner.query_frames(query).await
4453        }
4454
4455        async fn query_windowed_frames(
4456            &self,
4457            query: ConsoleTimelineWindowQuery,
4458        ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4459            self.inner.query_windowed_frames(query).await
4460        }
4461
4462        async fn frame_by_dedupe_key(
4463            &self,
4464            dedupe_key: &str,
4465        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4466            self.inner.frame_by_dedupe_key(dedupe_key).await
4467        }
4468
4469        async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4470            self.inner.latest_cursor().await
4471        }
4472
4473        async fn clear_frames(&self) -> ConsoleLogResult<()> {
4474            self.inner.clear_frames().await
4475        }
4476
4477        async fn record_source_watermark(
4478            &self,
4479            runtime_key: &str,
4480            source_kind: ConsoleFrameSourceKind,
4481            source_cursor: &str,
4482        ) -> ConsoleLogResult<()> {
4483            self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
4484            self.inner
4485                .record_source_watermark(runtime_key, source_kind, source_cursor)
4486                .await
4487        }
4488
4489        async fn source_watermark(
4490            &self,
4491            runtime_key: &str,
4492            source_kind: ConsoleFrameSourceKind,
4493        ) -> ConsoleLogResult<Option<String>> {
4494            self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
4495            self.inner.source_watermark(runtime_key, source_kind).await
4496        }
4497    }
4498
4499    async fn build_single_member_runtime() -> UnifiedRuntime {
4500        build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
4501    }
4502
4503    async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
4504        let definition = MobDefinition::from_toml(
4505            r#"
4506[mob]
4507id = "console-aggregator-perf-test"
4508
4509[profiles.worker]
4510model = "gpt-5.5"
4511external_addressable = true
4512
4513[profiles.worker.tools]
4514comms = true
4515"#,
4516        )
4517        .expect("definition parses");
4518        let runtime = UnifiedRuntime::builder()
4519            .definition(definition)
4520            .default_llm_client(client)
4521            .build()
4522            .await
4523            .expect("runtime builds");
4524        runtime
4525            .spawn(SpawnMemberSpec::from_wire(
4526                "worker".to_string(),
4527                "agent-a".to_string(),
4528                Some("You are agent-a.".into()),
4529                None,
4530                None,
4531            ))
4532            .await
4533            .expect("member spawns");
4534        runtime
4535    }
4536
4537    async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
4538        let definition = MobDefinition::from_toml(&format!(
4539            r#"
4540[mob]
4541id = "{mob_id}"
4542
4543[profiles.worker]
4544model = "gpt-5.5"
4545external_addressable = true
4546
4547[profiles.worker.tools]
4548comms = true
4549"#
4550        ))
4551        .expect("definition parses");
4552        UnifiedRuntime::builder()
4553            .definition(definition)
4554            .default_llm_client(Arc::new(TestClient::default()))
4555            .build()
4556            .await
4557            .expect("runtime builds")
4558    }
4559
4560    async fn build_stress_runtime(
4561        member_count: usize,
4562        history_delay: Duration,
4563    ) -> (
4564        tempfile::TempDir,
4565        Arc<UnifiedRuntime>,
4566        DelayedHistorySessionService,
4567    ) {
4568        let temp_dir = tempfile::tempdir().expect("temp dir");
4569        let session_path = temp_dir.path().join("sessions");
4570        std::fs::create_dir_all(&session_path).expect("session path");
4571        let factory = AgentFactory::new(&session_path).comms(true);
4572        let base_service = Arc::new(build_ephemeral_service(
4573            factory,
4574            Config::default(),
4575            member_count + 8,
4576        ));
4577        let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
4578        let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
4579        let definition = MobDefinition::from_toml(
4580            r#"
4581[mob]
4582id = "console-aggregator-stress-test"
4583
4584[profiles.worker]
4585model = "gpt-5.5"
4586external_addressable = true
4587
4588[profiles.worker.tools]
4589comms = true
4590"#,
4591        )
4592        .expect("definition parses");
4593        let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
4594            .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
4595                allow_ephemeral_sessions: true,
4596                notify_orchestrator_on_resume: true,
4597                default_llm_client: Some(Arc::new(TestClient::default())),
4598            });
4599        let runtime = Arc::new(
4600            UnifiedRuntime::bootstrap(
4601                spec,
4602                crate::types::MobKitConfig {
4603                    modules: Vec::new(),
4604                    discovery: crate::types::DiscoverySpec {
4605                        namespace: "stress".to_string(),
4606                        modules: Vec::new(),
4607                    },
4608                    pre_spawn: Vec::new(),
4609                },
4610                Duration::from_secs(5),
4611            )
4612            .await
4613            .expect("runtime boots"),
4614        );
4615        for idx in 0..member_count {
4616            runtime
4617                .spawn(SpawnMemberSpec::from_wire(
4618                    "worker".to_string(),
4619                    format!("agent-{idx}"),
4620                    Some(format!("You are agent-{idx}.").into()),
4621                    None,
4622                    None,
4623                ))
4624                .await
4625                .expect("member spawns");
4626        }
4627        (temp_dir, runtime, delayed_service)
4628    }
4629
4630    fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
4631        RuntimeEntry {
4632            runtime_key: runtime_key.to_string(),
4633            identity_namespace: "test".to_string(),
4634            runtime: runtime.mob_runtime().clone(),
4635            identity_runtime: runtime.identity_runtime().cloned(),
4636            console_events: runtime.console_events(),
4637            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4638        }
4639    }
4640
4641    fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
4642        ConsoleIdentityRecord {
4643            identity: identity.to_string(),
4644            display_name: identity.to_string(),
4645            runtime_key: "runtime-cache".to_string(),
4646            runtime_member_id: identity.to_string(),
4647            session_id: Some(format!("session-{identity}")),
4648            visibility: ConsoleVisibility::Addressable,
4649            addressable: true,
4650            health: "ready".to_string(),
4651            topology_peers: Vec::new(),
4652            labels: BTreeMap::new(),
4653        }
4654    }
4655
4656    #[test]
4657    fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
4658        let mut durable = identity_record_for_test("test/review:singleton");
4659        durable.runtime_member_id = "rt:review:singleton:1".to_string();
4660        durable.session_id = Some("session-stale".to_string());
4661        durable
4662            .labels
4663            .insert("agent_identity".to_string(), "review:singleton".to_string());
4664
4665        let mut live = identity_record_for_test("test/review:singleton");
4666        live.runtime_member_id = "rt:review:singleton:0".to_string();
4667        live.session_id = Some("session-live".to_string());
4668        live.labels
4669            .insert("agent_identity".to_string(), "review:singleton".to_string());
4670
4671        let records = dedupe_identity_records(vec![durable, live]);
4672
4673        assert_eq!(
4674            records.len(),
4675            2,
4676            "list projection must expose stale durable/live alias split-brain instead of picking one record"
4677        );
4678        assert!(
4679            records
4680                .iter()
4681                .any(|record| record.runtime_member_id == "rt:review:singleton:0")
4682        );
4683        assert!(
4684            records
4685                .iter()
4686                .any(|record| record.runtime_member_id == "rt:review:singleton:1")
4687        );
4688    }
4689
4690    #[test]
4691    fn dedupe_identity_records_preserves_duplicate_live_aliases() {
4692        let mut first = identity_record_for_test("test/review:singleton");
4693        first.runtime_member_id = "rt:review:singleton:0".to_string();
4694        first
4695            .labels
4696            .insert("agent_identity".to_string(), "review:singleton".to_string());
4697
4698        let mut second = identity_record_for_test("test/review:singleton");
4699        second.runtime_member_id = "rt:review:singleton:1".to_string();
4700        second
4701            .labels
4702            .insert("agent_identity".to_string(), "review:singleton".to_string());
4703
4704        let records = dedupe_identity_records(vec![first, second]);
4705
4706        assert_eq!(
4707            records.len(),
4708            2,
4709            "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
4710        );
4711    }
4712
4713    #[test]
4714    fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
4715        let mut first = identity_record_for_test("test/review:singleton");
4716        first.runtime_member_id = "rt:review:singleton:0".to_string();
4717        first.session_id = None;
4718        first
4719            .labels
4720            .insert("agent_identity".to_string(), "review:singleton".to_string());
4721        first
4722            .labels
4723            .insert("source_mob_id".to_string(), "mob-a".to_string());
4724
4725        let mut second = first.clone();
4726        second
4727            .labels
4728            .insert("source_mob_id".to_string(), "mob-b".to_string());
4729
4730        let records = dedupe_identity_records(vec![first, second]);
4731
4732        assert_eq!(
4733            records.len(),
4734            2,
4735            "same member/session aliases from different source mobs must remain visible"
4736        );
4737    }
4738
4739    #[test]
4740    fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
4741        let durable = identity_record_for_test("test/review:singleton");
4742        let live = durable.clone();
4743
4744        let records = dedupe_identity_records(vec![durable, live]);
4745
4746        assert_eq!(
4747            records.len(),
4748            1,
4749            "healthy durable and primary live projections should not duplicate the roster"
4750        );
4751    }
4752
4753    #[test]
4754    fn stale_durable_record_error_detects_session_split_brain() {
4755        let mut durable = identity_record_for_test("test/review:singleton");
4756        durable.runtime_member_id = "rt:review:singleton:0".to_string();
4757        durable.session_id = Some("session-durable".to_string());
4758
4759        let mut live = durable.clone();
4760        live.session_id = Some("session-live".to_string());
4761
4762        let err = stale_durable_record_error("review:singleton", &durable, &[live])
4763            .expect("session mismatch must be stale split-brain");
4764
4765        assert!(
4766            err.contains("stale durable identity alias"),
4767            "unexpected error: {err}"
4768        );
4769        assert!(
4770            err.contains("session-durable") && err.contains("session-live"),
4771            "error should name both sessions: {err}"
4772        );
4773    }
4774
4775    async fn identity_runtime_for_test(
4776        identities: &[&str],
4777    ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
4778    {
4779        let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
4780            crate::identity_first::IdentityRuntimeConfig {
4781                continuity_store: Arc::new(
4782                    crate::identity_first::LocalContinuityStore::in_memory()?
4783                ),
4784                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
4785                runtime_instance_id: "console-aggregator-identity-test".to_string(),
4786                has_runtime_store: true,
4787                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
4788                bridge: None,
4789                default_timeout: None,
4790            },
4791        ));
4792        for identity in identities {
4793            let identity = crate::identity_first::AgentIdentity::parse(identity)?;
4794            let record = crate::identity_first::ContinuityRecord {
4795                identity: identity.clone(),
4796                agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
4797                    "rt:{}:0",
4798                    identity.as_str()
4799                ))?,
4800                session_id: SessionId::new(),
4801                generation: crate::identity_first::ContinuityGeneration::new(0),
4802                checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
4803            };
4804            runtime
4805                .register(
4806                    crate::identity_first::DurableAgentSpec {
4807                        identity: identity.clone(),
4808                        profile: meerkat_mob::ProfileName::from("worker"),
4809                        addressability: crate::identity_first::AgentAddressability::Addressable,
4810                        display_name: None,
4811                        labels: BTreeMap::new(),
4812                        context: None,
4813                        additional_instructions: Vec::new(),
4814                        initial_message: None,
4815                        runtime_mode_override: None,
4816                    },
4817                    crate::identity_first::IdentityLifecycleState::Active,
4818                    Some(record),
4819                    Some(crate::identity_first::LeaseGrant {
4820                        identity,
4821                        fencing_token: crate::identity_first::FencingToken::new(1),
4822                        ttl: Duration::from_mins(5),
4823                    }),
4824                )
4825                .await;
4826        }
4827        Ok(runtime)
4828    }
4829
4830    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4831    async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
4832    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4833        let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
4834        let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
4835        let aggregator = MobKitConsoleAggregator::in_memory();
4836        aggregator.register_runtime_handles_with_policy(
4837            "runtime-a",
4838            "",
4839            runtime.mob_runtime().clone(),
4840            Some(identity_runtime),
4841            runtime.console_events(),
4842            Arc::new(AllowAllConsoleVisibilityPolicy),
4843        );
4844
4845        assert!(
4846            aggregator
4847                .inspect_identity("rt:review:singleton:0")
4848                .await?
4849                .is_none(),
4850            "runtime-member-id shaped requests must not inspect a durable rt:* identity"
4851        );
4852        assert!(
4853            !aggregator.retire_identity("rt:review:singleton:0").await?,
4854            "runtime-member-id shaped requests must not retire a durable rt:* identity"
4855        );
4856        let reserve_err = aggregator
4857            .reserve_identity_first_interaction(
4858                ConsoleSendRequest {
4859                    identity: "rt:review:singleton:0".to_string(),
4860                    content: json!("hello"),
4861                    origin: "console:test".to_string(),
4862                    idempotency_key: "rt-shadow-reserve".to_string(),
4863                    handling_mode: Some("queue".to_string()),
4864                },
4865                None,
4866            )
4867            .await
4868            .expect_err(
4869                "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
4870            );
4871        assert!(
4872            reserve_err.to_string().contains("unknown identity"),
4873            "unexpected reserve error: {reserve_err}"
4874        );
4875        let send_err = aggregator
4876            .send(ConsoleSendRequest {
4877                identity: "rt:review:singleton:0".to_string(),
4878                content: json!("hello"),
4879                origin: "console:test".to_string(),
4880                idempotency_key: "rt-shadow-send".to_string(),
4881                handling_mode: Some("queue".to_string()),
4882            })
4883            .await
4884            .expect_err(
4885                "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
4886            );
4887        assert!(
4888            send_err.to_string().contains("unknown identity"),
4889            "unexpected send error: {send_err}"
4890        );
4891        let blob_err = match aggregator
4892            .binary_blob_store_for_identity("rt:review:singleton:0")
4893            .await
4894        {
4895            Ok(_) => panic!(
4896                "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
4897            ),
4898            Err(err) => err,
4899        };
4900        assert!(
4901            blob_err.to_string().contains("unknown identity"),
4902            "unexpected blob error: {blob_err}"
4903        );
4904
4905        let _ = runtime.mob_handle().stop().await;
4906        Ok(())
4907    }
4908
4909    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4910    async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
4911    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4912        let runtime = build_empty_runtime("identity-label-test").await;
4913        let mut labels = BTreeMap::new();
4914        labels.insert(
4915            "agent_identity".to_string(),
4916            "channel:C0SMOKEOB3".to_string(),
4917        );
4918        labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
4919        runtime
4920            .spawn(
4921                SpawnMemberSpec::from_wire(
4922                    "worker".to_string(),
4923                    "rt:channel:C0SMOKEOB3:0".to_string(),
4924                    Some("You are C0SMOKEOB3.".into()),
4925                    None,
4926                    None,
4927                )
4928                .with_labels(labels),
4929            )
4930            .await
4931            .expect("member spawns");
4932
4933        let entry = RuntimeEntry {
4934            runtime_key: "runtime-a".to_string(),
4935            identity_namespace: String::new(),
4936            runtime: runtime.mob_runtime().clone(),
4937            identity_runtime: runtime.identity_runtime().cloned(),
4938            console_events: runtime.console_events(),
4939            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4940        };
4941        let aggregator = MobKitConsoleAggregator::in_memory();
4942        aggregator
4943            .inner
4944            .runtimes
4945            .write()
4946            .expect("runtime registry")
4947            .insert("runtime-a".to_string(), entry);
4948
4949        let records = aggregator.list_identities_fresh().await?;
4950        let record = records
4951            .iter()
4952            .find(|record| record.identity == "channel:C0SMOKEOB3")
4953            .expect("durable identity is exposed");
4954        assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
4955
4956        let inspection = aggregator
4957            .inspect_identity("channel:C0SMOKEOB3")
4958            .await?
4959            .expect("durable identity resolves back to runtime member");
4960        assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
4961        assert_eq!(
4962            inspection.identity.runtime_member_id,
4963            "rt:channel:C0SMOKEOB3:0"
4964        );
4965
4966        let _ = runtime.mob_handle().stop().await;
4967        Ok(())
4968    }
4969
4970    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4971    async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
4972    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4973        let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
4974        let mut labels = BTreeMap::new();
4975        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4976        runtime
4977            .spawn(
4978                SpawnMemberSpec::from_wire(
4979                    "worker".to_string(),
4980                    "rt:review:singleton:0".to_string(),
4981                    Some("You are the Review Agent.".into()),
4982                    None,
4983                    None,
4984                )
4985                .with_labels(labels),
4986            )
4987            .await?;
4988        let hidden_session_id = runtime
4989            .mob_handle()
4990            .resolve_bridge_session_id_observation(&meerkat_mob::ids::MeerkatId::from(
4991                "rt:review:singleton:0",
4992            ))
4993            .await
4994            .map(|sid| sid.to_string());
4995        let mut duplicate_labels = BTreeMap::new();
4996        duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4997        runtime
4998            .spawn(
4999                SpawnMemberSpec::from_wire(
5000                    "worker".to_string(),
5001                    "rt:review:singleton:1".to_string(),
5002                    Some("You are a visible duplicate Review Agent.".into()),
5003                    None,
5004                    None,
5005                )
5006                .with_labels(duplicate_labels),
5007            )
5008            .await?;
5009
5010        let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5011        let aggregator = MobKitConsoleAggregator::in_memory();
5012        aggregator.register_runtime_handles_with_policy(
5013            "runtime-a",
5014            "",
5015            runtime.mob_runtime().clone(),
5016            Some(identity_runtime),
5017            runtime.console_events(),
5018            Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5019        );
5020
5021        assert!(
5022            aggregator
5023                .inspect_identity("review:singleton")
5024                .await?
5025                .is_none(),
5026            "member-hidden live alias must not inspect through aggregator"
5027        );
5028        let records = aggregator.list_identities_fresh().await?;
5029        assert!(
5030            records
5031                .iter()
5032                .all(|record| record.identity != "review:singleton"),
5033            "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5034        );
5035        let appended = aggregator
5036            .store()
5037            .append_if_absent(NewConsoleFrame {
5038                id: None,
5039                dedupe_key: "member-hidden-frame".to_string(),
5040                timestamp_ms: 1,
5041                runtime_key: "runtime-a".to_string(),
5042                identity: "review:singleton".to_string(),
5043                conversation_id: Some("review:singleton".to_string()),
5044                session_id: hidden_session_id,
5045                kind: "text_delta".to_string(),
5046                status: ConsoleFrameStatus::Delivered,
5047                payload: json!({ "delta": "hidden" }),
5048                source: ConsoleFrameSource {
5049                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5050                    source_cursor: None,
5051                },
5052                source_event_id: Some("member-hidden-frame".to_string()),
5053                interaction_id: None,
5054                turn_id: None,
5055                run_id: None,
5056                parent_frame_id: None,
5057                caused_by_frame_id: None,
5058            })
5059            .await?;
5060        assert!(
5061            !aggregator
5062                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5063                    frame: appended.frame.clone()
5064                })
5065                .await,
5066            "member-hidden live alias must not pass timeline event visibility"
5067        );
5068        let identity_only = aggregator
5069            .store()
5070            .append_if_absent(NewConsoleFrame {
5071                id: None,
5072                dedupe_key: "member-hidden-identity-only-frame".to_string(),
5073                timestamp_ms: 2,
5074                runtime_key: "runtime-a".to_string(),
5075                identity: "review:singleton".to_string(),
5076                conversation_id: Some("review:singleton".to_string()),
5077                session_id: None,
5078                kind: "text_delta".to_string(),
5079                status: ConsoleFrameStatus::Delivered,
5080                payload: json!({ "delta": "identity-only-hidden" }),
5081                source: ConsoleFrameSource {
5082                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5083                    source_cursor: None,
5084                },
5085                source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5086                interaction_id: None,
5087                turn_id: None,
5088                run_id: None,
5089                parent_frame_id: None,
5090                caused_by_frame_id: None,
5091            })
5092            .await?;
5093        assert!(
5094            !aggregator
5095                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5096                    frame: identity_only.frame.clone()
5097                })
5098                .await,
5099            "member-hidden durable alias must not leak identity-only timeline frames"
5100        );
5101        let page = aggregator
5102            .query_timeline(ConsoleTimelineQuery {
5103                identity: Some("review:singleton".to_string()),
5104                limit: 10,
5105                ..ConsoleTimelineQuery::default()
5106            })
5107            .await?;
5108        assert!(
5109            page.frames.is_empty(),
5110            "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5111        );
5112        assert!(
5113            !aggregator.retire_identity("review:singleton").await?,
5114            "member-hidden live alias must not retire through aggregator"
5115        );
5116        let reserve_err = aggregator
5117            .reserve_identity_first_interaction(
5118                ConsoleSendRequest {
5119                    identity: "review:singleton".to_string(),
5120                    content: json!("hello"),
5121                    origin: "console:test".to_string(),
5122                    idempotency_key: "member-hidden-reserve".to_string(),
5123                    handling_mode: Some("queue".to_string()),
5124                },
5125                None,
5126            )
5127            .await
5128            .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5129        assert!(
5130            reserve_err.to_string().contains("unknown identity"),
5131            "unexpected reserve error: {reserve_err}"
5132        );
5133        let send_err = aggregator
5134            .send(ConsoleSendRequest {
5135                identity: "review:singleton".to_string(),
5136                content: json!("hello"),
5137                origin: "console:test".to_string(),
5138                idempotency_key: "member-hidden-send".to_string(),
5139                handling_mode: Some("queue".to_string()),
5140            })
5141            .await
5142            .expect_err("member-hidden live alias must not receive sends");
5143        assert!(
5144            send_err.to_string().contains("unknown identity"),
5145            "unexpected send error: {send_err}"
5146        );
5147        let blob_result = aggregator
5148            .binary_blob_store_for_identity("review:singleton")
5149            .await;
5150        let Err(blob_err) = blob_result else {
5151            panic!("member-hidden live alias must not expose blob store");
5152        };
5153        assert!(
5154            blob_err.to_string().contains("unknown identity"),
5155            "unexpected blob error: {blob_err}"
5156        );
5157
5158        let _ = runtime.mob_handle().stop().await;
5159        Ok(())
5160    }
5161
5162    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5163    async fn explicit_session_backfill_target_resolves_durable_identity_label()
5164    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5165        let runtime = build_empty_runtime("identity-label-backfill-test").await;
5166        let mut labels = BTreeMap::new();
5167        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5168        runtime
5169            .spawn(
5170                SpawnMemberSpec::from_wire(
5171                    "worker".to_string(),
5172                    "rt:review:singleton:0".to_string(),
5173                    Some("You are the Review Agent.".into()),
5174                    None,
5175                    None,
5176                )
5177                .with_labels(labels),
5178            )
5179            .await
5180            .expect("member spawns");
5181
5182        let entry = RuntimeEntry {
5183            runtime_key: "runtime-a".to_string(),
5184            identity_namespace: String::new(),
5185            runtime: runtime.mob_runtime().clone(),
5186            identity_runtime: runtime.identity_runtime().cloned(),
5187            console_events: runtime.console_events(),
5188            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5189        };
5190        let aggregator = MobKitConsoleAggregator::in_memory();
5191        aggregator
5192            .inner
5193            .runtimes
5194            .write()
5195            .expect("runtime registry")
5196            .insert("runtime-a".to_string(), entry);
5197
5198        let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5199            .await
5200            .expect("durable label should resolve targeted session backfill");
5201        assert_eq!(target.record.identity, "review:singleton");
5202        assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5203
5204        let _ = runtime.mob_handle().stop().await;
5205        Ok(())
5206    }
5207
5208    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5209    async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5210    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5211        let runtime = build_empty_runtime("identity-stale-durable-test").await;
5212        let mut labels = BTreeMap::new();
5213        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5214        runtime
5215            .spawn(
5216                SpawnMemberSpec::from_wire(
5217                    "worker".to_string(),
5218                    "rt:review:singleton:0".to_string(),
5219                    Some("You are the live Review Agent.".into()),
5220                    None,
5221                    None,
5222                )
5223                .with_labels(labels),
5224            )
5225            .await
5226            .expect("member spawns");
5227
5228        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5229            crate::identity_first::IdentityRuntimeConfig {
5230                continuity_store: Arc::new(
5231                    crate::identity_first::LocalContinuityStore::in_memory()?
5232                ),
5233                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5234                runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5235                has_runtime_store: true,
5236                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5237                bridge: None,
5238                default_timeout: None,
5239            },
5240        ));
5241        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5242        let record = crate::identity_first::ContinuityRecord {
5243            identity: identity.clone(),
5244            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5245                "rt:review:singleton:1",
5246            )?,
5247            session_id: SessionId::new(),
5248            generation: crate::identity_first::ContinuityGeneration::new(1),
5249            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5250        };
5251        identity_runtime
5252            .register(
5253                crate::identity_first::DurableAgentSpec {
5254                    identity: identity.clone(),
5255                    profile: meerkat_mob::ProfileName::from("worker"),
5256                    addressability: crate::identity_first::AgentAddressability::Addressable,
5257                    display_name: None,
5258                    labels: BTreeMap::new(),
5259                    context: None,
5260                    additional_instructions: Vec::new(),
5261                    initial_message: None,
5262                    runtime_mode_override: None,
5263                },
5264                crate::identity_first::IdentityLifecycleState::Active,
5265                Some(record),
5266                Some(crate::identity_first::LeaseGrant {
5267                    identity,
5268                    fencing_token: crate::identity_first::FencingToken::new(7),
5269                    ttl: Duration::from_mins(5),
5270                }),
5271            )
5272            .await;
5273
5274        let aggregator = MobKitConsoleAggregator::in_memory();
5275        aggregator
5276            .inner
5277            .runtimes
5278            .write()
5279            .expect("runtime registry")
5280            .insert(
5281                "runtime-a".to_string(),
5282                RuntimeEntry {
5283                    runtime_key: "runtime-a".to_string(),
5284                    identity_namespace: String::new(),
5285                    runtime: runtime.mob_runtime().clone(),
5286                    identity_runtime: Some(identity_runtime),
5287                    console_events: runtime.console_events(),
5288                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5289                },
5290            );
5291
5292        let err = aggregator
5293            .inspect_identity("review:singleton")
5294            .await
5295            .expect_err("stale durable binding must not inspect as healthy");
5296        assert!(
5297            err.to_string().contains("stale durable identity alias"),
5298            "unexpected error: {err}"
5299        );
5300        let retire_err = aggregator
5301            .retire_identity("review:singleton")
5302            .await
5303            .expect_err("stale durable binding must not retire as healthy");
5304        assert!(
5305            retire_err
5306                .to_string()
5307                .contains("stale durable identity alias"),
5308            "unexpected retire error: {retire_err}"
5309        );
5310        let send_err = aggregator
5311            .send(ConsoleSendRequest {
5312                identity: "review:singleton".to_string(),
5313                content: json!("hello"),
5314                origin: "console:test".to_string(),
5315                idempotency_key: "stale-durable-send".to_string(),
5316                handling_mode: Some("queue".to_string()),
5317            })
5318            .await
5319            .expect_err("stale durable binding must not send to the wrong live member");
5320        assert!(
5321            send_err
5322                .to_string()
5323                .contains("stale durable identity alias"),
5324            "unexpected send error: {send_err}"
5325        );
5326        let reserve_err = aggregator
5327            .reserve_identity_first_interaction(
5328                ConsoleSendRequest {
5329                    identity: "review:singleton".to_string(),
5330                    content: json!("hello"),
5331                    origin: "console:test".to_string(),
5332                    idempotency_key: "stale-durable-reserve".to_string(),
5333                    handling_mode: Some("queue".to_string()),
5334                },
5335                None,
5336            )
5337            .await
5338            .expect_err("identity-first reserve must not bypass stale durable binding");
5339        assert!(
5340            reserve_err
5341                .to_string()
5342                .contains("stale durable identity alias"),
5343            "unexpected reserve error: {reserve_err}"
5344        );
5345        let blob_err = match aggregator
5346            .binary_blob_store_for_identity("review:singleton")
5347            .await
5348        {
5349            Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5350            Err(err) => err,
5351        };
5352        assert!(
5353            blob_err
5354                .to_string()
5355                .contains("stale durable identity alias"),
5356            "unexpected blob error: {blob_err}"
5357        );
5358        let records = aggregator.list_identities_fresh().await?;
5359        assert!(
5360            records
5361                .iter()
5362                .any(|record| record.identity == "review:singleton"
5363                    && record.runtime_member_id == "rt:review:singleton:0"),
5364            "live alias should still be listed; records: {records:#?}"
5365        );
5366        assert!(
5367            records
5368                .iter()
5369                .any(|record| record.identity == "review:singleton"
5370                    && record.runtime_member_id == "rt:review:singleton:1"),
5371            "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5372        );
5373
5374        let _ = runtime.mob_handle().stop().await;
5375        Ok(())
5376    }
5377
5378    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5379    async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
5380    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5381        let runtime = build_empty_runtime("identity-wrong-projection-test").await;
5382        let mut labels = BTreeMap::new();
5383        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5384        runtime
5385            .spawn(
5386                SpawnMemberSpec::from_wire(
5387                    "worker".to_string(),
5388                    "rt:review:singleton:0".to_string(),
5389                    Some("You are the mislabeled Review Agent.".into()),
5390                    None,
5391                    None,
5392                )
5393                .with_labels(labels),
5394            )
5395            .await
5396            .expect("member spawns");
5397
5398        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5399            crate::identity_first::IdentityRuntimeConfig {
5400                continuity_store: Arc::new(
5401                    crate::identity_first::LocalContinuityStore::in_memory()?
5402                ),
5403                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5404                runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
5405                has_runtime_store: true,
5406                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5407                bridge: None,
5408                default_timeout: None,
5409            },
5410        ));
5411        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5412        let record = crate::identity_first::ContinuityRecord {
5413            identity: identity.clone(),
5414            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5415                "rt:review:singleton:0",
5416            )?,
5417            session_id: SessionId::new(),
5418            generation: crate::identity_first::ContinuityGeneration::new(1),
5419            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5420        };
5421        identity_runtime
5422            .register(
5423                crate::identity_first::DurableAgentSpec {
5424                    identity: identity.clone(),
5425                    profile: meerkat_mob::ProfileName::from("worker"),
5426                    addressability: crate::identity_first::AgentAddressability::Addressable,
5427                    display_name: None,
5428                    labels: BTreeMap::new(),
5429                    context: None,
5430                    additional_instructions: Vec::new(),
5431                    initial_message: None,
5432                    runtime_mode_override: None,
5433                },
5434                crate::identity_first::IdentityLifecycleState::Active,
5435                Some(record),
5436                Some(crate::identity_first::LeaseGrant {
5437                    identity,
5438                    fencing_token: crate::identity_first::FencingToken::new(7),
5439                    ttl: Duration::from_mins(5),
5440                }),
5441            )
5442            .await;
5443
5444        let aggregator = MobKitConsoleAggregator::in_memory();
5445        aggregator
5446            .inner
5447            .runtimes
5448            .write()
5449            .expect("runtime registry")
5450            .insert(
5451                "runtime-a".to_string(),
5452                RuntimeEntry {
5453                    runtime_key: "runtime-a".to_string(),
5454                    identity_namespace: String::new(),
5455                    runtime: runtime.mob_runtime().clone(),
5456                    identity_runtime: Some(identity_runtime),
5457                    console_events: runtime.console_events(),
5458                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5459                },
5460            );
5461
5462        let err = aggregator
5463            .inspect_identity("review:singleton")
5464            .await
5465            .expect_err("wrong projected live identity must not inspect as healthy");
5466        assert!(
5467            err.to_string()
5468                .contains("projects identity other:singleton"),
5469            "unexpected error: {err}"
5470        );
5471        let retire_err = aggregator
5472            .retire_identity("review:singleton")
5473            .await
5474            .expect_err("wrong projected live identity must not retire as healthy");
5475        assert!(
5476            retire_err
5477                .to_string()
5478                .contains("projects identity other:singleton"),
5479            "unexpected retire error: {retire_err}"
5480        );
5481        let send_err = aggregator
5482            .send(ConsoleSendRequest {
5483                identity: "review:singleton".to_string(),
5484                content: json!("hello"),
5485                origin: "console:test".to_string(),
5486                idempotency_key: "wrong-projection-send".to_string(),
5487                handling_mode: Some("queue".to_string()),
5488            })
5489            .await
5490            .expect_err("wrong projected live identity must not send as unknown");
5491        assert!(
5492            send_err
5493                .to_string()
5494                .contains("projects identity other:singleton"),
5495            "unexpected send error: {send_err}"
5496        );
5497        let reserve_err = aggregator
5498            .reserve_identity_first_interaction(
5499                ConsoleSendRequest {
5500                    identity: "review:singleton".to_string(),
5501                    content: json!("hello"),
5502                    origin: "console:test".to_string(),
5503                    idempotency_key: "wrong-projection-reserve".to_string(),
5504                    handling_mode: Some("queue".to_string()),
5505                },
5506                None,
5507            )
5508            .await
5509            .expect_err("identity-first reserve must not bypass wrong live projection");
5510        assert!(
5511            reserve_err
5512                .to_string()
5513                .contains("projects identity other:singleton"),
5514            "unexpected reserve error: {reserve_err}"
5515        );
5516        let blob_err = match aggregator
5517            .binary_blob_store_for_identity("review:singleton")
5518            .await
5519        {
5520            Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
5521            Err(err) => err,
5522        };
5523        assert!(
5524            blob_err
5525                .to_string()
5526                .contains("projects identity other:singleton"),
5527            "unexpected blob error: {blob_err}"
5528        );
5529        let projected_err = aggregator
5530            .inspect_identity("other:singleton")
5531            .await
5532            .expect_err("wrong projected alias must not inspect through live-only fallback");
5533        assert!(
5534            projected_err
5535                .to_string()
5536                .contains("identity runtime binding for review:singleton"),
5537            "unexpected projected-alias error: {projected_err}"
5538        );
5539        let projected_retire_err = aggregator
5540            .retire_identity("other:singleton")
5541            .await
5542            .expect_err("wrong projected alias must not retire through live-only fallback");
5543        assert!(
5544            projected_retire_err
5545                .to_string()
5546                .contains("identity runtime binding for review:singleton"),
5547            "unexpected projected-alias retire error: {projected_retire_err}"
5548        );
5549
5550        let _ = runtime.mob_handle().stop().await;
5551        Ok(())
5552    }
5553
5554    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5555    async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
5556    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5557        let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
5558        let mut labels = BTreeMap::new();
5559        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5560        runtime
5561            .spawn(
5562                SpawnMemberSpec::from_wire(
5563                    "worker".to_string(),
5564                    "rt:review:singleton:0".to_string(),
5565                    Some("You are a hidden wrong projection.".into()),
5566                    None,
5567                    None,
5568                )
5569                .with_labels(labels),
5570            )
5571            .await
5572            .expect("member spawns");
5573
5574        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5575            crate::identity_first::IdentityRuntimeConfig {
5576                continuity_store: Arc::new(
5577                    crate::identity_first::LocalContinuityStore::in_memory()?
5578                ),
5579                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5580                runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
5581                has_runtime_store: true,
5582                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5583                bridge: None,
5584                default_timeout: None,
5585            },
5586        ));
5587        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5588        identity_runtime
5589            .register(
5590                crate::identity_first::DurableAgentSpec {
5591                    identity: identity.clone(),
5592                    profile: meerkat_mob::ProfileName::from("worker"),
5593                    addressability: crate::identity_first::AgentAddressability::Addressable,
5594                    display_name: None,
5595                    labels: BTreeMap::new(),
5596                    context: None,
5597                    additional_instructions: Vec::new(),
5598                    initial_message: None,
5599                    runtime_mode_override: None,
5600                },
5601                crate::identity_first::IdentityLifecycleState::Active,
5602                Some(crate::identity_first::ContinuityRecord {
5603                    identity: identity.clone(),
5604                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5605                        "rt:review:singleton:0",
5606                    )?,
5607                    session_id: SessionId::new(),
5608                    generation: crate::identity_first::ContinuityGeneration::new(1),
5609                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5610                }),
5611                Some(crate::identity_first::LeaseGrant {
5612                    identity,
5613                    fencing_token: crate::identity_first::FencingToken::new(7),
5614                    ttl: Duration::from_mins(5),
5615                }),
5616            )
5617            .await;
5618
5619        let aggregator = MobKitConsoleAggregator::in_memory();
5620        aggregator
5621            .inner
5622            .runtimes
5623            .write()
5624            .expect("runtime registry")
5625            .insert(
5626                "runtime-a".to_string(),
5627                RuntimeEntry {
5628                    runtime_key: "runtime-a".to_string(),
5629                    identity_namespace: String::new(),
5630                    runtime: runtime.mob_runtime().clone(),
5631                    identity_runtime: Some(identity_runtime),
5632                    console_events: runtime.console_events(),
5633                    visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
5634                },
5635            );
5636
5637        let err = aggregator
5638            .inspect_identity("review:singleton")
5639            .await
5640            .expect_err("hidden wrong projection must still fail durable binding validation");
5641        assert!(
5642            err.to_string()
5643                .contains("projects identity other:singleton"),
5644            "unexpected hidden wrong-projection error: {err}"
5645        );
5646        let reserve_err = aggregator
5647            .reserve_identity_first_interaction(
5648                ConsoleSendRequest {
5649                    identity: "review:singleton".to_string(),
5650                    content: json!("hello"),
5651                    origin: "console:test".to_string(),
5652                    idempotency_key: "hidden-wrong-projection-reserve".to_string(),
5653                    handling_mode: Some("queue".to_string()),
5654                },
5655                None,
5656            )
5657            .await
5658            .expect_err("reserve must not hide wrong projection from durable validation");
5659        assert!(
5660            reserve_err
5661                .to_string()
5662                .contains("projects identity other:singleton"),
5663            "unexpected reserve error: {reserve_err}"
5664        );
5665
5666        let _ = runtime.mob_handle().stop().await;
5667        Ok(())
5668    }
5669
5670    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5671    async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
5672    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5673        let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
5674        let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
5675        let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5676            crate::identity_first::IdentityRuntimeConfig {
5677                continuity_store: Arc::new(
5678                    crate::identity_first::LocalContinuityStore::in_memory()?
5679                ),
5680                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5681                runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
5682                has_runtime_store: true,
5683                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5684                bridge: None,
5685                default_timeout: None,
5686            },
5687        ));
5688        let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5689            crate::identity_first::IdentityRuntimeConfig {
5690                continuity_store: Arc::new(
5691                    crate::identity_first::LocalContinuityStore::in_memory()?
5692                ),
5693                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5694                runtime_instance_id: "runtime-id-visible-match-test".to_string(),
5695                has_runtime_store: true,
5696                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5697                bridge: None,
5698                default_timeout: None,
5699            },
5700        ));
5701        for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
5702            let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5703            identity_runtime
5704                .register(
5705                    crate::identity_first::DurableAgentSpec {
5706                        identity: identity.clone(),
5707                        profile: meerkat_mob::ProfileName::from("worker"),
5708                        addressability: crate::identity_first::AgentAddressability::Addressable,
5709                        display_name: None,
5710                        labels: BTreeMap::new(),
5711                        context: None,
5712                        additional_instructions: Vec::new(),
5713                        initial_message: None,
5714                        runtime_mode_override: None,
5715                    },
5716                    crate::identity_first::IdentityLifecycleState::Active,
5717                    Some(crate::identity_first::ContinuityRecord {
5718                        identity: identity.clone(),
5719                        agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5720                            "rt:review:singleton:0",
5721                        )?,
5722                        session_id: SessionId::new(),
5723                        generation: crate::identity_first::ContinuityGeneration::new(1),
5724                        checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5725                    }),
5726                    Some(crate::identity_first::LeaseGrant {
5727                        identity,
5728                        fencing_token: crate::identity_first::FencingToken::new(7),
5729                        ttl: Duration::from_mins(5),
5730                    }),
5731                )
5732                .await;
5733        }
5734
5735        let aggregator = MobKitConsoleAggregator::in_memory();
5736        {
5737            let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
5738            runtimes.insert(
5739                "a-hidden".to_string(),
5740                RuntimeEntry {
5741                    runtime_key: "a-hidden".to_string(),
5742                    identity_namespace: String::new(),
5743                    runtime: hidden_runtime.mob_runtime().clone(),
5744                    identity_runtime: Some(hidden_identity_runtime),
5745                    console_events: hidden_runtime.console_events(),
5746                    visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
5747                },
5748            );
5749            runtimes.insert(
5750                "b-visible".to_string(),
5751                RuntimeEntry {
5752                    runtime_key: "b-visible".to_string(),
5753                    identity_namespace: String::new(),
5754                    runtime: visible_runtime.mob_runtime().clone(),
5755                    identity_runtime: Some(visible_identity_runtime),
5756                    console_events: visible_runtime.console_events(),
5757                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5758                },
5759            );
5760        }
5761
5762        let inspection = aggregator
5763            .inspect_identity("rt:review:singleton:0")
5764            .await?
5765            .expect("visible runtime-id match should be returned");
5766        assert_eq!(inspection.identity.runtime_key, "b-visible");
5767
5768        let _ = hidden_runtime.mob_handle().stop().await;
5769        let _ = visible_runtime.mob_handle().stop().await;
5770        Ok(())
5771    }
5772
5773    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5774    async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
5775    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5776        let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
5777        let mut primary_labels = BTreeMap::new();
5778        primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5779        runtime
5780            .spawn(
5781                SpawnMemberSpec::from_wire(
5782                    "worker".to_string(),
5783                    "rt:review:singleton:0".to_string(),
5784                    Some("You are the primary Review Agent.".into()),
5785                    None,
5786                    None,
5787                )
5788                .with_labels(primary_labels),
5789            )
5790            .await
5791            .expect("primary member spawns");
5792
5793        let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
5794        let mut delegate_labels = BTreeMap::new();
5795        delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5796        delegate_runtime
5797            .spawn(
5798                SpawnMemberSpec::from_wire(
5799                    "worker".to_string(),
5800                    "rt:review:singleton:0".to_string(),
5801                    Some("You are the wrong projected Review Agent.".into()),
5802                    None,
5803                    None,
5804                )
5805                .with_labels(delegate_labels),
5806            )
5807            .await
5808            .expect("delegate member spawns");
5809        runtime
5810            .mob_runtime()
5811            .agent_mob_mcp_state()
5812            .expect("runtime should expose mob MCP state")
5813            .mob_insert_handle(
5814                meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
5815                delegate_runtime.mob_handle().clone(),
5816            )
5817            .await;
5818
5819        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5820            crate::identity_first::IdentityRuntimeConfig {
5821                continuity_store: Arc::new(
5822                    crate::identity_first::LocalContinuityStore::in_memory()?
5823                ),
5824                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5825                runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
5826                has_runtime_store: true,
5827                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5828                bridge: None,
5829                default_timeout: None,
5830            },
5831        ));
5832        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5833        let record = crate::identity_first::ContinuityRecord {
5834            identity: identity.clone(),
5835            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5836                "rt:review:singleton:0",
5837            )?,
5838            session_id: SessionId::new(),
5839            generation: crate::identity_first::ContinuityGeneration::new(1),
5840            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5841        };
5842        identity_runtime
5843            .register(
5844                crate::identity_first::DurableAgentSpec {
5845                    identity: identity.clone(),
5846                    profile: meerkat_mob::ProfileName::from("worker"),
5847                    addressability: crate::identity_first::AgentAddressability::Addressable,
5848                    display_name: None,
5849                    labels: BTreeMap::new(),
5850                    context: None,
5851                    additional_instructions: Vec::new(),
5852                    initial_message: None,
5853                    runtime_mode_override: None,
5854                },
5855                crate::identity_first::IdentityLifecycleState::Active,
5856                Some(record),
5857                Some(crate::identity_first::LeaseGrant {
5858                    identity,
5859                    fencing_token: crate::identity_first::FencingToken::new(7),
5860                    ttl: Duration::from_mins(5),
5861                }),
5862            )
5863            .await;
5864
5865        let aggregator = MobKitConsoleAggregator::in_memory();
5866        aggregator
5867            .inner
5868            .runtimes
5869            .write()
5870            .expect("runtime registry")
5871            .insert(
5872                "runtime-a".to_string(),
5873                RuntimeEntry {
5874                    runtime_key: "runtime-a".to_string(),
5875                    identity_namespace: String::new(),
5876                    runtime: runtime.mob_runtime().clone(),
5877                    identity_runtime: Some(identity_runtime),
5878                    console_events: runtime.console_events(),
5879                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5880                },
5881            );
5882
5883        let err = aggregator
5884            .inspect_identity("review:singleton")
5885            .await
5886            .expect_err("cross-source wrong projection must not be hidden by primary source");
5887        assert!(
5888            err.to_string()
5889                .contains("projects identity other:singleton")
5890                || err.to_string().contains("ambiguous live identity alias"),
5891            "unexpected error: {err}"
5892        );
5893
5894        let _ = delegate_runtime.mob_handle().stop().await;
5895        let _ = runtime.mob_handle().stop().await;
5896        Ok(())
5897    }
5898
5899    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5900    async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
5901    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5902        let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
5903        let mut labels = BTreeMap::new();
5904        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5905        runtime
5906            .spawn(
5907                SpawnMemberSpec::from_wire(
5908                    "worker".to_string(),
5909                    "rt:review:singleton:0".to_string(),
5910                    Some("You are the first Review Agent.".into()),
5911                    None,
5912                    None,
5913                )
5914                .with_labels(labels.clone()),
5915            )
5916            .await
5917            .expect("first member spawns");
5918        runtime
5919            .spawn(
5920                SpawnMemberSpec::from_wire(
5921                    "worker".to_string(),
5922                    "rt:review:singleton:1".to_string(),
5923                    Some("You are the second Review Agent.".into()),
5924                    None,
5925                    None,
5926                )
5927                .with_labels(labels),
5928            )
5929            .await
5930            .expect("second member spawns");
5931
5932        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5933            crate::identity_first::IdentityRuntimeConfig {
5934                continuity_store: Arc::new(
5935                    crate::identity_first::LocalContinuityStore::in_memory()?
5936                ),
5937                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5938                runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
5939                has_runtime_store: true,
5940                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5941                bridge: None,
5942                default_timeout: None,
5943            },
5944        ));
5945        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5946        identity_runtime
5947            .register(
5948                crate::identity_first::DurableAgentSpec {
5949                    identity: identity.clone(),
5950                    profile: meerkat_mob::ProfileName::from("worker"),
5951                    addressability: crate::identity_first::AgentAddressability::Addressable,
5952                    display_name: None,
5953                    labels: BTreeMap::new(),
5954                    context: None,
5955                    additional_instructions: Vec::new(),
5956                    initial_message: None,
5957                    runtime_mode_override: None,
5958                },
5959                crate::identity_first::IdentityLifecycleState::Active,
5960                Some(crate::identity_first::ContinuityRecord {
5961                    identity: identity.clone(),
5962                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5963                        "rt:review:singleton:0",
5964                    )?,
5965                    session_id: SessionId::new(),
5966                    generation: crate::identity_first::ContinuityGeneration::new(1),
5967                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5968                }),
5969                Some(crate::identity_first::LeaseGrant {
5970                    identity,
5971                    fencing_token: crate::identity_first::FencingToken::new(8),
5972                    ttl: Duration::from_mins(5),
5973                }),
5974            )
5975            .await;
5976
5977        let aggregator = MobKitConsoleAggregator::in_memory();
5978        aggregator
5979            .inner
5980            .runtimes
5981            .write()
5982            .expect("runtime registry")
5983            .insert(
5984                "runtime-a".to_string(),
5985                RuntimeEntry {
5986                    runtime_key: "runtime-a".to_string(),
5987                    identity_namespace: String::new(),
5988                    runtime: runtime.mob_runtime().clone(),
5989                    identity_runtime: Some(identity_runtime),
5990                    console_events: runtime.console_events(),
5991                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5992                },
5993            );
5994
5995        let err = aggregator
5996            .inspect_identity("review:singleton")
5997            .await
5998            .expect_err("duplicate live alias must not inspect via durable fast path");
5999        assert!(
6000            err.to_string().contains("ambiguous live identity alias"),
6001            "unexpected error: {err}"
6002        );
6003        let retire_err = aggregator
6004            .retire_identity("review:singleton")
6005            .await
6006            .expect_err("duplicate live alias must not retire via durable fast path");
6007        assert!(
6008            retire_err
6009                .to_string()
6010                .contains("ambiguous live identity alias"),
6011            "unexpected retire error: {retire_err}"
6012        );
6013        let runtime_id_err = aggregator
6014            .inspect_identity("rt:review:singleton:0")
6015            .await
6016            .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6017        assert!(
6018            runtime_id_err
6019                .to_string()
6020                .contains("ambiguous live identity alias"),
6021            "unexpected runtime-id inspect error: {runtime_id_err}"
6022        );
6023        let runtime_id_retire_err = aggregator
6024            .retire_identity("rt:review:singleton:0")
6025            .await
6026            .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6027        assert!(
6028            runtime_id_retire_err
6029                .to_string()
6030                .contains("ambiguous live identity alias")
6031                || runtime_id_retire_err
6032                    .to_string()
6033                    .contains("stale durable identity alias"),
6034            "unexpected runtime-id retire error: {runtime_id_retire_err}"
6035        );
6036
6037        let _ = runtime.mob_handle().stop().await;
6038        Ok(())
6039    }
6040
6041    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6042    async fn inspect_identity_ignores_hidden_live_alias_candidates()
6043    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6044        let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6045        let mut labels = BTreeMap::new();
6046        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6047        runtime
6048            .spawn(
6049                SpawnMemberSpec::from_wire(
6050                    "worker".to_string(),
6051                    "rt:review:singleton:0".to_string(),
6052                    Some("You are the visible Review Agent.".into()),
6053                    None,
6054                    None,
6055                )
6056                .with_labels(labels.clone()),
6057            )
6058            .await
6059            .expect("visible member spawns");
6060        runtime
6061            .spawn(
6062                SpawnMemberSpec::from_wire(
6063                    "worker".to_string(),
6064                    "rt:review:singleton:1".to_string(),
6065                    Some("You are hidden maintenance noise.".into()),
6066                    None,
6067                    None,
6068                )
6069                .with_labels(labels),
6070            )
6071            .await
6072            .expect("hidden member spawns");
6073
6074        let aggregator = MobKitConsoleAggregator::in_memory();
6075        aggregator.register_runtime_handles_with_policy(
6076            "runtime-a",
6077            "",
6078            runtime.mob_runtime().clone(),
6079            None,
6080            runtime.console_events(),
6081            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6082        );
6083
6084        let inspection = aggregator
6085            .inspect_identity("review:singleton")
6086            .await?
6087            .expect("visible live alias should inspect without hidden ambiguity");
6088        assert_eq!(inspection.identity.identity, "review:singleton");
6089        assert_eq!(
6090            inspection.identity.runtime_member_id,
6091            "rt:review:singleton:0"
6092        );
6093
6094        let retired = aggregator.retire_identity("review:singleton").await?;
6095        assert!(
6096            retired,
6097            "visible live alias should retire without hidden ambiguity"
6098        );
6099
6100        let _ = runtime.mob_handle().stop().await;
6101        Ok(())
6102    }
6103
6104    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6105    async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6106    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6107        let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6108        let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6109        let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6110        let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6111
6112        let aggregator = MobKitConsoleAggregator::in_memory();
6113        aggregator.register_runtime_handles_with_policy(
6114            "a-hidden",
6115            "",
6116            hidden_runtime.mob_runtime().clone(),
6117            Some(hidden_identity_runtime),
6118            hidden_runtime.console_events(),
6119            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6120        );
6121        aggregator.register_runtime_handles_with_policy(
6122            "b-visible",
6123            "",
6124            visible_runtime.mob_runtime().clone(),
6125            Some(visible_identity_runtime),
6126            visible_runtime.console_events(),
6127            Arc::new(AllowAllConsoleVisibilityPolicy),
6128        );
6129
6130        let accepted = aggregator
6131            .reserve_identity_first_interaction(
6132                ConsoleSendRequest {
6133                    identity: "review:singleton".to_string(),
6134                    content: json!("hello"),
6135                    origin: "console:test".to_string(),
6136                    idempotency_key: "visible-after-hidden-reserve".to_string(),
6137                    handling_mode: Some("queue".to_string()),
6138                },
6139                None,
6140            )
6141            .await?;
6142        let page = aggregator
6143            .query_timeline(ConsoleTimelineQuery {
6144                identity: Some("review:singleton".to_string()),
6145                ..ConsoleTimelineQuery::default()
6146            })
6147            .await?;
6148        let frame = page
6149            .frames
6150            .iter()
6151            .find(|frame| frame.id == accepted.input_frame_id)
6152            .ok_or("accepted frame missing from timeline")?;
6153        assert_eq!(frame.runtime_key, "b-visible");
6154
6155        let _ = hidden_runtime.mob_handle().stop().await;
6156        let _ = visible_runtime.mob_handle().stop().await;
6157        Ok(())
6158    }
6159
6160    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6161    async fn identity_first_list_identities_projects_cached_topology_peers()
6162    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6163        let runtime = build_empty_runtime("identity-topology-cache-test").await;
6164        let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6165        identity_runtime
6166            .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6167                crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6168                crate::identity_first::AgentIdentity::parse("agent:beta")?,
6169            )?])
6170            .await;
6171
6172        let aggregator = MobKitConsoleAggregator::in_memory();
6173        aggregator.register_runtime_handles_with_policy(
6174            "identity-first",
6175            "",
6176            runtime.mob_runtime().clone(),
6177            Some(identity_runtime),
6178            runtime.console_events(),
6179            Arc::new(AllowAllConsoleVisibilityPolicy),
6180        );
6181
6182        let identities = aggregator.list_identities().await?;
6183        let alpha = identities
6184            .iter()
6185            .find(|record| record.identity == "agent:alpha")
6186            .ok_or("agent:alpha identity missing")?;
6187        assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6188        let inspection = aggregator
6189            .inspect_identity("agent:alpha")
6190            .await?
6191            .ok_or("agent:alpha inspection missing")?;
6192        assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6193
6194        let _ = runtime.mob_handle().stop().await;
6195        Ok(())
6196    }
6197
6198    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6199    async fn identity_first_list_identities_includes_member_only_spawned_workers()
6200    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6201        let runtime = build_empty_runtime("identity-member-only-test").await;
6202        let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6203        let aggregator = MobKitConsoleAggregator::in_memory();
6204        aggregator.register_runtime_handles_with_policy(
6205            "identity-first",
6206            "",
6207            runtime.mob_runtime().clone(),
6208            Some(identity_runtime),
6209            runtime.console_events(),
6210            Arc::new(AllowAllConsoleVisibilityPolicy),
6211        );
6212
6213        runtime
6214            .spawn(SpawnMemberSpec::from_wire(
6215                "worker".to_string(),
6216                "agent:beta".to_string(),
6217                Some("You are beta.".into()),
6218                None,
6219                None,
6220            ))
6221            .await?;
6222
6223        let identities = aggregator.list_identities().await?;
6224        assert!(
6225            identities
6226                .iter()
6227                .any(|record| record.identity == "agent:beta"),
6228            "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6229        );
6230
6231        let _ = runtime.mob_handle().stop().await;
6232        Ok(())
6233    }
6234
6235    #[tokio::test]
6236    async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6237        let aggregator = MobKitConsoleAggregator::in_memory();
6238        let record = identity_record_for_test("agent-cached");
6239        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6240        aggregator
6241            .inner
6242            .identity_read_model
6243            .primed
6244            .store(true, Ordering::Release);
6245        let _guard = aggregator
6246            .inner
6247            .identity_read_model
6248            .refresh_lock
6249            .clone()
6250            .lock_owned()
6251            .await;
6252
6253        let identities =
6254            tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6255                .await
6256                .expect("hot identity list should not wait for refresh lock")
6257                .expect("identity list succeeds");
6258
6259        assert_eq!(identities, vec![record]);
6260    }
6261
6262    #[tokio::test]
6263    async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6264        let aggregator = MobKitConsoleAggregator::in_memory();
6265        let guard = aggregator
6266            .inner
6267            .identity_read_model
6268            .refresh_lock
6269            .clone()
6270            .lock_owned()
6271            .await;
6272        let waiter = tokio::spawn({
6273            let aggregator = aggregator.clone();
6274            async move { aggregator.list_identities().await }
6275        });
6276
6277        tokio::time::sleep(Duration::from_millis(20)).await;
6278        assert!(
6279            !waiter.is_finished(),
6280            "cold identity list should wait for the in-flight refresh to finish"
6281        );
6282
6283        let record = identity_record_for_test("agent-primed");
6284        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6285        aggregator
6286            .inner
6287            .identity_read_model
6288            .primed
6289            .store(true, Ordering::Release);
6290        drop(guard);
6291
6292        let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6293            .await
6294            .expect("cold identity list waiter should resume")
6295            .expect("waiter joins")
6296            .expect("identity list succeeds");
6297        assert_eq!(identities, vec![record]);
6298    }
6299
6300    #[tokio::test]
6301    async fn query_timeline_reads_from_aggregate_store() {
6302        let aggregator = MobKitConsoleAggregator::in_memory();
6303        let frame = NewConsoleFrame {
6304            id: None,
6305            dedupe_key: "event-1".to_string(),
6306            timestamp_ms: 1,
6307            runtime_key: "runtime-a".to_string(),
6308            identity: "agent-a".to_string(),
6309            conversation_id: Some("agent-a".to_string()),
6310            session_id: None,
6311            kind: "text_delta".to_string(),
6312            status: ConsoleFrameStatus::Delivered,
6313            payload: json!({ "delta": "hello" }),
6314            source: ConsoleFrameSource {
6315                kind: ConsoleFrameSourceKind::ConsoleEvent,
6316                source_cursor: None,
6317            },
6318            source_event_id: Some("event-1".to_string()),
6319            interaction_id: None,
6320            turn_id: None,
6321            run_id: None,
6322            parent_frame_id: None,
6323            caused_by_frame_id: None,
6324        };
6325        aggregator
6326            .store()
6327            .append_if_absent(frame)
6328            .await
6329            .expect("append frame");
6330
6331        let page = aggregator
6332            .query_timeline(ConsoleTimelineQuery {
6333                identity: Some("agent-a".to_string()),
6334                limit: 10,
6335                ..ConsoleTimelineQuery::default()
6336            })
6337            .await
6338            .expect("query timeline");
6339        assert_eq!(page.frames.len(), 1);
6340        assert_eq!(page.frames[0].kind, "text_delta");
6341    }
6342
6343    #[tokio::test]
6344    async fn identity_recent_anchor_respects_query_limit() {
6345        let aggregator = MobKitConsoleAggregator::in_memory();
6346        aggregator
6347            .store()
6348            .append_if_absent(NewConsoleFrame {
6349                id: None,
6350                dedupe_key: "anchored-user-input".to_string(),
6351                timestamp_ms: 1,
6352                runtime_key: "runtime-a".to_string(),
6353                identity: "agent-a".to_string(),
6354                conversation_id: Some("agent-a".to_string()),
6355                session_id: None,
6356                kind: "user_input".to_string(),
6357                status: ConsoleFrameStatus::Delivered,
6358                payload: json!({ "content": "anchor me" }),
6359                source: ConsoleFrameSource {
6360                    kind: ConsoleFrameSourceKind::Synthetic,
6361                    source_cursor: None,
6362                },
6363                source_event_id: Some("anchored-user-input".to_string()),
6364                interaction_id: Some("turn-a".to_string()),
6365                turn_id: None,
6366                run_id: None,
6367                parent_frame_id: None,
6368                caused_by_frame_id: None,
6369            })
6370            .await
6371            .expect("append anchor");
6372        for idx in 2..=40 {
6373            aggregator
6374                .store()
6375                .append_if_absent(NewConsoleFrame {
6376                    id: None,
6377                    dedupe_key: format!("noisy-tail-{idx}"),
6378                    timestamp_ms: idx,
6379                    runtime_key: "runtime-a".to_string(),
6380                    identity: "agent-a".to_string(),
6381                    conversation_id: Some("agent-a".to_string()),
6382                    session_id: None,
6383                    kind: "reasoning_delta".to_string(),
6384                    status: ConsoleFrameStatus::Delivered,
6385                    payload: json!({ "delta": idx }),
6386                    source: ConsoleFrameSource {
6387                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6388                        source_cursor: None,
6389                    },
6390                    source_event_id: Some(format!("noisy-tail-{idx}")),
6391                    interaction_id: Some("turn-a".to_string()),
6392                    turn_id: None,
6393                    run_id: None,
6394                    parent_frame_id: None,
6395                    caused_by_frame_id: None,
6396                })
6397                .await
6398                .expect("append noisy tail");
6399        }
6400
6401        let page = aggregator
6402            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6403                identity: Some("agent-a".to_string()),
6404                mode: ConsoleTimelineMode::Recent,
6405                limit: 5,
6406                ..ConsoleTimelineWindowQuery::default()
6407            })
6408            .await
6409            .expect("query recent identity timeline");
6410
6411        assert_eq!(
6412            page.frames.len(),
6413            5,
6414            "identity anchor merge must not exceed the requested limit"
6415        );
6416        assert!(
6417            page.frames
6418                .iter()
6419                .any(|frame| frame.dedupe_key == "anchored-user-input"),
6420            "the bounded result should still retain the useful turn anchor: {:#?}",
6421            page.frames
6422        );
6423        assert_eq!(
6424            page.frames.last().and_then(|frame| frame.cursor.seq()),
6425            Some(40)
6426        );
6427    }
6428
6429    #[tokio::test]
6430    async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
6431        let aggregator = MobKitConsoleAggregator::in_memory();
6432        let runtime = build_single_member_runtime().await;
6433        let mut entry = runtime_entry_for_test("runtime-a", &runtime);
6434        entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
6435        aggregator
6436            .inner
6437            .runtimes
6438            .write()
6439            .expect("runtime registry")
6440            .insert("runtime-a".to_string(), entry);
6441
6442        for idx in 0..1_500 {
6443            aggregator
6444                .store()
6445                .append_if_absent(NewConsoleFrame {
6446                    id: None,
6447                    dedupe_key: format!("hidden-gap-{idx}"),
6448                    timestamp_ms: idx,
6449                    runtime_key: "runtime-a".to_string(),
6450                    identity: "agent-a".to_string(),
6451                    conversation_id: Some("agent-a".to_string()),
6452                    session_id: None,
6453                    kind: "hidden_noise".to_string(),
6454                    status: ConsoleFrameStatus::Delivered,
6455                    payload: json!({ "delta": idx }),
6456                    source: ConsoleFrameSource {
6457                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6458                        source_cursor: None,
6459                    },
6460                    source_event_id: Some(format!("hidden-gap-{idx}")),
6461                    interaction_id: None,
6462                    turn_id: None,
6463                    run_id: None,
6464                    parent_frame_id: None,
6465                    caused_by_frame_id: None,
6466                })
6467                .await
6468                .expect("append hidden frame");
6469        }
6470        aggregator
6471            .store()
6472            .append_if_absent(NewConsoleFrame {
6473                id: None,
6474                dedupe_key: "visible-after-gap".to_string(),
6475                timestamp_ms: 1_501,
6476                runtime_key: "runtime-a".to_string(),
6477                identity: "agent-a".to_string(),
6478                conversation_id: Some("agent-a".to_string()),
6479                session_id: None,
6480                kind: "interaction_complete".to_string(),
6481                status: ConsoleFrameStatus::Delivered,
6482                payload: json!({ "text": "visible after hidden gap" }),
6483                source: ConsoleFrameSource {
6484                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6485                    source_cursor: None,
6486                },
6487                source_event_id: Some("visible-after-gap".to_string()),
6488                interaction_id: Some("turn-visible".to_string()),
6489                turn_id: None,
6490                run_id: None,
6491                parent_frame_id: None,
6492                caused_by_frame_id: None,
6493            })
6494            .await
6495            .expect("append visible frame");
6496
6497        let page = aggregator
6498            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6499                identity: Some("agent-a".to_string()),
6500                mode: ConsoleTimelineMode::Since,
6501                limit: 1,
6502                ..ConsoleTimelineWindowQuery::default()
6503            })
6504            .await
6505            .expect("query timeline");
6506        assert_eq!(page.frames.len(), 1);
6507        assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
6508        assert_eq!(
6509            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6510            Some(1_501)
6511        );
6512        let _ = runtime.mob_handle().stop().await;
6513    }
6514
6515    #[tokio::test]
6516    async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
6517        let aggregator = MobKitConsoleAggregator::in_memory();
6518        for idx in 1..=300 {
6519            aggregator
6520                .store()
6521                .append_if_absent(NewConsoleFrame {
6522                    id: None,
6523                    dedupe_key: format!("visible-{idx}"),
6524                    timestamp_ms: idx,
6525                    runtime_key: "runtime-a".to_string(),
6526                    identity: "agent-a".to_string(),
6527                    conversation_id: Some("agent-a".to_string()),
6528                    session_id: None,
6529                    kind: "interaction_complete".to_string(),
6530                    status: ConsoleFrameStatus::Delivered,
6531                    payload: json!({ "text": format!("visible {idx}") }),
6532                    source: ConsoleFrameSource {
6533                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6534                        source_cursor: None,
6535                    },
6536                    source_event_id: Some(format!("visible-{idx}")),
6537                    interaction_id: Some(format!("turn-{idx}")),
6538                    turn_id: None,
6539                    run_id: None,
6540                    parent_frame_id: None,
6541                    caused_by_frame_id: None,
6542                })
6543                .await
6544                .expect("append visible frame");
6545        }
6546
6547        let first = aggregator
6548            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6549                identity: Some("agent-a".to_string()),
6550                mode: ConsoleTimelineMode::Since,
6551                limit: 10,
6552                ..ConsoleTimelineWindowQuery::default()
6553            })
6554            .await
6555            .expect("query first page");
6556        assert_eq!(first.frames.len(), 10);
6557        assert_eq!(
6558            first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6559            Some(10)
6560        );
6561
6562        let second = aggregator
6563            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6564                identity: Some("agent-a".to_string()),
6565                mode: ConsoleTimelineMode::Since,
6566                after: first.next_cursor,
6567                limit: 10,
6568                ..ConsoleTimelineWindowQuery::default()
6569            })
6570            .await
6571            .expect("query second page");
6572        assert_eq!(second.frames.len(), 10);
6573        assert_eq!(second.frames[0].dedupe_key, "visible-11");
6574        assert_eq!(
6575            second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6576            Some(20)
6577        );
6578    }
6579
6580    #[tokio::test]
6581    async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
6582        let store = Arc::new(CountingConsoleLogStore::new());
6583        let aggregator = MobKitConsoleAggregator::new(store.clone());
6584        let runtime = build_single_member_runtime().await;
6585        aggregator
6586            .inner
6587            .runtimes
6588            .write()
6589            .expect("runtime registry")
6590            .insert(
6591                "runtime-a".to_string(),
6592                runtime_entry_for_test("runtime-a", &runtime),
6593            );
6594        let inserted = store
6595            .append_if_absent(NewConsoleFrame {
6596                id: None,
6597                dedupe_key: "event-1".to_string(),
6598                timestamp_ms: 1,
6599                runtime_key: "runtime-a".to_string(),
6600                identity: "agent-a".to_string(),
6601                conversation_id: Some("agent-a".to_string()),
6602                session_id: None,
6603                kind: "text_delta".to_string(),
6604                status: ConsoleFrameStatus::Delivered,
6605                payload: json!({ "delta": "hello" }),
6606                source: ConsoleFrameSource {
6607                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6608                    source_cursor: None,
6609                },
6610                source_event_id: Some("event-1".to_string()),
6611                interaction_id: None,
6612                turn_id: None,
6613                run_id: None,
6614                parent_frame_id: None,
6615                caused_by_frame_id: None,
6616            })
6617            .await
6618            .expect("append frame");
6619
6620        let page = aggregator
6621            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6622                identity: Some("agent-a".to_string()),
6623                mode: ConsoleTimelineMode::Since,
6624                after: Some(inserted.frame.cursor),
6625                limit: 10,
6626                ..ConsoleTimelineWindowQuery::default()
6627            })
6628            .await
6629            .expect("query continuation");
6630
6631        assert!(page.frames.is_empty());
6632        assert_eq!(
6633            store.source_watermark_calls(),
6634            0,
6635            "empty since continuation must not synchronously force session-history backfill"
6636        );
6637        let _ = runtime.mob_handle().stop().await;
6638    }
6639
6640    #[test]
6641    fn legacy_timeline_struct_literals_remain_source_compatible() {
6642        let query = ConsoleTimelineQuery {
6643            identity: Some("agent-a".to_string()),
6644            conversation_id: None,
6645            after: Some(ConsoleCursor::from("console:1")),
6646            limit: 10,
6647        };
6648        let page = ConsoleTimelinePage {
6649            frames: Vec::new(),
6650            next_cursor: query.after.clone(),
6651        };
6652
6653        assert_eq!(query.limit, 10);
6654        assert_eq!(
6655            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6656            Some(1)
6657        );
6658    }
6659
6660    #[tokio::test]
6661    async fn query_timeline_is_store_local_for_registered_runtimes() {
6662        let store = Arc::new(CountingConsoleLogStore::new());
6663        let aggregator = MobKitConsoleAggregator::new(store.clone());
6664        let runtime = build_single_member_runtime().await;
6665        aggregator
6666            .inner
6667            .runtimes
6668            .write()
6669            .expect("runtime registry")
6670            .insert(
6671                "runtime-a".to_string(),
6672                runtime_entry_for_test("runtime-a", &runtime),
6673            );
6674        store
6675            .append_if_absent(NewConsoleFrame {
6676                id: None,
6677                dedupe_key: "event-1".to_string(),
6678                timestamp_ms: 1,
6679                runtime_key: "runtime-a".to_string(),
6680                identity: "agent-a".to_string(),
6681                conversation_id: Some("agent-a".to_string()),
6682                session_id: None,
6683                kind: "text_delta".to_string(),
6684                status: ConsoleFrameStatus::Delivered,
6685                payload: json!({ "delta": "hello" }),
6686                source: ConsoleFrameSource {
6687                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6688                    source_cursor: None,
6689                },
6690                source_event_id: Some("event-1".to_string()),
6691                interaction_id: None,
6692                turn_id: None,
6693                run_id: None,
6694                parent_frame_id: None,
6695                caused_by_frame_id: None,
6696            })
6697            .await
6698            .expect("append frame");
6699
6700        let page = tokio::time::timeout(
6701            Duration::from_millis(250),
6702            aggregator.query_timeline(ConsoleTimelineQuery {
6703                identity: Some("agent-a".to_string()),
6704                limit: 10,
6705                ..ConsoleTimelineQuery::default()
6706            }),
6707        )
6708        .await
6709        .expect("timeline query should not wait for session history")
6710        .expect("timeline query succeeds");
6711
6712        assert_eq!(page.frames.len(), 1);
6713        assert_eq!(
6714            store.source_watermark_calls(),
6715            0,
6716            "query_timeline must not synchronously touch session-history watermarks"
6717        );
6718        let _ = runtime.mob_handle().stop().await;
6719    }
6720
6721    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6722    async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
6723        let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
6724            delay: Duration::from_secs(2),
6725        }))
6726        .await;
6727        let aggregator = MobKitConsoleAggregator::in_memory();
6728        aggregator
6729            .inner
6730            .runtimes
6731            .write()
6732            .expect("runtime registry")
6733            .insert(
6734                "runtime-a".to_string(),
6735                runtime_entry_for_test("runtime-a", &runtime),
6736            );
6737
6738        let start = Instant::now();
6739        let accepted = tokio::time::timeout(
6740            Duration::from_millis(300),
6741            aggregator.send(ConsoleSendRequest {
6742                identity: "test/agent-a".to_string(),
6743                content: json!("hello slow agent"),
6744                origin: "console:test".to_string(),
6745                idempotency_key: "nonblocking-send".to_string(),
6746                handling_mode: Some("queue".to_string()),
6747            }),
6748        )
6749        .await
6750        .expect("console send should return once the input is accepted")
6751        .expect("send succeeds");
6752
6753        assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
6754        assert!(
6755            start.elapsed() < Duration::from_secs(1),
6756            "console send should not wait for the delayed LLM turn"
6757        );
6758
6759        wait_for_session_history_text(
6760            &aggregator,
6761            "test/agent-a",
6762            "slow ok",
6763            Duration::from_secs(5),
6764        )
6765        .await
6766        .expect("background dispatch should still complete and project history");
6767        let _ = runtime.mob_handle().stop().await;
6768    }
6769
6770    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6771    async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
6772    {
6773        let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
6774        let aggregator = MobKitConsoleAggregator::in_memory();
6775        aggregator.register_runtime(ConsoleRuntimeRegistration {
6776            runtime_key: "runtime-late".to_string(),
6777            runtime: runtime.clone(),
6778            identity_namespace: "late".to_string(),
6779            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6780        });
6781
6782        runtime
6783            .spawn(SpawnMemberSpec::from_wire(
6784                "worker".to_string(),
6785                "agent-late".to_string(),
6786                Some("You are agent-late.".into()),
6787                None,
6788                None,
6789            ))
6790            .await
6791            .expect("late member spawns");
6792        let session_id = send_message_on_mob_with_mode(
6793            &runtime.mob_handle(),
6794            "agent-late",
6795            ContentInput::Text("hello after registration".to_string()),
6796            meerkat_core::types::HandlingMode::Queue,
6797        )
6798        .await
6799        .expect("direct member send succeeds");
6800        wait_for_identity_record(
6801            &aggregator,
6802            "late/agent-late",
6803            Some(session_id.as_str()),
6804            Duration::from_secs(5),
6805        )
6806        .await?;
6807
6808        wait_for_session_history_text(
6809            &aggregator,
6810            "late/agent-late",
6811            "You are agent-late.",
6812            Duration::from_secs(5),
6813        )
6814        .await?;
6815        let _ = runtime.mob_handle().stop().await;
6816        Ok(())
6817    }
6818
6819    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6820    async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
6821    -> Result<(), String> {
6822        let runtime = build_single_member_runtime().await;
6823        let entry = runtime_entry_for_test("runtime-a", &runtime);
6824        let resolved = member_sources_for_entry(&entry)
6825            .await
6826            .into_iter()
6827            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6828            .expect("agent-a member exists");
6829        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6830            .await
6831            .expect("identity record exists");
6832        let session_id = record.session_id.expect("agent-a has a session");
6833        wait_for_runtime_session_history_text(
6834            &runtime,
6835            &session_id,
6836            "You are agent-a.",
6837            Duration::from_secs(5),
6838        )
6839        .await?;
6840
6841        let aggregator = MobKitConsoleAggregator::in_memory();
6842        aggregator
6843            .inner
6844            .runtimes
6845            .write()
6846            .expect("runtime registry")
6847            .insert("runtime-a".to_string(), entry);
6848        let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
6849        aggregator
6850            .store()
6851            .record_source_watermark(
6852                &watermark_key,
6853                ConsoleFrameSourceKind::SessionHistory,
6854                &format_session_history_watermark(&session_id, 0, current_time_ms()),
6855            )
6856            .await
6857            .expect("record fresh empty watermark");
6858
6859        let page = tokio::time::timeout(
6860            Duration::from_secs(2),
6861            aggregator.query_timeline(ConsoleTimelineQuery {
6862                identity: Some("test/agent-a".to_string()),
6863                limit: 20,
6864                ..ConsoleTimelineQuery::default()
6865            }),
6866        )
6867        .await
6868        .expect("explicit identity query should not stall")
6869        .expect("query succeeds");
6870
6871        assert!(
6872            page.frames.is_empty(),
6873            "empty fresh-watermark query should return promptly before async backfill; frames: {:#?}",
6874            page.frames
6875        );
6876        wait_for_session_history_text(
6877            &aggregator,
6878            "test/agent-a",
6879            "You are agent-a.",
6880            Duration::from_secs(5),
6881        )
6882        .await?;
6883        let _ = runtime.mob_handle().stop().await;
6884        Ok(())
6885    }
6886
6887    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6888    async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
6889    {
6890        let store = Arc::new(CountingConsoleLogStore::new());
6891        let aggregator = MobKitConsoleAggregator::new(store.clone());
6892        let runtime = build_single_member_runtime().await;
6893        let entry = runtime_entry_for_test("runtime-a", &runtime);
6894        let resolved = member_sources_for_entry(&entry)
6895            .await
6896            .into_iter()
6897            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6898            .expect("agent-a member exists");
6899        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6900            .await
6901            .expect("identity record exists");
6902        let session_id = record.session_id.expect("agent-a has a session");
6903        wait_for_runtime_session_history_text(
6904            &runtime,
6905            &session_id,
6906            "You are agent-a.",
6907            Duration::from_secs(5),
6908        )
6909        .await?;
6910        aggregator
6911            .inner
6912            .runtimes
6913            .write()
6914            .expect("runtime registry")
6915            .insert("runtime-a".to_string(), entry);
6916
6917        store
6918            .append_if_absent(NewConsoleFrame {
6919                id: None,
6920                dedupe_key: "stale-session-history-agent-a".to_string(),
6921                timestamp_ms: 10,
6922                runtime_key: "runtime-a".to_string(),
6923                identity: "test/agent-a".to_string(),
6924                conversation_id: Some("test/agent-a".to_string()),
6925                session_id: Some(session_id.clone()),
6926                kind: "user_input".to_string(),
6927                status: ConsoleFrameStatus::Delivered,
6928                payload: json!({
6929                    "text": "stale projected history",
6930                    "type": "user_input",
6931                }),
6932                source: ConsoleFrameSource {
6933                    kind: ConsoleFrameSourceKind::SessionHistory,
6934                    source_cursor: Some("stale-session-history-agent-a".to_string()),
6935                },
6936                source_event_id: Some("stale-session-history-agent-a".to_string()),
6937                interaction_id: None,
6938                turn_id: None,
6939                run_id: None,
6940                parent_frame_id: None,
6941                caused_by_frame_id: None,
6942            })
6943            .await
6944            .expect("append stale history frame");
6945
6946        let fresh_prompt = "fresh prompt after stale history";
6947        let sent_session_id = send_message_on_mob_with_mode(
6948            &runtime.mob_handle(),
6949            "agent-a",
6950            ContentInput::Text(fresh_prompt.to_string()),
6951            meerkat_core::types::HandlingMode::Queue,
6952        )
6953        .await
6954        .expect("direct member send succeeds");
6955        assert_eq!(sent_session_id, session_id);
6956        wait_for_runtime_session_history_text(
6957            &runtime,
6958            &session_id,
6959            fresh_prompt,
6960            Duration::from_secs(5),
6961        )
6962        .await?;
6963
6964        let page = aggregator
6965            .query_timeline(ConsoleTimelineQuery {
6966                identity: Some("test/agent-a".to_string()),
6967                limit: 50,
6968                ..ConsoleTimelineQuery::default()
6969            })
6970            .await
6971            .expect("query child timeline");
6972
6973        assert!(
6974            page.frames.iter().any(|frame| {
6975                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
6976                    && session_history_content_text(frame).as_deref()
6977                        == Some("stale projected history")
6978            }),
6979            "explicit identity query should return existing history before async refresh; frames: {:#?}",
6980            page.frames
6981        );
6982        wait_for_session_history_text(
6983            &aggregator,
6984            "test/agent-a",
6985            fresh_prompt,
6986            Duration::from_secs(5),
6987        )
6988        .await?;
6989        assert!(
6990            store.source_watermark_calls() > 0,
6991            "stale existing session history should force a targeted source refresh"
6992        );
6993        let _ = runtime.mob_handle().stop().await;
6994        Ok(())
6995    }
6996
6997    async fn wait_for_session_history_text(
6998        aggregator: &MobKitConsoleAggregator,
6999        identity: &str,
7000        expected: &str,
7001        timeout: Duration,
7002    ) -> Result<(), String> {
7003        let deadline = Instant::now() + timeout;
7004        let mut observed = Vec::new();
7005        while Instant::now() < deadline {
7006            let _ = aggregator.list_identities().await;
7007            let page = aggregator
7008                .query_timeline(ConsoleTimelineQuery {
7009                    identity: Some(identity.to_string()),
7010                    limit: 20,
7011                    ..ConsoleTimelineQuery::default()
7012                })
7013                .await
7014                .expect("query timeline");
7015            observed = page.frames;
7016            if observed.iter().any(|frame| {
7017                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7018                    && session_history_content_text(frame).as_deref() == Some(expected)
7019            }) {
7020                return Ok(());
7021            }
7022            tokio::time::sleep(Duration::from_millis(25)).await;
7023        }
7024
7025        Err(format!(
7026            "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7027        ))
7028    }
7029
7030    async fn wait_for_session_backfill_target(
7031        aggregator: &MobKitConsoleAggregator,
7032        identity: &str,
7033        timeout: Duration,
7034    ) -> Result<String, String> {
7035        let deadline = Instant::now() + timeout;
7036        while Instant::now() < deadline {
7037            if let Some(target) =
7038                session_backfill_target_for_identity(&aggregator.inner, identity).await
7039            {
7040                return Ok(target.session_id);
7041            }
7042            tokio::time::sleep(Duration::from_millis(25)).await;
7043        }
7044        Err(format!(
7045            "session backfill target for {identity:?} was not resolvable"
7046        ))
7047    }
7048
7049    async fn wait_for_runtime_session_history_text(
7050        runtime: &UnifiedRuntime,
7051        session_id: &str,
7052        expected: &str,
7053        timeout: Duration,
7054    ) -> Result<(), String> {
7055        let deadline = Instant::now() + timeout;
7056        let mut observed = Vec::new();
7057        while Instant::now() < deadline {
7058            let page = runtime
7059                .mob_runtime()
7060                .read_session_history(session_id, 0, Some(20))
7061                .await
7062                .map_err(|err| err.to_string())?;
7063            observed = page.messages;
7064            if observed.iter().enumerate().any(|(idx, message)| {
7065                let Some(message) = serde_json::to_value(message).ok() else {
7066                    return false;
7067                };
7068                frames_from_session_history_message(
7069                    "runtime-a",
7070                    "test/agent-a",
7071                    session_id,
7072                    idx,
7073                    message,
7074                )
7075                .iter()
7076                .any(|frame| {
7077                    matches!(
7078                        frame.kind.as_str(),
7079                        "user_input" | "system_notice" | "interaction_complete"
7080                    ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7081                })
7082            }) {
7083                return Ok(());
7084            }
7085            tokio::time::sleep(Duration::from_millis(25)).await;
7086        }
7087
7088        Err(format!(
7089            "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7090        ))
7091    }
7092
7093    async fn wait_for_identity_record(
7094        aggregator: &MobKitConsoleAggregator,
7095        identity: &str,
7096        session_id: Option<&str>,
7097        timeout: Duration,
7098    ) -> Result<(), String> {
7099        let deadline = Instant::now() + timeout;
7100        let mut observed = Vec::new();
7101        while Instant::now() < deadline {
7102            observed = aggregator
7103                .list_identities()
7104                .await
7105                .map_err(|err| err.to_string())?;
7106            if observed.iter().any(|record| {
7107                record.identity == identity && record.session_id.as_deref() == session_id
7108            }) {
7109                return Ok(());
7110            }
7111            tokio::time::sleep(Duration::from_millis(25)).await;
7112        }
7113
7114        Err(format!(
7115            "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7116        ))
7117    }
7118
7119    fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7120        session_history_payload_text(&frame.payload)
7121    }
7122
7123    fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7124        session_history_payload_text(&frame.payload)
7125    }
7126
7127    fn session_history_payload_text(payload: &Value) -> Option<String> {
7128        if let Some(text) = payload.get("text").and_then(Value::as_str) {
7129            return Some(text.to_string());
7130        }
7131        if let Some(text) = payload.get("result").and_then(Value::as_str) {
7132            return Some(text.to_string());
7133        }
7134        if let Some(text) = payload.get("body").and_then(Value::as_str) {
7135            return Some(text.to_string());
7136        }
7137        match payload.get("content")? {
7138            Value::String(text) => Some(text.clone()),
7139            Value::Array(blocks) => Some(
7140                blocks
7141                    .iter()
7142                    .filter_map(|block| block.get("text").and_then(Value::as_str))
7143                    .collect::<Vec<_>>()
7144                    .join(""),
7145            ),
7146            _ => None,
7147        }
7148    }
7149
7150    #[tokio::test]
7151    async fn query_timeline_handles_large_store_without_backfill_calls() {
7152        let store = Arc::new(CountingConsoleLogStore::new());
7153        let aggregator = MobKitConsoleAggregator::new(store.clone());
7154        for idx in 0..5_000 {
7155            store
7156                .append_if_absent(NewConsoleFrame {
7157                    id: None,
7158                    dedupe_key: format!("event-{idx}"),
7159                    timestamp_ms: idx,
7160                    runtime_key: "runtime-a".to_string(),
7161                    identity: "agent-a".to_string(),
7162                    conversation_id: Some("agent-a".to_string()),
7163                    session_id: Some("session-a".to_string()),
7164                    kind: "text_delta".to_string(),
7165                    status: ConsoleFrameStatus::Delivered,
7166                    payload: json!({ "delta": idx }),
7167                    source: ConsoleFrameSource {
7168                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7169                        source_cursor: None,
7170                    },
7171                    source_event_id: Some(format!("event-{idx}")),
7172                    interaction_id: None,
7173                    turn_id: None,
7174                    run_id: None,
7175                    parent_frame_id: None,
7176                    caused_by_frame_id: None,
7177                })
7178                .await
7179                .expect("append frame");
7180        }
7181
7182        let page = aggregator
7183            .query_timeline(ConsoleTimelineQuery {
7184                identity: Some("agent-a".to_string()),
7185                limit: 1_000,
7186                ..ConsoleTimelineQuery::default()
7187            })
7188            .await
7189            .expect("large query");
7190
7191        assert_eq!(page.frames.len(), 1_000);
7192        assert_eq!(store.source_watermark_calls(), 0);
7193    }
7194
7195    #[tokio::test]
7196    async fn query_timeline_rejects_future_cursor_after_store_reset() {
7197        let aggregator = MobKitConsoleAggregator::in_memory();
7198        let err = aggregator
7199            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7200                after: Some(ConsoleCursor::from("console:99")),
7201                limit: 10,
7202                ..ConsoleTimelineWindowQuery::default()
7203            })
7204            .await
7205            .expect_err("future cursor on empty/reset store must be replay-unavailable");
7206
7207        assert!(
7208            err.to_string()
7209                .contains("beyond the current store frontier"),
7210            "unexpected error: {err}"
7211        );
7212    }
7213
7214    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7215    async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7216        let store = Arc::new(CountingConsoleLogStore::new());
7217        let aggregator = MobKitConsoleAggregator::new(store.clone());
7218        let runtime = build_single_member_runtime().await;
7219        aggregator
7220            .inner
7221            .runtimes
7222            .write()
7223            .expect("runtime registry")
7224            .insert(
7225                "runtime-a".to_string(),
7226                runtime_entry_for_test("runtime-a", &runtime),
7227            );
7228        let session_id =
7229            wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
7230                .await
7231                .expect("spawned member is resolvable for targeted backfill");
7232        wait_for_runtime_session_history_text(
7233            &runtime,
7234            &session_id,
7235            "You are agent-a.",
7236            Duration::from_secs(5),
7237        )
7238        .await
7239        .expect("spawned member kickoff is readable before live-frame race");
7240        store
7241            .append_if_absent(NewConsoleFrame {
7242                id: None,
7243                dedupe_key: "live-tool-before-history".to_string(),
7244                timestamp_ms: 10,
7245                runtime_key: "runtime-a".to_string(),
7246                identity: "test/agent-a".to_string(),
7247                conversation_id: Some("test/agent-a".to_string()),
7248                session_id: None,
7249                kind: "tool_execution_started".to_string(),
7250                status: ConsoleFrameStatus::Delivered,
7251                payload: json!({
7252                    "id": "call-live",
7253                    "name": "king_search",
7254                    "source_event_type": "tool_execution_started",
7255                    "type": "tool_execution_started",
7256                }),
7257                source: ConsoleFrameSource {
7258                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7259                    source_cursor: Some("live-tool-before-history".to_string()),
7260                },
7261                source_event_id: Some("live-tool-before-history".to_string()),
7262                interaction_id: None,
7263                turn_id: None,
7264                run_id: None,
7265                parent_frame_id: None,
7266                caused_by_frame_id: None,
7267            })
7268            .await
7269            .expect("append live tool frame");
7270
7271        let page = aggregator
7272            .query_timeline(ConsoleTimelineQuery {
7273                identity: Some("test/agent-a".to_string()),
7274                limit: 20,
7275                ..ConsoleTimelineQuery::default()
7276            })
7277            .await
7278            .expect("query child timeline");
7279
7280        assert!(
7281            page.frames
7282                .iter()
7283                .any(|frame| frame.kind == "tool_execution_started"),
7284            "initial explicit child timeline should return existing store frames without waiting for session history"
7285        );
7286
7287        let mut observed_backfill = false;
7288        for _ in 0..80 {
7289            let page = aggregator
7290                .query_timeline(ConsoleTimelineQuery {
7291                    identity: Some("test/agent-a".to_string()),
7292                    limit: 20,
7293                    ..ConsoleTimelineQuery::default()
7294                })
7295                .await
7296                .expect("query child timeline after scheduled backfill");
7297            if page.frames.iter().any(|frame| {
7298                frame.kind == "user_input"
7299                    && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7300                    && frame.payload.to_string().contains("You are agent-a.")
7301            }) {
7302                observed_backfill = true;
7303                break;
7304            }
7305            tokio::time::sleep(Duration::from_millis(25)).await;
7306        }
7307        assert!(
7308            observed_backfill,
7309            "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
7310        );
7311        let _ = runtime.mob_handle().stop().await;
7312    }
7313
7314    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7315    async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
7316        let store = Arc::new(CountingConsoleLogStore::new());
7317        let aggregator = MobKitConsoleAggregator::new(store.clone());
7318        let (_temp, runtime, _delayed_service) =
7319            build_stress_runtime(64, Duration::from_millis(0)).await;
7320        aggregator
7321            .inner
7322            .runtimes
7323            .write()
7324            .expect("runtime registry")
7325            .insert(
7326                "runtime-a".to_string(),
7327                runtime_entry_for_test("runtime-a", runtime.as_ref()),
7328            );
7329        for idx in 0..1_000 {
7330            store
7331                .append_if_absent(NewConsoleFrame {
7332                    id: None,
7333                    dedupe_key: format!("event-{idx}"),
7334                    timestamp_ms: idx,
7335                    runtime_key: "runtime-a".to_string(),
7336                    identity: "test/agent-0".to_string(),
7337                    conversation_id: Some("test/agent-0".to_string()),
7338                    session_id: Some("session-a".to_string()),
7339                    kind: "text_delta".to_string(),
7340                    status: ConsoleFrameStatus::Delivered,
7341                    payload: json!({ "delta": idx }),
7342                    source: ConsoleFrameSource {
7343                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7344                        source_cursor: None,
7345                    },
7346                    source_event_id: Some(format!("event-{idx}")),
7347                    interaction_id: None,
7348                    turn_id: None,
7349                    run_id: None,
7350                    parent_frame_id: None,
7351                    caused_by_frame_id: None,
7352                })
7353                .await
7354                .expect("append frame");
7355        }
7356
7357        let page = tokio::time::timeout(
7358            Duration::from_secs(2),
7359            aggregator.query_timeline(ConsoleTimelineQuery {
7360                identity: Some("test/agent-0".to_string()),
7361                limit: 1_000,
7362                ..ConsoleTimelineQuery::default()
7363            }),
7364        )
7365        .await
7366        .expect("identity timeline query should not rediscover the roster per frame")
7367        .expect("timeline query succeeds");
7368
7369        assert_eq!(page.frames.len(), 1_000);
7370        assert_eq!(store.source_watermark_calls(), 0);
7371        let _ = runtime.mob_handle().stop().await;
7372    }
7373
7374    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7375    async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
7376        const MEMBER_COUNT: usize = 32;
7377        let (_temp, runtime, delayed_service) =
7378            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
7379        let aggregator = MobKitConsoleAggregator::in_memory();
7380        aggregator
7381            .inner
7382            .runtimes
7383            .write()
7384            .expect("runtime registry")
7385            .insert(
7386                "runtime-stress".to_string(),
7387                runtime_entry_for_test("runtime-stress", &runtime),
7388            );
7389
7390        let started = Instant::now();
7391        aggregator
7392            .refresh_session_history()
7393            .await
7394            .expect("stress refresh");
7395        let elapsed = started.elapsed();
7396
7397        assert!(
7398            delayed_service.read_calls() >= MEMBER_COUNT,
7399            "expected at least one history read per member, saw {}",
7400            delayed_service.read_calls()
7401        );
7402        assert!(
7403            delayed_service.max_active_reads() > 1,
7404            "session history backfill should fan out instead of reading members serially"
7405        );
7406        assert!(
7407            delayed_service.max_active_reads()
7408                <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
7409            "session history backfill should respect the default concurrency limit"
7410        );
7411        assert!(
7412            elapsed < Duration::from_millis(600),
7413            "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
7414            MEMBER_COUNT * 40
7415        );
7416        let _ = runtime.mob_handle().stop().await;
7417    }
7418
7419    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7420    async fn session_history_backfill_respects_configured_concurrency_limit() {
7421        const MEMBER_COUNT: usize = 16;
7422        let (_temp, runtime, delayed_service) =
7423            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
7424        let aggregator =
7425            MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
7426                max_concurrent_session_backfills: 4,
7427                ..ConsoleAggregatorOptions::default()
7428            });
7429        aggregator
7430            .inner
7431            .runtimes
7432            .write()
7433            .expect("runtime registry")
7434            .insert(
7435                "runtime-stress".to_string(),
7436                runtime_entry_for_test("runtime-stress", &runtime),
7437            );
7438
7439        aggregator
7440            .refresh_session_history()
7441            .await
7442            .expect("stress refresh");
7443
7444        assert!(
7445            delayed_service.max_active_reads() <= 4,
7446            "configured concurrency limit should bound session history reads, saw {}",
7447            delayed_service.max_active_reads()
7448        );
7449        let _ = runtime.mob_handle().stop().await;
7450    }
7451
7452    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7453    async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
7454        const MEMBER_COUNT: usize = 24;
7455        const LIVE_EVENT_COUNT: usize = 2_048;
7456        let (_temp, runtime, _delayed_service) =
7457            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
7458        let aggregator = MobKitConsoleAggregator::in_memory();
7459        aggregator.register_runtime(ConsoleRuntimeRegistration {
7460            runtime_key: "runtime-burst".to_string(),
7461            runtime: runtime.clone(),
7462            identity_namespace: "stress".to_string(),
7463            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7464        });
7465        let console_events = runtime.console_events();
7466        for idx in 0..LIVE_EVENT_COUNT {
7467            console_events
7468                .append(
7469                    "agent-0",
7470                    Some("burst-turn".to_string()),
7471                    "text_delta",
7472                    json!({ "delta": format!("frame-{idx}") }),
7473                )
7474                .await;
7475        }
7476
7477        let deadline = Instant::now() + Duration::from_secs(5);
7478        let mut observed = 0;
7479        while Instant::now() < deadline {
7480            observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
7481            if observed >= LIVE_EVENT_COUNT {
7482                break;
7483            }
7484            tokio::time::sleep(Duration::from_millis(25)).await;
7485        }
7486
7487        assert_eq!(
7488            observed, LIVE_EVENT_COUNT,
7489            "live pump should not drop frames while slow background backfill is running"
7490        );
7491        let _ = runtime.mob_handle().stop().await;
7492    }
7493
7494    async fn count_console_event_frames(
7495        aggregator: &MobKitConsoleAggregator,
7496        identity: &str,
7497    ) -> usize {
7498        let mut after = None;
7499        let mut count = 0;
7500        loop {
7501            let page = aggregator
7502                .query_timeline(ConsoleTimelineQuery {
7503                    identity: Some(identity.to_string()),
7504                    after,
7505                    limit: 1_000,
7506                    ..ConsoleTimelineQuery::default()
7507                })
7508                .await
7509                .expect("query burst timeline");
7510            if page.frames.is_empty() {
7511                break;
7512            }
7513            count += page
7514                .frames
7515                .iter()
7516                .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
7517                .count();
7518            after = page.next_cursor;
7519            if after.is_none() {
7520                break;
7521            }
7522        }
7523        count
7524    }
7525
7526    #[tokio::test]
7527    async fn status_updates_get_replayable_aggregate_cursors() {
7528        let aggregator = MobKitConsoleAggregator::in_memory();
7529        let frame = NewConsoleFrame {
7530            id: None,
7531            dedupe_key: "send-1".to_string(),
7532            timestamp_ms: 1,
7533            runtime_key: "runtime-a".to_string(),
7534            identity: "agent-a".to_string(),
7535            conversation_id: Some("agent-a".to_string()),
7536            session_id: Some("session-1".to_string()),
7537            kind: "user_input".to_string(),
7538            status: ConsoleFrameStatus::Accepted,
7539            payload: json!({ "content": "hello" }),
7540            source: ConsoleFrameSource {
7541                kind: ConsoleFrameSourceKind::Send,
7542                source_cursor: None,
7543            },
7544            source_event_id: None,
7545            interaction_id: Some("interaction-1".to_string()),
7546            turn_id: None,
7547            run_id: None,
7548            parent_frame_id: None,
7549            caused_by_frame_id: None,
7550        };
7551        let inserted = aggregator
7552            .store()
7553            .append_if_absent(frame)
7554            .await
7555            .expect("append frame");
7556
7557        update_frame_status_and_emit(
7558            &aggregator.inner,
7559            &inserted.frame.id,
7560            ConsoleFrameStatus::Delivered,
7561        )
7562        .await
7563        .expect("update status");
7564
7565        let page = aggregator
7566            .query_timeline(ConsoleTimelineQuery {
7567                identity: Some("agent-a".to_string()),
7568                after: Some(inserted.frame.cursor.clone()),
7569                limit: 10,
7570                ..ConsoleTimelineQuery::default()
7571            })
7572            .await
7573            .expect("query timeline");
7574        assert_eq!(page.frames.len(), 1);
7575        assert_eq!(page.frames[0].kind, "frame_updated");
7576        assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
7577        assert_eq!(
7578            page.frames[0]
7579                .payload
7580                .get("frame")
7581                .and_then(|frame| frame.get("status"))
7582                .and_then(Value::as_str),
7583            Some("delivered")
7584        );
7585    }
7586
7587    #[tokio::test]
7588    async fn steer_delivery_appends_terminal_control_frame() {
7589        let aggregator = MobKitConsoleAggregator::in_memory();
7590        let frame = NewConsoleFrame {
7591            id: None,
7592            dedupe_key: "send-steer-1".to_string(),
7593            timestamp_ms: 1,
7594            runtime_key: "runtime-a".to_string(),
7595            identity: "agent-a".to_string(),
7596            conversation_id: Some("agent-a".to_string()),
7597            session_id: Some("session-1".to_string()),
7598            kind: "user_input".to_string(),
7599            status: ConsoleFrameStatus::Delivered,
7600            payload: json!({
7601                "content": "operator steer",
7602                "handling_mode": "steer",
7603            }),
7604            source: ConsoleFrameSource {
7605                kind: ConsoleFrameSourceKind::Send,
7606                source_cursor: None,
7607            },
7608            source_event_id: None,
7609            interaction_id: Some("interaction-steer-1".to_string()),
7610            turn_id: None,
7611            run_id: None,
7612            parent_frame_id: None,
7613            caused_by_frame_id: None,
7614        };
7615        let inserted = aggregator
7616            .store()
7617            .append_if_absent(frame)
7618            .await
7619            .expect("append steer input");
7620
7621        append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
7622            .await
7623            .expect("append steer terminal");
7624
7625        let page = aggregator
7626            .query_timeline(ConsoleTimelineQuery {
7627                identity: Some("agent-a".to_string()),
7628                after: Some(inserted.frame.cursor.clone()),
7629                limit: 10,
7630                ..ConsoleTimelineQuery::default()
7631            })
7632            .await
7633            .expect("query timeline");
7634
7635        assert_eq!(page.frames.len(), 1);
7636        let terminal = &page.frames[0];
7637        assert_eq!(terminal.kind, "interaction_complete");
7638        assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
7639        assert_eq!(
7640            terminal.interaction_id.as_deref(),
7641            Some("interaction-steer-1")
7642        );
7643        assert_eq!(
7644            terminal.parent_frame_id.as_deref(),
7645            Some(inserted.frame.id.as_str())
7646        );
7647        assert_eq!(
7648            terminal.payload.get("reason").and_then(Value::as_str),
7649            Some("steer_delivered")
7650        );
7651    }
7652
7653    #[tokio::test]
7654    async fn history_counterpart_scan_is_not_capped_to_one_page() {
7655        let aggregator = MobKitConsoleAggregator::in_memory();
7656        for idx in 0..1_005 {
7657            aggregator
7658                .store()
7659                .append_if_absent(NewConsoleFrame {
7660                    id: None,
7661                    dedupe_key: format!("filler-{idx}"),
7662                    timestamp_ms: idx,
7663                    runtime_key: "runtime-a".to_string(),
7664                    identity: "agent-a".to_string(),
7665                    conversation_id: Some("agent-a".to_string()),
7666                    session_id: Some("session-a".to_string()),
7667                    kind: "text_delta".to_string(),
7668                    status: ConsoleFrameStatus::Completed,
7669                    payload: json!({ "delta": idx }),
7670                    source: ConsoleFrameSource {
7671                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7672                        source_cursor: None,
7673                    },
7674                    source_event_id: Some(format!("filler-{idx}")),
7675                    interaction_id: None,
7676                    turn_id: None,
7677                    run_id: None,
7678                    parent_frame_id: None,
7679                    caused_by_frame_id: None,
7680                })
7681                .await
7682                .expect("append filler");
7683        }
7684        aggregator
7685            .store()
7686            .append_if_absent(NewConsoleFrame {
7687                id: None,
7688                dedupe_key: "live-user-input".to_string(),
7689                timestamp_ms: 2_000,
7690                runtime_key: "runtime-a".to_string(),
7691                identity: "agent-a".to_string(),
7692                conversation_id: Some("agent-a".to_string()),
7693                session_id: Some("session-a".to_string()),
7694                kind: "user_input".to_string(),
7695                status: ConsoleFrameStatus::Delivered,
7696                payload: json!({ "content": "already here" }),
7697                source: ConsoleFrameSource {
7698                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7699                    source_cursor: None,
7700                },
7701                source_event_id: Some("live-user-input".to_string()),
7702                interaction_id: None,
7703                turn_id: None,
7704                run_id: None,
7705                parent_frame_id: None,
7706                caused_by_frame_id: None,
7707            })
7708            .await
7709            .expect("append live input");
7710
7711        let history = NewConsoleFrame {
7712            id: None,
7713            dedupe_key: "history-user-input".to_string(),
7714            timestamp_ms: 3_000,
7715            runtime_key: "runtime-a".to_string(),
7716            identity: "agent-a".to_string(),
7717            conversation_id: Some("agent-a".to_string()),
7718            session_id: Some("session-a".to_string()),
7719            kind: "user_input".to_string(),
7720            status: ConsoleFrameStatus::Completed,
7721            payload: json!({ "content": "already here" }),
7722            source: ConsoleFrameSource {
7723                kind: ConsoleFrameSourceKind::SessionHistory,
7724                source_cursor: Some("session-a:1006".to_string()),
7725            },
7726            source_event_id: None,
7727            interaction_id: None,
7728            turn_id: None,
7729            run_id: None,
7730            parent_frame_id: None,
7731            caused_by_frame_id: None,
7732        };
7733
7734        assert!(
7735            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7736                .await
7737                .expect("counterpart scan")
7738        );
7739    }
7740
7741    #[tokio::test]
7742    async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
7743        let aggregator = MobKitConsoleAggregator::in_memory();
7744        aggregator
7745            .store()
7746            .append_if_absent(NewConsoleFrame {
7747                id: None,
7748                dedupe_key: "live-user-input".to_string(),
7749                timestamp_ms: 2_000,
7750                runtime_key: "runtime-a".to_string(),
7751                identity: "agent-a".to_string(),
7752                conversation_id: Some("agent-a".to_string()),
7753                session_id: Some("session-a".to_string()),
7754                kind: "user_input".to_string(),
7755                status: ConsoleFrameStatus::Delivered,
7756                payload: json!({ "content": "hello from operator" }),
7757                source: ConsoleFrameSource {
7758                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7759                    source_cursor: None,
7760                },
7761                source_event_id: Some("live-user-input".to_string()),
7762                interaction_id: None,
7763                turn_id: None,
7764                run_id: None,
7765                parent_frame_id: None,
7766                caused_by_frame_id: None,
7767            })
7768            .await
7769            .expect("append live input");
7770
7771        let history = NewConsoleFrame {
7772            id: None,
7773            dedupe_key: "history-user-input".to_string(),
7774            timestamp_ms: 3_000,
7775            runtime_key: "runtime-a".to_string(),
7776            identity: "agent-a".to_string(),
7777            conversation_id: Some("agent-a".to_string()),
7778            session_id: Some("session-a".to_string()),
7779            kind: "user_input".to_string(),
7780            status: ConsoleFrameStatus::Completed,
7781            payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
7782            source: ConsoleFrameSource {
7783                kind: ConsoleFrameSourceKind::SessionHistory,
7784                source_cursor: Some("session-a:2".to_string()),
7785            },
7786            source_event_id: None,
7787            interaction_id: None,
7788            turn_id: None,
7789            run_id: None,
7790            parent_frame_id: None,
7791            caused_by_frame_id: None,
7792        };
7793
7794        assert!(
7795            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7796                .await
7797                .expect("counterpart scan")
7798        );
7799    }
7800
7801    #[tokio::test]
7802    async fn history_counterpart_scan_matches_content_block_user_prompts() {
7803        let aggregator = MobKitConsoleAggregator::in_memory();
7804        aggregator
7805            .store()
7806            .append_if_absent(NewConsoleFrame {
7807                id: None,
7808                dedupe_key: "live-user-input".to_string(),
7809                timestamp_ms: 2_000,
7810                runtime_key: "runtime-a".to_string(),
7811                identity: "agent-a".to_string(),
7812                conversation_id: Some("agent-a".to_string()),
7813                session_id: Some("session-a".to_string()),
7814                kind: "user_input".to_string(),
7815                status: ConsoleFrameStatus::Delivered,
7816                payload: json!({ "content": "hello from operator" }),
7817                source: ConsoleFrameSource {
7818                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7819                    source_cursor: None,
7820                },
7821                source_event_id: Some("live-user-input".to_string()),
7822                interaction_id: None,
7823                turn_id: None,
7824                run_id: None,
7825                parent_frame_id: None,
7826                caused_by_frame_id: None,
7827            })
7828            .await
7829            .expect("append live input");
7830
7831        let history = NewConsoleFrame {
7832            id: None,
7833            dedupe_key: "history-user-input".to_string(),
7834            timestamp_ms: 3_000,
7835            runtime_key: "runtime-a".to_string(),
7836            identity: "agent-a".to_string(),
7837            conversation_id: Some("agent-a".to_string()),
7838            session_id: Some("session-a".to_string()),
7839            kind: "user_input".to_string(),
7840            status: ConsoleFrameStatus::Completed,
7841            payload: json!({
7842                "content": [{ "type": "text", "text": "hello from operator" }]
7843            }),
7844            source: ConsoleFrameSource {
7845                kind: ConsoleFrameSourceKind::SessionHistory,
7846                source_cursor: Some("session-a:2".to_string()),
7847            },
7848            source_event_id: None,
7849            interaction_id: None,
7850            turn_id: None,
7851            run_id: None,
7852            parent_frame_id: None,
7853            caused_by_frame_id: None,
7854        };
7855
7856        assert!(
7857            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7858                .await
7859                .expect("counterpart scan")
7860        );
7861    }
7862
7863    #[tokio::test]
7864    async fn history_counterpart_scan_matches_live_tool_results() {
7865        let aggregator = MobKitConsoleAggregator::in_memory();
7866        aggregator
7867            .store()
7868            .append_if_absent(NewConsoleFrame {
7869                id: None,
7870                dedupe_key: "live-tool-result".to_string(),
7871                timestamp_ms: 2_000,
7872                runtime_key: "runtime-a".to_string(),
7873                identity: "agent-a".to_string(),
7874                conversation_id: Some("agent-a".to_string()),
7875                session_id: Some("session-a".to_string()),
7876                kind: "tool_execution_completed".to_string(),
7877                status: ConsoleFrameStatus::Delivered,
7878                payload: json!({
7879                    "id": "call-1",
7880                    "tool_call_id": "call-1",
7881                    "result": "{ \"count\": 70 }"
7882                }),
7883                source: ConsoleFrameSource {
7884                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7885                    source_cursor: None,
7886                },
7887                source_event_id: Some("live-tool-result".to_string()),
7888                interaction_id: None,
7889                turn_id: None,
7890                run_id: None,
7891                parent_frame_id: None,
7892                caused_by_frame_id: None,
7893            })
7894            .await
7895            .expect("append live tool result");
7896
7897        let history = NewConsoleFrame {
7898            id: None,
7899            dedupe_key: "history-tool-result".to_string(),
7900            timestamp_ms: 3_000,
7901            runtime_key: "runtime-a".to_string(),
7902            identity: "agent-a".to_string(),
7903            conversation_id: Some("agent-a".to_string()),
7904            session_id: Some("session-a".to_string()),
7905            kind: "tool_execution_completed".to_string(),
7906            status: ConsoleFrameStatus::Completed,
7907            payload: json!({
7908                "id": "call-1",
7909                "tool_call_id": "call-1",
7910                "result": "{ \"count\": 70 }",
7911                "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
7912            }),
7913            source: ConsoleFrameSource {
7914                kind: ConsoleFrameSourceKind::SessionHistory,
7915                source_cursor: Some("session-a:3:0".to_string()),
7916            },
7917            source_event_id: None,
7918            interaction_id: None,
7919            turn_id: None,
7920            run_id: None,
7921            parent_frame_id: None,
7922            caused_by_frame_id: None,
7923        };
7924
7925        assert!(
7926            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7927                .await
7928                .expect("counterpart scan")
7929        );
7930    }
7931
7932    #[tokio::test]
7933    async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
7934        let aggregator = MobKitConsoleAggregator::in_memory();
7935        for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
7936            aggregator
7937                .store()
7938                .append_if_absent(NewConsoleFrame {
7939                    id: None,
7940                    dedupe_key: format!("live-delta-{idx}"),
7941                    timestamp_ms: 2_000 + idx as u64,
7942                    runtime_key: "runtime-a".to_string(),
7943                    identity: "agent-a".to_string(),
7944                    conversation_id: Some("agent-a".to_string()),
7945                    session_id: Some("session-a".to_string()),
7946                    kind: "text_delta".to_string(),
7947                    status: ConsoleFrameStatus::Delivered,
7948                    payload: json!({ "delta": delta }),
7949                    source: ConsoleFrameSource {
7950                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7951                        source_cursor: None,
7952                    },
7953                    source_event_id: Some(format!("live-delta-{idx}")),
7954                    interaction_id: Some("turn-a".to_string()),
7955                    turn_id: None,
7956                    run_id: None,
7957                    parent_frame_id: None,
7958                    caused_by_frame_id: None,
7959                })
7960                .await
7961                .expect("append live delta");
7962        }
7963
7964        let history = NewConsoleFrame {
7965            id: None,
7966            dedupe_key: "history-assistant-complete".to_string(),
7967            timestamp_ms: 3_000,
7968            runtime_key: "runtime-a".to_string(),
7969            identity: "agent-a".to_string(),
7970            conversation_id: Some("agent-a".to_string()),
7971            session_id: Some("session-a".to_string()),
7972            kind: "interaction_complete".to_string(),
7973            status: ConsoleFrameStatus::Completed,
7974            payload: json!({ "result": "Ready and standing by." }),
7975            source: ConsoleFrameSource {
7976                kind: ConsoleFrameSourceKind::SessionHistory,
7977                source_cursor: Some("session-a:3".to_string()),
7978            },
7979            source_event_id: None,
7980            interaction_id: None,
7981            turn_id: None,
7982            run_id: None,
7983            parent_frame_id: None,
7984            caused_by_frame_id: None,
7985        };
7986
7987        assert!(
7988            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7989                .await
7990                .expect("counterpart scan")
7991        );
7992    }
7993
7994    #[test]
7995    fn session_history_watermark_key_is_session_scoped() {
7996        assert_ne!(
7997            session_history_watermark_runtime_key("runtime-a", "session-1"),
7998            session_history_watermark_runtime_key("runtime-a", "session-2")
7999        );
8000    }
8001
8002    #[test]
8003    fn session_history_watermarks_are_cursor_and_ttl_aware() {
8004        let legacy = "session:with:colon:42";
8005        let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8006        let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8007
8008        assert_eq!(
8009            parse_session_history_watermark(legacy, "session:with:colon"),
8010            Some(42)
8011        );
8012        assert_eq!(
8013            parse_session_history_watermark(&checked, "session:with:colon"),
8014            Some(43)
8015        );
8016        assert!(session_history_watermark_is_fresh(
8017            &checked,
8018            "session:with:colon",
8019            1_500
8020        ));
8021        assert!(!session_history_watermark_is_fresh(
8022            &checked,
8023            "session:with:colon",
8024            1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8025        ));
8026        assert!(session_history_watermark_is_fresh(
8027            &empty_checked,
8028            "session:with:colon",
8029            1_500
8030        ));
8031        assert!(!session_history_watermark_is_fresh(
8032            &empty_checked,
8033            "session:with:colon",
8034            1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8035        ));
8036    }
8037
8038    #[test]
8039    fn session_history_messages_project_to_renderable_frames() {
8040        let user = frame_from_session_history_message(
8041            "runtime-a",
8042            "agent-a",
8043            "session-a",
8044            0,
8045            json!({
8046                "role": "user",
8047                "content": "hello",
8048                "timestamp_ms": 10
8049            }),
8050        )
8051        .expect("user history frame");
8052        let assistant = frame_from_session_history_message(
8053            "runtime-a",
8054            "agent-a",
8055            "session-a",
8056            1,
8057            json!({
8058                "role": "assistant",
8059                "content": "hi there",
8060                "stop_reason": "end_turn",
8061                "usage": { "input_tokens": 1, "output_tokens": 1, "total_tokens": 2 },
8062                "timestamp_ms": 11
8063            }),
8064        )
8065        .expect("assistant history frame");
8066
8067        assert_eq!(user.kind, "user_input");
8068        assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8069        assert_eq!(
8070            user.payload["content"],
8071            json!([{ "type": "text", "text": "hello" }])
8072        );
8073        assert_eq!(assistant.kind, "interaction_complete");
8074        assert_eq!(assistant.payload["text"], json!("hi there"));
8075        assert!(
8076            assistant
8077                .dedupe_key
8078                .starts_with("session-history:runtime-a:session-a:1:")
8079        );
8080    }
8081
8082    #[test]
8083    fn session_history_projection_filters_scaffold_user_messages() {
8084        let spawn_notice = frame_from_session_history_message(
8085            "runtime-a",
8086            "agent-a",
8087            "session-a",
8088            0,
8089            json!({
8090                "role": "user",
8091                "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8092                "timestamp_ms": 10
8093            }),
8094        );
8095        let peer_update = frame_from_session_history_message(
8096            "runtime-a",
8097            "agent-a",
8098            "session-a",
8099            1,
8100            json!({
8101                "role": "user",
8102                "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8103                "timestamp_ms": 11
8104            }),
8105        );
8106        let real_user = frame_from_session_history_message(
8107            "runtime-a",
8108            "agent-a",
8109            "session-a",
8110            2,
8111            json!({
8112                "role": "user",
8113                "content": "Please review the incident notes.",
8114                "timestamp_ms": 12
8115            }),
8116        );
8117
8118        assert!(spawn_notice.is_none());
8119        assert!(peer_update.is_none());
8120        assert!(real_user.is_some());
8121    }
8122
8123    #[test]
8124    fn session_history_projection_skips_non_transcript_messages() {
8125        let skipped = frame_from_session_history_message(
8126            "runtime-a",
8127            "agent-a",
8128            "session-a",
8129            0,
8130            json!({
8131                "content": "internal system prompt"
8132            }),
8133        );
8134        assert!(skipped.is_none());
8135    }
8136
8137    #[test]
8138    fn session_history_projection_extracts_assistant_blocks() {
8139        let frame = frame_from_session_history_message(
8140            "runtime-a",
8141            "agent-a",
8142            "session-a",
8143            0,
8144            json!({
8145                "role": "block_assistant",
8146                "blocks": [
8147                    { "block_type": "text", "data": { "text": "hello " } },
8148                    { "block_type": "text", "data": { "text": "there" } }
8149                ],
8150                "stop_reason": "end_turn"
8151            }),
8152        )
8153        .expect("assistant block history frame");
8154        assert_eq!(frame.payload["text"], json!("hello there"));
8155    }
8156
8157    #[test]
8158    fn session_history_projection_extracts_nested_text_block_data() {
8159        let frame = frame_from_session_history_message(
8160            "runtime-a",
8161            "agent-a",
8162            "session-a",
8163            0,
8164            json!({
8165                "role": "block_assistant",
8166                "blocks": [
8167                    {
8168                        "block_type": "text",
8169                        "data": { "text": "Ready and standing by." }
8170                    }
8171                ],
8172                "stop_reason": "end_turn",
8173                "created_at": "1970-01-01T00:00:00.010Z"
8174            }),
8175        )
8176        .expect("assistant block history frame");
8177
8178        assert_eq!(frame.kind, "interaction_complete");
8179        assert_eq!(frame.payload["result"], json!("Ready and standing by."));
8180    }
8181
8182    #[test]
8183    fn session_history_projection_drops_reasoning_blocks_from_result_text() {
8184        let frame = frame_from_session_history_message(
8185            "runtime-a",
8186            "agent-a",
8187            "session-a",
8188            0,
8189            json!({
8190                "role": "block_assistant",
8191                "blocks": [
8192                    {
8193                        "block_type": "reasoning",
8194                        "data": { "text": "**Planning**\nI should not be rendered." }
8195                    },
8196                    {
8197                        "block_type": "text",
8198                        "data": { "text": "Visible answer." }
8199                    }
8200                ],
8201                "stop_reason": "end_turn",
8202                "created_at": "1970-01-01T00:00:00.010Z"
8203            }),
8204        )
8205        .expect("assistant block history frame");
8206
8207        assert_eq!(frame.kind, "interaction_complete");
8208        assert_eq!(frame.payload["result"], json!("Visible answer."));
8209        assert_eq!(frame.payload["text"], json!("Visible answer."));
8210    }
8211
8212    #[test]
8213    fn session_history_projection_leaves_reasoning_only_result_empty() {
8214        let frame = frame_from_session_history_message(
8215            "runtime-a",
8216            "agent-a",
8217            "session-a",
8218            0,
8219            json!({
8220                "role": "block_assistant",
8221                "blocks": [
8222                    {
8223                        "block_type": "reasoning",
8224                        "data": { "text": "Private planning text." }
8225                    },
8226                    {
8227                        "block_type": "tool_use",
8228                        "data": { "id": "toolu-1", "name": "peers", "args": {} }
8229                    }
8230                ],
8231                "stop_reason": "end_turn",
8232                "created_at": "1970-01-01T00:00:00.010Z"
8233            }),
8234        )
8235        .expect("assistant block history frame");
8236
8237        assert_eq!(frame.kind, "interaction_complete");
8238        assert_eq!(frame.payload["result"], json!(""));
8239        assert_eq!(frame.payload["text"], json!(""));
8240    }
8241
8242    #[test]
8243    fn session_history_projection_preserves_tool_results() {
8244        let frames = frames_from_session_history_message(
8245            "runtime-a",
8246            "agent-a",
8247            "session-a",
8248            5,
8249            json!({
8250                "role": "tool_results",
8251                "results": [
8252                    {
8253                        "tool_use_id": "call-peers",
8254                        "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
8255                        "is_error": false
8256                    }
8257                ],
8258                "created_at": "1970-01-01T00:00:00.050Z"
8259            }),
8260        );
8261
8262        assert_eq!(frames.len(), 1);
8263        let frame = &frames[0];
8264        assert_eq!(frame.kind, "tool_execution_completed");
8265        assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
8266        assert_eq!(
8267            frame.payload["result"],
8268            json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
8269        );
8270        assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
8271        assert_eq!(frame.timestamp_ms, 50);
8272    }
8273
8274    #[test]
8275    fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
8276        let frames = frames_from_session_history_message(
8277            "runtime-a",
8278            "review:singleton",
8279            "session-a",
8280            7,
8281            json!({
8282                "role": "block_assistant",
8283                "blocks": [
8284                    {
8285                        "block_type": "tool_use",
8286                        "data": {
8287                            "id": "call-spawn",
8288                            "name": "mob_spawn_member",
8289                            "args": {
8290                                "member_id": "review-worker-alpha",
8291                                "profile": "review-worker",
8292                                "initial_message": "Review Initiative Alpha and save the result."
8293                            }
8294                        }
8295                    }
8296                ],
8297                "stop_reason": "tool_use",
8298                "created_at": "1970-01-01T00:00:00.070Z"
8299            }),
8300        );
8301
8302        assert_eq!(frames.len(), 2);
8303        let child = frames
8304            .iter()
8305            .find(|frame| frame.identity == "review-worker-alpha")
8306            .expect("spawned member initial message frame");
8307        assert_eq!(child.kind, "user_input");
8308        assert_eq!(child.timestamp_ms, 70);
8309        assert_eq!(
8310            child.payload["message"]["content"],
8311            "Review Initiative Alpha and save the result."
8312        );
8313        assert_eq!(child.payload["parent_identity"], "review:singleton");
8314        assert_eq!(child.payload["via_tool"], "mob_spawn_member");
8315        assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
8316    }
8317
8318    #[test]
8319    fn session_history_projection_surfaces_specs_spawn_initial_messages() {
8320        let frames = frames_from_session_history_message(
8321            "runtime-a",
8322            "review:singleton",
8323            "session-a",
8324            8,
8325            json!({
8326                "role": "block_assistant",
8327                "blocks": [
8328                    {
8329                        "block_type": "tool_use",
8330                        "data": {
8331                            "id": "call-spawn-many",
8332                            "name": "mob_spawn_member",
8333                            "args": {
8334                                "specs": [
8335                                    {
8336                                        "agent_identity": "review-worker-alpha",
8337                                        "profile": "review-worker",
8338                                        "initial_message": "Review Initiative Alpha."
8339                                    },
8340                                    {
8341                                        "agent_identity": "review-worker-beta",
8342                                        "profile": "review-worker",
8343                                        "initial_message": "Review Initiative Beta."
8344                                    }
8345                                ]
8346                            }
8347                        }
8348                    }
8349                ],
8350                "stop_reason": "tool_use",
8351                "created_at": "1970-01-01T00:00:00.080Z"
8352            }),
8353        );
8354
8355        assert_eq!(frames.len(), 3);
8356        let child_messages = frames
8357            .iter()
8358            .filter(|frame| frame.kind == "user_input")
8359            .map(|frame| {
8360                (
8361                    frame.identity.as_str(),
8362                    frame.payload["message"]["content"]
8363                        .as_str()
8364                        .unwrap_or_default(),
8365                )
8366            })
8367            .collect::<Vec<_>>();
8368        assert_eq!(
8369            child_messages,
8370            vec![
8371                ("review-worker-alpha", "Review Initiative Alpha."),
8372                ("review-worker-beta", "Review Initiative Beta."),
8373            ]
8374        );
8375    }
8376
8377    #[tokio::test]
8378    async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
8379        let aggregator = MobKitConsoleAggregator::in_memory();
8380        let frames = frames_from_session_history_message(
8381            "runtime-a",
8382            "review:singleton",
8383            "session-a",
8384            9,
8385            json!({
8386                "role": "block_assistant",
8387                "blocks": [
8388                    {
8389                        "block_type": "tool_use",
8390                        "data": {
8391                            "id": "call-spawn-many",
8392                            "name": "mob_spawn_member",
8393                            "args": {
8394                                "specs": [
8395                                    {
8396                                        "agent_identity": "review-worker-alpha",
8397                                        "profile": "review-worker",
8398                                        "initial_message": "Review Initiative Alpha."
8399                                    },
8400                                    {
8401                                        "agent_identity": "review-worker-beta",
8402                                        "profile": "review-worker",
8403                                        "initial_message": "Review Initiative Beta."
8404                                    }
8405                                ]
8406                            }
8407                        }
8408                    }
8409                ],
8410                "stop_reason": "tool_use",
8411                "created_at": "1970-01-01T00:00:00.090Z"
8412            }),
8413        );
8414        for frame in frames {
8415            aggregator
8416                .store()
8417                .append_if_absent(frame)
8418                .await
8419                .expect("append projected session-history frame");
8420        }
8421
8422        let child_page = aggregator
8423            .query_timeline(ConsoleTimelineQuery {
8424                identity: Some("review-worker-alpha".to_string()),
8425                limit: 20,
8426                ..ConsoleTimelineQuery::default()
8427            })
8428            .await
8429            .expect("query child timeline");
8430        assert_eq!(child_page.frames.len(), 1);
8431        assert_eq!(child_page.frames[0].kind, "user_input");
8432        assert_eq!(
8433            child_page.frames[0].payload["message"]["content"],
8434            "Review Initiative Alpha."
8435        );
8436        assert_eq!(
8437            child_page.frames[0].payload["parent_identity"],
8438            "review:singleton"
8439        );
8440
8441        let parent_page = aggregator
8442            .query_timeline(ConsoleTimelineQuery {
8443                identity: Some("review:singleton".to_string()),
8444                limit: 20,
8445                ..ConsoleTimelineQuery::default()
8446            })
8447            .await
8448            .expect("query parent timeline");
8449        assert!(
8450            parent_page
8451                .frames
8452                .iter()
8453                .all(|frame| frame.identity == "review:singleton"),
8454            "child initial messages must not leak into parent timeline: {:#?}",
8455            parent_page.frames
8456        );
8457    }
8458
8459    #[test]
8460    fn session_history_projection_applies_namespace_to_spawned_child_identity() {
8461        let frames = frames_from_session_history_message_with_namespace(
8462            "runtime-a",
8463            "test/review:singleton",
8464            "test",
8465            "session-a",
8466            10,
8467            json!({
8468                "role": "block_assistant",
8469                "blocks": [
8470                    {
8471                        "block_type": "tool_use",
8472                        "data": {
8473                            "id": "call-spawn-many",
8474                            "name": "mob_spawn_member",
8475                            "args": {
8476                                "specs": [
8477                                    {
8478                                        "agent_identity": "review-worker-alpha",
8479                                        "profile": "review-worker",
8480                                        "initial_message": "Review Initiative Alpha."
8481                                    },
8482                                    {
8483                                        "agent_identity": "test/review-worker-beta",
8484                                        "profile": "review-worker",
8485                                        "initial_message": "Review Initiative Beta."
8486                                    }
8487                                ]
8488                            }
8489                        }
8490                    }
8491                ],
8492                "stop_reason": "tool_use",
8493                "created_at": "1970-01-01T00:00:00.100Z"
8494            }),
8495        );
8496
8497        let child_identities = frames
8498            .iter()
8499            .filter(|frame| frame.kind == "user_input")
8500            .map(|frame| frame.identity.as_str())
8501            .collect::<Vec<_>>();
8502        assert_eq!(
8503            child_identities,
8504            vec!["test/review-worker-alpha", "test/review-worker-beta"]
8505        );
8506    }
8507
8508    #[tokio::test]
8509    async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
8510        let aggregator = MobKitConsoleAggregator::in_memory();
8511        aggregator
8512            .inner
8513            .identity_read_model
8514            .replace(vec![
8515                identity_record_for_test("test/review:singleton"),
8516                identity_record_for_test("test/review-worker-alpha"),
8517            ])
8518            .await;
8519        let frames = frames_from_session_history_message_with_namespace(
8520            "runtime-a",
8521            "test/review:singleton",
8522            "test",
8523            "session-a",
8524            11,
8525            json!({
8526                "role": "block_assistant",
8527                "blocks": [
8528                    {
8529                        "block_type": "tool_use",
8530                        "data": {
8531                            "id": "call-spawn",
8532                            "name": "mob_spawn_member",
8533                            "args": {
8534                                "agent_identity": "review-worker-alpha",
8535                                "profile": "review-worker",
8536                                "initial_message": "Review Initiative Alpha."
8537                            }
8538                        }
8539                    }
8540                ],
8541                "stop_reason": "tool_use",
8542                "created_at": "1970-01-01T00:00:00.110Z"
8543            }),
8544        );
8545        for frame in frames {
8546            aggregator
8547                .store()
8548                .append_if_absent(frame)
8549                .await
8550                .expect("append projected namespaced session-history frame");
8551        }
8552
8553        let namespaced_child_page = aggregator
8554            .query_timeline(ConsoleTimelineQuery {
8555                identity: Some("test/review-worker-alpha".to_string()),
8556                limit: 20,
8557                ..ConsoleTimelineQuery::default()
8558            })
8559            .await
8560            .expect("query namespaced child timeline");
8561        assert_eq!(namespaced_child_page.frames.len(), 1);
8562        assert_eq!(
8563            namespaced_child_page.frames[0].identity,
8564            "test/review-worker-alpha"
8565        );
8566        assert_eq!(
8567            namespaced_child_page.frames[0].payload["message"]["content"],
8568            "Review Initiative Alpha."
8569        );
8570
8571        let raw_child_page = aggregator
8572            .query_timeline(ConsoleTimelineQuery {
8573                identity: Some("review-worker-alpha".to_string()),
8574                limit: 20,
8575                ..ConsoleTimelineQuery::default()
8576            })
8577            .await
8578            .expect("query raw child timeline");
8579        assert!(
8580            raw_child_page.frames.is_empty(),
8581            "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
8582            raw_child_page.frames
8583        );
8584    }
8585
8586    #[test]
8587    fn namespace_helpers_preserve_namespace_named_member_identity() {
8588        assert_eq!(apply_namespace("test", "test"), "test/test");
8589        assert_eq!(
8590            strip_namespace("test/test", "test").as_deref(),
8591            Some("test")
8592        );
8593        assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
8594        assert_eq!(
8595            strip_namespace("test/worker", "test").as_deref(),
8596            Some("worker")
8597        );
8598        assert_eq!(strip_namespace("test", "test"), None);
8599    }
8600
8601    #[test]
8602    fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
8603        assert!(!member_id_matches_durable_identity(
8604            "rt:review:singleton:0",
8605            "review:singleton"
8606        ));
8607        assert!(!member_id_matches_durable_identity(
8608            "review:singleton:gen1",
8609            "review:singleton"
8610        ));
8611        assert!(!member_id_matches_durable_identity(
8612            "review:singleton:1",
8613            "review:singleton"
8614        ));
8615        assert!(!member_id_matches_durable_identity(
8616            "rt:review:singleton:qa:0",
8617            "review:singleton"
8618        ));
8619        assert!(!member_id_matches_durable_identity(
8620            "review:singleton:qa",
8621            "review:singleton"
8622        ));
8623    }
8624
8625    #[test]
8626    fn session_history_projection_uses_rfc3339_created_at_timestamp() {
8627        let frame = frame_from_session_history_message(
8628            "runtime-a",
8629            "agent-a",
8630            "session-a",
8631            0,
8632            json!({
8633                "role": "user",
8634                "content": "hello",
8635                "created_at": "2026-05-12T05:00:06.564227Z"
8636            }),
8637        )
8638        .expect("user history frame");
8639
8640        assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
8641    }
8642}