Skip to main content

meerkat_mobkit/console_aggregator/
mod.rs

1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23    MobRuntime, MobRuntimeError, assert_member_accepts_images,
24    is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30    ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31    SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32    SourceIngestionTransition,
33};
34pub use store::{
35    ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36    SqliteConsoleLogStore,
37};
38pub use types::{
39    AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40    ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41    ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42    ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43    ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59    CachedOnly,
60    IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65    inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70    pub session_history_backfill_enabled: bool,
71    pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75    fn default() -> Self {
76        Self {
77            session_history_backfill_enabled: true,
78            max_concurrent_session_backfills: 16,
79        }
80    }
81}
82
83struct AggregatorInner {
84    store: Arc<dyn ConsoleLogStore>,
85    runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86    event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87    active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88    opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89    session_backfill_permits: Arc<Semaphore>,
90    identity_read_model: ConsoleIdentityReadModel,
91    options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96    inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97    refresh_lock: Arc<tokio::sync::Mutex<()>>,
98    primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102    fn default() -> Self {
103        Self {
104            inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106            primed: Arc::new(AtomicBool::new(false)),
107        }
108    }
109}
110
111impl ConsoleIdentityReadModel {
112    async fn snapshot(
113        &self,
114        inner: Arc<AggregatorInner>,
115    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116        if !self.primed.load(Ordering::Acquire) {
117            Box::pin(self.prime_now(inner)).await?;
118        }
119        Ok(self.inner.read().await.clone())
120    }
121
122    async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123        self.inner.read().await.clone()
124    }
125
126    async fn refresh_now_if_idle(
127        &self,
128        inner: Arc<AggregatorInner>,
129        mode: IdentityCollectionMode,
130    ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132            return None;
133        };
134        let result = Box::pin(collect_identity_records(&inner, mode)).await;
135        drop(guard);
136        match result {
137            Ok(identities) => {
138                self.replace(identities.clone()).await;
139                Some(Ok(identities))
140            }
141            Err(err) => Some(Err(err)),
142        }
143    }
144
145    async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146        if self.primed.load(Ordering::Acquire) {
147            return Ok(());
148        }
149        let _guard = self.refresh_lock.clone().lock_owned().await;
150        if self.primed.load(Ordering::Acquire) {
151            return Ok(());
152        }
153        let identities = Box::pin(collect_identity_records(
154            &inner,
155            IdentityCollectionMode::CachedOnly,
156        ))
157        .await?;
158        *self.inner.write().await = identities;
159        self.primed.store(true, Ordering::Release);
160        Ok(())
161    }
162
163    fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
164        let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
165            return;
166        };
167        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
168            return;
169        };
170        let read_model = self.clone();
171        runtime_handle.spawn(async move {
172            let _guard = guard;
173            match Box::pin(collect_identity_records(
174                &inner,
175                IdentityCollectionMode::IncludeLiveMembers,
176            ))
177            .await
178            {
179                Ok(identities) => {
180                    *read_model.inner.write().await = identities;
181                    read_model.primed.store(true, Ordering::Release);
182                }
183                Err(err) => {
184                    tracing::warn!(error = %err, "console identity read-model refresh failed");
185                }
186            }
187        });
188    }
189
190    async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
191        *self.inner.write().await = identities;
192        self.primed.store(true, Ordering::Release);
193    }
194}
195
196#[derive(Clone)]
197struct RuntimeEntry {
198    runtime_key: String,
199    identity_namespace: String,
200    runtime: MobRuntime,
201    identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
202    console_events: ConsoleEventStore,
203    visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
204}
205
206#[derive(Clone)]
207struct ResolvedConsoleMember {
208    entry: RuntimeEntry,
209    handle: MobHandle,
210    member: MobMemberListEntry,
211    source_mob_id: String,
212    runtime_identity: String,
213}
214
215fn console_member_for_resolved_member(
216    resolved: &ResolvedConsoleMember,
217    record: &ConsoleIdentityRecord,
218) -> ConsoleMember {
219    ConsoleMember {
220        agent_identity: crate::member_comms_id::runtime_alias_str(
221            resolved.member.agent_identity.as_str(),
222        )
223        .into_owned(),
224        role: resolved.member.role.to_string(),
225        state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
226        model_capabilities: Default::default(),
227        runtime_mode: Some(resolved.member.runtime_mode.to_string()),
228        session_id: record.session_id.clone(),
229        wired_to: resolved
230            .member
231            .wired_to
232            .iter()
233            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
234            .collect(),
235        labels: record.labels.clone(),
236    }
237}
238
239async fn resolved_member_visible(
240    resolved: &ResolvedConsoleMember,
241    record: &ConsoleIdentityRecord,
242) -> bool {
243    if !raw_resolved_member_visible(resolved, record) {
244        return false;
245    }
246    !Box::pin(live_record_shadowed_by_hidden_durable_binding(
247        resolved, record,
248    ))
249    .await
250}
251
252fn raw_resolved_member_visible(
253    resolved: &ResolvedConsoleMember,
254    record: &ConsoleIdentityRecord,
255) -> bool {
256    resolved.entry.visibility_policy.identity_visible(record)
257        && resolved
258            .entry
259            .visibility_policy
260            .member_visible(&console_member_for_resolved_member(resolved, record))
261}
262
263pub trait ConsoleVisibilityPolicy: Send + Sync {
264    fn include_implicit_delegate_members(&self) -> bool {
265        true
266    }
267
268    fn member_visible(&self, _member: &ConsoleMember) -> bool {
269        true
270    }
271
272    fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
273        true
274    }
275
276    fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
277        true
278    }
279
280    fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
281        None
282    }
283}
284
285#[derive(Debug, Default)]
286pub struct AllowAllConsoleVisibilityPolicy;
287
288impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
289
290#[derive(Debug, Default)]
291pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
292
293impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
294    fn include_implicit_delegate_members(&self) -> bool {
295        false
296    }
297
298    fn member_visible(&self, member: &ConsoleMember) -> bool {
299        !is_implicit_delegate_member(member.role.as_str(), &member.labels)
300    }
301
302    fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
303        !is_implicit_delegate_member(
304            record
305                .labels
306                .get("role")
307                .map(String::as_str)
308                .unwrap_or_default(),
309            &record.labels,
310        )
311    }
312}
313
314#[derive(Clone)]
315pub struct ConsoleRuntimeRegistration {
316    pub runtime_key: String,
317    pub runtime: Arc<UnifiedRuntime>,
318    pub identity_namespace: String,
319    pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
320}
321
322impl MobKitConsoleAggregator {
323    pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
324        Self::new_with_options(store, ConsoleAggregatorOptions::default())
325    }
326
327    pub fn new_with_options(
328        store: Arc<dyn ConsoleLogStore>,
329        mut options: ConsoleAggregatorOptions,
330    ) -> Self {
331        options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
332        let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
333        Self {
334            inner: Arc::new(AggregatorInner {
335                store,
336                runtimes: RwLock::new(BTreeMap::new()),
337                event_tx,
338                active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
339                opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
340                session_backfill_permits: Arc::new(Semaphore::new(
341                    options.max_concurrent_session_backfills,
342                )),
343                identity_read_model: ConsoleIdentityReadModel::default(),
344                options,
345            }),
346        }
347    }
348
349    pub fn in_memory() -> Self {
350        Self::new(Arc::new(InMemoryConsoleLogStore::new()))
351    }
352
353    pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
354        Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
355    }
356
357    pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
358        self.inner.event_tx.subscribe()
359    }
360
361    pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
362        self.inner.store.clone()
363    }
364
365    pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
366        let identity_runtime = registration.runtime.identity_runtime().cloned();
367        self.register_runtime_handles_with_policy(
368            registration.runtime_key,
369            registration.identity_namespace,
370            registration.runtime.mob_runtime().clone(),
371            identity_runtime,
372            registration.runtime.console_events(),
373            registration.visibility_policy,
374        );
375    }
376
377    pub(crate) fn register_runtime_handles_with_policy(
378        &self,
379        runtime_key: impl Into<String>,
380        identity_namespace: impl Into<String>,
381        runtime: MobRuntime,
382        identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
383        console_events: ConsoleEventStore,
384        visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
385    ) {
386        let runtime_key = runtime_key.into();
387        let identity_namespace = identity_namespace.into();
388        let entry = RuntimeEntry {
389            runtime_key: runtime_key.clone(),
390            identity_namespace,
391            runtime,
392            identity_runtime,
393            console_events: console_events.clone(),
394            visibility_policy,
395        };
396        if let Ok(mut runtimes) = self.inner.runtimes.write() {
397            runtimes.insert(runtime_key.clone(), entry);
398        }
399        self.inner
400            .identity_read_model
401            .refresh_soon(self.inner.clone());
402        let inner = self.inner.clone();
403        let events_for_live = console_events.clone();
404        let events_for_live_recovery = console_events.clone();
405        let runtime_key_for_live = runtime_key.clone();
406        tokio::spawn(async move {
407            let mut rx = events_for_live.subscribe();
408            loop {
409                match rx.recv().await {
410                    Ok(envelope) => {
411                        let _ =
412                            project_console_event(inner.clone(), &runtime_key_for_live, envelope)
413                                .await;
414                    }
415                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
416                        let _ = recover_lagged_source_events(
417                            inner.clone(),
418                            &runtime_key_for_live,
419                            &events_for_live_recovery,
420                        )
421                        .await;
422                    }
423                    Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
424                }
425            }
426        });
427
428        let inner = self.inner.clone();
429        let events_for_replay = console_events;
430        let runtime_key_for_replay = runtime_key;
431        tokio::spawn(async move {
432            let mut ingestion_state = SourceIngestionState::Registered;
433            if let Ok((next, _effects)) =
434                ingestion_state.apply(SourceIngestionTransition::StartBackfill)
435            {
436                ingestion_state = next;
437            }
438            if let Ok(events) = events_for_replay.replay_all(None).await {
439                for envelope in events {
440                    let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
441                        .await;
442                }
443            }
444            spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
445            spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
446            if let Ok((next, _effects)) =
447                ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
448            {
449                ingestion_state = next;
450            }
451            let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
452        });
453    }
454
455    pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
456        if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
457            self.inner.clone(),
458            IdentityCollectionMode::IncludeLiveMembers,
459        ))
460        .await
461        {
462            let identities = identities?;
463            spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464            return Ok(identities);
465        }
466        self.inner
467            .identity_read_model
468            .refresh_soon(self.inner.clone());
469        let identities =
470            Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
471        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
472        Ok(identities)
473    }
474
475    #[cfg(test)]
476    pub(crate) async fn list_identities_fresh(
477        &self,
478    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
479        let _guard = self
480            .inner
481            .identity_read_model
482            .refresh_lock
483            .clone()
484            .lock_owned()
485            .await;
486        let identities =
487            collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
488                .await?;
489        self.inner
490            .identity_read_model
491            .replace(identities.clone())
492            .await;
493        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
494        Ok(identities)
495    }
496
497    pub async fn inspect_identity(
498        &self,
499        identity: &str,
500    ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
501        let entries = self
502            .inner
503            .runtimes
504            .read()
505            .map_err(|_| runtime_registry_lock_error())?
506            .values()
507            .cloned()
508            .collect::<Vec<_>>();
509        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
510        let mut durable_matches = Vec::new();
511        if !requested_identity_has_runtime_member_alias(identity, &entries) {
512            for entry in &entries {
513                let Some(identity_runtime) = entry.identity_runtime.clone() else {
514                    continue;
515                };
516                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
517                {
518                    let Ok(parsed_identity) =
519                        crate::identity_first::AgentIdentity::parse(&raw_identity)
520                    else {
521                        continue;
522                    };
523                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
524                        continue;
525                    };
526                    let mut record = identity_record_for_status(entry, &status);
527                    let topology_peers =
528                        identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
529                    record.topology_peers = topology_peers
530                        .get(&record.identity)
531                        .cloned()
532                        .unwrap_or_default();
533                    if Box::pin(console_identity_record_visible(entry, &record)).await {
534                        durable_matches.push((
535                            entry.clone(),
536                            identity_runtime.clone(),
537                            parsed_identity,
538                            record,
539                        ));
540                    }
541                }
542            }
543        }
544        if durable_matches.len() > 1 {
545            let candidates = durable_matches
546                .iter()
547                .map(|(_, _, _, record)| record.runtime_member_id.clone())
548                .collect::<Vec<_>>()
549                .join(", ");
550            return Err(format!(
551                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
552            )
553            .into());
554        }
555        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
556            durable_matches.into_iter().next()
557        {
558            let durable_live_records = Box::pin(self.live_records_for_durable_record(
559                &entry,
560                identity,
561                &record,
562                &live_records,
563            ))
564            .await;
565            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
566                &entry,
567                &durable_live_records,
568            ))
569            .await;
570            if let Some(ambiguous_error) =
571                ambiguous_live_alias_error(identity, &visible_durable_live_records)
572            {
573                return Err(ambiguous_error.into());
574            }
575            if let Some(rebound_record) = self_heal_durable_session_mismatch(
576                &entry,
577                identity_runtime.as_ref(),
578                &parsed_identity,
579                identity,
580                &record,
581                &durable_live_records,
582            )
583            .await
584            .map_err(ConsoleLogError::from)?
585            {
586                record = rebound_record;
587            }
588            if let Some(stale_error) =
589                stale_durable_record_error(identity, &record, &durable_live_records)
590            {
591                return Err(stale_error.into());
592            }
593            if let Some(session_id) = record.session_id.clone() {
594                spawn_session_history_backfill_target(
595                    self.inner.clone(),
596                    SessionBackfillTarget {
597                        entry,
598                        record: record.clone(),
599                        session_id,
600                    },
601                    false,
602                );
603            }
604            return Ok(Some(ConsoleIdentityInspection {
605                peers: record.topology_peers.clone(),
606                identity: record,
607            }));
608        }
609        let mut runtime_id_matches = Vec::new();
610        for entry in &entries {
611            let Some(identity_runtime) = entry.identity_runtime.clone() else {
612                continue;
613            };
614            let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
615                continue;
616            };
617            let Some(status) = identity_runtime
618                .statuses()
619                .await
620                .into_iter()
621                .find(|status| {
622                    status
623                        .agent_runtime_id
624                        .as_ref()
625                        .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
626                })
627            else {
628                continue;
629            };
630            let mut record = identity_record_for_status(entry, &status);
631            let topology_peers =
632                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
633            record.topology_peers = topology_peers
634                .get(&record.identity)
635                .cloned()
636                .unwrap_or_default();
637            if Box::pin(console_identity_record_visible(entry, &record)).await {
638                runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
639            }
640        }
641        if runtime_id_matches.len() > 1 {
642            let candidates = runtime_id_matches
643                .iter()
644                .map(|(_, _, _, record)| record.identity.clone())
645                .collect::<Vec<_>>()
646                .join(", ");
647            return Err(format!(
648                "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
649            )
650            .into());
651        }
652        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
653            runtime_id_matches.into_iter().next()
654        {
655            let durable_live_records = Box::pin(self.live_records_for_durable_record(
656                &entry,
657                identity,
658                &record,
659                &live_records,
660            ))
661            .await;
662            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
663                &entry,
664                &durable_live_records,
665            ))
666            .await;
667            if let Some(ambiguous_error) =
668                ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
669            {
670                return Err(ambiguous_error.into());
671            }
672            if let Some(rebound_record) = self_heal_durable_session_mismatch(
673                &entry,
674                identity_runtime.as_ref(),
675                &parsed_identity,
676                identity,
677                &record,
678                &durable_live_records,
679            )
680            .await
681            .map_err(ConsoleLogError::from)?
682            {
683                record = rebound_record;
684            }
685            if let Some(stale_error) =
686                stale_durable_record_error(identity, &record, &durable_live_records)
687            {
688                return Err(stale_error.into());
689            }
690            if let Some(session_id) = record.session_id.clone() {
691                spawn_session_history_backfill_target(
692                    self.inner.clone(),
693                    SessionBackfillTarget {
694                        entry: entry.clone(),
695                        record: record.clone(),
696                        session_id,
697                    },
698                    false,
699                );
700            }
701            return Ok(Some(ConsoleIdentityInspection {
702                peers: record.topology_peers.clone(),
703                identity: record,
704            }));
705        }
706
707        let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
708        if live_matches.len() > 1 {
709            let candidates = live_matches
710                .iter()
711                .map(|resolved| resolved.runtime_identity.clone())
712                .collect::<Vec<_>>()
713                .join(", ");
714            return Err(format!(
715                "ambiguous live identity alias {identity}: candidates [{candidates}]"
716            )
717            .into());
718        }
719        if let Some(resolved) = live_matches.into_iter().next() {
720            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
721                return Ok(None);
722            };
723            if let Some(stale_error) =
724                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
725            {
726                return Err(stale_error.into());
727            }
728            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
729                && let Ok(parsed_identity) =
730                    crate::identity_first::AgentIdentity::parse(&record.identity)
731                && let Ok(status) = identity_runtime.status(&parsed_identity).await
732            {
733                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
734                let durable_live_records = Box::pin(self.live_records_for_durable_record(
735                    &resolved.entry,
736                    identity,
737                    &durable_record,
738                    &live_records,
739                ))
740                .await;
741                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
742                    &resolved.entry,
743                    &durable_live_records,
744                ))
745                .await;
746                if let Some(ambiguous_error) =
747                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
748                {
749                    return Err(ambiguous_error.into());
750                }
751                if let Some(rebound_record) = self_heal_durable_session_mismatch(
752                    &resolved.entry,
753                    identity_runtime.as_ref(),
754                    &parsed_identity,
755                    identity,
756                    &durable_record,
757                    &durable_live_records,
758                )
759                .await
760                .map_err(ConsoleLogError::from)?
761                {
762                    durable_record = rebound_record;
763                }
764                if let Some(stale_error) =
765                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
766                {
767                    return Err(stale_error.into());
768                }
769                if status
770                    .agent_runtime_id
771                    .as_ref()
772                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
773                {
774                    return Ok(None);
775                }
776            }
777            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
778                return Ok(None);
779            }
780            if let Some(session_id) = record.session_id.clone() {
781                spawn_session_history_backfill_target(
782                    self.inner.clone(),
783                    SessionBackfillTarget {
784                        entry: resolved.entry.clone(),
785                        record: record.clone(),
786                        session_id,
787                    },
788                    false,
789                );
790            }
791            let peers = resolved
792                .member
793                .wired_to
794                .iter()
795                .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
796                .collect();
797            return Ok(Some(ConsoleIdentityInspection {
798                identity: record,
799                peers,
800            }));
801        }
802
803        Ok(None)
804    }
805
806    pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
807        let entries = self
808            .inner
809            .runtimes
810            .read()
811            .map_err(|_| runtime_registry_lock_error())?
812            .clone();
813        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
814        let entries_vec = entries.values().cloned().collect::<Vec<_>>();
815        let mut durable_matches = Vec::new();
816        if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
817            for entry in entries.values() {
818                let Some(identity_runtime) = entry.identity_runtime.clone() else {
819                    continue;
820                };
821                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
822                {
823                    let Ok(parsed_identity) =
824                        crate::identity_first::AgentIdentity::parse(&raw_identity)
825                    else {
826                        continue;
827                    };
828                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
829                        continue;
830                    };
831                    let record = identity_record_for_status(entry, &status);
832                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
833                        continue;
834                    }
835                    durable_matches.push((
836                        entry.clone(),
837                        identity_runtime.clone(),
838                        parsed_identity,
839                        record,
840                    ));
841                }
842            }
843        }
844        if durable_matches.len() > 1 {
845            let candidates = durable_matches
846                .iter()
847                .map(|(_, _, identity, _)| identity.as_str().to_string())
848                .collect::<Vec<_>>()
849                .join(", ");
850            return Err(format!(
851                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
852            )
853            .into());
854        }
855        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
856            durable_matches.into_iter().next()
857        {
858            let durable_live_records = Box::pin(self.live_records_for_durable_record(
859                &entry,
860                identity,
861                &record,
862                &live_records,
863            ))
864            .await;
865            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
866                &entry,
867                &durable_live_records,
868            ))
869            .await;
870            if let Some(ambiguous_error) =
871                ambiguous_live_alias_error(identity, &visible_durable_live_records)
872            {
873                return Err(ambiguous_error.into());
874            }
875            if let Some(rebound_record) = self_heal_durable_session_mismatch(
876                &entry,
877                identity_runtime.as_ref(),
878                &parsed_identity,
879                identity,
880                &record,
881                &durable_live_records,
882            )
883            .await
884            .map_err(ConsoleLogError::from)?
885            {
886                record = rebound_record;
887            }
888            if let Some(stale_error) =
889                stale_durable_record_error(identity, &record, &durable_live_records)
890            {
891                return Err(stale_error.into());
892            }
893            identity_runtime
894                .retire(&parsed_identity)
895                .await
896                .map_err(|err| -> ConsoleLogError {
897                    format!("retire failed for {identity}: {err}").into()
898                })?;
899            {
900                let mut durable_identities =
901                    namespace_match_candidates(identity, &entry.identity_namespace);
902                durable_identities.push(parsed_identity.as_str().to_string());
903                durable_identities.sort();
904                durable_identities.dedup();
905                if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
906                    &entry,
907                    &durable_identities,
908                ))
909                .await
910                {
911                    tracing::warn!(
912                        identity,
913                        error = %err,
914                        "stale console member cleanup failed after durable identity retire"
915                    );
916                }
917            }
918            return Ok(true);
919        }
920        let matches = Box::pin(self.resolve_visible_members(identity)).await;
921        if matches.len() > 1 {
922            let candidates = matches
923                .iter()
924                .map(|resolved| resolved.runtime_identity.clone())
925                .collect::<Vec<_>>()
926                .join(", ");
927            return Err(format!(
928                "ambiguous live identity alias {identity}: candidates [{candidates}]"
929            )
930            .into());
931        }
932        let mut live_retired_any = false;
933        for resolved in matches {
934            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
935                continue;
936            };
937            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
938                continue;
939            }
940            if let Some(stale_error) =
941                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
942            {
943                return Err(stale_error.into());
944            }
945            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
946                && let Ok(parsed_identity) =
947                    crate::identity_first::AgentIdentity::parse(&record.identity)
948                && let Ok(status) = identity_runtime.status(&parsed_identity).await
949            {
950                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
951                let durable_live_records = Box::pin(self.live_records_for_durable_record(
952                    &resolved.entry,
953                    identity,
954                    &durable_record,
955                    &live_records,
956                ))
957                .await;
958                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
959                    &resolved.entry,
960                    &durable_live_records,
961                ))
962                .await;
963                if let Some(ambiguous_error) =
964                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
965                {
966                    return Err(ambiguous_error.into());
967                }
968                if let Some(rebound_record) = self_heal_durable_session_mismatch(
969                    &resolved.entry,
970                    identity_runtime.as_ref(),
971                    &parsed_identity,
972                    identity,
973                    &durable_record,
974                    &durable_live_records,
975                )
976                .await
977                .map_err(ConsoleLogError::from)?
978                {
979                    durable_record = rebound_record;
980                }
981                if let Some(stale_error) =
982                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
983                {
984                    return Err(stale_error.into());
985                }
986                if status
987                    .agent_runtime_id
988                    .as_ref()
989                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
990                {
991                    continue;
992                }
993            }
994            resolved
995                .handle
996                .retire(crate::member_comms_id::mob_member_id(
997                    resolved.runtime_identity.as_str(),
998                ))
999                .await
1000                .map_err(|err| -> ConsoleLogError {
1001                    format!("retire failed for {identity}: {err}").into()
1002                })
1003                .or_else(|err| {
1004                    if lifecycle_archive_cleanup_completed(&err.to_string()) {
1005                        Ok(())
1006                    } else {
1007                        Err(err)
1008                    }
1009                })?;
1010            live_retired_any = true;
1011        }
1012        Ok(live_retired_any)
1013    }
1014
1015    pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1016        self.inner.store.clear_frames().await
1017    }
1018
1019    pub async fn query_timeline(
1020        &self,
1021        query: ConsoleTimelineQuery,
1022    ) -> ConsoleLogResult<ConsoleTimelinePage> {
1023        let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1024        Ok(ConsoleTimelinePage {
1025            frames: page.frames,
1026            next_cursor: page.next_cursor,
1027        })
1028    }
1029
1030    pub async fn query_timeline_windowed(
1031        &self,
1032        query: ConsoleTimelineWindowQuery,
1033    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1034        self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1035            .await?;
1036        if query.after.is_none()
1037            && query.before.is_none()
1038            && let Some(identity) = query.identity.clone()
1039        {
1040            let mut probe_query = query.clone();
1041            probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1042            let page = self.inner.store.query_windowed_frames(probe_query).await?;
1043            if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1044                spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1045            }
1046        }
1047        Box::pin(self.query_timeline_visible(query)).await
1048    }
1049
1050    async fn reject_after_cursor_beyond_store_frontier(
1051        &self,
1052        after: Option<&ConsoleCursor>,
1053    ) -> ConsoleLogResult<()> {
1054        let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1055            return Ok(());
1056        };
1057        let latest_seq = self
1058            .inner
1059            .store
1060            .latest_cursor()
1061            .await?
1062            .and_then(|cursor| cursor.seq())
1063            .unwrap_or(0);
1064        if after_seq > latest_seq {
1065            return Err(std::io::Error::other(
1066                "timeline replay cursor is beyond the current store frontier",
1067            )
1068            .into());
1069        }
1070        Ok(())
1071    }
1072
1073    async fn query_timeline_visible(
1074        &self,
1075        query: ConsoleTimelineWindowQuery,
1076    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1077        let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1078        let mut scan_query = query.clone();
1079        scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1080        let mut visible_frames = Vec::with_capacity(requested_limit);
1081        let mut anchor_frames = Vec::new();
1082        let mut identity_visibility_cache = HashMap::new();
1083        let identity_records = self.inner.identity_read_model.current().await;
1084        let mut next_cursor = query.after.clone();
1085        let mut since_last_scanned_cursor = query.after.clone();
1086        let mut since_last_delivered_cursor = None;
1087        let mut since_stopped_at_visible_limit = false;
1088        let mut latest_cursor = None;
1089        let mut exhausted = false;
1090        let mut scanned = 0usize;
1091
1092        loop {
1093            let page = self
1094                .inner
1095                .store
1096                .query_windowed_frames(scan_query.clone())
1097                .await?;
1098            latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1099            if page.frames.is_empty() {
1100                exhausted = true;
1101                break;
1102            }
1103            let raw_len = page.frames.len();
1104            scanned = scanned.saturating_add(raw_len);
1105            match query.mode {
1106                ConsoleTimelineMode::Since => {
1107                    if let Some(cursor) = page.next_cursor.clone() {
1108                        since_last_scanned_cursor = Some(cursor.clone());
1109                        scan_query.after = Some(cursor);
1110                    }
1111                }
1112                ConsoleTimelineMode::Recent => {
1113                    if let Some(first) = page.frames.first() {
1114                        scan_query.before = Some(first.cursor.clone());
1115                    }
1116                    next_cursor = page.next_cursor.clone().or(next_cursor);
1117                }
1118            }
1119
1120            for frame in page.frames {
1121                let allow_historical_identity =
1122                    query.identity.as_deref() == Some(frame.identity.as_str());
1123                if Box::pin(frame_is_visible_cached(
1124                    &self.inner,
1125                    &frame,
1126                    allow_historical_identity,
1127                    &mut identity_visibility_cache,
1128                    &identity_records,
1129                ))
1130                .await
1131                .unwrap_or(false)
1132                {
1133                    if query.mode == ConsoleTimelineMode::Recent
1134                        && query.identity.is_some()
1135                        && is_identity_timeline_anchor_frame(&frame)
1136                        && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1137                    {
1138                        anchor_frames.push(frame.clone());
1139                    }
1140                    match query.mode {
1141                        ConsoleTimelineMode::Since => {
1142                            if visible_frames.len() >= requested_limit {
1143                                since_stopped_at_visible_limit = true;
1144                                break;
1145                            }
1146                            since_last_delivered_cursor = Some(frame.cursor.clone());
1147                            visible_frames.push(frame);
1148                            if visible_frames.len() >= requested_limit {
1149                                since_stopped_at_visible_limit = true;
1150                                exhausted = false;
1151                                break;
1152                            }
1153                        }
1154                        ConsoleTimelineMode::Recent => {
1155                            visible_frames.push(frame);
1156                        }
1157                    }
1158                }
1159            }
1160            let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1161                && query.identity.is_some()
1162                && anchor_frames.is_empty()
1163                && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1164            if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1165                break;
1166            }
1167            if since_stopped_at_visible_limit {
1168                break;
1169            }
1170            if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1171                exhausted = true;
1172                break;
1173            }
1174            if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1175                break;
1176            }
1177        }
1178
1179        if query.mode == ConsoleTimelineMode::Recent {
1180            visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1181            if visible_frames.len() > requested_limit {
1182                visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1183            }
1184            if !anchor_frames.is_empty() {
1185                let anchor_cursors = anchor_frames
1186                    .iter()
1187                    .map(|frame| frame.cursor.clone())
1188                    .collect::<Vec<_>>();
1189                let mut merged = anchor_frames;
1190                for frame in visible_frames {
1191                    if !merged
1192                        .iter()
1193                        .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1194                    {
1195                        merged.push(frame);
1196                    }
1197                }
1198                merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1199                visible_frames = merged;
1200                if visible_frames.len() > requested_limit {
1201                    let mut anchors = Vec::new();
1202                    let mut tail = Vec::new();
1203                    for frame in visible_frames {
1204                        if anchor_cursors.contains(&frame.cursor) {
1205                            anchors.push(frame);
1206                        } else {
1207                            tail.push(frame);
1208                        }
1209                    }
1210                    visible_frames = if anchors.len() >= requested_limit {
1211                        anchors.split_off(anchors.len() - requested_limit)
1212                    } else {
1213                        let keep_tail = requested_limit.saturating_sub(anchors.len());
1214                        if tail.len() > keep_tail {
1215                            tail = tail.split_off(tail.len() - keep_tail);
1216                        }
1217                        anchors.extend(tail);
1218                        anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1219                        anchors
1220                    };
1221                }
1222            }
1223        }
1224
1225        Ok(ConsoleTimelineWindowPage {
1226            frames: visible_frames,
1227            next_cursor: match query.mode {
1228                ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1229                    since_last_delivered_cursor.or(since_last_scanned_cursor)
1230                }
1231                ConsoleTimelineMode::Since => since_last_scanned_cursor,
1232                ConsoleTimelineMode::Recent => next_cursor,
1233            },
1234            latest_cursor,
1235            exhausted,
1236        })
1237    }
1238
1239    pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1240        let runtime_keys = self
1241            .inner
1242            .runtimes
1243            .read()
1244            .map_err(|_| runtime_registry_lock_error())?
1245            .keys()
1246            .cloned()
1247            .collect::<Vec<_>>();
1248        let results =
1249            join_all(runtime_keys.into_iter().map(|runtime_key| {
1250                backfill_session_history(self.inner.clone(), runtime_key, true)
1251            }))
1252            .await;
1253        for result in results {
1254            result?;
1255        }
1256        Ok(())
1257    }
1258
1259    pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1260        self.inner.store.latest_cursor().await
1261    }
1262
1263    pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1264        match event {
1265            ConsoleTimelineEvent::ConsoleFrame { frame }
1266            | ConsoleTimelineEvent::FrameUpdated { frame } => {
1267                let identity_records = self.inner.identity_read_model.current().await;
1268                Box::pin(frame_is_visible(
1269                    &self.inner,
1270                    frame,
1271                    false,
1272                    &identity_records,
1273                ))
1274                .await
1275                .unwrap_or(false)
1276            }
1277            ConsoleTimelineEvent::SnapshotStarted { .. }
1278            | ConsoleTimelineEvent::SnapshotComplete { .. }
1279            | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1280        }
1281    }
1282
1283    pub async fn timeline_frame_visible_for_query(
1284        &self,
1285        frame: &ConsoleFrame,
1286        identity: Option<&str>,
1287    ) -> bool {
1288        let allow_historical_identity = identity == Some(frame.identity.as_str());
1289        let identity_records = self.inner.identity_read_model.current().await;
1290        Box::pin(frame_is_visible(
1291            &self.inner,
1292            frame,
1293            allow_historical_identity,
1294            &identity_records,
1295        ))
1296        .await
1297        .unwrap_or(false)
1298    }
1299
1300    pub async fn send(
1301        &self,
1302        request: ConsoleSendRequest,
1303    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1304        validate_send_request(&request)?;
1305        let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1306            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1307        };
1308        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1309            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1310        };
1311        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1312            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1313        }
1314        if !member_is_addressable(&resolved.member) {
1315            return Err(ConsoleSendError::NotAddressable(request.identity));
1316        }
1317        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1318            return Err(ConsoleSendError::Retired(request.identity));
1319        }
1320
1321        let content = content_input_from_value(&request.content)?;
1322        let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1323        assert_member_accepts_images(
1324            &resolved.handle,
1325            resolved.entry.runtime.session_service(),
1326            &resolved.runtime_identity,
1327            &content,
1328        )
1329        .await
1330        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1331
1332        let dedupe_key = send_dedupe_key(
1333            &resolved.entry.runtime_key,
1334            &request.identity,
1335            &request.origin,
1336            &request.idempotency_key,
1337        );
1338        let handling_mode_value = request
1339            .handling_mode
1340            .as_deref()
1341            .unwrap_or("queue")
1342            .to_string();
1343        let request_fingerprint =
1344            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1345        if let Some(existing) = self
1346            .inner
1347            .store
1348            .frame_by_dedupe_key(&dedupe_key)
1349            .await
1350            .map_err(ConsoleSendError::Log)?
1351        {
1352            let same_request = existing.source.source_cursor.as_deref()
1353                == Some(request_fingerprint.as_str())
1354                || existing.source.source_cursor.is_none()
1355                    && existing.payload.get("origin").and_then(Value::as_str)
1356                        == Some(request.origin.as_str())
1357                    && existing.payload.get("content") == Some(&request.content)
1358                    && existing
1359                        .payload
1360                        .get("handling_mode")
1361                        .and_then(Value::as_str)
1362                        == Some(handling_mode_value.as_str());
1363            if !same_request {
1364                return Err(ConsoleSendError::IdempotencyConflict(
1365                    request.idempotency_key,
1366                ));
1367            }
1368            return Ok(accepted_from_frame(&existing));
1369        }
1370
1371        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1372        resolved
1373            .entry
1374            .console_events
1375            .reserve_interaction_value(
1376                &resolved.runtime_identity,
1377                Some(resolved.runtime_identity.as_str()),
1378                &interaction_id,
1379                &request.origin,
1380                request.content.clone(),
1381            )
1382            .await
1383            .map_err(ConsoleSendError::State)?;
1384        let session_id = resolved
1385            .handle
1386            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1387                resolved.runtime_identity.as_str(),
1388            ))
1389            .await
1390            .map(|sid| sid.to_string());
1391        let mut new_frame = NewConsoleFrame {
1392            id: None,
1393            dedupe_key,
1394            timestamp_ms: current_time_ms(),
1395            runtime_key: resolved.entry.runtime_key.clone(),
1396            identity: request.identity.clone(),
1397            conversation_id: Some(request.identity.clone()),
1398            session_id: session_id.clone(),
1399            kind: "user_input".to_string(),
1400            status: ConsoleFrameStatus::Accepted,
1401            payload: json!({
1402                "content": request.content,
1403                "origin": request.origin,
1404                "idempotency_key": request.idempotency_key,
1405                "handling_mode": handling_mode_value,
1406            }),
1407            source: ConsoleFrameSource {
1408                kind: ConsoleFrameSourceKind::Send,
1409                source_cursor: Some(request_fingerprint),
1410            },
1411            source_event_id: None,
1412            interaction_id: Some(interaction_id.clone()),
1413            turn_id: None,
1414            run_id: None,
1415            parent_frame_id: None,
1416            caused_by_frame_id: None,
1417        };
1418        if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1419            new_frame.payload = redacted;
1420            new_frame.status = ConsoleFrameStatus::Redacted;
1421        }
1422
1423        let _ = SendState::Requested
1424            .apply(SendTransition::PersistAccepted)
1425            .map_err(ConsoleSendError::State)?;
1426        let outcome = self
1427            .inner
1428            .store
1429            .append_if_absent(new_frame)
1430            .await
1431            .map_err(ConsoleSendError::Log)?;
1432        if outcome.disposition == AppendDisposition::Existing {
1433            return Ok(accepted_from_frame(&outcome.frame));
1434        }
1435        let _ = self
1436            .inner
1437            .event_tx
1438            .send(ConsoleTimelineEvent::ConsoleFrame {
1439                frame: outcome.frame.clone(),
1440            });
1441        let accepted = accepted_from_frame(&outcome.frame);
1442
1443        let (dispatching, _effects) = SendState::AcceptedPersisted
1444            .apply(SendTransition::StartDispatch)
1445            .map_err(ConsoleSendError::State)?;
1446        update_frame_status_and_emit(
1447            &self.inner,
1448            &outcome.frame.id,
1449            ConsoleFrameStatus::Dispatching,
1450        )
1451        .await
1452        .map_err(ConsoleSendError::Log)?;
1453
1454        spawn_console_send_dispatch(
1455            self.inner.clone(),
1456            resolved,
1457            content,
1458            handling_mode,
1459            dispatching,
1460            outcome.frame,
1461            interaction_id,
1462        );
1463        Ok(accepted)
1464    }
1465
1466    pub async fn reserve_identity_first_interaction(
1467        &self,
1468        request: ConsoleSendRequest,
1469        session_id: Option<&str>,
1470    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1471        validate_send_request(&request)?;
1472        let _content = content_input_from_value(&request.content)?;
1473        let runtime_key =
1474            Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1475        let handling_mode_value = request
1476            .handling_mode
1477            .as_deref()
1478            .unwrap_or("queue")
1479            .to_string();
1480        let dedupe_key = send_dedupe_key(
1481            &runtime_key,
1482            &request.identity,
1483            &request.origin,
1484            &request.idempotency_key,
1485        );
1486        let request_fingerprint =
1487            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1488        if let Some(existing) = self
1489            .inner
1490            .store
1491            .frame_by_dedupe_key(&dedupe_key)
1492            .await
1493            .map_err(ConsoleSendError::Log)?
1494        {
1495            let same_request = existing.source.source_cursor.as_deref()
1496                == Some(request_fingerprint.as_str())
1497                || existing.source.source_cursor.is_none()
1498                    && existing.payload.get("origin").and_then(Value::as_str)
1499                        == Some(request.origin.as_str())
1500                    && existing.payload.get("content") == Some(&request.content)
1501                    && existing
1502                        .payload
1503                        .get("handling_mode")
1504                        .and_then(Value::as_str)
1505                        == Some(handling_mode_value.as_str());
1506            if !same_request {
1507                return Err(ConsoleSendError::IdempotencyConflict(
1508                    request.idempotency_key,
1509                ));
1510            }
1511            return Ok(accepted_from_frame(&existing));
1512        }
1513
1514        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1515        let new_frame = NewConsoleFrame {
1516            id: None,
1517            dedupe_key,
1518            timestamp_ms: current_time_ms(),
1519            runtime_key,
1520            identity: request.identity.clone(),
1521            conversation_id: Some(request.identity.clone()),
1522            session_id: session_id.map(ToString::to_string),
1523            kind: "user_input".to_string(),
1524            status: ConsoleFrameStatus::Accepted,
1525            payload: json!({
1526                "content": request.content,
1527                "origin": request.origin,
1528                "idempotency_key": request.idempotency_key,
1529                "handling_mode": handling_mode_value,
1530            }),
1531            source: ConsoleFrameSource {
1532                kind: ConsoleFrameSourceKind::Send,
1533                source_cursor: Some(request_fingerprint),
1534            },
1535            source_event_id: None,
1536            interaction_id: Some(interaction_id),
1537            turn_id: None,
1538            run_id: None,
1539            parent_frame_id: None,
1540            caused_by_frame_id: None,
1541        };
1542        let outcome = self
1543            .inner
1544            .store
1545            .append_if_absent(new_frame)
1546            .await
1547            .map_err(ConsoleSendError::Log)?;
1548        let _ = self
1549            .inner
1550            .event_tx
1551            .send(ConsoleTimelineEvent::ConsoleFrame {
1552                frame: outcome.frame.clone(),
1553            });
1554        Ok(accepted_from_frame(&outcome.frame))
1555    }
1556
1557    async fn runtime_key_for_identity_first_send(
1558        &self,
1559        identity: &str,
1560    ) -> Result<String, ConsoleSendError> {
1561        let entries = self
1562            .inner
1563            .runtimes
1564            .read()
1565            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1566            .clone();
1567        if entries.is_empty() {
1568            return Ok("identity-first".to_string());
1569        }
1570
1571        let mut identity_runtime_keys = Vec::new();
1572        let mut hidden_durable_match = false;
1573        let mut durable_matches = Vec::new();
1574        for entry in entries.values() {
1575            let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1576                continue;
1577            };
1578            identity_runtime_keys.push(entry.runtime_key.clone());
1579            if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1580                continue;
1581            }
1582            for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1583            {
1584                let Ok(parsed_identity) =
1585                    crate::identity_first::AgentIdentity::parse(&runtime_identity)
1586                else {
1587                    continue;
1588                };
1589                if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1590                    let record = identity_record_for_status(entry, &status);
1591                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
1592                        hidden_durable_match = true;
1593                        continue;
1594                    }
1595                    durable_matches.push((
1596                        entry.clone(),
1597                        identity_runtime.clone(),
1598                        parsed_identity,
1599                        record,
1600                    ));
1601                }
1602            }
1603        }
1604        if durable_matches.len() > 1 {
1605            let candidates = durable_matches
1606                .iter()
1607                .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1608                .collect::<Vec<_>>()
1609                .join(", ");
1610            return Err(ConsoleSendError::InvalidRequest(format!(
1611                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1612            )));
1613        }
1614        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1615            durable_matches.into_iter().next()
1616        {
1617            let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1618            let durable_live_records = Box::pin(self.live_records_for_durable_record(
1619                &entry,
1620                identity,
1621                &record,
1622                &live_records,
1623            ))
1624            .await;
1625            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1626                &entry,
1627                &durable_live_records,
1628            ))
1629            .await;
1630            if let Some(ambiguous_error) =
1631                ambiguous_live_alias_error(identity, &visible_durable_live_records)
1632            {
1633                return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1634            }
1635            if let Some(rebound_record) = self_heal_durable_session_mismatch(
1636                &entry,
1637                identity_runtime.as_ref(),
1638                &parsed_identity,
1639                identity,
1640                &record,
1641                &durable_live_records,
1642            )
1643            .await
1644            .map_err(ConsoleSendError::InvalidRequest)?
1645            {
1646                record = rebound_record;
1647            }
1648            if let Some(stale_error) =
1649                stale_durable_record_error(identity, &record, &durable_live_records)
1650            {
1651                return Err(ConsoleSendError::InvalidRequest(stale_error));
1652            }
1653            return Ok(entry.runtime_key);
1654        }
1655        if entries.values().any(|entry| {
1656            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1657        }) {
1658            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1659        }
1660        if hidden_durable_match {
1661            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1662        }
1663
1664        match identity_runtime_keys.as_slice() {
1665            [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1666            [runtime_key] => Ok(runtime_key.clone()),
1667            _ => Err(ConsoleSendError::InvalidRequest(format!(
1668                "identity-first send for '{identity}' did not match exactly one registered runtime"
1669            ))),
1670        }
1671    }
1672
1673    pub async fn mark_interaction_delivery_failed(
1674        &self,
1675        input_frame_id: &str,
1676    ) -> Result<(), ConsoleSendError> {
1677        update_frame_status_and_emit(
1678            &self.inner,
1679            input_frame_id,
1680            ConsoleFrameStatus::DeliveryFailed,
1681        )
1682        .await
1683        .map_err(ConsoleSendError::Log)?;
1684        Ok(())
1685    }
1686
1687    pub async fn mark_interaction_delivered(
1688        &self,
1689        input_frame_id: &str,
1690    ) -> Result<(), ConsoleSendError> {
1691        update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1692            .await
1693            .map_err(ConsoleSendError::Log)?;
1694        Ok(())
1695    }
1696
1697    pub async fn mark_steer_interaction_delivered(
1698        &self,
1699        input_frame_id: &str,
1700        interaction_id: &str,
1701    ) -> Result<(), ConsoleSendError> {
1702        let Some(updated) = update_frame_status_and_emit(
1703            &self.inner,
1704            input_frame_id,
1705            ConsoleFrameStatus::Delivered,
1706        )
1707        .await
1708        .map_err(ConsoleSendError::Log)?
1709        else {
1710            return Ok(());
1711        };
1712        append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1713            .await
1714            .map_err(ConsoleSendError::Log)?;
1715        Ok(())
1716    }
1717
1718    pub async fn binary_blob_store_for_identity(
1719        &self,
1720        identity: &str,
1721    ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1722        if identity.trim().is_empty() {
1723            return Err(ConsoleSendError::InvalidRequest(
1724                "identity must be non-empty".to_string(),
1725            ));
1726        }
1727        let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1728            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1729        };
1730        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1731            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1732        };
1733        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1734            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1735        }
1736        if !member_is_addressable(&resolved.member) {
1737            return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1738        }
1739        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1740            return Err(ConsoleSendError::Retired(identity.to_string()));
1741        }
1742        Ok(resolved.entry.runtime.binary_blob_store())
1743    }
1744
1745    pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1746        self.inner
1747            .runtimes
1748            .read()
1749            .map(|entries| {
1750                entries
1751                    .values()
1752                    .filter_map(|entry| entry.runtime.binary_blob_store())
1753                    .collect()
1754            })
1755            .unwrap_or_default()
1756    }
1757
1758    async fn resolve_send_member(
1759        &self,
1760        identity: &str,
1761    ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1762        let matches = Box::pin(self.resolve_visible_members(identity)).await;
1763        if matches.len() > 1 {
1764            let candidates = matches
1765                .iter()
1766                .map(|resolved| resolved.runtime_identity.clone())
1767                .collect::<Vec<_>>()
1768                .join(", ");
1769            return Err(ConsoleSendError::AmbiguousIdentity {
1770                identity: identity.to_string(),
1771                candidates,
1772            });
1773        }
1774        let Some(resolved) = matches.into_iter().next() else {
1775            if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1776                return Err(ConsoleSendError::InvalidRequest(stale_error));
1777            }
1778            return Ok(None);
1779        };
1780        if let Some(record) = identity_record_for_resolved_member(&resolved).await
1781            && let Some(stale_error) =
1782                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1783        {
1784            return Err(ConsoleSendError::InvalidRequest(stale_error));
1785        }
1786        if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1787            return Err(ConsoleSendError::InvalidRequest(stale_error));
1788        }
1789        Ok(Some(resolved))
1790    }
1791
1792    async fn durable_send_binding_error(
1793        &self,
1794        identity: &str,
1795    ) -> Result<Option<String>, ConsoleSendError> {
1796        let entries = self
1797            .inner
1798            .runtimes
1799            .read()
1800            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1801            .clone();
1802        if entries.values().any(|entry| {
1803            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1804        }) {
1805            return Ok(None);
1806        }
1807        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1808        let mut durable_matches = Vec::new();
1809        for entry in entries.values() {
1810            let Some(identity_runtime) = entry.identity_runtime.clone() else {
1811                continue;
1812            };
1813            for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1814                let Ok(parsed_identity) =
1815                    crate::identity_first::AgentIdentity::parse(&raw_identity)
1816                else {
1817                    continue;
1818                };
1819                let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1820                    continue;
1821                };
1822                let record = identity_record_for_status(entry, &status);
1823                if !Box::pin(console_identity_record_visible(entry, &record)).await {
1824                    continue;
1825                }
1826                durable_matches.push((
1827                    entry.clone(),
1828                    identity_runtime.clone(),
1829                    parsed_identity,
1830                    record,
1831                ));
1832            }
1833        }
1834        if durable_matches.len() > 1 {
1835            let candidates = durable_matches
1836                .iter()
1837                .map(|(_, _, _, record)| record.runtime_member_id.clone())
1838                .collect::<Vec<_>>()
1839                .join(", ");
1840            return Err(ConsoleSendError::AmbiguousIdentity {
1841                identity: identity.to_string(),
1842                candidates,
1843            });
1844        }
1845        let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1846            durable_matches.into_iter().next()
1847        else {
1848            return Ok(None);
1849        };
1850        let durable_live_records = Box::pin(self.live_records_for_durable_record(
1851            &entry,
1852            identity,
1853            &durable_record,
1854            &live_records,
1855        ))
1856        .await;
1857        let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1858            &entry,
1859            &durable_live_records,
1860        ))
1861        .await;
1862        if let Some(ambiguous_error) =
1863            ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1864        {
1865            return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1866        }
1867        if let Some(rebound_record) = self_heal_durable_session_mismatch(
1868            &entry,
1869            identity_runtime.as_ref(),
1870            &parsed_identity,
1871            identity,
1872            &durable_record,
1873            &durable_live_records,
1874        )
1875        .await
1876        .map_err(ConsoleSendError::InvalidRequest)?
1877        {
1878            durable_record = rebound_record;
1879        }
1880        Ok(stale_durable_record_error(
1881            identity,
1882            &durable_record,
1883            &durable_live_records,
1884        ))
1885    }
1886
1887    async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1888        let entries = self
1889            .inner
1890            .runtimes
1891            .read()
1892            .ok()
1893            .map(|entries| entries.clone())
1894            .unwrap_or_default();
1895        let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1896        let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1897        for entry in entries.values() {
1898            let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1899            if raw_identities.is_empty() {
1900                continue;
1901            }
1902            let mids = raw_identities
1903                .iter()
1904                .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
1905                .collect::<Vec<_>>();
1906            for resolved in Box::pin(member_sources_for_entry(entry)).await {
1907                let session_id = resolved
1908                    .handle
1909                    .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1910                    .await
1911                    .map(|sid| sid.to_string())
1912                    .unwrap_or_default();
1913                if mids.contains(&resolved.member.agent_identity) {
1914                    exact_matches.push((session_id, resolved));
1915                } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1916                {
1917                    label_matches.push((session_id, resolved));
1918                }
1919            }
1920        }
1921        let mut matches = exact_matches;
1922        matches.extend(label_matches);
1923        let mut seen_members = BTreeSet::new();
1924        matches.retain(|(session_id, resolved)| {
1925            seen_members.insert((
1926                resolved.entry.runtime_key.clone(),
1927                resolved.source_mob_id.clone(),
1928                session_id.clone(),
1929                resolved.runtime_identity.clone(),
1930            ))
1931        });
1932        matches.sort_by(|left, right| right.0.cmp(&left.0));
1933        matches.into_iter().map(|(_, resolved)| resolved).collect()
1934    }
1935
1936    async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1937        let mut visible = Vec::new();
1938        for resolved in Box::pin(self.resolve_members(identity)).await {
1939            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1940                continue;
1941            };
1942            if Box::pin(resolved_member_visible(&resolved, &record)).await {
1943                visible.push(resolved);
1944            }
1945        }
1946        visible
1947    }
1948
1949    async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1950        let mut records = Vec::new();
1951        for resolved in Box::pin(self.resolve_members(identity)).await {
1952            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1953                continue;
1954            };
1955            if Box::pin(resolved_member_visible(&resolved, &record)).await {
1956                records.push(record);
1957            }
1958        }
1959        records
1960    }
1961
1962    async fn live_records_for_durable_record(
1963        &self,
1964        entry: &RuntimeEntry,
1965        requested_identity: &str,
1966        durable: &ConsoleIdentityRecord,
1967        requested_live_records: &[ConsoleIdentityRecord],
1968    ) -> Vec<ConsoleIdentityRecord> {
1969        let mut records = if durable.identity == requested_identity {
1970            requested_live_records.to_vec()
1971        } else {
1972            Box::pin(self.live_records_for_identity(&durable.identity)).await
1973        };
1974        for runtime_record in Box::pin(live_records_for_runtime_member(
1975            entry,
1976            &durable.runtime_member_id,
1977        ))
1978        .await
1979        {
1980            if records.iter().any(|record| {
1981                record.runtime_key == runtime_record.runtime_key
1982                    && record.runtime_member_id == runtime_record.runtime_member_id
1983                    && record.session_id == runtime_record.session_id
1984                    && record.identity == runtime_record.identity
1985            }) {
1986                continue;
1987            }
1988            records.push(runtime_record);
1989        }
1990        records
1991    }
1992}
1993
1994async fn console_identity_record_visible(
1995    entry: &RuntimeEntry,
1996    record: &ConsoleIdentityRecord,
1997) -> bool {
1998    if !entry.visibility_policy.identity_visible(record) {
1999        return false;
2000    }
2001    let mut bound_live_member_seen = false;
2002    let mut wrong_live_projection_seen = false;
2003    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2004        if resolved.runtime_identity != record.runtime_member_id {
2005            continue;
2006        }
2007        bound_live_member_seen = true;
2008        let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2009            continue;
2010        };
2011        if live_record.identity != record.identity {
2012            wrong_live_projection_seen = true;
2013            continue;
2014        }
2015        if raw_resolved_member_visible(&resolved, &live_record) {
2016            return true;
2017        }
2018    }
2019    if wrong_live_projection_seen {
2020        return true;
2021    }
2022    !bound_live_member_seen
2023}
2024
2025async fn live_record_shadowed_by_hidden_durable_binding(
2026    resolved: &ResolvedConsoleMember,
2027    record: &ConsoleIdentityRecord,
2028) -> bool {
2029    let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2030        return false;
2031    };
2032    let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2033        return false;
2034    };
2035    let Ok(status) = identity_runtime.status(&identity).await else {
2036        return false;
2037    };
2038    let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2039        return false;
2040    };
2041    if bound_runtime_id.as_str() == resolved.runtime_identity {
2042        return false;
2043    }
2044    for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2045        if candidate.runtime_identity != bound_runtime_id.as_str() {
2046            continue;
2047        }
2048        let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2049            continue;
2050        };
2051        if bound_record.identity == record.identity
2052            && !raw_resolved_member_visible(&candidate, &bound_record)
2053        {
2054            return true;
2055        }
2056    }
2057    false
2058}
2059
2060async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2061    let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2062        .unwrap_or_else(|| frame.identity.clone());
2063    let frame_session_id = frame.session_id.as_deref();
2064    let mut saw_hidden = false;
2065    let mut saw_visible = false;
2066    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2067        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2068            continue;
2069        };
2070        let session_matches =
2071            frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2072        let runtime_matches = record.runtime_member_id == runtime_member_id;
2073        let identity_matches = record.identity == frame.identity;
2074        if !session_matches && !runtime_matches && !identity_matches {
2075            continue;
2076        }
2077        if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2078            return true;
2079        }
2080        if Box::pin(resolved_member_visible(&resolved, &record)).await {
2081            saw_visible = true;
2082        } else {
2083            saw_hidden = true;
2084        }
2085    }
2086    saw_hidden && !saw_visible
2087}
2088
2089async fn live_records_for_runtime_member(
2090    entry: &RuntimeEntry,
2091    runtime_member_id: &str,
2092) -> Vec<ConsoleIdentityRecord> {
2093    let mut records = Vec::new();
2094    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2095        if resolved.runtime_identity != runtime_member_id {
2096            continue;
2097        }
2098        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2099            continue;
2100        };
2101        records.push(record);
2102    }
2103    records
2104}
2105
2106async fn visible_live_records_for_entry(
2107    entry: &RuntimeEntry,
2108    records: &[ConsoleIdentityRecord],
2109) -> Vec<ConsoleIdentityRecord> {
2110    let mut visible = Vec::new();
2111    for record in records {
2112        if record.runtime_key != entry.runtime_key
2113            || Box::pin(console_identity_record_visible(entry, record)).await
2114        {
2115            visible.push(record.clone());
2116        }
2117    }
2118    visible
2119}
2120
2121async fn reconcile_stale_live_record_binding(
2122    entry: &RuntimeEntry,
2123    requested_identity: &str,
2124    live_record: &ConsoleIdentityRecord,
2125) -> Option<String> {
2126    let identity_runtime = entry.identity_runtime.as_ref()?;
2127    for status in identity_runtime.statuses().await {
2128        let matches_runtime = status
2129            .agent_runtime_id
2130            .as_ref()
2131            .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2132        if !matches_runtime {
2133            continue;
2134        }
2135        let durable_record = identity_record_for_status(entry, &status);
2136        if let Some(session_mismatch) =
2137            stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2138        {
2139            let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2140                continue;
2141            };
2142            let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2143                return Some(format!(
2144                    "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2145                ));
2146            };
2147            return match identity_runtime
2148                .rebind_session_after_live_respawn(&status.identity, session_id)
2149                .await
2150            {
2151                Ok(_) => {
2152                    tracing::warn!(
2153                        requested_identity,
2154                        identity = %durable_record.identity,
2155                        runtime_member_id = %durable_record.runtime_member_id,
2156                        old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2157                        new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2158                        "self-healed stale durable identity alias session mismatch"
2159                    );
2160                    None
2161                }
2162                Err(err) => Some(format!(
2163                    "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2164                    durable_record.identity,
2165                    session_mismatch.session_id.as_deref().unwrap_or("<none>")
2166                )),
2167            };
2168        }
2169        if let Some(stale_error) = stale_durable_record_error(
2170            requested_identity,
2171            &durable_record,
2172            std::slice::from_ref(live_record),
2173        ) {
2174            return Some(stale_error);
2175        }
2176    }
2177    None
2178}
2179
2180async fn self_heal_durable_session_mismatch(
2181    entry: &RuntimeEntry,
2182    identity_runtime: &crate::identity_first::IdentityRuntime,
2183    parsed_identity: &crate::identity_first::AgentIdentity,
2184    requested_identity: &str,
2185    durable: &ConsoleIdentityRecord,
2186    live_records: &[ConsoleIdentityRecord],
2187) -> Result<Option<ConsoleIdentityRecord>, String> {
2188    let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2189        return Ok(None);
2190    };
2191    let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2192        return Ok(None);
2193    };
2194    let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2195        format!(
2196            "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2197        )
2198    })?;
2199    identity_runtime
2200        .rebind_session_after_live_respawn(parsed_identity, session_id)
2201        .await
2202        .map_err(|err| {
2203            format!(
2204                "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2205                durable.identity, live_session_id
2206            )
2207        })?;
2208    tracing::warn!(
2209        requested_identity,
2210        identity = %durable.identity,
2211        runtime_member_id = %durable.runtime_member_id,
2212        old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2213        new_session_id = live_session_id,
2214        "self-healed stale durable identity alias session mismatch"
2215    );
2216    let status = identity_runtime
2217        .status(parsed_identity)
2218        .await
2219        .map_err(|err| {
2220            format!(
2221                "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2222                durable.identity
2223            )
2224        })?;
2225    Ok(Some(identity_record_for_status(entry, &status)))
2226}
2227
2228fn stale_durable_session_mismatch<'a>(
2229    durable: &ConsoleIdentityRecord,
2230    live_records: &'a [ConsoleIdentityRecord],
2231) -> Option<&'a ConsoleIdentityRecord> {
2232    live_records.iter().find(|record| {
2233        record.identity == durable.identity
2234            && record.runtime_member_id == durable.runtime_member_id
2235            && durable.session_id.is_some()
2236            && record.session_id.is_some()
2237            && durable.session_id != record.session_id
2238    })
2239}
2240
2241fn stale_durable_record_error(
2242    requested_identity: &str,
2243    durable: &ConsoleIdentityRecord,
2244    live_records: &[ConsoleIdentityRecord],
2245) -> Option<String> {
2246    let matching_live = live_records
2247        .iter()
2248        .filter(|record| record.identity == durable.identity)
2249        .collect::<Vec<_>>();
2250    if let Some(wrong_projection) = live_records.iter().find(|record| {
2251        record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2252    }) {
2253        return Some(format!(
2254            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2255            durable.identity, durable.runtime_member_id, wrong_projection.identity
2256        ));
2257    }
2258    if matching_live.is_empty() {
2259        return None;
2260    }
2261    if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2262        return Some(format!(
2263            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2264            durable.identity,
2265            durable.runtime_member_id,
2266            durable.session_id.as_deref().unwrap_or("<none>"),
2267            session_mismatch.session_id.as_deref().unwrap_or("<none>")
2268        ));
2269    }
2270    if matching_live
2271        .iter()
2272        .any(|record| record.runtime_member_id == durable.runtime_member_id)
2273    {
2274        return None;
2275    }
2276    let live_candidates = matching_live
2277        .iter()
2278        .map(|record| record.runtime_member_id.as_str())
2279        .collect::<Vec<_>>()
2280        .join(", ");
2281    Some(format!(
2282        "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2283        durable.identity, durable.runtime_member_id, live_candidates
2284    ))
2285}
2286
2287fn ambiguous_live_alias_error(
2288    requested_identity: &str,
2289    live_records: &[ConsoleIdentityRecord],
2290) -> Option<String> {
2291    if live_records.len() <= 1 {
2292        return None;
2293    }
2294    let candidates = live_records
2295        .iter()
2296        .map(|record| {
2297            format!(
2298                "{}@{}",
2299                record.runtime_member_id,
2300                record
2301                    .labels
2302                    .get("source_mob_id")
2303                    .map(String::as_str)
2304                    .unwrap_or(record.runtime_key.as_str())
2305            )
2306        })
2307        .collect::<Vec<_>>()
2308        .join(", ");
2309    Some(format!(
2310        "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2311    ))
2312}
2313
2314fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2315    // Roster ids are comms-safe encodings of public aliases (meerkat 0.7
2316    // MemberCommsName); compare in the public alias space.
2317    crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2318}
2319
2320async fn retire_stale_console_members_for_identity(
2321    handle: &MobHandle,
2322    durable_identities: &[String],
2323    keep_runtime_member_id: Option<&str>,
2324) -> Result<(), String> {
2325    let stale_members = handle
2326        .list_members_including_retiring()
2327        .await
2328        .into_iter()
2329        .filter(|member| {
2330            (durable_identities.iter().any(|durable_identity| {
2331                member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2332            }) || member.labels.get("agent_identity").is_some_and(|identity| {
2333                durable_identities
2334                    .iter()
2335                    .any(|durable_identity| identity == durable_identity)
2336            })) && keep_runtime_member_id
2337                .map(|keep| {
2338                    // `keep` is a public alias; compare decoded.
2339                    crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2340                        != keep
2341                })
2342                .unwrap_or(true)
2343        })
2344        .map(|member| member.agent_identity)
2345        .collect::<Vec<_>>();
2346    for member_id in stale_members {
2347        match handle.retire(member_id).await {
2348            Ok(()) => {}
2349            Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2350            Err(err) => return Err(err.to_string()),
2351        }
2352    }
2353    Ok(())
2354}
2355
2356async fn retire_stale_console_members_for_runtime_entry(
2357    entry: &RuntimeEntry,
2358    durable_identities: &[String],
2359) -> Result<(), String> {
2360    let primary_handle = entry.runtime.handle();
2361    let primary_mob_id = primary_handle.mob_id().to_string();
2362    let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2363    if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2364        for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2365            .await
2366            .unwrap_or_default()
2367        {
2368            if mob_id.as_str() != primary_mob_id {
2369                handles.push((mob_id.to_string(), handle));
2370            }
2371        }
2372    }
2373    let mut seen_handles = BTreeSet::new();
2374    for (mob_id, handle) in handles {
2375        if !seen_handles.insert(mob_id) {
2376            continue;
2377        }
2378        retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2379    }
2380    Ok(())
2381}
2382
2383fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2384    is_recoverable_lifecycle_cleanup_error(error)
2385}
2386
2387fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2388    if frames.is_empty() {
2389        return true;
2390    }
2391    let latest_session_history_timestamp_ms = frames
2392        .iter()
2393        .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2394        .map(|frame| frame.timestamp_ms)
2395        .max();
2396    if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2397        return current_time_ms().saturating_sub(latest_timestamp_ms)
2398            >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2399    }
2400    frames.iter().any(|frame| {
2401        matches!(
2402            frame.kind.as_str(),
2403            "turn_started"
2404                | "run_started"
2405                | "reasoning_delta"
2406                | "reasoning_complete"
2407                | "tool_call_requested"
2408                | "tool_call"
2409                | "tool_execution_started"
2410                | "tool_execution_completed"
2411                | "tool_result_received"
2412        )
2413    })
2414}
2415
2416fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2417    match frame.kind.as_str() {
2418        "user_input" | "run_started" => true,
2419        "interaction_started" => frame
2420            .payload
2421            .get("content")
2422            .or_else(|| frame.payload.get("prompt"))
2423            .is_some(),
2424        _ => false,
2425    }
2426}
2427
2428fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2429    let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2430    for record in records {
2431        by_identity
2432            .entry(identity_record_dedupe_key(&record))
2433            .and_modify(|current| {
2434                if identity_record_prefer(&record, current) {
2435                    *current = record.clone();
2436                }
2437            })
2438            .or_insert(record);
2439    }
2440    by_identity.into_values().collect()
2441}
2442
2443fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2444    format!(
2445        "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2446        record.identity,
2447        record.runtime_key,
2448        record
2449            .labels
2450            .get("source_mob_id")
2451            .map(String::as_str)
2452            .unwrap_or(""),
2453        record.runtime_member_id,
2454        record.session_id.as_deref().unwrap_or("")
2455    )
2456}
2457
2458fn identity_record_prefer(
2459    candidate: &ConsoleIdentityRecord,
2460    current: &ConsoleIdentityRecord,
2461) -> bool {
2462    let candidate_is_live_label_projection =
2463        candidate
2464            .labels
2465            .get("agent_identity")
2466            .is_some_and(|identity| {
2467                identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2468                    || candidate.runtime_member_id != candidate.identity
2469            });
2470    let current_is_live_label_projection =
2471        current
2472            .labels
2473            .get("agent_identity")
2474            .is_some_and(|identity| {
2475                identity == &strip_namespace(&current.identity, "").unwrap_or_default()
2476                    || current.runtime_member_id != current.identity
2477            });
2478    if candidate_is_live_label_projection != current_is_live_label_projection {
2479        return candidate_is_live_label_projection;
2480    }
2481    let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2482    let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2483    if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2484        return candidate_has_distinct_runtime_binding;
2485    }
2486    let candidate_live = candidate.addressable && candidate.health != "retired";
2487    let current_live = current.addressable && current.health != "retired";
2488    if candidate_live != current_live {
2489        return candidate_live;
2490    }
2491    candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2492}
2493
2494async fn collect_identity_records(
2495    inner: &Arc<AggregatorInner>,
2496    mode: IdentityCollectionMode,
2497) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2498    let entries = inner
2499        .runtimes
2500        .read()
2501        .map_err(|_| runtime_registry_lock_error())?
2502        .clone();
2503    let mut identities = Vec::new();
2504    for entry in entries.values() {
2505        if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2506            let topology_peers =
2507                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2508            for status in identity_runtime.statuses().await {
2509                let mut record = identity_record_for_status(entry, &status);
2510                record.topology_peers = topology_peers
2511                    .get(&record.identity)
2512                    .cloned()
2513                    .unwrap_or_default();
2514                if Box::pin(console_identity_record_visible(entry, &record)).await {
2515                    identities.push(record);
2516                }
2517            }
2518            if mode == IdentityCollectionMode::CachedOnly {
2519                continue;
2520            }
2521        }
2522        let members = if entry.identity_runtime.is_some() {
2523            match Box::pin(tokio::time::timeout(
2524                IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2525                member_sources_for_entry(entry),
2526            ))
2527            .await
2528            {
2529                Ok(members) => members,
2530                Err(_) => {
2531                    tracing::warn!(
2532                        runtime_key = %entry.runtime_key,
2533                        "identity-first live member refresh timed out; keeping cached identity read model"
2534                    );
2535                    Vec::new()
2536                }
2537            }
2538        } else {
2539            Box::pin(member_sources_for_entry(entry)).await
2540        };
2541        for resolved in members {
2542            if let Some(record) = identity_record_for_resolved_member(&resolved).await
2543                && Box::pin(resolved_member_visible(&resolved, &record)).await
2544            {
2545                identities.push(record);
2546            }
2547        }
2548    }
2549    Ok(dedupe_identity_records(identities))
2550}
2551
2552async fn identity_runtime_topology_peers(
2553    entry: &RuntimeEntry,
2554    identity_runtime: &crate::identity_first::IdentityRuntime,
2555) -> BTreeMap<String, Vec<String>> {
2556    let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2557    for edge in identity_runtime.desired_peer_edges().await {
2558        let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2559        let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2560        peers.entry(a.clone()).or_default().insert(b.clone());
2561        peers.entry(b).or_default().insert(a);
2562    }
2563    peers
2564        .into_iter()
2565        .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2566        .collect()
2567}
2568
2569fn spawn_identity_backfills_for_records(
2570    inner: Arc<AggregatorInner>,
2571    records: &[ConsoleIdentityRecord],
2572) {
2573    if !inner.options.session_history_backfill_enabled {
2574        return;
2575    }
2576    let entries = match inner.runtimes.read() {
2577        Ok(entries) => entries.clone(),
2578        Err(_) => return,
2579    };
2580    for record in records {
2581        if record.health == "dormant" || record.health == "uninitialized" {
2582            continue;
2583        }
2584        let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2585            continue;
2586        };
2587        let Some(session_id) = record.session_id.clone() else {
2588            continue;
2589        };
2590        spawn_session_history_backfill_target(
2591            inner.clone(),
2592            SessionBackfillTarget {
2593                entry,
2594                record: record.clone(),
2595                session_id,
2596            },
2597            false,
2598        );
2599    }
2600}
2601
2602async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2603    let mut resolved = Vec::new();
2604    let primary_handle = entry.runtime.handle();
2605    let primary_mob_id = primary_handle.mob_id().to_string();
2606    for member in primary_handle.list_members_observation_snapshot().await {
2607        resolved.push(ResolvedConsoleMember {
2608            entry: entry.clone(),
2609            handle: primary_handle.clone(),
2610            // Roster ids are comms-safe encodings (meerkat 0.7); the
2611            // aggregator's runtime identity is the public alias.
2612            runtime_identity: crate::member_comms_id::runtime_alias_str(
2613                member.agent_identity.as_str(),
2614            )
2615            .into_owned(),
2616            source_mob_id: primary_mob_id.clone(),
2617            member,
2618        });
2619    }
2620
2621    let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2622        return resolved;
2623    };
2624    if !entry.visibility_policy.include_implicit_delegate_members() {
2625        return resolved;
2626    }
2627    for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2628        .await
2629        .unwrap_or_default()
2630    {
2631        if mob_id.as_str() == primary_mob_id {
2632            continue;
2633        }
2634        for member in handle.list_members_observation_snapshot().await {
2635            resolved.push(ResolvedConsoleMember {
2636                entry: entry.clone(),
2637                handle: handle.clone(),
2638                // Roster ids are comms-safe encodings (meerkat 0.7); the
2639                // aggregator's runtime identity is the public alias.
2640                runtime_identity: crate::member_comms_id::runtime_alias_str(
2641                    member.agent_identity.as_str(),
2642                )
2643                .into_owned(),
2644                source_mob_id: mob_id.to_string(),
2645                member,
2646            });
2647        }
2648    }
2649    resolved
2650}
2651
2652async fn dispatch_message_to_resolved_member(
2653    resolved: &ResolvedConsoleMember,
2654    content: ContentInput,
2655    handling_mode: meerkat_core::types::HandlingMode,
2656) -> Result<String, String> {
2657    let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2658    match send_message_on_mob_with_mode(
2659        &resolved.handle,
2660        &resolved.runtime_identity,
2661        content.clone(),
2662        handling_mode,
2663    )
2664    .await
2665    {
2666        Ok(session_id) => Ok(session_id),
2667        Err(err) if is_not_externally_addressable(&err) => {
2668            let member = resolved
2669                .handle
2670                .member(&mid)
2671                .await
2672                .map_err(|err| err.to_string())?;
2673            let _receipt = member
2674                .internal_turn(content)
2675                .await
2676                .map_err(|err| err.to_string())?;
2677            resolved
2678                .handle
2679                .resolve_bridge_session_id_observation(&mid)
2680                .await
2681                .map(|sid| sid.to_string())
2682                .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2683        }
2684        Err(err) => Err(err.to_string()),
2685    }
2686}
2687
2688fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2689    matches!(
2690        err,
2691        MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2692    )
2693}
2694
2695fn spawn_console_send_dispatch(
2696    inner: Arc<AggregatorInner>,
2697    resolved: ResolvedConsoleMember,
2698    content: ContentInput,
2699    handling_mode: meerkat_core::types::HandlingMode,
2700    dispatching: SendState,
2701    user_frame: ConsoleFrame,
2702    interaction_id: String,
2703) {
2704    tokio::spawn(async move {
2705        match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2706            Ok(_) => {
2707                let _ = dispatching.apply(SendTransition::MarkDelivered);
2708                if let Err(err) = update_frame_status_and_emit(
2709                    &inner,
2710                    &user_frame.id,
2711                    ConsoleFrameStatus::Delivered,
2712                )
2713                .await
2714                {
2715                    tracing::warn!(
2716                        frame_id = %user_frame.id,
2717                        error = %err,
2718                        "failed to update console send delivery status"
2719                    );
2720                }
2721                if handling_mode == HandlingMode::Steer
2722                    && let Err(err) =
2723                        append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2724                {
2725                    tracing::warn!(
2726                        frame_id = %user_frame.id,
2727                        interaction_id = %interaction_id,
2728                        error = %err,
2729                        "failed to append console steer terminal frame"
2730                    );
2731                }
2732            }
2733            Err(err) => {
2734                let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2735                if let Err(update_err) = update_frame_status_and_emit(
2736                    &inner,
2737                    &user_frame.id,
2738                    ConsoleFrameStatus::DeliveryFailed,
2739                )
2740                .await
2741                {
2742                    tracing::warn!(
2743                        frame_id = %user_frame.id,
2744                        error = %update_err,
2745                        "failed to update console send failure status"
2746                    );
2747                }
2748                let failure_frame = NewConsoleFrame {
2749                    id: None,
2750                    dedupe_key: format!("delivery-failed:{}", user_frame.id),
2751                    timestamp_ms: current_time_ms(),
2752                    runtime_key: user_frame.runtime_key,
2753                    identity: user_frame.identity,
2754                    conversation_id: user_frame.conversation_id,
2755                    session_id: user_frame.session_id,
2756                    kind: "message_delivery_failed".to_string(),
2757                    status: ConsoleFrameStatus::DeliveryFailed,
2758                    payload: json!({ "reason": err }),
2759                    source: ConsoleFrameSource {
2760                        kind: ConsoleFrameSourceKind::Synthetic,
2761                        source_cursor: None,
2762                    },
2763                    source_event_id: None,
2764                    interaction_id: Some(interaction_id),
2765                    turn_id: None,
2766                    run_id: None,
2767                    parent_frame_id: Some(user_frame.id.clone()),
2768                    caused_by_frame_id: Some(user_frame.id),
2769                };
2770                if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2771                    tracing::warn!(
2772                        error = %append_err,
2773                        "failed to append console send failure frame"
2774                    );
2775                }
2776            }
2777        }
2778    });
2779}
2780
2781async fn append_steer_delivery_terminal(
2782    inner: &AggregatorInner,
2783    user_frame: &ConsoleFrame,
2784    interaction_id: &str,
2785) -> ConsoleLogResult<AppendOutcome> {
2786    append_and_emit(
2787        inner,
2788        NewConsoleFrame {
2789            id: None,
2790            dedupe_key: format!("steer-delivered:{}", user_frame.id),
2791            timestamp_ms: current_time_ms(),
2792            runtime_key: user_frame.runtime_key.clone(),
2793            identity: user_frame.identity.clone(),
2794            conversation_id: user_frame.conversation_id.clone(),
2795            session_id: user_frame.session_id.clone(),
2796            kind: "interaction_complete".to_string(),
2797            status: ConsoleFrameStatus::Completed,
2798            payload: json!({
2799                "reason": "steer_delivered",
2800                "handling_mode": "steer",
2801            }),
2802            source: ConsoleFrameSource {
2803                kind: ConsoleFrameSourceKind::Synthetic,
2804                source_cursor: None,
2805            },
2806            source_event_id: None,
2807            interaction_id: Some(interaction_id.to_string()),
2808            turn_id: None,
2809            run_id: None,
2810            parent_frame_id: Some(user_frame.id.clone()),
2811            caused_by_frame_id: Some(user_frame.id.clone()),
2812        },
2813    )
2814    .await
2815}
2816
2817#[derive(Debug)]
2818pub enum ConsoleSendError {
2819    UnknownIdentity(String),
2820    AmbiguousIdentity {
2821        identity: String,
2822        candidates: String,
2823    },
2824    NotAddressable(String),
2825    Retired(String),
2826    InvalidContent(String),
2827    InvalidHandlingMode(String),
2828    InvalidRequest(String),
2829    IdempotencyConflict(String),
2830    State(&'static str),
2831    Dispatch(String),
2832    Log(ConsoleLogError),
2833}
2834
2835impl std::fmt::Display for ConsoleSendError {
2836    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2837        match self {
2838            Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2839            Self::AmbiguousIdentity {
2840                identity,
2841                candidates,
2842            } => write!(
2843                f,
2844                "ambiguous live identity alias {identity}: candidates [{candidates}]"
2845            ),
2846            Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2847            Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2848            Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2849            Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2850            Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2851            Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2852            Self::State(message) => write!(f, "console send state error: {message}"),
2853            Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2854            Self::Log(err) => write!(f, "console log error: {err}"),
2855        }
2856    }
2857}
2858
2859impl std::error::Error for ConsoleSendError {}
2860
2861async fn backfill_session_history(
2862    inner: Arc<AggregatorInner>,
2863    runtime_key: String,
2864    force_refresh: bool,
2865) -> ConsoleLogResult<()> {
2866    if !inner.options.session_history_backfill_enabled {
2867        return Ok(());
2868    }
2869    let Some(entry) = inner
2870        .runtimes
2871        .read()
2872        .ok()
2873        .and_then(|entries| entries.get(&runtime_key).cloned())
2874    else {
2875        return Ok(());
2876    };
2877    let members = Box::pin(member_sources_for_entry(&entry)).await;
2878    let mut targets = Vec::new();
2879    for resolved in members {
2880        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2881            continue;
2882        };
2883        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
2884            continue;
2885        }
2886        let Some(session_id) = record.session_id.clone() else {
2887            continue;
2888        };
2889        targets.push(SessionBackfillTarget {
2890            entry: entry.clone(),
2891            record,
2892            session_id,
2893        });
2894    }
2895    backfill_session_history_targets(inner, targets, force_refresh).await
2896}
2897
2898#[derive(Clone)]
2899struct SessionBackfillTarget {
2900    entry: RuntimeEntry,
2901    record: ConsoleIdentityRecord,
2902    session_id: String,
2903}
2904
2905async fn backfill_session_history_targets(
2906    inner: Arc<AggregatorInner>,
2907    targets: Vec<SessionBackfillTarget>,
2908    force_refresh: bool,
2909) -> ConsoleLogResult<()> {
2910    let mut tasks = tokio::task::JoinSet::new();
2911    for target in targets {
2912        tasks.spawn(backfill_one_session_history(
2913            inner.clone(),
2914            target,
2915            force_refresh,
2916        ));
2917    }
2918    let mut first_error = None;
2919    while let Some(result) = tasks.join_next().await {
2920        match result {
2921            Ok(Ok(())) => {}
2922            Ok(Err(err)) => {
2923                if first_error.is_none() {
2924                    first_error = Some(err);
2925                }
2926            }
2927            Err(err) => {
2928                if first_error.is_none() {
2929                    first_error = Some(Box::new(std::io::Error::other(format!(
2930                        "session backfill task failed: {err}"
2931                    ))) as ConsoleLogError);
2932                }
2933            }
2934        }
2935    }
2936    if let Some(err) = first_error {
2937        Err(err)
2938    } else {
2939        Ok(())
2940    }
2941}
2942
2943async fn backfill_one_session_history(
2944    inner: Arc<AggregatorInner>,
2945    target: SessionBackfillTarget,
2946    force_refresh: bool,
2947) -> ConsoleLogResult<()> {
2948    let _permit = inner
2949        .session_backfill_permits
2950        .clone()
2951        .acquire_owned()
2952        .await
2953        .map_err(|err| -> ConsoleLogError {
2954            Box::new(std::io::Error::other(format!(
2955                "session backfill limiter closed: {err}"
2956            )))
2957        })?;
2958    let SessionBackfillTarget {
2959        entry,
2960        record,
2961        session_id,
2962    } = target;
2963    let watermark_runtime_key =
2964        session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2965    let watermark = inner
2966        .store
2967        .source_watermark(
2968            &watermark_runtime_key,
2969            ConsoleFrameSourceKind::SessionHistory,
2970        )
2971        .await?;
2972    let now_ms = current_time_ms();
2973    let mut offset = watermark
2974        .as_deref()
2975        .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2976        .unwrap_or(0);
2977    if !force_refresh
2978        && watermark.as_deref().is_some_and(|watermark| {
2979            session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2980        })
2981    {
2982        return Ok(());
2983    }
2984    loop {
2985        let page = match entry
2986            .runtime
2987            .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2988            .await
2989        {
2990            Ok(page) => page,
2991            Err(err) => {
2992                record_session_history_backfill_failure(
2993                    &inner,
2994                    &watermark_runtime_key,
2995                    &entry.runtime_key,
2996                    &record.identity,
2997                    &session_id,
2998                    offset,
2999                    err.to_string(),
3000                )
3001                .await?;
3002                break;
3003            }
3004        };
3005        let page_value = match serde_json::to_value(page) {
3006            Ok(value) => value,
3007            Err(err) => {
3008                record_session_history_backfill_failure(
3009                    &inner,
3010                    &watermark_runtime_key,
3011                    &entry.runtime_key,
3012                    &record.identity,
3013                    &session_id,
3014                    offset,
3015                    err.to_string(),
3016                )
3017                .await?;
3018                break;
3019            }
3020        };
3021        let base_offset = page_value
3022            .get("offset")
3023            .and_then(Value::as_u64)
3024            .unwrap_or(offset as u64) as usize;
3025        let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3026            record_session_history_backfill_failure(
3027                &inner,
3028                &watermark_runtime_key,
3029                &entry.runtime_key,
3030                &record.identity,
3031                &session_id,
3032                offset,
3033                "session history page missing messages".to_string(),
3034            )
3035            .await?;
3036            break;
3037        };
3038        if messages.is_empty() {
3039            if offset > 0 {
3040                record_session_history_watermark(
3041                    &inner,
3042                    &watermark_runtime_key,
3043                    &session_id,
3044                    offset,
3045                )
3046                .await?;
3047            }
3048            break;
3049        }
3050        for (idx, message) in messages.iter().enumerate() {
3051            let absolute_offset = base_offset + idx;
3052            let frames = frames_from_session_history_message_with_namespace(
3053                &entry.runtime_key,
3054                &record.identity,
3055                &entry.identity_namespace,
3056                &session_id,
3057                absolute_offset,
3058                message.clone(),
3059            );
3060            for mut frame in frames {
3061                if history_frame_has_existing_counterpart(&inner, &frame).await? {
3062                    continue;
3063                }
3064                if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3065                    frame.payload = redacted;
3066                    frame.status = ConsoleFrameStatus::Redacted;
3067                }
3068                append_and_emit(&inner, frame).await?;
3069            }
3070        }
3071        let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3072            // Forward-progress guard: a session store that returns a page whose
3073            // (base_offset + len) does not advance past the offset we requested
3074            // — e.g. a cursor that resets to 0 while still reporting has_more —
3075            // would make this loop re-read the same page forever, re-recording
3076            // the same watermark (a livelock observed on an orphaned gateway).
3077            // Stop instead of spinning; the last good watermark stands.
3078            tracing::warn!(
3079                session_id = %session_id,
3080                requested_offset = offset,
3081                page_base_offset = base_offset,
3082                page_len = messages.len(),
3083                "session history backfill made no forward progress; stopping to avoid a livelock"
3084            );
3085            break;
3086        };
3087        offset = next_offset;
3088        record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3089            .await?;
3090        let has_more = page_value
3091            .get("has_more")
3092            .and_then(Value::as_bool)
3093            .unwrap_or(false);
3094        if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3095            break;
3096        }
3097    }
3098    Ok(())
3099}
3100
3101/// Compute the next session-history backfill offset, enforcing strict forward
3102/// progress. Returns `None` when the page (`base_offset + page_len`) does not
3103/// advance past `requested_offset` — a non-advancing/looping cursor — which
3104/// signals the caller to stop rather than re-read the same page forever.
3105fn advance_backfill_offset(
3106    requested_offset: usize,
3107    base_offset: usize,
3108    page_len: usize,
3109) -> Option<usize> {
3110    let next = base_offset.saturating_add(page_len);
3111    (next > requested_offset).then_some(next)
3112}
3113
3114async fn record_session_history_watermark(
3115    inner: &AggregatorInner,
3116    watermark_runtime_key: &str,
3117    session_id: &str,
3118    offset: usize,
3119) -> ConsoleLogResult<()> {
3120    inner
3121        .store
3122        .record_source_watermark(
3123            watermark_runtime_key,
3124            ConsoleFrameSourceKind::SessionHistory,
3125            &format_session_history_watermark(session_id, offset, current_time_ms()),
3126        )
3127        .await
3128}
3129
3130fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3131    if !inner.options.session_history_backfill_enabled {
3132        return;
3133    }
3134    tokio::spawn(async move {
3135        {
3136            let mut active = inner.active_session_backfills.lock().await;
3137            if !active.insert(runtime_key.clone()) {
3138                return;
3139            }
3140        }
3141        let result = Box::pin(backfill_session_history(
3142            inner.clone(),
3143            runtime_key.clone(),
3144            false,
3145        ))
3146        .await;
3147        let mut active = inner.active_session_backfills.lock().await;
3148        active.remove(&runtime_key);
3149        drop(active);
3150        if let Err(err) = result {
3151            tracing::warn!(
3152                runtime_key = %runtime_key,
3153                error = %err,
3154                "console session-history backfill failed"
3155            );
3156        }
3157    });
3158}
3159
3160fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3161    if !inner.options.session_history_backfill_enabled {
3162        return;
3163    }
3164    tokio::spawn(async move {
3165        let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3166        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3167        loop {
3168            interval.tick().await;
3169            let runtime_still_registered = inner
3170                .runtimes
3171                .read()
3172                .ok()
3173                .is_some_and(|entries| entries.contains_key(&runtime_key));
3174            if !runtime_still_registered {
3175                break;
3176            }
3177            spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3178        }
3179    });
3180}
3181
3182fn spawn_session_history_backfill_target(
3183    inner: Arc<AggregatorInner>,
3184    target: SessionBackfillTarget,
3185    force_refresh: bool,
3186) {
3187    if !inner.options.session_history_backfill_enabled {
3188        return;
3189    }
3190    tokio::spawn(async move {
3191        let active_key = targeted_session_history_active_key(&target, force_refresh);
3192        let result =
3193            run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3194        if let Err(err) = result {
3195            tracing::warn!(
3196                active_key = %active_key,
3197                error = %err,
3198                "console targeted session-history backfill failed"
3199            );
3200        }
3201    });
3202}
3203
3204async fn run_targeted_session_history_backfill(
3205    inner: Arc<AggregatorInner>,
3206    target: SessionBackfillTarget,
3207    force_refresh: bool,
3208) -> ConsoleLogResult<()> {
3209    let active_key = targeted_session_history_active_key(&target, force_refresh);
3210    {
3211        let mut active = inner.active_session_backfills.lock().await;
3212        if !active.insert(active_key.clone()) {
3213            return Ok(());
3214        }
3215    }
3216    let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3217    let mut active = inner.active_session_backfills.lock().await;
3218    active.remove(&active_key);
3219    result
3220}
3221
3222fn targeted_session_history_active_key(
3223    target: &SessionBackfillTarget,
3224    force_refresh: bool,
3225) -> String {
3226    let mode = if force_refresh { "force" } else { "refresh" };
3227    format!(
3228        "{}:session-history:{}:{mode}",
3229        target.entry.runtime_key, target.session_id
3230    )
3231}
3232
3233fn spawn_session_history_backfill_for_identity(
3234    inner: Arc<AggregatorInner>,
3235    identity: String,
3236    force_refresh: bool,
3237) {
3238    if !inner.options.session_history_backfill_enabled {
3239        return;
3240    }
3241    tokio::spawn(async move {
3242        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3243            spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3244        }
3245    });
3246}
3247
3248fn spawn_opportunistic_session_history_backfill_for_identity(
3249    inner: Arc<AggregatorInner>,
3250    identity: String,
3251) {
3252    if !inner.options.session_history_backfill_enabled {
3253        return;
3254    }
3255    tokio::spawn(async move {
3256        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3257            let active_key = format!(
3258                "{}:session-history:{}",
3259                target.entry.runtime_key, target.session_id
3260            );
3261            {
3262                let mut seen = inner.opportunistic_session_backfills.lock().await;
3263                if !seen.insert(active_key) {
3264                    continue;
3265                }
3266            }
3267            spawn_session_history_backfill_target(inner.clone(), target, false);
3268        }
3269    });
3270}
3271
3272#[cfg(test)]
3273async fn session_backfill_target_for_identity(
3274    inner: &AggregatorInner,
3275    identity: &str,
3276) -> Option<SessionBackfillTarget> {
3277    session_backfill_targets_for_identity(inner, identity)
3278        .await
3279        .into_iter()
3280        .next()
3281}
3282
3283async fn session_backfill_targets_for_identity(
3284    inner: &AggregatorInner,
3285    identity: &str,
3286) -> Vec<SessionBackfillTarget> {
3287    let entries = inner
3288        .runtimes
3289        .read()
3290        .ok()
3291        .map(|entries| entries.clone())
3292        .unwrap_or_default();
3293    let mut targets = Vec::new();
3294    for entry in entries.values() {
3295        let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3296        if raw_identities.is_empty() {
3297            continue;
3298        }
3299        let mids = raw_identities
3300            .iter()
3301            .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3302            .collect::<Vec<_>>();
3303        for resolved in Box::pin(member_sources_for_entry(entry))
3304            .await
3305            .into_iter()
3306            .filter(|candidate| {
3307                resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3308            })
3309        {
3310            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3311                continue;
3312            };
3313            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3314                continue;
3315            }
3316            let Some(session_id) = record.session_id.clone() else {
3317                continue;
3318            };
3319            targets.push(SessionBackfillTarget {
3320                entry: entry.clone(),
3321                record,
3322                session_id,
3323            });
3324        }
3325    }
3326    targets
3327}
3328
3329fn resolved_member_matches_raw_identities(
3330    resolved: &ResolvedConsoleMember,
3331    raw_identities: &[String],
3332    mids: &[AgentIdentity],
3333) -> bool {
3334    mids.contains(&resolved.member.agent_identity)
3335        || resolved
3336            .member
3337            .labels
3338            .get("agent_identity")
3339            .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3340}
3341
3342async fn recover_lagged_source_events(
3343    inner: Arc<AggregatorInner>,
3344    runtime_key: &str,
3345    console_events: &ConsoleEventStore,
3346) -> ConsoleLogResult<()> {
3347    let watermark = inner
3348        .store
3349        .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3350        .await?;
3351    match console_events.replay_all(watermark.as_deref()).await {
3352        Ok(events) => {
3353            for envelope in events {
3354                project_console_event(inner.clone(), runtime_key, envelope).await?;
3355            }
3356        }
3357        Err(err) => {
3358            append_source_gap(
3359                &inner,
3360                runtime_key,
3361                format!(
3362                    "{}:{}:{}",
3363                    err.error, err.stream, err.requested_last_event_id
3364                ),
3365            )
3366            .await?;
3367        }
3368    }
3369    Ok(())
3370}
3371
3372async fn append_source_gap(
3373    inner: &AggregatorInner,
3374    runtime_key: &str,
3375    reason: String,
3376) -> ConsoleLogResult<()> {
3377    append_and_emit(
3378        inner,
3379        NewConsoleFrame {
3380            id: None,
3381            dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3382            timestamp_ms: current_time_ms(),
3383            runtime_key: runtime_key.to_string(),
3384            identity: "__console__".to_string(),
3385            conversation_id: None,
3386            session_id: None,
3387            kind: "replay_unavailable".to_string(),
3388            status: ConsoleFrameStatus::DeliveryFailed,
3389            payload: json!({
3390                "reason": reason,
3391                "source_kind": "console_event",
3392            }),
3393            source: ConsoleFrameSource {
3394                kind: ConsoleFrameSourceKind::Synthetic,
3395                source_cursor: None,
3396            },
3397            source_event_id: None,
3398            interaction_id: None,
3399            turn_id: None,
3400            run_id: None,
3401            parent_frame_id: None,
3402            caused_by_frame_id: None,
3403        },
3404    )
3405    .await?;
3406    let _ = inner
3407        .event_tx
3408        .send(ConsoleTimelineEvent::ReplayUnavailable {
3409            requested_cursor: format!("source-gap:{runtime_key}"),
3410            latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3411        });
3412    Ok(())
3413}
3414
3415/// Record a session-history backfill failure: stamp a watermark at the current
3416/// offset so the freshness gate throttles retries to once per
3417/// `SESSION_HISTORY_REFRESH_TTL_MS` (without it the 5s discovery loop re-reads
3418/// a persistently-failing session forever), and append a single gap frame.
3419async fn record_session_history_backfill_failure(
3420    inner: &AggregatorInner,
3421    watermark_runtime_key: &str,
3422    runtime_key: &str,
3423    identity: &str,
3424    session_id: &str,
3425    offset: usize,
3426    reason: String,
3427) -> ConsoleLogResult<()> {
3428    record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3429    append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3430}
3431
3432/// Stable dedupe key for a session-history backfill gap frame: a persistently
3433/// failing session collapses to a single gap frame across retries.
3434fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3435    format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3436}
3437
3438async fn append_backfill_gap(
3439    inner: &AggregatorInner,
3440    runtime_key: &str,
3441    identity: &str,
3442    session_id: &str,
3443    reason: String,
3444) -> ConsoleLogResult<()> {
3445    append_and_emit(
3446        inner,
3447        NewConsoleFrame {
3448            id: None,
3449            // Stable per (runtime, identity, session): repeated failures for
3450            // the same session collapse to one gap frame instead of minting a
3451            // fresh one (timestamped) on every retry — which grew the timeline
3452            // store unbounded.
3453            dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3454            timestamp_ms: current_time_ms(),
3455            runtime_key: runtime_key.to_string(),
3456            identity: identity.to_string(),
3457            conversation_id: Some(identity.to_string()),
3458            session_id: Some(session_id.to_string()),
3459            kind: "replay_unavailable".to_string(),
3460            status: ConsoleFrameStatus::DeliveryFailed,
3461            payload: json!({
3462                "reason": reason,
3463                "source_kind": "session_history",
3464            }),
3465            source: ConsoleFrameSource {
3466                kind: ConsoleFrameSourceKind::Synthetic,
3467                source_cursor: None,
3468            },
3469            source_event_id: None,
3470            interaction_id: None,
3471            turn_id: None,
3472            run_id: None,
3473            parent_frame_id: None,
3474            caused_by_frame_id: None,
3475        },
3476    )
3477    .await?;
3478    Ok(())
3479}
3480
3481async fn project_console_event(
3482    inner: Arc<AggregatorInner>,
3483    runtime_key: &str,
3484    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3485) -> ConsoleLogResult<()> {
3486    let Some(entry) = inner
3487        .runtimes
3488        .read()
3489        .ok()
3490        .and_then(|entries| entries.get(runtime_key).cloned())
3491    else {
3492        return Ok(());
3493    };
3494    let mut frame = frame_from_console_event(&entry, envelope);
3495    if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3496        frame.payload = redacted;
3497        frame.status = ConsoleFrameStatus::Redacted;
3498    }
3499    let source_cursor = frame
3500        .source_event_id
3501        .clone()
3502        .unwrap_or_else(|| frame.dedupe_key.clone());
3503    let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3504        Some(frame.identity.clone())
3505    } else {
3506        None
3507    };
3508    let opportunistic_refresh_identity = if refresh_identity.is_none()
3509        && console_event_should_start_session_history_backfill(&frame)
3510    {
3511        Some(frame.identity.clone())
3512    } else {
3513        None
3514    };
3515    append_and_emit(&inner, frame).await?;
3516    inner
3517        .store
3518        .record_source_watermark(
3519            &entry.runtime_key,
3520            ConsoleFrameSourceKind::ConsoleEvent,
3521            &source_cursor,
3522        )
3523        .await?;
3524    if let Some(identity) = refresh_identity {
3525        spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3526    } else if let Some(identity) = opportunistic_refresh_identity {
3527        spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3528    }
3529    Ok(())
3530}
3531
3532fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3533    matches!(
3534        frame.kind.as_str(),
3535        "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3536    ) || frame.session_id.is_some()
3537}
3538
3539fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3540    if frame.identity == SYSTEM_EVENT_IDENTITY {
3541        return false;
3542    }
3543    matches!(
3544        frame.kind.as_str(),
3545        "turn_started"
3546            | "run_started"
3547            | "reasoning_complete"
3548            | "tool_call_requested"
3549            | "tool_call"
3550            | "tool_execution_started"
3551            | "text_delta"
3552            | "system_notice"
3553    )
3554}
3555
3556async fn append_and_emit(
3557    inner: &AggregatorInner,
3558    frame: NewConsoleFrame,
3559) -> ConsoleLogResult<AppendOutcome> {
3560    let outcome = inner.store.append_if_absent(frame).await?;
3561    if outcome.disposition == AppendDisposition::Inserted {
3562        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3563            frame: outcome.frame.clone(),
3564        });
3565    }
3566    Ok(outcome)
3567}
3568
3569async fn update_frame_status_and_emit(
3570    inner: &AggregatorInner,
3571    frame_id: &str,
3572    status: ConsoleFrameStatus,
3573) -> ConsoleLogResult<Option<ConsoleFrame>> {
3574    let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3575        return Ok(None);
3576    };
3577    let update_marker = NewConsoleFrame {
3578        id: None,
3579        dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3580        timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3581        runtime_key: updated.runtime_key.clone(),
3582        identity: updated.identity.clone(),
3583        conversation_id: updated.conversation_id.clone(),
3584        session_id: updated.session_id.clone(),
3585        kind: "frame_updated".to_string(),
3586        status: updated.status,
3587        payload: json!({ "frame": updated.clone() }),
3588        source: ConsoleFrameSource {
3589            kind: ConsoleFrameSourceKind::Synthetic,
3590            source_cursor: None,
3591        },
3592        source_event_id: None,
3593        interaction_id: updated.interaction_id.clone(),
3594        turn_id: updated.turn_id.clone(),
3595        run_id: updated.run_id.clone(),
3596        parent_frame_id: Some(updated.id.clone()),
3597        caused_by_frame_id: Some(updated.id.clone()),
3598    };
3599    let outcome = inner.store.append_if_absent(update_marker).await?;
3600    if outcome.disposition == AppendDisposition::Inserted {
3601        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3602            frame: outcome.frame,
3603        });
3604    }
3605    Ok(Some(updated))
3606}
3607
3608fn frame_from_console_event(
3609    entry: &RuntimeEntry,
3610    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3611) -> NewConsoleFrame {
3612    let event_id = envelope.event_id;
3613    let interaction_id = envelope.interaction_id;
3614    let event_type = envelope.event_type;
3615    let payload = envelope.data;
3616    let turn_id = payload
3617        .get("turn_id")
3618        .and_then(Value::as_str)
3619        .map(ToString::to_string);
3620    let run_id = payload
3621        .get("run_id")
3622        .and_then(Value::as_str)
3623        .map(ToString::to_string);
3624    let status = match event_type.as_str() {
3625        "interaction_started" => ConsoleFrameStatus::Accepted,
3626        "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3627        "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3628        _ => ConsoleFrameStatus::Delivered,
3629    };
3630    let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3631    let dedupe_key = reasoning_payload_text(&event_type, &payload)
3632        .map(|reasoning_text| {
3633            // Reasoning hashes assume console events carry full/cumulative text; if deltas become
3634            // fragmentary, key by a stable reasoning segment id or emit only the complete frame.
3635            let turn_scope = interaction_id
3636                .as_deref()
3637                .or(turn_id.as_deref())
3638                .or(run_id.as_deref())
3639                .unwrap_or(event_id.as_str());
3640            format!(
3641                "console-reasoning:{}:{}:{}",
3642                entry.runtime_key,
3643                turn_scope,
3644                hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3645            )
3646        })
3647        .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3648    NewConsoleFrame {
3649        id: Some(event_id.clone()),
3650        dedupe_key,
3651        timestamp_ms: envelope.timestamp_ms,
3652        runtime_key: entry.runtime_key.clone(),
3653        identity: identity.clone(),
3654        conversation_id: Some(identity),
3655        session_id: payload
3656            .get("session_id")
3657            .and_then(Value::as_str)
3658            .map(ToString::to_string),
3659        kind: event_type,
3660        status,
3661        payload,
3662        source: ConsoleFrameSource {
3663            kind: ConsoleFrameSourceKind::ConsoleEvent,
3664            source_cursor: None,
3665        },
3666        source_event_id: Some(event_id),
3667        interaction_id,
3668        turn_id,
3669        run_id,
3670        parent_frame_id: None,
3671        caused_by_frame_id: None,
3672    }
3673}
3674
3675#[cfg(test)]
3676fn frame_from_session_history_message(
3677    runtime_key: &str,
3678    identity: &str,
3679    session_id: &str,
3680    offset: usize,
3681    message: Value,
3682) -> Option<NewConsoleFrame> {
3683    frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3684        .into_iter()
3685        .next()
3686}
3687
3688#[cfg(test)]
3689fn frames_from_session_history_message(
3690    runtime_key: &str,
3691    identity: &str,
3692    session_id: &str,
3693    offset: usize,
3694    message: Value,
3695) -> Vec<NewConsoleFrame> {
3696    frames_from_session_history_message_with_namespace(
3697        runtime_key,
3698        identity,
3699        "",
3700        session_id,
3701        offset,
3702        message,
3703    )
3704}
3705
3706/// Derive `assistant_image` frames from a session-history tool result's text,
3707/// mirroring the live ingest edge (`parse_generate_image_tool_result` ->
3708/// per-image `assistant_image` envelopes). The serialized
3709/// `ImageGenerationToolResult` is self-describing, so — unlike the live path —
3710/// no tool name is needed: we parse the result text directly. Returns an empty
3711/// vec for non-image tool results. The console dedupes images by blob/reference,
3712/// so an overlap with a live `assistant_image` frame collapses to one image.
3713#[allow(clippy::too_many_arguments)]
3714fn session_history_assistant_image_frames(
3715    runtime_key: &str,
3716    identity: &str,
3717    session_id: &str,
3718    offset: usize,
3719    result_idx: usize,
3720    timestamp_ms: u64,
3721    tool_use_id: &str,
3722    result_text: &str,
3723) -> Vec<NewConsoleFrame> {
3724    let Ok(image_result) = serde_json::from_str::<
3725        meerkat_core::image_generation::ImageGenerationToolResult,
3726    >(result_text) else {
3727        return Vec::new();
3728    };
3729    image_result
3730        .images
3731        .iter()
3732        .enumerate()
3733        .map(|(image_idx, image)| NewConsoleFrame {
3734            id: None,
3735            // Key on the stable image identity so a backfill frame and any live
3736            // `assistant_image` frame for the same image collapse on dedupe.
3737            dedupe_key: format!(
3738                "console-assistant-image:{runtime_key}:{}",
3739                image.blob_ref.blob_id
3740            ),
3741            timestamp_ms,
3742            runtime_key: runtime_key.to_string(),
3743            identity: identity.to_string(),
3744            conversation_id: Some(identity.to_string()),
3745            session_id: Some(session_id.to_string()),
3746            kind: "assistant_image".to_string(),
3747            status: ConsoleFrameStatus::Completed,
3748            payload: json!({
3749                "source_event_type": "session_history",
3750                "tool_call_id": tool_use_id,
3751                "image_id": image.image_id.0.to_string(),
3752                "blob_id": image.blob_ref.blob_id,
3753                "media_type": image.media_type.as_str(),
3754                "width": image.width,
3755                "height": image.height,
3756                "revised_prompt": image_result.revised_prompt.clone(),
3757                "type": "session_history",
3758            }),
3759            source: ConsoleFrameSource {
3760                kind: ConsoleFrameSourceKind::SessionHistory,
3761                source_cursor: Some(format!(
3762                    "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3763                )),
3764            },
3765            source_event_id: None,
3766            interaction_id: None,
3767            turn_id: None,
3768            run_id: None,
3769            parent_frame_id: None,
3770            caused_by_frame_id: None,
3771        })
3772        .collect()
3773}
3774
3775fn frames_from_session_history_message_with_namespace(
3776    runtime_key: &str,
3777    identity: &str,
3778    identity_namespace: &str,
3779    session_id: &str,
3780    offset: usize,
3781    message: Value,
3782) -> Vec<NewConsoleFrame> {
3783    let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3784    let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3785        return Vec::new();
3786    };
3787    if let Message::ToolResults {
3788        results,
3789        created_at,
3790    } = parsed
3791    {
3792        return results
3793            .into_iter()
3794            .enumerate()
3795            .flat_map(|(idx, result)| {
3796                let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3797                let result_text = result.text_content();
3798                let tool_use_id = result.tool_use_id.clone();
3799                let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3800                let mut frames = vec![NewConsoleFrame {
3801                    id: None,
3802                    dedupe_key: format!(
3803                        "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3804                    ),
3805                    timestamp_ms,
3806                    runtime_key: runtime_key.to_string(),
3807                    identity: identity.to_string(),
3808                    conversation_id: Some(identity.to_string()),
3809                    session_id: Some(session_id.to_string()),
3810                    kind: "tool_execution_completed".to_string(),
3811                    status: ConsoleFrameStatus::Completed,
3812                    payload: json!({
3813                        "id": tool_use_id,
3814                        "tool_call_id": tool_use_id,
3815                        "result": result_text,
3816                        "content": content,
3817                        "is_error": result.is_error,
3818                        "source_event_type": "session_history",
3819                        "type": "session_history",
3820                    }),
3821                    source: ConsoleFrameSource {
3822                        kind: ConsoleFrameSourceKind::SessionHistory,
3823                        source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3824                    },
3825                    source_event_id: None,
3826                    interaction_id: None,
3827                    turn_id: None,
3828                    run_id: None,
3829                    parent_frame_id: None,
3830                    caused_by_frame_id: None,
3831                }];
3832                // Backfill parity with the live ingest edge: a generate_image
3833                // tool result serializes an `ImageGenerationToolResult` into the
3834                // result text (which carries NO tool name on meerkat 0.7.x). The
3835                // live path emits per-image `assistant_image` frames; without
3836                // this the image renders as raw tool JSON after a reload /
3837                // history backfill. Parse the result directly and mirror those
3838                // frames (the console dedupes images by blob/reference).
3839                frames.extend(session_history_assistant_image_frames(
3840                    runtime_key,
3841                    identity,
3842                    session_id,
3843                    offset,
3844                    idx,
3845                    timestamp_ms,
3846                    &tool_use_id,
3847                    &result_text,
3848                ));
3849                frames
3850            })
3851            .collect();
3852    }
3853    let (kind, timestamp_ms, payload) = match &parsed {
3854        Message::User(user) => {
3855            if session_history_user_message_is_scaffold(&message) {
3856                return Vec::new();
3857            }
3858            (
3859                "user_input",
3860                user.created_at.timestamp_millis().max(0) as u64,
3861                json!({
3862                    "content": user.content,
3863                    "message": message,
3864                }),
3865            )
3866        }
3867        // Meerkat 0.7 removed the legacy flat-text `Message::Assistant`
3868        // variant; all assistant history is block-shaped now.
3869        Message::BlockAssistant(assistant) => {
3870            let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3871            (
3872                "interaction_complete",
3873                assistant.created_at.timestamp_millis().max(0) as u64,
3874                json!({
3875                    "result": text,
3876                    "text": text,
3877                    "message": message,
3878                    "source_event_type": "session_history",
3879                    "type": "session_history",
3880                }),
3881            )
3882        }
3883        Message::SystemNotice(notice) => (
3884            "system_notice",
3885            notice.created_at.timestamp_millis().max(0) as u64,
3886            json!({
3887                "message": message,
3888                "kind": notice.kind,
3889                "render_class": notice.kind.render_class(),
3890                "body": notice.body,
3891                "blocks": notice.blocks,
3892                "source_event_type": "session_history",
3893                "type": "session_history",
3894            }),
3895        ),
3896        Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3897    };
3898    let mut frames = vec![NewConsoleFrame {
3899        id: None,
3900        dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3901        timestamp_ms,
3902        runtime_key: runtime_key.to_string(),
3903        identity: identity.to_string(),
3904        conversation_id: Some(identity.to_string()),
3905        session_id: Some(session_id.to_string()),
3906        kind: kind.to_string(),
3907        status: ConsoleFrameStatus::Completed,
3908        payload,
3909        source: ConsoleFrameSource {
3910            kind: ConsoleFrameSourceKind::SessionHistory,
3911            source_cursor: Some(format!("{session_id}:{offset}")),
3912        },
3913        source_event_id: None,
3914        interaction_id: None,
3915        turn_id: None,
3916        run_id: None,
3917        parent_frame_id: None,
3918        caused_by_frame_id: None,
3919    }];
3920    if let Message::BlockAssistant(assistant) = &parsed {
3921        frames.extend(spawn_initial_message_frames_from_assistant(
3922            runtime_key,
3923            identity,
3924            identity_namespace,
3925            session_id,
3926            offset,
3927            assistant,
3928            &payload_hash,
3929        ));
3930    }
3931    frames
3932}
3933
3934fn spawn_initial_message_frames_from_assistant(
3935    runtime_key: &str,
3936    parent_identity: &str,
3937    identity_namespace: &str,
3938    session_id: &str,
3939    offset: usize,
3940    assistant: &meerkat_core::types::BlockAssistantMessage,
3941    payload_hash: &str,
3942) -> Vec<NewConsoleFrame> {
3943    let mut frames = Vec::new();
3944    for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3945        if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3946            continue;
3947        }
3948        let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3949            continue;
3950        };
3951        for (spawn_idx, (target_identity, initial_message)) in
3952            spawn_initial_messages_from_tool_args(&args)
3953                .into_iter()
3954                .enumerate()
3955        {
3956            let target_identity = apply_namespace(&target_identity, identity_namespace);
3957            let message_content = match &initial_message {
3958                Value::String(text) => json!([
3959                    {
3960                        "type": "text",
3961                        "text": text,
3962                    }
3963                ]),
3964                other => other.clone(),
3965            };
3966            let message = match &initial_message {
3967                Value::String(text) => json!({
3968                    "role": "user",
3969                    "content": text,
3970                    "created_at": assistant.created_at,
3971                }),
3972                other => json!({
3973                    "role": "user",
3974                    "content": other,
3975                    "created_at": assistant.created_at,
3976                }),
3977            };
3978            frames.push(NewConsoleFrame {
3979                id: None,
3980                dedupe_key: format!(
3981                    "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3982                ),
3983                timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3984                runtime_key: runtime_key.to_string(),
3985                identity: target_identity.clone(),
3986                conversation_id: Some(target_identity),
3987                session_id: None,
3988                kind: "user_input".to_string(),
3989                status: ConsoleFrameStatus::Completed,
3990                payload: json!({
3991                    "content": message_content,
3992                    "message": message,
3993                    "source_event_type": "session_history_spawn_initial_message",
3994                    "type": "session_history",
3995                    "tool_call_id": tool.id,
3996                    "parent_identity": parent_identity,
3997                    "via_tool": tool.name,
3998                }),
3999                source: ConsoleFrameSource {
4000                    kind: ConsoleFrameSourceKind::SessionHistory,
4001                    source_cursor: Some(format!(
4002                        "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4003                    )),
4004                },
4005                source_event_id: None,
4006                interaction_id: None,
4007                turn_id: None,
4008                run_id: None,
4009                parent_frame_id: None,
4010                caused_by_frame_id: None,
4011            });
4012        }
4013    }
4014    frames
4015}
4016
4017fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4018    let mut messages = Vec::new();
4019    if let Some(message) = spawn_initial_message_from_record(args) {
4020        messages.push(message);
4021    }
4022    if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4023        for spec in specs {
4024            if let Some(message) = spawn_initial_message_from_record(spec) {
4025                messages.push(message);
4026            }
4027        }
4028    }
4029    messages
4030}
4031
4032fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4033    let target_identity = record
4034        .get("member_id")
4035        .or_else(|| record.get("agent_identity"))
4036        .and_then(Value::as_str)?
4037        .trim();
4038    if target_identity.is_empty() {
4039        return None;
4040    }
4041    let initial_message = record
4042        .get("initial_message")
4043        .or_else(|| record.get("task"))?
4044        .clone();
4045    Some((target_identity.to_string(), initial_message))
4046}
4047
4048fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4049    message
4050        .get("content")
4051        .is_some_and(scaffold_message_content_is_noise)
4052}
4053
4054fn scaffold_message_content_is_noise(value: &Value) -> bool {
4055    match value {
4056        Value::String(text) => scaffold_message_text_is_noise(text),
4057        Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4058        Value::Object(map) => ["text", "content", "message"]
4059            .iter()
4060            .filter_map(|key| map.get(*key))
4061            .any(scaffold_message_content_is_noise),
4062        _ => false,
4063    }
4064}
4065
4066fn scaffold_message_text_is_noise(text: &str) -> bool {
4067    let trimmed = text.trim_start();
4068    trimmed.starts_with("[PEER UPDATE]")
4069        || trimmed
4070            .to_ascii_lowercase()
4071            .starts_with("you have been spawned")
4072}
4073
4074fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4075    let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4076    rest.split(':').next()?.parse().ok()
4077}
4078
4079fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4080    format!("{session_id}:{offset}:{checked_at_ms}")
4081}
4082
4083fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4084    let Some(checked_at_ms) = watermark
4085        .rsplit_once(':')
4086        .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4087    else {
4088        return false;
4089    };
4090    let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4091    let ttl_ms = if offset > 0 {
4092        SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4093    } else {
4094        SESSION_HISTORY_REFRESH_TTL_MS
4095    };
4096    now_ms.saturating_sub(checked_at_ms) < ttl_ms
4097}
4098
4099async fn history_frame_has_existing_counterpart(
4100    inner: &AggregatorInner,
4101    frame: &NewConsoleFrame,
4102) -> ConsoleLogResult<bool> {
4103    let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4104    let Some(fingerprint) = fingerprint else {
4105        return Ok(false);
4106    };
4107    let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4108    let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4109    let mut after = None;
4110    loop {
4111        let page = inner
4112            .store
4113            .query_frames(ConsoleTimelineQuery {
4114                identity: Some(frame.identity.clone()),
4115                conversation_id: frame.conversation_id.clone(),
4116                after,
4117                limit: 1_000,
4118            })
4119            .await?;
4120        for existing in &page.frames {
4121            let same_session = existing.session_id == frame.session_id
4122                || existing.session_id.is_none()
4123                || frame.session_id.is_none();
4124            if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4125                continue;
4126            }
4127            if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4128                == Some(&fingerprint)
4129            {
4130                return Ok(true);
4131            }
4132            if assistant_terminal
4133                && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4134            {
4135                let turn_key = existing
4136                    .interaction_id
4137                    .as_deref()
4138                    .or(existing.turn_id.as_deref())
4139                    .or(existing.run_id.as_deref())
4140                    .unwrap_or("session");
4141                let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4142                aggregated.push_str(delta);
4143                if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4144                    return Ok(true);
4145                }
4146            }
4147        }
4148        if page.frames.is_empty() || page.next_cursor.is_none() {
4149            return Ok(false);
4150        }
4151        after = page.next_cursor;
4152    }
4153}
4154
4155fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4156    format!("{runtime_key}:session-history:{session_id}")
4157}
4158
4159fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4160    match kind {
4161        "user_input" | "interaction_started" => payload
4162            .get("content")
4163            .map(stable_value_fingerprint)
4164            .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4165        "tool_execution_completed" => tool_result_fingerprint(payload),
4166        "text_delta" => {
4167            text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4168        }
4169        kind if is_reasoning_event_kind(kind) => {
4170            reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4171        }
4172        "text_complete" | "interaction_complete" | "run_completed" => {
4173            assistant_terminal_fingerprint(kind, payload)
4174        }
4175        _ => None,
4176    }
4177}
4178
4179fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4180    let id = payload
4181        .get("tool_call_id")
4182        .or_else(|| payload.get("id"))
4183        .and_then(Value::as_str)
4184        .unwrap_or("");
4185    let result = payload
4186        .get("result")
4187        .or_else(|| payload.get("content"))
4188        .map(stable_value_fingerprint)?;
4189    Some(format!("{id}:{result}"))
4190}
4191
4192fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4193    match kind {
4194        "text_complete" | "interaction_complete" | "run_completed" => payload
4195            .get("text")
4196            .or_else(|| payload.get("result"))
4197            .or_else(|| payload.get("content"))
4198            .map(stable_value_fingerprint),
4199        _ => None,
4200    }
4201}
4202
4203fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4204    if kind != "text_delta" {
4205        return None;
4206    }
4207    payload
4208        .get("delta")
4209        .or_else(|| payload.get("text"))
4210        .or_else(|| payload.get("content"))
4211        .and_then(Value::as_str)
4212        .or_else(|| payload.as_str())
4213}
4214
4215fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4216    if !is_reasoning_event_kind(kind) {
4217        return None;
4218    }
4219    if kind == "reasoning_delta" {
4220        return payload
4221            .get("delta")
4222            .or_else(|| payload.get("text"))
4223            .or_else(|| payload.get("content"))
4224            .or_else(|| payload.get("summary"))
4225            .or_else(|| payload.get("reasoning"))
4226            .and_then(Value::as_str)
4227            .or_else(|| payload.as_str());
4228    }
4229    payload
4230        .get("summary")
4231        .or_else(|| payload.get("text"))
4232        .or_else(|| payload.get("content"))
4233        .or_else(|| payload.get("delta"))
4234        .or_else(|| payload.get("reasoning"))
4235        .and_then(Value::as_str)
4236        .or_else(|| payload.as_str())
4237}
4238
4239fn is_reasoning_event_kind(kind: &str) -> bool {
4240    matches!(
4241        kind,
4242        "reasoning_delta"
4243            | "reasoning_complete"
4244            | "reasoning_summary"
4245            | "reasoning_summary_delta"
4246            | "reasoning_summary_complete"
4247    )
4248}
4249
4250fn stable_value_fingerprint(value: &Value) -> String {
4251    if let Some(text) = content_value_text(value) {
4252        return normalize_transcript_fingerprint_text(&text);
4253    }
4254    match value {
4255        Value::String(text) => normalize_transcript_fingerprint_text(text),
4256        other => serde_json::to_string(other).unwrap_or_default(),
4257    }
4258}
4259
4260fn content_value_text(value: &Value) -> Option<String> {
4261    match value {
4262        Value::String(text) => Some(text.clone()),
4263        Value::Array(items) => {
4264            let text = items
4265                .iter()
4266                .filter_map(content_value_text)
4267                .collect::<String>();
4268            (!text.is_empty()).then_some(text)
4269        }
4270        Value::Object(map) => ["text", "content", "message", "blocks"]
4271            .iter()
4272            .filter_map(|key| map.get(*key))
4273            .find_map(content_value_text),
4274        _ => None,
4275    }
4276}
4277
4278fn normalize_transcript_fingerprint_text(text: &str) -> String {
4279    let trimmed = text.trim();
4280    trimmed
4281        .strip_prefix("[EVENT via rpc] ")
4282        .unwrap_or(trimmed)
4283        .trim()
4284        .to_string()
4285}
4286
4287#[derive(Debug, Clone, Copy)]
4288enum CachedIdentityVisibility {
4289    Visible,
4290    Hidden,
4291    Missing,
4292}
4293
4294async fn frame_is_visible(
4295    inner: &AggregatorInner,
4296    frame: &ConsoleFrame,
4297    allow_historical_identity: bool,
4298    identity_records: &[ConsoleIdentityRecord],
4299) -> ConsoleLogResult<bool> {
4300    let mut identity_visibility_cache = HashMap::new();
4301    Box::pin(frame_is_visible_cached(
4302        inner,
4303        frame,
4304        allow_historical_identity,
4305        &mut identity_visibility_cache,
4306        identity_records,
4307    ))
4308    .await
4309}
4310
4311async fn frame_is_visible_cached(
4312    inner: &AggregatorInner,
4313    frame: &ConsoleFrame,
4314    allow_historical_identity: bool,
4315    identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4316    identity_records: &[ConsoleIdentityRecord],
4317) -> ConsoleLogResult<bool> {
4318    let entry = {
4319        let entries = inner
4320            .runtimes
4321            .read()
4322            .map_err(|_| runtime_registry_lock_error())?;
4323        if entries.is_empty() {
4324            return Ok(true);
4325        }
4326        let Some(entry) = entries.get(&frame.runtime_key) else {
4327            return Ok(false);
4328        };
4329        entry.clone()
4330    };
4331    if frame.identity != "__console__" {
4332        let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4333        let identity_visibility =
4334            if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4335                cached
4336            } else {
4337                let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4338                    .unwrap_or_else(|| frame.identity.clone());
4339                let visibility = match identity_records.iter().find(|record| {
4340                    record.runtime_key == frame.runtime_key
4341                        && (record.identity == frame.identity
4342                            || record.runtime_member_id == runtime_member_id)
4343                }) {
4344                    Some(record) => {
4345                        if Box::pin(console_identity_record_visible(&entry, record)).await {
4346                            CachedIdentityVisibility::Visible
4347                        } else {
4348                            CachedIdentityVisibility::Hidden
4349                        }
4350                    }
4351                    None => CachedIdentityVisibility::Missing,
4352                };
4353                identity_visibility_cache.insert(cache_key, visibility);
4354                visibility
4355            };
4356        match identity_visibility {
4357            CachedIdentityVisibility::Visible => {
4358                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4359                    return Ok(false);
4360                }
4361            }
4362            CachedIdentityVisibility::Hidden => return Ok(false),
4363            CachedIdentityVisibility::Missing => {
4364                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4365                    return Ok(false);
4366                }
4367                return Ok(
4368                    allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4369                );
4370            }
4371        }
4372    }
4373    Ok(entry.visibility_policy.frame_visible(frame))
4374}
4375
4376async fn identity_record_for_member(
4377    entry: &RuntimeEntry,
4378    handle: &MobHandle,
4379    member: &MobMemberListEntry,
4380) -> Option<ConsoleIdentityRecord> {
4381    // Roster ids are comms-safe encodings (meerkat 0.7 MemberCommsName);
4382    // identity records carry the public alias.
4383    let runtime_member_id =
4384        crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4385    let durable_identity = member
4386        .labels
4387        .get("agent_identity")
4388        .filter(|value| !value.trim().is_empty())
4389        .map_or(runtime_member_id.as_str(), String::as_str);
4390    let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4391    let addressable = member_is_addressable(member);
4392    let visibility = match member.status {
4393        meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4394            ConsoleVisibility::RetiredReadable
4395        }
4396        // Broken/unknown members have no live runtime binding; mirror the
4397        // identity-first projection, which marks them unreachable.
4398        meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4399            ConsoleVisibility::Unreachable
4400        }
4401        _ if addressable => ConsoleVisibility::Addressable,
4402        _ => ConsoleVisibility::Hidden,
4403    };
4404    let session_id = handle
4405        .resolve_bridge_session_id_observation(&member.agent_identity)
4406        .await
4407        .map(|sid| sid.to_string());
4408    let mut labels = member.labels.clone();
4409    labels
4410        .entry("role".to_string())
4411        .or_insert_with(|| member.role.to_string());
4412    // Spawn-time console metadata (group/display labels, spawned_by lineage)
4413    // registered by the agent-tool spawn path fills gaps the roster does not
4414    // carry; roster labels win on conflict — except lineage, which only the
4415    // spawn registry may assert (these records feed aggregate-mode ABAC
4416    // attribute priming).
4417    crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4418    if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4419        crate::console_spawn::merge_registered_labels(&mut labels, &registered);
4420    }
4421    let display_name = labels
4422        .get("display_name")
4423        .cloned()
4424        .unwrap_or_else(|| runtime_member_id.clone());
4425    Some(ConsoleIdentityRecord {
4426        identity,
4427        display_name,
4428        runtime_key: entry.runtime_key.clone(),
4429        runtime_member_id,
4430        session_id,
4431        visibility,
4432        addressable,
4433        health: if addressable {
4434            "ready"
4435        } else {
4436            "hidden_by_policy"
4437        }
4438        .to_string(),
4439        topology_peers: member
4440            .wired_to
4441            .iter()
4442            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4443            .collect(),
4444        labels,
4445    })
4446}
4447
4448async fn identity_record_for_resolved_member(
4449    resolved: &ResolvedConsoleMember,
4450) -> Option<ConsoleIdentityRecord> {
4451    let mut record =
4452        identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4453    if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4454        record
4455            .labels
4456            .entry("source_mob_id".to_string())
4457            .or_insert_with(|| resolved.source_mob_id.clone());
4458    }
4459    Some(record)
4460}
4461
4462fn identity_record_for_status(
4463    entry: &RuntimeEntry,
4464    status: &crate::identity_first::IdentityStatus,
4465) -> ConsoleIdentityRecord {
4466    let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4467    let runtime_member_id = status
4468        .agent_runtime_id
4469        .as_ref()
4470        .map(crate::identity_first::AgentRuntimeId::as_str)
4471        .unwrap_or_else(|| status.identity.as_str())
4472        .to_string();
4473    let addressable = status.addressability
4474        == crate::identity_first::AgentAddressability::Addressable
4475        && matches!(
4476            status.state,
4477            crate::identity_first::IdentityLifecycleState::Active
4478                | crate::identity_first::IdentityLifecycleState::Dormant
4479                | crate::identity_first::IdentityLifecycleState::Uninitialized
4480        );
4481    let visibility = match status.state {
4482        crate::identity_first::IdentityLifecycleState::Retiring => {
4483            ConsoleVisibility::RetiredReadable
4484        }
4485        crate::identity_first::IdentityLifecycleState::Broken
4486        | crate::identity_first::IdentityLifecycleState::Suspended => {
4487            ConsoleVisibility::Unreachable
4488        }
4489        _ if addressable => ConsoleVisibility::Addressable,
4490        _ => ConsoleVisibility::Hidden,
4491    };
4492    let health = match status.state {
4493        crate::identity_first::IdentityLifecycleState::Active => "ready",
4494        crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4495        crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4496        crate::identity_first::IdentityLifecycleState::Broken => "broken",
4497        crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4498        crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4499    }
4500    .to_string();
4501    let display_name = status
4502        .display_name
4503        .as_ref()
4504        .map(crate::identity_first::DisplayName::as_str)
4505        .unwrap_or_else(|| status.identity.as_str())
4506        .to_string();
4507    ConsoleIdentityRecord {
4508        identity,
4509        display_name,
4510        runtime_key: entry.runtime_key.clone(),
4511        runtime_member_id,
4512        session_id: status.session_id.as_ref().map(ToString::to_string),
4513        visibility,
4514        addressable,
4515        health,
4516        topology_peers: Vec::new(),
4517        labels: status.labels.clone(),
4518    }
4519}
4520
4521pub(crate) fn is_implicit_delegate_member(
4522    role: &str,
4523    labels: &std::collections::BTreeMap<String, String>,
4524) -> bool {
4525    role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4526}
4527
4528fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4529    member
4530        .labels
4531        .get("addressable")
4532        .map(|value| !value.eq_ignore_ascii_case("false"))
4533        .unwrap_or(true)
4534}
4535
4536fn apply_namespace(identity: &str, namespace: &str) -> String {
4537    let namespace = namespace.trim().trim_matches('/');
4538    if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4539        identity.to_string()
4540    } else {
4541        format!("{namespace}/{identity}")
4542    }
4543}
4544
4545fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4546    let namespace = namespace.trim().trim_matches('/');
4547    if namespace.is_empty() {
4548        return Some(identity.to_string());
4549    }
4550    identity
4551        .strip_prefix(namespace)
4552        .and_then(|rest| rest.strip_prefix('/'))
4553        .map(ToString::to_string)
4554}
4555
4556fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4557    let namespace = namespace.trim().trim_matches('/');
4558    if namespace.is_empty() {
4559        return vec![identity.to_string()];
4560    }
4561    let mut candidates = Vec::new();
4562    if let Some(stripped) = strip_namespace(identity, namespace) {
4563        candidates.push(stripped);
4564        candidates.push(identity.to_string());
4565    }
4566    candidates.sort();
4567    candidates.dedup();
4568    candidates
4569}
4570
4571fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4572    namespace_match_candidates(identity, namespace)
4573        .iter()
4574        .any(|candidate| candidate.starts_with("rt:"))
4575}
4576
4577fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4578    entries.iter().any(|entry| {
4579        requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4580    })
4581}
4582
4583fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4584    if request.identity.trim().is_empty() {
4585        return Err(ConsoleSendError::InvalidRequest(
4586            "identity must be non-empty".to_string(),
4587        ));
4588    }
4589    if request.origin.trim().is_empty() {
4590        return Err(ConsoleSendError::InvalidRequest(
4591            "origin must be non-empty".to_string(),
4592        ));
4593    }
4594    if request.idempotency_key.trim().is_empty() {
4595        return Err(ConsoleSendError::InvalidRequest(
4596            "idempotency_key must be non-empty".to_string(),
4597        ));
4598    }
4599    Ok(())
4600}
4601
4602fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4603    let content: ContentInput = serde_json::from_value(value.clone())
4604        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4605    match &content {
4606        ContentInput::Text(text) if text.trim().is_empty() => Err(
4607            ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4608        ),
4609        ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4610            "content blocks must be non-empty".to_string(),
4611        )),
4612        _ => Ok(content),
4613    }
4614}
4615
4616fn parse_handling_mode(
4617    value: Option<&str>,
4618) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4619    match value.unwrap_or("queue") {
4620        "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4621        "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4622        other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4623    }
4624}
4625
4626fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4627    ConsoleInteractionAccepted {
4628        interaction_id: frame
4629            .interaction_id
4630            .clone()
4631            .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4632        identity: frame.identity.clone(),
4633        conversation_id: frame.conversation_id.clone(),
4634        session_id: frame.session_id.clone(),
4635        input_frame_id: frame.id.clone(),
4636        cursor: frame.cursor.clone(),
4637        status: frame.status,
4638    }
4639}
4640
4641fn send_dedupe_key(
4642    runtime_key: &str,
4643    identity: &str,
4644    origin: &str,
4645    idempotency_key: &str,
4646) -> String {
4647    format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4648}
4649
4650fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4651    let content_json = serde_json::to_string(content).unwrap_or_default();
4652    hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4653}
4654
4655fn hash_short(value: &str) -> String {
4656    let mut hasher = Sha256::new();
4657    hasher.update(value.as_bytes());
4658    let digest = hasher.finalize();
4659    to_hex(&digest[..8])
4660}
4661
4662fn to_hex(bytes: &[u8]) -> String {
4663    const HEX: &[u8; 16] = b"0123456789abcdef";
4664    let mut out = String::with_capacity(bytes.len() * 2);
4665    for byte in bytes {
4666        out.push(HEX[(byte >> 4) as usize] as char);
4667        out.push(HEX[(byte & 0x0f) as usize] as char);
4668    }
4669    out
4670}
4671
4672fn current_time_ms() -> u64 {
4673    match SystemTime::now().duration_since(UNIX_EPOCH) {
4674        Ok(duration) => duration.as_millis() as u64,
4675        Err(_) => 0,
4676    }
4677}
4678
4679fn runtime_registry_lock_error() -> ConsoleLogError {
4680    Box::new(std::io::Error::other(
4681        "console runtime registry lock poisoned",
4682    ))
4683}
4684
4685#[cfg(test)]
4686#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4687mod tests {
4688    /// Regression: a session that keeps failing backfill must collapse to ONE
4689    /// gap frame. The dedupe key is stable per (runtime, identity, session) —
4690    /// independent of wall-clock — so repeated failures don't mint a fresh
4691    /// timestamped frame each retry and grow the timeline store unbounded.
4692    #[test]
4693    fn backfill_gap_dedupe_key_is_stable_per_session() {
4694        use super::backfill_gap_dedupe_key;
4695        let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4696        let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4697        assert_eq!(a, b, "same target must reuse one gap frame");
4698        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4699        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4700        // The key carries no wall-clock component (the pre-fix bug appended
4701        // current_time_ms, so every retry was a unique frame).
4702        assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4703    }
4704
4705    use std::sync::Arc;
4706    use std::sync::atomic::{AtomicUsize, Ordering};
4707    use std::time::Duration;
4708    use std::time::Instant;
4709
4710    use futures::StreamExt;
4711    use meerkat::{AgentFactory, Config, build_ephemeral_service};
4712    use meerkat_client::types::LlmStream;
4713    use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4714    use meerkat_core::{
4715        AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4716        RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4717        SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4718        SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4719        StreamError,
4720    };
4721    use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4722    use serde_json::json;
4723
4724    use super::*;
4725    use crate::mob_handle_runtime::MobBootstrapSpec;
4726
4727    #[derive(Debug)]
4728    struct HideHiddenNoiseFrames;
4729
4730    impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4731        fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4732            frame.kind != "hidden_noise"
4733        }
4734    }
4735
4736    #[derive(Debug)]
4737    struct HideRuntimeMemberIdentity(&'static str);
4738
4739    impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4740        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4741            record.runtime_member_id != self.0
4742        }
4743    }
4744
4745    #[derive(Debug)]
4746    struct HideRuntimeMemberOnly(&'static str);
4747
4748    impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4749        fn member_visible(&self, member: &ConsoleMember) -> bool {
4750            member.agent_identity != self.0
4751        }
4752    }
4753
4754    #[derive(Debug)]
4755    struct HideConsoleIdentity(&'static str);
4756
4757    impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4758        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4759            record.identity != self.0
4760        }
4761    }
4762
4763    struct CountingConsoleLogStore {
4764        inner: InMemoryConsoleLogStore,
4765        source_watermark_calls: AtomicUsize,
4766        record_watermark_calls: AtomicUsize,
4767    }
4768
4769    impl CountingConsoleLogStore {
4770        fn new() -> Self {
4771            Self {
4772                inner: InMemoryConsoleLogStore::new(),
4773                source_watermark_calls: AtomicUsize::new(0),
4774                record_watermark_calls: AtomicUsize::new(0),
4775            }
4776        }
4777
4778        fn source_watermark_calls(&self) -> usize {
4779            self.source_watermark_calls.load(Ordering::SeqCst)
4780        }
4781    }
4782
4783    struct SlowTestClient {
4784        delay: Duration,
4785    }
4786
4787    #[async_trait::async_trait]
4788    impl LlmClient for SlowTestClient {
4789        fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4790            Ok(messages.to_vec())
4791        }
4792
4793        fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4794            let delay = self.delay;
4795            let delayed_text = futures::stream::once(async move {
4796                tokio::time::sleep(delay).await;
4797                Ok(LlmEvent::TextDelta {
4798                    delta: "slow ok".to_string(),
4799                    meta: None,
4800                })
4801            });
4802            let done = futures::stream::once(async {
4803                Ok(LlmEvent::Done {
4804                    outcome: LlmDoneOutcome::Success {
4805                        stop_reason: StopReason::EndTurn,
4806                    },
4807                })
4808            });
4809            Box::pin(delayed_text.chain(done))
4810        }
4811
4812        fn provider(&self) -> meerkat_core::Provider {
4813            meerkat_core::Provider::Other
4814        }
4815
4816        async fn health_check(&self) -> Result<(), LlmError> {
4817            Ok(())
4818        }
4819    }
4820
4821    #[derive(Clone)]
4822    struct DelayedHistorySessionService {
4823        inner: Arc<dyn MobSessionService>,
4824        delay: Duration,
4825        read_calls: Arc<AtomicUsize>,
4826        active_reads: Arc<AtomicUsize>,
4827        max_active_reads: Arc<AtomicUsize>,
4828    }
4829
4830    impl DelayedHistorySessionService {
4831        fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4832            Self {
4833                inner,
4834                delay,
4835                read_calls: Arc::new(AtomicUsize::new(0)),
4836                active_reads: Arc::new(AtomicUsize::new(0)),
4837                max_active_reads: Arc::new(AtomicUsize::new(0)),
4838            }
4839        }
4840
4841        fn read_calls(&self) -> usize {
4842            self.read_calls.load(Ordering::SeqCst)
4843        }
4844
4845        fn max_active_reads(&self) -> usize {
4846            self.max_active_reads.load(Ordering::SeqCst)
4847        }
4848    }
4849
4850    #[async_trait::async_trait]
4851    impl SessionService for DelayedHistorySessionService {
4852        async fn create_session(
4853            &self,
4854            req: meerkat_core::CreateSessionRequest,
4855        ) -> Result<RunResult, SessionError> {
4856            self.inner.create_session(req).await
4857        }
4858
4859        async fn start_turn(
4860            &self,
4861            id: &SessionId,
4862            req: StartTurnRequest,
4863        ) -> Result<RunResult, SessionError> {
4864            self.inner.start_turn(id, req).await
4865        }
4866
4867        async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4868            self.inner.interrupt(id).await
4869        }
4870
4871        async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4872            self.inner.read(id).await
4873        }
4874
4875        async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4876            self.inner.list(query).await
4877        }
4878
4879        async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4880            self.inner.archive(id).await
4881        }
4882
4883        async fn subscribe_session_events(
4884            &self,
4885            id: &SessionId,
4886        ) -> Result<EventStream, StreamError> {
4887            SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4888        }
4889    }
4890
4891    #[async_trait::async_trait]
4892    impl SessionServiceCommsExt for DelayedHistorySessionService {
4893        async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4894            self.inner.comms_runtime(session_id).await
4895        }
4896    }
4897
4898    #[async_trait::async_trait]
4899    impl SessionServiceControlExt for DelayedHistorySessionService {
4900        async fn append_system_context(
4901            &self,
4902            id: &SessionId,
4903            req: AppendSystemContextRequest,
4904        ) -> Result<AppendSystemContextResult, SessionControlError> {
4905            self.inner.append_system_context(id, req).await
4906        }
4907    }
4908
4909    #[async_trait::async_trait]
4910    impl SessionServiceHistoryExt for DelayedHistorySessionService {
4911        async fn read_history(
4912            &self,
4913            id: &SessionId,
4914            query: SessionHistoryQuery,
4915        ) -> Result<SessionHistoryPage, SessionError> {
4916            self.read_calls.fetch_add(1, Ordering::SeqCst);
4917            let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4918            self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4919            tokio::time::sleep(self.delay).await;
4920            let result = self.inner.read_history(id, query).await;
4921            self.active_reads.fetch_sub(1, Ordering::SeqCst);
4922            result
4923        }
4924    }
4925
4926    #[async_trait::async_trait]
4927    impl MobSessionService for DelayedHistorySessionService {
4928        fn supports_persistent_sessions(&self) -> bool {
4929            self.inner.supports_persistent_sessions()
4930        }
4931
4932        fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4933            self.inner.runtime_adapter()
4934        }
4935
4936        async fn session_belongs_to_mob(
4937            &self,
4938            session_id: &SessionId,
4939            mob_id: &meerkat_mob::MobId,
4940        ) -> bool {
4941            self.inner.session_belongs_to_mob(session_id, mob_id).await
4942        }
4943
4944        async fn cancel_all_checkpointers(&self) {
4945            self.inner.cancel_all_checkpointers().await;
4946        }
4947
4948        async fn rearm_all_checkpointers(&self) {
4949            self.inner.rearm_all_checkpointers().await;
4950        }
4951    }
4952
4953    #[async_trait::async_trait]
4954    impl ConsoleLogStore for CountingConsoleLogStore {
4955        async fn append_if_absent(
4956            &self,
4957            frame: NewConsoleFrame,
4958        ) -> ConsoleLogResult<AppendOutcome> {
4959            self.inner.append_if_absent(frame).await
4960        }
4961
4962        async fn update_frame_status(
4963            &self,
4964            frame_id: &str,
4965            status: ConsoleFrameStatus,
4966        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4967            self.inner.update_frame_status(frame_id, status).await
4968        }
4969
4970        async fn query_frames(
4971            &self,
4972            query: ConsoleTimelineQuery,
4973        ) -> ConsoleLogResult<ConsoleTimelinePage> {
4974            self.inner.query_frames(query).await
4975        }
4976
4977        async fn query_windowed_frames(
4978            &self,
4979            query: ConsoleTimelineWindowQuery,
4980        ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4981            self.inner.query_windowed_frames(query).await
4982        }
4983
4984        async fn frame_by_dedupe_key(
4985            &self,
4986            dedupe_key: &str,
4987        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4988            self.inner.frame_by_dedupe_key(dedupe_key).await
4989        }
4990
4991        async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4992            self.inner.latest_cursor().await
4993        }
4994
4995        async fn clear_frames(&self) -> ConsoleLogResult<()> {
4996            self.inner.clear_frames().await
4997        }
4998
4999        async fn record_source_watermark(
5000            &self,
5001            runtime_key: &str,
5002            source_kind: ConsoleFrameSourceKind,
5003            source_cursor: &str,
5004        ) -> ConsoleLogResult<()> {
5005            self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5006            self.inner
5007                .record_source_watermark(runtime_key, source_kind, source_cursor)
5008                .await
5009        }
5010
5011        async fn source_watermark(
5012            &self,
5013            runtime_key: &str,
5014            source_kind: ConsoleFrameSourceKind,
5015        ) -> ConsoleLogResult<Option<String>> {
5016            self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5017            self.inner.source_watermark(runtime_key, source_kind).await
5018        }
5019    }
5020
5021    async fn build_single_member_runtime() -> UnifiedRuntime {
5022        build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5023    }
5024
5025    async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5026        let definition = MobDefinition::from_toml(
5027            r#"
5028[mob]
5029id = "console-aggregator-perf-test"
5030
5031[profiles.worker]
5032model = "gpt-5.5"
5033external_addressable = true
5034
5035[profiles.worker.tools]
5036comms = true
5037"#,
5038        )
5039        .expect("definition parses");
5040        let runtime = UnifiedRuntime::builder()
5041            .definition(definition)
5042            .default_llm_client(client)
5043            .build()
5044            .await
5045            .expect("runtime builds");
5046        runtime
5047            .spawn(SpawnMemberSpec::from_wire(
5048                "worker".to_string(),
5049                "agent-a".to_string(),
5050                Some("You are agent-a.".into()),
5051                None,
5052                None,
5053            ))
5054            .await
5055            .expect("member spawns");
5056        runtime
5057    }
5058
5059    async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5060        let definition = MobDefinition::from_toml(&format!(
5061            r#"
5062[mob]
5063id = "{mob_id}"
5064
5065[profiles.worker]
5066model = "gpt-5.5"
5067external_addressable = true
5068
5069[profiles.worker.tools]
5070comms = true
5071"#
5072        ))
5073        .expect("definition parses");
5074        UnifiedRuntime::builder()
5075            .definition(definition)
5076            .default_llm_client(Arc::new(TestClient::default()))
5077            .build()
5078            .await
5079            .expect("runtime builds")
5080    }
5081
5082    async fn build_stress_runtime(
5083        member_count: usize,
5084        history_delay: Duration,
5085    ) -> (
5086        tempfile::TempDir,
5087        Arc<UnifiedRuntime>,
5088        DelayedHistorySessionService,
5089    ) {
5090        let temp_dir = tempfile::tempdir().expect("temp dir");
5091        let session_path = temp_dir.path().join("sessions");
5092        std::fs::create_dir_all(&session_path).expect("session path");
5093        let factory = AgentFactory::new(&session_path).comms(true);
5094        let base_service = Arc::new(build_ephemeral_service(
5095            factory,
5096            Config::default(),
5097            member_count + 8,
5098        ));
5099        let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5100        let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5101        let definition = MobDefinition::from_toml(
5102            r#"
5103[mob]
5104id = "console-aggregator-stress-test"
5105
5106[profiles.worker]
5107model = "gpt-5.5"
5108external_addressable = true
5109
5110[profiles.worker.tools]
5111comms = true
5112"#,
5113        )
5114        .expect("definition parses");
5115        let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5116            .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5117                allow_ephemeral_sessions: true,
5118                notify_orchestrator_on_resume: true,
5119                default_llm_client: Some(Arc::new(TestClient::default())),
5120            });
5121        let runtime = Arc::new(
5122            UnifiedRuntime::bootstrap(
5123                spec,
5124                crate::types::MobKitConfig {
5125                    modules: Vec::new(),
5126                    discovery: crate::types::DiscoverySpec {
5127                        namespace: "stress".to_string(),
5128                        modules: Vec::new(),
5129                    },
5130                    pre_spawn: Vec::new(),
5131                },
5132                Duration::from_secs(5),
5133            )
5134            .await
5135            .expect("runtime boots"),
5136        );
5137        for idx in 0..member_count {
5138            runtime
5139                .spawn(SpawnMemberSpec::from_wire(
5140                    "worker".to_string(),
5141                    format!("agent-{idx}"),
5142                    Some(format!("You are agent-{idx}.").into()),
5143                    None,
5144                    None,
5145                ))
5146                .await
5147                .expect("member spawns");
5148        }
5149        (temp_dir, runtime, delayed_service)
5150    }
5151
5152    fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5153        RuntimeEntry {
5154            runtime_key: runtime_key.to_string(),
5155            identity_namespace: "test".to_string(),
5156            runtime: runtime.mob_runtime().clone(),
5157            identity_runtime: runtime.identity_runtime().cloned(),
5158            console_events: runtime.console_events(),
5159            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5160        }
5161    }
5162
5163    fn console_envelope_for_test(
5164        event_id: &str,
5165        interaction_id: Option<&str>,
5166        event_type: &str,
5167        data: Value,
5168    ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5169        crate::console_contracts::ConsoleIdentityEventEnvelope {
5170            event_id: event_id.to_string(),
5171            interaction_id: interaction_id.map(ToString::to_string),
5172            identity: "agent-a".to_string(),
5173            event_type: event_type.to_string(),
5174            timestamp_ms: 1_000,
5175            data,
5176        }
5177    }
5178
5179    fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5180        ConsoleIdentityRecord {
5181            identity: identity.to_string(),
5182            display_name: identity.to_string(),
5183            runtime_key: "runtime-cache".to_string(),
5184            runtime_member_id: identity.to_string(),
5185            session_id: Some(format!("session-{identity}")),
5186            visibility: ConsoleVisibility::Addressable,
5187            addressable: true,
5188            health: "ready".to_string(),
5189            topology_peers: Vec::new(),
5190            labels: BTreeMap::new(),
5191        }
5192    }
5193
5194    #[test]
5195    fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5196        let mut durable = identity_record_for_test("test/review:singleton");
5197        durable.runtime_member_id = "rt:review:singleton:1".to_string();
5198        durable.session_id = Some("session-stale".to_string());
5199        durable
5200            .labels
5201            .insert("agent_identity".to_string(), "review:singleton".to_string());
5202
5203        let mut live = identity_record_for_test("test/review:singleton");
5204        live.runtime_member_id = "rt:review:singleton:0".to_string();
5205        live.session_id = Some("session-live".to_string());
5206        live.labels
5207            .insert("agent_identity".to_string(), "review:singleton".to_string());
5208
5209        let records = dedupe_identity_records(vec![durable, live]);
5210
5211        assert_eq!(
5212            records.len(),
5213            2,
5214            "list projection must expose stale durable/live alias split-brain instead of picking one record"
5215        );
5216        assert!(
5217            records
5218                .iter()
5219                .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5220        );
5221        assert!(
5222            records
5223                .iter()
5224                .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5225        );
5226    }
5227
5228    #[test]
5229    fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5230        let mut first = identity_record_for_test("test/review:singleton");
5231        first.runtime_member_id = "rt:review:singleton:0".to_string();
5232        first
5233            .labels
5234            .insert("agent_identity".to_string(), "review:singleton".to_string());
5235
5236        let mut second = identity_record_for_test("test/review:singleton");
5237        second.runtime_member_id = "rt:review:singleton:1".to_string();
5238        second
5239            .labels
5240            .insert("agent_identity".to_string(), "review:singleton".to_string());
5241
5242        let records = dedupe_identity_records(vec![first, second]);
5243
5244        assert_eq!(
5245            records.len(),
5246            2,
5247            "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5248        );
5249    }
5250
5251    #[test]
5252    fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5253        let mut first = identity_record_for_test("test/review:singleton");
5254        first.runtime_member_id = "rt:review:singleton:0".to_string();
5255        first.session_id = None;
5256        first
5257            .labels
5258            .insert("agent_identity".to_string(), "review:singleton".to_string());
5259        first
5260            .labels
5261            .insert("source_mob_id".to_string(), "mob-a".to_string());
5262
5263        let mut second = first.clone();
5264        second
5265            .labels
5266            .insert("source_mob_id".to_string(), "mob-b".to_string());
5267
5268        let records = dedupe_identity_records(vec![first, second]);
5269
5270        assert_eq!(
5271            records.len(),
5272            2,
5273            "same member/session aliases from different source mobs must remain visible"
5274        );
5275    }
5276
5277    #[test]
5278    fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5279        let durable = identity_record_for_test("test/review:singleton");
5280        let live = durable.clone();
5281
5282        let records = dedupe_identity_records(vec![durable, live]);
5283
5284        assert_eq!(
5285            records.len(),
5286            1,
5287            "healthy durable and primary live projections should not duplicate the roster"
5288        );
5289    }
5290
5291    #[test]
5292    fn stale_durable_record_error_detects_session_split_brain() {
5293        let mut durable = identity_record_for_test("test/review:singleton");
5294        durable.runtime_member_id = "rt:review:singleton:0".to_string();
5295        durable.session_id = Some("session-durable".to_string());
5296
5297        let mut live = durable.clone();
5298        live.session_id = Some("session-live".to_string());
5299
5300        let err = stale_durable_record_error("review:singleton", &durable, &[live])
5301            .expect("session mismatch must be stale split-brain");
5302
5303        assert!(
5304            err.contains("stale durable identity alias"),
5305            "unexpected error: {err}"
5306        );
5307        assert!(
5308            err.contains("session-durable") && err.contains("session-live"),
5309            "error should name both sessions: {err}"
5310        );
5311    }
5312
5313    async fn identity_runtime_for_test(
5314        identities: &[&str],
5315    ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5316    {
5317        let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5318            crate::identity_first::IdentityRuntimeConfig {
5319                continuity_store: Arc::new(
5320                    crate::identity_first::LocalContinuityStore::in_memory()?
5321                ),
5322                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5323                runtime_instance_id: "console-aggregator-identity-test".to_string(),
5324                has_runtime_store: true,
5325                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5326                bridge: None,
5327                default_timeout: None,
5328            },
5329        ));
5330        for identity in identities {
5331            let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5332            let record = crate::identity_first::ContinuityRecord {
5333                identity: identity.clone(),
5334                agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5335                    "rt:{}:0",
5336                    identity.as_str()
5337                ))?,
5338                session_id: SessionId::new(),
5339                generation: crate::identity_first::ContinuityGeneration::new(0),
5340                checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5341            };
5342            runtime
5343                .register(
5344                    crate::identity_first::DurableAgentSpec {
5345                        identity: identity.clone(),
5346                        profile: meerkat_mob::ProfileName::from("worker"),
5347                        addressability: crate::identity_first::AgentAddressability::Addressable,
5348                        display_name: None,
5349                        labels: BTreeMap::new(),
5350                        context: None,
5351                        additional_instructions: Vec::new(),
5352                        initial_message: None,
5353                        runtime_mode_override: None,
5354                        backend: None,
5355                        binding: None,
5356                    },
5357                    crate::identity_first::IdentityLifecycleState::Active,
5358                    Some(record),
5359                    Some(crate::identity_first::LeaseGrant {
5360                        identity,
5361                        fencing_token: crate::identity_first::FencingToken::new(1),
5362                        ttl: Duration::from_mins(5),
5363                    }),
5364                )
5365                .await;
5366        }
5367        Ok(runtime)
5368    }
5369
5370    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5371    async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5372    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5373        let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5374        let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5375        let aggregator = MobKitConsoleAggregator::in_memory();
5376        aggregator.register_runtime_handles_with_policy(
5377            "runtime-a",
5378            "",
5379            runtime.mob_runtime().clone(),
5380            Some(identity_runtime),
5381            runtime.console_events(),
5382            Arc::new(AllowAllConsoleVisibilityPolicy),
5383        );
5384
5385        assert!(
5386            aggregator
5387                .inspect_identity("rt:review:singleton:0")
5388                .await?
5389                .is_none(),
5390            "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5391        );
5392        assert!(
5393            !aggregator.retire_identity("rt:review:singleton:0").await?,
5394            "runtime-member-id shaped requests must not retire a durable rt:* identity"
5395        );
5396        let reserve_err = aggregator
5397            .reserve_identity_first_interaction(
5398                ConsoleSendRequest {
5399                    identity: "rt:review:singleton:0".to_string(),
5400                    content: json!("hello"),
5401                    origin: "console:test".to_string(),
5402                    idempotency_key: "rt-shadow-reserve".to_string(),
5403                    handling_mode: Some("queue".to_string()),
5404                },
5405                None,
5406            )
5407            .await
5408            .expect_err(
5409                "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5410            );
5411        assert!(
5412            reserve_err.to_string().contains("unknown identity"),
5413            "unexpected reserve error: {reserve_err}"
5414        );
5415        let send_err = aggregator
5416            .send(ConsoleSendRequest {
5417                identity: "rt:review:singleton:0".to_string(),
5418                content: json!("hello"),
5419                origin: "console:test".to_string(),
5420                idempotency_key: "rt-shadow-send".to_string(),
5421                handling_mode: Some("queue".to_string()),
5422            })
5423            .await
5424            .expect_err(
5425                "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5426            );
5427        assert!(
5428            send_err.to_string().contains("unknown identity"),
5429            "unexpected send error: {send_err}"
5430        );
5431        let blob_err = match aggregator
5432            .binary_blob_store_for_identity("rt:review:singleton:0")
5433            .await
5434        {
5435            Ok(_) => panic!(
5436                "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5437            ),
5438            Err(err) => err,
5439        };
5440        assert!(
5441            blob_err.to_string().contains("unknown identity"),
5442            "unexpected blob error: {blob_err}"
5443        );
5444
5445        let _ = runtime.mob_handle().stop().await;
5446        Ok(())
5447    }
5448
5449    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5450    async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5451    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5452        let runtime = build_empty_runtime("identity-label-test").await;
5453        let mut labels = BTreeMap::new();
5454        labels.insert(
5455            "agent_identity".to_string(),
5456            "channel:C0SMOKEOB3".to_string(),
5457        );
5458        labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5459        runtime
5460            .spawn(
5461                SpawnMemberSpec::from_wire(
5462                    "worker".to_string(),
5463                    "rt:channel:C0SMOKEOB3:0".to_string(),
5464                    Some("You are C0SMOKEOB3.".into()),
5465                    None,
5466                    None,
5467                )
5468                .with_labels(labels),
5469            )
5470            .await
5471            .expect("member spawns");
5472
5473        let entry = RuntimeEntry {
5474            runtime_key: "runtime-a".to_string(),
5475            identity_namespace: String::new(),
5476            runtime: runtime.mob_runtime().clone(),
5477            identity_runtime: runtime.identity_runtime().cloned(),
5478            console_events: runtime.console_events(),
5479            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5480        };
5481        let aggregator = MobKitConsoleAggregator::in_memory();
5482        aggregator
5483            .inner
5484            .runtimes
5485            .write()
5486            .expect("runtime registry")
5487            .insert("runtime-a".to_string(), entry);
5488
5489        let records = aggregator.list_identities_fresh().await?;
5490        let record = records
5491            .iter()
5492            .find(|record| record.identity == "channel:C0SMOKEOB3")
5493            .expect("durable identity is exposed");
5494        assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5495
5496        let inspection = aggregator
5497            .inspect_identity("channel:C0SMOKEOB3")
5498            .await?
5499            .expect("durable identity resolves back to runtime member");
5500        assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5501        assert_eq!(
5502            inspection.identity.runtime_member_id,
5503            "rt:channel:C0SMOKEOB3:0"
5504        );
5505
5506        let _ = runtime.mob_handle().stop().await;
5507        Ok(())
5508    }
5509
5510    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5511    async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
5512    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5513        let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
5514        let mut labels = BTreeMap::new();
5515        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5516        runtime
5517            .spawn(
5518                SpawnMemberSpec::from_wire(
5519                    "worker".to_string(),
5520                    "rt:review:singleton:0".to_string(),
5521                    Some("You are the Review Agent.".into()),
5522                    None,
5523                    None,
5524                )
5525                .with_labels(labels),
5526            )
5527            .await?;
5528        let hidden_session_id = runtime
5529            .mob_handle()
5530            .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
5531                "rt:review:singleton:0",
5532            ))
5533            .await
5534            .map(|sid| sid.to_string());
5535        let mut duplicate_labels = BTreeMap::new();
5536        duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5537        runtime
5538            .spawn(
5539                SpawnMemberSpec::from_wire(
5540                    "worker".to_string(),
5541                    "rt:review:singleton:1".to_string(),
5542                    Some("You are a visible duplicate Review Agent.".into()),
5543                    None,
5544                    None,
5545                )
5546                .with_labels(duplicate_labels),
5547            )
5548            .await?;
5549
5550        let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5551        let aggregator = MobKitConsoleAggregator::in_memory();
5552        aggregator.register_runtime_handles_with_policy(
5553            "runtime-a",
5554            "",
5555            runtime.mob_runtime().clone(),
5556            Some(identity_runtime),
5557            runtime.console_events(),
5558            Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5559        );
5560
5561        assert!(
5562            aggregator
5563                .inspect_identity("review:singleton")
5564                .await?
5565                .is_none(),
5566            "member-hidden live alias must not inspect through aggregator"
5567        );
5568        let records = aggregator.list_identities_fresh().await?;
5569        assert!(
5570            records
5571                .iter()
5572                .all(|record| record.identity != "review:singleton"),
5573            "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5574        );
5575        let appended = aggregator
5576            .store()
5577            .append_if_absent(NewConsoleFrame {
5578                id: None,
5579                dedupe_key: "member-hidden-frame".to_string(),
5580                timestamp_ms: 1,
5581                runtime_key: "runtime-a".to_string(),
5582                identity: "review:singleton".to_string(),
5583                conversation_id: Some("review:singleton".to_string()),
5584                session_id: hidden_session_id,
5585                kind: "text_delta".to_string(),
5586                status: ConsoleFrameStatus::Delivered,
5587                payload: json!({ "delta": "hidden" }),
5588                source: ConsoleFrameSource {
5589                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5590                    source_cursor: None,
5591                },
5592                source_event_id: Some("member-hidden-frame".to_string()),
5593                interaction_id: None,
5594                turn_id: None,
5595                run_id: None,
5596                parent_frame_id: None,
5597                caused_by_frame_id: None,
5598            })
5599            .await?;
5600        assert!(
5601            !aggregator
5602                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5603                    frame: appended.frame.clone()
5604                })
5605                .await,
5606            "member-hidden live alias must not pass timeline event visibility"
5607        );
5608        let identity_only = aggregator
5609            .store()
5610            .append_if_absent(NewConsoleFrame {
5611                id: None,
5612                dedupe_key: "member-hidden-identity-only-frame".to_string(),
5613                timestamp_ms: 2,
5614                runtime_key: "runtime-a".to_string(),
5615                identity: "review:singleton".to_string(),
5616                conversation_id: Some("review:singleton".to_string()),
5617                session_id: None,
5618                kind: "text_delta".to_string(),
5619                status: ConsoleFrameStatus::Delivered,
5620                payload: json!({ "delta": "identity-only-hidden" }),
5621                source: ConsoleFrameSource {
5622                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5623                    source_cursor: None,
5624                },
5625                source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5626                interaction_id: None,
5627                turn_id: None,
5628                run_id: None,
5629                parent_frame_id: None,
5630                caused_by_frame_id: None,
5631            })
5632            .await?;
5633        assert!(
5634            !aggregator
5635                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5636                    frame: identity_only.frame.clone()
5637                })
5638                .await,
5639            "member-hidden durable alias must not leak identity-only timeline frames"
5640        );
5641        let page = aggregator
5642            .query_timeline(ConsoleTimelineQuery {
5643                identity: Some("review:singleton".to_string()),
5644                limit: 10,
5645                ..ConsoleTimelineQuery::default()
5646            })
5647            .await?;
5648        assert!(
5649            page.frames.is_empty(),
5650            "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5651        );
5652        assert!(
5653            !aggregator.retire_identity("review:singleton").await?,
5654            "member-hidden live alias must not retire through aggregator"
5655        );
5656        let reserve_err = aggregator
5657            .reserve_identity_first_interaction(
5658                ConsoleSendRequest {
5659                    identity: "review:singleton".to_string(),
5660                    content: json!("hello"),
5661                    origin: "console:test".to_string(),
5662                    idempotency_key: "member-hidden-reserve".to_string(),
5663                    handling_mode: Some("queue".to_string()),
5664                },
5665                None,
5666            )
5667            .await
5668            .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5669        assert!(
5670            reserve_err.to_string().contains("unknown identity"),
5671            "unexpected reserve error: {reserve_err}"
5672        );
5673        let send_err = aggregator
5674            .send(ConsoleSendRequest {
5675                identity: "review:singleton".to_string(),
5676                content: json!("hello"),
5677                origin: "console:test".to_string(),
5678                idempotency_key: "member-hidden-send".to_string(),
5679                handling_mode: Some("queue".to_string()),
5680            })
5681            .await
5682            .expect_err("member-hidden live alias must not receive sends");
5683        assert!(
5684            send_err.to_string().contains("unknown identity"),
5685            "unexpected send error: {send_err}"
5686        );
5687        let blob_result = aggregator
5688            .binary_blob_store_for_identity("review:singleton")
5689            .await;
5690        let Err(blob_err) = blob_result else {
5691            panic!("member-hidden live alias must not expose blob store");
5692        };
5693        assert!(
5694            blob_err.to_string().contains("unknown identity"),
5695            "unexpected blob error: {blob_err}"
5696        );
5697
5698        let _ = runtime.mob_handle().stop().await;
5699        Ok(())
5700    }
5701
5702    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5703    async fn explicit_session_backfill_target_resolves_durable_identity_label()
5704    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5705        let runtime = build_empty_runtime("identity-label-backfill-test").await;
5706        let mut labels = BTreeMap::new();
5707        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5708        runtime
5709            .spawn(
5710                SpawnMemberSpec::from_wire(
5711                    "worker".to_string(),
5712                    "rt:review:singleton:0".to_string(),
5713                    Some("You are the Review Agent.".into()),
5714                    None,
5715                    None,
5716                )
5717                .with_labels(labels),
5718            )
5719            .await
5720            .expect("member spawns");
5721
5722        let entry = RuntimeEntry {
5723            runtime_key: "runtime-a".to_string(),
5724            identity_namespace: String::new(),
5725            runtime: runtime.mob_runtime().clone(),
5726            identity_runtime: runtime.identity_runtime().cloned(),
5727            console_events: runtime.console_events(),
5728            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5729        };
5730        let aggregator = MobKitConsoleAggregator::in_memory();
5731        aggregator
5732            .inner
5733            .runtimes
5734            .write()
5735            .expect("runtime registry")
5736            .insert("runtime-a".to_string(), entry);
5737
5738        let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5739            .await
5740            .expect("durable label should resolve targeted session backfill");
5741        assert_eq!(target.record.identity, "review:singleton");
5742        assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5743
5744        let _ = runtime.mob_handle().stop().await;
5745        Ok(())
5746    }
5747
5748    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5749    async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5750    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5751        let runtime = build_empty_runtime("identity-stale-durable-test").await;
5752        let mut labels = BTreeMap::new();
5753        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5754        runtime
5755            .spawn(
5756                SpawnMemberSpec::from_wire(
5757                    "worker".to_string(),
5758                    "rt:review:singleton:0".to_string(),
5759                    Some("You are the live Review Agent.".into()),
5760                    None,
5761                    None,
5762                )
5763                .with_labels(labels),
5764            )
5765            .await
5766            .expect("member spawns");
5767
5768        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5769            crate::identity_first::IdentityRuntimeConfig {
5770                continuity_store: Arc::new(
5771                    crate::identity_first::LocalContinuityStore::in_memory()?
5772                ),
5773                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5774                runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5775                has_runtime_store: true,
5776                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5777                bridge: None,
5778                default_timeout: None,
5779            },
5780        ));
5781        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5782        let record = crate::identity_first::ContinuityRecord {
5783            identity: identity.clone(),
5784            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5785                "rt:review:singleton:1",
5786            )?,
5787            session_id: SessionId::new(),
5788            generation: crate::identity_first::ContinuityGeneration::new(1),
5789            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5790        };
5791        identity_runtime
5792            .register(
5793                crate::identity_first::DurableAgentSpec {
5794                    identity: identity.clone(),
5795                    profile: meerkat_mob::ProfileName::from("worker"),
5796                    addressability: crate::identity_first::AgentAddressability::Addressable,
5797                    display_name: None,
5798                    labels: BTreeMap::new(),
5799                    context: None,
5800                    additional_instructions: Vec::new(),
5801                    initial_message: None,
5802                    runtime_mode_override: None,
5803                    backend: None,
5804                    binding: None,
5805                },
5806                crate::identity_first::IdentityLifecycleState::Active,
5807                Some(record),
5808                Some(crate::identity_first::LeaseGrant {
5809                    identity,
5810                    fencing_token: crate::identity_first::FencingToken::new(7),
5811                    ttl: Duration::from_mins(5),
5812                }),
5813            )
5814            .await;
5815
5816        let aggregator = MobKitConsoleAggregator::in_memory();
5817        aggregator
5818            .inner
5819            .runtimes
5820            .write()
5821            .expect("runtime registry")
5822            .insert(
5823                "runtime-a".to_string(),
5824                RuntimeEntry {
5825                    runtime_key: "runtime-a".to_string(),
5826                    identity_namespace: String::new(),
5827                    runtime: runtime.mob_runtime().clone(),
5828                    identity_runtime: Some(identity_runtime),
5829                    console_events: runtime.console_events(),
5830                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5831                },
5832            );
5833
5834        let err = aggregator
5835            .inspect_identity("review:singleton")
5836            .await
5837            .expect_err("stale durable binding must not inspect as healthy");
5838        assert!(
5839            err.to_string().contains("stale durable identity alias"),
5840            "unexpected error: {err}"
5841        );
5842        let retire_err = aggregator
5843            .retire_identity("review:singleton")
5844            .await
5845            .expect_err("stale durable binding must not retire as healthy");
5846        assert!(
5847            retire_err
5848                .to_string()
5849                .contains("stale durable identity alias"),
5850            "unexpected retire error: {retire_err}"
5851        );
5852        let send_err = aggregator
5853            .send(ConsoleSendRequest {
5854                identity: "review:singleton".to_string(),
5855                content: json!("hello"),
5856                origin: "console:test".to_string(),
5857                idempotency_key: "stale-durable-send".to_string(),
5858                handling_mode: Some("queue".to_string()),
5859            })
5860            .await
5861            .expect_err("stale durable binding must not send to the wrong live member");
5862        assert!(
5863            send_err
5864                .to_string()
5865                .contains("stale durable identity alias"),
5866            "unexpected send error: {send_err}"
5867        );
5868        let reserve_err = aggregator
5869            .reserve_identity_first_interaction(
5870                ConsoleSendRequest {
5871                    identity: "review:singleton".to_string(),
5872                    content: json!("hello"),
5873                    origin: "console:test".to_string(),
5874                    idempotency_key: "stale-durable-reserve".to_string(),
5875                    handling_mode: Some("queue".to_string()),
5876                },
5877                None,
5878            )
5879            .await
5880            .expect_err("identity-first reserve must not bypass stale durable binding");
5881        assert!(
5882            reserve_err
5883                .to_string()
5884                .contains("stale durable identity alias"),
5885            "unexpected reserve error: {reserve_err}"
5886        );
5887        let blob_err = match aggregator
5888            .binary_blob_store_for_identity("review:singleton")
5889            .await
5890        {
5891            Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5892            Err(err) => err,
5893        };
5894        assert!(
5895            blob_err
5896                .to_string()
5897                .contains("stale durable identity alias"),
5898            "unexpected blob error: {blob_err}"
5899        );
5900        let records = aggregator.list_identities_fresh().await?;
5901        assert!(
5902            records
5903                .iter()
5904                .any(|record| record.identity == "review:singleton"
5905                    && record.runtime_member_id == "rt:review:singleton:0"),
5906            "live alias should still be listed; records: {records:#?}"
5907        );
5908        assert!(
5909            records
5910                .iter()
5911                .any(|record| record.identity == "review:singleton"
5912                    && record.runtime_member_id == "rt:review:singleton:1"),
5913            "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5914        );
5915
5916        let _ = runtime.mob_handle().stop().await;
5917        Ok(())
5918    }
5919
5920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5921    async fn identity_first_alias_self_heals_session_only_split_brain()
5922    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5923        let runtime = build_empty_runtime("identity-session-rebind-test").await;
5924        let mut labels = BTreeMap::new();
5925        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5926        runtime
5927            .spawn(
5928                SpawnMemberSpec::from_wire(
5929                    "worker".to_string(),
5930                    "rt:review:singleton:0".to_string(),
5931                    Some("You are the live Review Agent.".into()),
5932                    None,
5933                    None,
5934                )
5935                .with_labels(labels),
5936            )
5937            .await
5938            .expect("member spawns");
5939        let live_session_id = runtime
5940            .mob_handle()
5941            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
5942                "rt:review:singleton:0",
5943            ))
5944            .await
5945            .expect("live member has bridge session");
5946
5947        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5948            crate::identity_first::IdentityRuntimeConfig {
5949                continuity_store: Arc::new(
5950                    crate::identity_first::LocalContinuityStore::in_memory()?
5951                ),
5952                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5953                runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
5954                has_runtime_store: true,
5955                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5956                bridge: None,
5957                default_timeout: None,
5958            },
5959        ));
5960        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5961        let stale_session_id = SessionId::new();
5962        assert_ne!(stale_session_id, live_session_id);
5963        let record = crate::identity_first::ContinuityRecord {
5964            identity: identity.clone(),
5965            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5966                "rt:review:singleton:0",
5967            )?,
5968            session_id: stale_session_id,
5969            generation: crate::identity_first::ContinuityGeneration::new(1),
5970            checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
5971        };
5972        identity_runtime
5973            .register(
5974                crate::identity_first::DurableAgentSpec {
5975                    identity: identity.clone(),
5976                    profile: meerkat_mob::ProfileName::from("worker"),
5977                    addressability: crate::identity_first::AgentAddressability::Addressable,
5978                    display_name: None,
5979                    labels: BTreeMap::new(),
5980                    context: None,
5981                    additional_instructions: Vec::new(),
5982                    initial_message: None,
5983                    runtime_mode_override: None,
5984                    backend: None,
5985                    binding: None,
5986                },
5987                crate::identity_first::IdentityLifecycleState::Active,
5988                Some(record),
5989                Some(crate::identity_first::LeaseGrant {
5990                    identity: identity.clone(),
5991                    fencing_token: crate::identity_first::FencingToken::new(7),
5992                    ttl: Duration::from_mins(5),
5993                }),
5994            )
5995            .await;
5996
5997        let aggregator = MobKitConsoleAggregator::in_memory();
5998        aggregator
5999            .inner
6000            .runtimes
6001            .write()
6002            .expect("runtime registry")
6003            .insert(
6004                "runtime-a".to_string(),
6005                RuntimeEntry {
6006                    runtime_key: "runtime-a".to_string(),
6007                    identity_namespace: String::new(),
6008                    runtime: runtime.mob_runtime().clone(),
6009                    identity_runtime: Some(identity_runtime.clone()),
6010                    console_events: runtime.console_events(),
6011                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6012                },
6013            );
6014
6015        aggregator
6016            .reserve_identity_first_interaction(
6017                ConsoleSendRequest {
6018                    identity: "review:singleton".to_string(),
6019                    content: json!("hello"),
6020                    origin: "console:test".to_string(),
6021                    idempotency_key: "session-rebind-reserve".to_string(),
6022                    handling_mode: Some("queue".to_string()),
6023                },
6024                None,
6025            )
6026            .await
6027            .expect("session-only split-brain should self-heal before reserve");
6028
6029        let rebound_status = identity_runtime.status(&identity).await?;
6030        assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6031
6032        let inspection = aggregator
6033            .inspect_identity("review:singleton")
6034            .await?
6035            .expect("rebound identity should inspect");
6036        let live_session_string = live_session_id.to_string();
6037        assert_eq!(
6038            inspection.identity.session_id.as_deref(),
6039            Some(live_session_string.as_str())
6040        );
6041
6042        let _ = runtime.mob_handle().stop().await;
6043        Ok(())
6044    }
6045
6046    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6047    async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6048    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6049        let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6050        let mut labels = BTreeMap::new();
6051        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6052        runtime
6053            .spawn(
6054                SpawnMemberSpec::from_wire(
6055                    "worker".to_string(),
6056                    "rt:review:singleton:0".to_string(),
6057                    Some("You are the mislabeled Review Agent.".into()),
6058                    None,
6059                    None,
6060                )
6061                .with_labels(labels),
6062            )
6063            .await
6064            .expect("member spawns");
6065
6066        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6067            crate::identity_first::IdentityRuntimeConfig {
6068                continuity_store: Arc::new(
6069                    crate::identity_first::LocalContinuityStore::in_memory()?
6070                ),
6071                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6072                runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6073                has_runtime_store: true,
6074                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6075                bridge: None,
6076                default_timeout: None,
6077            },
6078        ));
6079        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6080        let record = crate::identity_first::ContinuityRecord {
6081            identity: identity.clone(),
6082            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6083                "rt:review:singleton:0",
6084            )?,
6085            session_id: SessionId::new(),
6086            generation: crate::identity_first::ContinuityGeneration::new(1),
6087            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6088        };
6089        identity_runtime
6090            .register(
6091                crate::identity_first::DurableAgentSpec {
6092                    identity: identity.clone(),
6093                    profile: meerkat_mob::ProfileName::from("worker"),
6094                    addressability: crate::identity_first::AgentAddressability::Addressable,
6095                    display_name: None,
6096                    labels: BTreeMap::new(),
6097                    context: None,
6098                    additional_instructions: Vec::new(),
6099                    initial_message: None,
6100                    runtime_mode_override: None,
6101                    backend: None,
6102                    binding: None,
6103                },
6104                crate::identity_first::IdentityLifecycleState::Active,
6105                Some(record),
6106                Some(crate::identity_first::LeaseGrant {
6107                    identity,
6108                    fencing_token: crate::identity_first::FencingToken::new(7),
6109                    ttl: Duration::from_mins(5),
6110                }),
6111            )
6112            .await;
6113
6114        let aggregator = MobKitConsoleAggregator::in_memory();
6115        aggregator
6116            .inner
6117            .runtimes
6118            .write()
6119            .expect("runtime registry")
6120            .insert(
6121                "runtime-a".to_string(),
6122                RuntimeEntry {
6123                    runtime_key: "runtime-a".to_string(),
6124                    identity_namespace: String::new(),
6125                    runtime: runtime.mob_runtime().clone(),
6126                    identity_runtime: Some(identity_runtime),
6127                    console_events: runtime.console_events(),
6128                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6129                },
6130            );
6131
6132        let err = aggregator
6133            .inspect_identity("review:singleton")
6134            .await
6135            .expect_err("wrong projected live identity must not inspect as healthy");
6136        assert!(
6137            err.to_string()
6138                .contains("projects identity other:singleton"),
6139            "unexpected error: {err}"
6140        );
6141        let retire_err = aggregator
6142            .retire_identity("review:singleton")
6143            .await
6144            .expect_err("wrong projected live identity must not retire as healthy");
6145        assert!(
6146            retire_err
6147                .to_string()
6148                .contains("projects identity other:singleton"),
6149            "unexpected retire error: {retire_err}"
6150        );
6151        let send_err = aggregator
6152            .send(ConsoleSendRequest {
6153                identity: "review:singleton".to_string(),
6154                content: json!("hello"),
6155                origin: "console:test".to_string(),
6156                idempotency_key: "wrong-projection-send".to_string(),
6157                handling_mode: Some("queue".to_string()),
6158            })
6159            .await
6160            .expect_err("wrong projected live identity must not send as unknown");
6161        assert!(
6162            send_err
6163                .to_string()
6164                .contains("projects identity other:singleton"),
6165            "unexpected send error: {send_err}"
6166        );
6167        let reserve_err = aggregator
6168            .reserve_identity_first_interaction(
6169                ConsoleSendRequest {
6170                    identity: "review:singleton".to_string(),
6171                    content: json!("hello"),
6172                    origin: "console:test".to_string(),
6173                    idempotency_key: "wrong-projection-reserve".to_string(),
6174                    handling_mode: Some("queue".to_string()),
6175                },
6176                None,
6177            )
6178            .await
6179            .expect_err("identity-first reserve must not bypass wrong live projection");
6180        assert!(
6181            reserve_err
6182                .to_string()
6183                .contains("projects identity other:singleton"),
6184            "unexpected reserve error: {reserve_err}"
6185        );
6186        let blob_err = match aggregator
6187            .binary_blob_store_for_identity("review:singleton")
6188            .await
6189        {
6190            Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6191            Err(err) => err,
6192        };
6193        assert!(
6194            blob_err
6195                .to_string()
6196                .contains("projects identity other:singleton"),
6197            "unexpected blob error: {blob_err}"
6198        );
6199        let projected_err = aggregator
6200            .inspect_identity("other:singleton")
6201            .await
6202            .expect_err("wrong projected alias must not inspect through live-only fallback");
6203        assert!(
6204            projected_err
6205                .to_string()
6206                .contains("identity runtime binding for review:singleton"),
6207            "unexpected projected-alias error: {projected_err}"
6208        );
6209        let projected_retire_err = aggregator
6210            .retire_identity("other:singleton")
6211            .await
6212            .expect_err("wrong projected alias must not retire through live-only fallback");
6213        assert!(
6214            projected_retire_err
6215                .to_string()
6216                .contains("identity runtime binding for review:singleton"),
6217            "unexpected projected-alias retire error: {projected_retire_err}"
6218        );
6219
6220        let _ = runtime.mob_handle().stop().await;
6221        Ok(())
6222    }
6223
6224    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6225    async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6226    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6227        let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6228        let mut labels = BTreeMap::new();
6229        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6230        runtime
6231            .spawn(
6232                SpawnMemberSpec::from_wire(
6233                    "worker".to_string(),
6234                    "rt:review:singleton:0".to_string(),
6235                    Some("You are a hidden wrong projection.".into()),
6236                    None,
6237                    None,
6238                )
6239                .with_labels(labels),
6240            )
6241            .await
6242            .expect("member spawns");
6243
6244        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6245            crate::identity_first::IdentityRuntimeConfig {
6246                continuity_store: Arc::new(
6247                    crate::identity_first::LocalContinuityStore::in_memory()?
6248                ),
6249                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6250                runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6251                has_runtime_store: true,
6252                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6253                bridge: None,
6254                default_timeout: None,
6255            },
6256        ));
6257        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6258        identity_runtime
6259            .register(
6260                crate::identity_first::DurableAgentSpec {
6261                    identity: identity.clone(),
6262                    profile: meerkat_mob::ProfileName::from("worker"),
6263                    addressability: crate::identity_first::AgentAddressability::Addressable,
6264                    display_name: None,
6265                    labels: BTreeMap::new(),
6266                    context: None,
6267                    additional_instructions: Vec::new(),
6268                    initial_message: None,
6269                    runtime_mode_override: None,
6270                    backend: None,
6271                    binding: None,
6272                },
6273                crate::identity_first::IdentityLifecycleState::Active,
6274                Some(crate::identity_first::ContinuityRecord {
6275                    identity: identity.clone(),
6276                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6277                        "rt:review:singleton:0",
6278                    )?,
6279                    session_id: SessionId::new(),
6280                    generation: crate::identity_first::ContinuityGeneration::new(1),
6281                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6282                }),
6283                Some(crate::identity_first::LeaseGrant {
6284                    identity,
6285                    fencing_token: crate::identity_first::FencingToken::new(7),
6286                    ttl: Duration::from_mins(5),
6287                }),
6288            )
6289            .await;
6290
6291        let aggregator = MobKitConsoleAggregator::in_memory();
6292        aggregator
6293            .inner
6294            .runtimes
6295            .write()
6296            .expect("runtime registry")
6297            .insert(
6298                "runtime-a".to_string(),
6299                RuntimeEntry {
6300                    runtime_key: "runtime-a".to_string(),
6301                    identity_namespace: String::new(),
6302                    runtime: runtime.mob_runtime().clone(),
6303                    identity_runtime: Some(identity_runtime),
6304                    console_events: runtime.console_events(),
6305                    visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6306                },
6307            );
6308
6309        let err = aggregator
6310            .inspect_identity("review:singleton")
6311            .await
6312            .expect_err("hidden wrong projection must still fail durable binding validation");
6313        assert!(
6314            err.to_string()
6315                .contains("projects identity other:singleton"),
6316            "unexpected hidden wrong-projection error: {err}"
6317        );
6318        let reserve_err = aggregator
6319            .reserve_identity_first_interaction(
6320                ConsoleSendRequest {
6321                    identity: "review:singleton".to_string(),
6322                    content: json!("hello"),
6323                    origin: "console:test".to_string(),
6324                    idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6325                    handling_mode: Some("queue".to_string()),
6326                },
6327                None,
6328            )
6329            .await
6330            .expect_err("reserve must not hide wrong projection from durable validation");
6331        assert!(
6332            reserve_err
6333                .to_string()
6334                .contains("projects identity other:singleton"),
6335            "unexpected reserve error: {reserve_err}"
6336        );
6337
6338        let _ = runtime.mob_handle().stop().await;
6339        Ok(())
6340    }
6341
6342    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6343    async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6344    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6345        let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6346        let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6347        let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6348            crate::identity_first::IdentityRuntimeConfig {
6349                continuity_store: Arc::new(
6350                    crate::identity_first::LocalContinuityStore::in_memory()?
6351                ),
6352                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6353                runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6354                has_runtime_store: true,
6355                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6356                bridge: None,
6357                default_timeout: None,
6358            },
6359        ));
6360        let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6361            crate::identity_first::IdentityRuntimeConfig {
6362                continuity_store: Arc::new(
6363                    crate::identity_first::LocalContinuityStore::in_memory()?
6364                ),
6365                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6366                runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6367                has_runtime_store: true,
6368                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6369                bridge: None,
6370                default_timeout: None,
6371            },
6372        ));
6373        for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6374            let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6375            identity_runtime
6376                .register(
6377                    crate::identity_first::DurableAgentSpec {
6378                        identity: identity.clone(),
6379                        profile: meerkat_mob::ProfileName::from("worker"),
6380                        addressability: crate::identity_first::AgentAddressability::Addressable,
6381                        display_name: None,
6382                        labels: BTreeMap::new(),
6383                        context: None,
6384                        additional_instructions: Vec::new(),
6385                        initial_message: None,
6386                        runtime_mode_override: None,
6387                        backend: None,
6388                        binding: None,
6389                    },
6390                    crate::identity_first::IdentityLifecycleState::Active,
6391                    Some(crate::identity_first::ContinuityRecord {
6392                        identity: identity.clone(),
6393                        agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6394                            "rt:review:singleton:0",
6395                        )?,
6396                        session_id: SessionId::new(),
6397                        generation: crate::identity_first::ContinuityGeneration::new(1),
6398                        checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6399                    }),
6400                    Some(crate::identity_first::LeaseGrant {
6401                        identity,
6402                        fencing_token: crate::identity_first::FencingToken::new(7),
6403                        ttl: Duration::from_mins(5),
6404                    }),
6405                )
6406                .await;
6407        }
6408
6409        let aggregator = MobKitConsoleAggregator::in_memory();
6410        {
6411            let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6412            runtimes.insert(
6413                "a-hidden".to_string(),
6414                RuntimeEntry {
6415                    runtime_key: "a-hidden".to_string(),
6416                    identity_namespace: String::new(),
6417                    runtime: hidden_runtime.mob_runtime().clone(),
6418                    identity_runtime: Some(hidden_identity_runtime),
6419                    console_events: hidden_runtime.console_events(),
6420                    visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6421                },
6422            );
6423            runtimes.insert(
6424                "b-visible".to_string(),
6425                RuntimeEntry {
6426                    runtime_key: "b-visible".to_string(),
6427                    identity_namespace: String::new(),
6428                    runtime: visible_runtime.mob_runtime().clone(),
6429                    identity_runtime: Some(visible_identity_runtime),
6430                    console_events: visible_runtime.console_events(),
6431                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6432                },
6433            );
6434        }
6435
6436        let inspection = aggregator
6437            .inspect_identity("rt:review:singleton:0")
6438            .await?
6439            .expect("visible runtime-id match should be returned");
6440        assert_eq!(inspection.identity.runtime_key, "b-visible");
6441
6442        let _ = hidden_runtime.mob_handle().stop().await;
6443        let _ = visible_runtime.mob_handle().stop().await;
6444        Ok(())
6445    }
6446
6447    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6448    async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6449    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6450        let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6451        let mut primary_labels = BTreeMap::new();
6452        primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6453        runtime
6454            .spawn(
6455                SpawnMemberSpec::from_wire(
6456                    "worker".to_string(),
6457                    "rt:review:singleton:0".to_string(),
6458                    Some("You are the primary Review Agent.".into()),
6459                    None,
6460                    None,
6461                )
6462                .with_labels(primary_labels),
6463            )
6464            .await
6465            .expect("primary member spawns");
6466
6467        let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6468        let mut delegate_labels = BTreeMap::new();
6469        delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6470        delegate_runtime
6471            .spawn(
6472                SpawnMemberSpec::from_wire(
6473                    "worker".to_string(),
6474                    "rt:review:singleton:0".to_string(),
6475                    Some("You are the wrong projected Review Agent.".into()),
6476                    None,
6477                    None,
6478                )
6479                .with_labels(delegate_labels),
6480            )
6481            .await
6482            .expect("delegate member spawns");
6483        runtime
6484            .mob_runtime()
6485            .agent_mob_mcp_state()
6486            .expect("runtime should expose mob MCP state")
6487            .mob_insert_handle(
6488                meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6489                delegate_runtime.mob_handle().clone(),
6490            )
6491            .await;
6492
6493        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6494            crate::identity_first::IdentityRuntimeConfig {
6495                continuity_store: Arc::new(
6496                    crate::identity_first::LocalContinuityStore::in_memory()?
6497                ),
6498                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6499                runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
6500                has_runtime_store: true,
6501                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6502                bridge: None,
6503                default_timeout: None,
6504            },
6505        ));
6506        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6507        let record = crate::identity_first::ContinuityRecord {
6508            identity: identity.clone(),
6509            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6510                "rt:review:singleton:0",
6511            )?,
6512            session_id: SessionId::new(),
6513            generation: crate::identity_first::ContinuityGeneration::new(1),
6514            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6515        };
6516        identity_runtime
6517            .register(
6518                crate::identity_first::DurableAgentSpec {
6519                    identity: identity.clone(),
6520                    profile: meerkat_mob::ProfileName::from("worker"),
6521                    addressability: crate::identity_first::AgentAddressability::Addressable,
6522                    display_name: None,
6523                    labels: BTreeMap::new(),
6524                    context: None,
6525                    additional_instructions: Vec::new(),
6526                    initial_message: None,
6527                    runtime_mode_override: None,
6528                    backend: None,
6529                    binding: None,
6530                },
6531                crate::identity_first::IdentityLifecycleState::Active,
6532                Some(record),
6533                Some(crate::identity_first::LeaseGrant {
6534                    identity,
6535                    fencing_token: crate::identity_first::FencingToken::new(7),
6536                    ttl: Duration::from_mins(5),
6537                }),
6538            )
6539            .await;
6540
6541        let aggregator = MobKitConsoleAggregator::in_memory();
6542        aggregator
6543            .inner
6544            .runtimes
6545            .write()
6546            .expect("runtime registry")
6547            .insert(
6548                "runtime-a".to_string(),
6549                RuntimeEntry {
6550                    runtime_key: "runtime-a".to_string(),
6551                    identity_namespace: String::new(),
6552                    runtime: runtime.mob_runtime().clone(),
6553                    identity_runtime: Some(identity_runtime),
6554                    console_events: runtime.console_events(),
6555                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6556                },
6557            );
6558
6559        let err = aggregator
6560            .inspect_identity("review:singleton")
6561            .await
6562            .expect_err("cross-source wrong projection must not be hidden by primary source");
6563        assert!(
6564            err.to_string()
6565                .contains("projects identity other:singleton")
6566                || err.to_string().contains("ambiguous live identity alias"),
6567            "unexpected error: {err}"
6568        );
6569
6570        let _ = delegate_runtime.mob_handle().stop().await;
6571        let _ = runtime.mob_handle().stop().await;
6572        Ok(())
6573    }
6574
6575    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6576    async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
6577    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6578        let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
6579        let mut labels = BTreeMap::new();
6580        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6581        runtime
6582            .spawn(
6583                SpawnMemberSpec::from_wire(
6584                    "worker".to_string(),
6585                    "rt:review:singleton:0".to_string(),
6586                    Some("You are the first Review Agent.".into()),
6587                    None,
6588                    None,
6589                )
6590                .with_labels(labels.clone()),
6591            )
6592            .await
6593            .expect("first member spawns");
6594        runtime
6595            .spawn(
6596                SpawnMemberSpec::from_wire(
6597                    "worker".to_string(),
6598                    "rt:review:singleton:1".to_string(),
6599                    Some("You are the second Review Agent.".into()),
6600                    None,
6601                    None,
6602                )
6603                .with_labels(labels),
6604            )
6605            .await
6606            .expect("second member spawns");
6607
6608        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6609            crate::identity_first::IdentityRuntimeConfig {
6610                continuity_store: Arc::new(
6611                    crate::identity_first::LocalContinuityStore::in_memory()?
6612                ),
6613                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6614                runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
6615                has_runtime_store: true,
6616                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6617                bridge: None,
6618                default_timeout: None,
6619            },
6620        ));
6621        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6622        identity_runtime
6623            .register(
6624                crate::identity_first::DurableAgentSpec {
6625                    identity: identity.clone(),
6626                    profile: meerkat_mob::ProfileName::from("worker"),
6627                    addressability: crate::identity_first::AgentAddressability::Addressable,
6628                    display_name: None,
6629                    labels: BTreeMap::new(),
6630                    context: None,
6631                    additional_instructions: Vec::new(),
6632                    initial_message: None,
6633                    runtime_mode_override: None,
6634                    backend: None,
6635                    binding: None,
6636                },
6637                crate::identity_first::IdentityLifecycleState::Active,
6638                Some(crate::identity_first::ContinuityRecord {
6639                    identity: identity.clone(),
6640                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6641                        "rt:review:singleton:0",
6642                    )?,
6643                    session_id: SessionId::new(),
6644                    generation: crate::identity_first::ContinuityGeneration::new(1),
6645                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6646                }),
6647                Some(crate::identity_first::LeaseGrant {
6648                    identity,
6649                    fencing_token: crate::identity_first::FencingToken::new(8),
6650                    ttl: Duration::from_mins(5),
6651                }),
6652            )
6653            .await;
6654
6655        let aggregator = MobKitConsoleAggregator::in_memory();
6656        aggregator
6657            .inner
6658            .runtimes
6659            .write()
6660            .expect("runtime registry")
6661            .insert(
6662                "runtime-a".to_string(),
6663                RuntimeEntry {
6664                    runtime_key: "runtime-a".to_string(),
6665                    identity_namespace: String::new(),
6666                    runtime: runtime.mob_runtime().clone(),
6667                    identity_runtime: Some(identity_runtime),
6668                    console_events: runtime.console_events(),
6669                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6670                },
6671            );
6672
6673        let err = aggregator
6674            .inspect_identity("review:singleton")
6675            .await
6676            .expect_err("duplicate live alias must not inspect via durable fast path");
6677        assert!(
6678            err.to_string().contains("ambiguous live identity alias"),
6679            "unexpected error: {err}"
6680        );
6681        let retire_err = aggregator
6682            .retire_identity("review:singleton")
6683            .await
6684            .expect_err("duplicate live alias must not retire via durable fast path");
6685        assert!(
6686            retire_err
6687                .to_string()
6688                .contains("ambiguous live identity alias"),
6689            "unexpected retire error: {retire_err}"
6690        );
6691        let runtime_id_err = aggregator
6692            .inspect_identity("rt:review:singleton:0")
6693            .await
6694            .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6695        assert!(
6696            runtime_id_err
6697                .to_string()
6698                .contains("ambiguous live identity alias"),
6699            "unexpected runtime-id inspect error: {runtime_id_err}"
6700        );
6701        let runtime_id_retire_err = aggregator
6702            .retire_identity("rt:review:singleton:0")
6703            .await
6704            .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6705        assert!(
6706            runtime_id_retire_err
6707                .to_string()
6708                .contains("ambiguous live identity alias")
6709                || runtime_id_retire_err
6710                    .to_string()
6711                    .contains("stale durable identity alias"),
6712            "unexpected runtime-id retire error: {runtime_id_retire_err}"
6713        );
6714
6715        let _ = runtime.mob_handle().stop().await;
6716        Ok(())
6717    }
6718
6719    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6720    async fn inspect_identity_ignores_hidden_live_alias_candidates()
6721    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6722        let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6723        let mut labels = BTreeMap::new();
6724        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6725        runtime
6726            .spawn(
6727                SpawnMemberSpec::from_wire(
6728                    "worker".to_string(),
6729                    "rt:review:singleton:0".to_string(),
6730                    Some("You are the visible Review Agent.".into()),
6731                    None,
6732                    None,
6733                )
6734                .with_labels(labels.clone()),
6735            )
6736            .await
6737            .expect("visible member spawns");
6738        runtime
6739            .spawn(
6740                SpawnMemberSpec::from_wire(
6741                    "worker".to_string(),
6742                    "rt:review:singleton:1".to_string(),
6743                    Some("You are hidden maintenance noise.".into()),
6744                    None,
6745                    None,
6746                )
6747                .with_labels(labels),
6748            )
6749            .await
6750            .expect("hidden member spawns");
6751
6752        let aggregator = MobKitConsoleAggregator::in_memory();
6753        aggregator.register_runtime_handles_with_policy(
6754            "runtime-a",
6755            "",
6756            runtime.mob_runtime().clone(),
6757            None,
6758            runtime.console_events(),
6759            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6760        );
6761
6762        let inspection = aggregator
6763            .inspect_identity("review:singleton")
6764            .await?
6765            .expect("visible live alias should inspect without hidden ambiguity");
6766        assert_eq!(inspection.identity.identity, "review:singleton");
6767        assert_eq!(
6768            inspection.identity.runtime_member_id,
6769            "rt:review:singleton:0"
6770        );
6771
6772        let retired = aggregator.retire_identity("review:singleton").await?;
6773        assert!(
6774            retired,
6775            "visible live alias should retire without hidden ambiguity"
6776        );
6777
6778        let _ = runtime.mob_handle().stop().await;
6779        Ok(())
6780    }
6781
6782    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6783    async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6784    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6785        let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6786        let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6787        let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6788        let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6789
6790        let aggregator = MobKitConsoleAggregator::in_memory();
6791        aggregator.register_runtime_handles_with_policy(
6792            "a-hidden",
6793            "",
6794            hidden_runtime.mob_runtime().clone(),
6795            Some(hidden_identity_runtime),
6796            hidden_runtime.console_events(),
6797            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6798        );
6799        aggregator.register_runtime_handles_with_policy(
6800            "b-visible",
6801            "",
6802            visible_runtime.mob_runtime().clone(),
6803            Some(visible_identity_runtime),
6804            visible_runtime.console_events(),
6805            Arc::new(AllowAllConsoleVisibilityPolicy),
6806        );
6807
6808        let accepted = aggregator
6809            .reserve_identity_first_interaction(
6810                ConsoleSendRequest {
6811                    identity: "review:singleton".to_string(),
6812                    content: json!("hello"),
6813                    origin: "console:test".to_string(),
6814                    idempotency_key: "visible-after-hidden-reserve".to_string(),
6815                    handling_mode: Some("queue".to_string()),
6816                },
6817                None,
6818            )
6819            .await?;
6820        let page = aggregator
6821            .query_timeline(ConsoleTimelineQuery {
6822                identity: Some("review:singleton".to_string()),
6823                ..ConsoleTimelineQuery::default()
6824            })
6825            .await?;
6826        let frame = page
6827            .frames
6828            .iter()
6829            .find(|frame| frame.id == accepted.input_frame_id)
6830            .ok_or("accepted frame missing from timeline")?;
6831        assert_eq!(frame.runtime_key, "b-visible");
6832
6833        let _ = hidden_runtime.mob_handle().stop().await;
6834        let _ = visible_runtime.mob_handle().stop().await;
6835        Ok(())
6836    }
6837
6838    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6839    async fn identity_first_list_identities_projects_cached_topology_peers()
6840    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6841        let runtime = build_empty_runtime("identity-topology-cache-test").await;
6842        let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6843        identity_runtime
6844            .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6845                crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6846                crate::identity_first::AgentIdentity::parse("agent:beta")?,
6847            )?])
6848            .await;
6849
6850        let aggregator = MobKitConsoleAggregator::in_memory();
6851        aggregator.register_runtime_handles_with_policy(
6852            "identity-first",
6853            "",
6854            runtime.mob_runtime().clone(),
6855            Some(identity_runtime),
6856            runtime.console_events(),
6857            Arc::new(AllowAllConsoleVisibilityPolicy),
6858        );
6859
6860        let identities = aggregator.list_identities().await?;
6861        let alpha = identities
6862            .iter()
6863            .find(|record| record.identity == "agent:alpha")
6864            .ok_or("agent:alpha identity missing")?;
6865        assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6866        let inspection = aggregator
6867            .inspect_identity("agent:alpha")
6868            .await?
6869            .ok_or("agent:alpha inspection missing")?;
6870        assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6871
6872        let _ = runtime.mob_handle().stop().await;
6873        Ok(())
6874    }
6875
6876    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6877    async fn identity_first_list_identities_includes_member_only_spawned_workers()
6878    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6879        let runtime = build_empty_runtime("identity-member-only-test").await;
6880        let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6881        let aggregator = MobKitConsoleAggregator::in_memory();
6882        aggregator.register_runtime_handles_with_policy(
6883            "identity-first",
6884            "",
6885            runtime.mob_runtime().clone(),
6886            Some(identity_runtime),
6887            runtime.console_events(),
6888            Arc::new(AllowAllConsoleVisibilityPolicy),
6889        );
6890
6891        runtime
6892            .spawn(SpawnMemberSpec::from_wire(
6893                "worker".to_string(),
6894                "agent:beta".to_string(),
6895                Some("You are beta.".into()),
6896                None,
6897                None,
6898            ))
6899            .await?;
6900
6901        let identities = aggregator.list_identities().await?;
6902        assert!(
6903            identities
6904                .iter()
6905                .any(|record| record.identity == "agent:beta"),
6906            "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6907        );
6908
6909        let _ = runtime.mob_handle().stop().await;
6910        Ok(())
6911    }
6912
6913    #[tokio::test]
6914    async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6915        let aggregator = MobKitConsoleAggregator::in_memory();
6916        let record = identity_record_for_test("agent-cached");
6917        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6918        aggregator
6919            .inner
6920            .identity_read_model
6921            .primed
6922            .store(true, Ordering::Release);
6923        let _guard = aggregator
6924            .inner
6925            .identity_read_model
6926            .refresh_lock
6927            .clone()
6928            .lock_owned()
6929            .await;
6930
6931        let identities =
6932            tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6933                .await
6934                .expect("hot identity list should not wait for refresh lock")
6935                .expect("identity list succeeds");
6936
6937        assert_eq!(identities, vec![record]);
6938    }
6939
6940    #[tokio::test]
6941    async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6942        let aggregator = MobKitConsoleAggregator::in_memory();
6943        let guard = aggregator
6944            .inner
6945            .identity_read_model
6946            .refresh_lock
6947            .clone()
6948            .lock_owned()
6949            .await;
6950        let waiter = tokio::spawn({
6951            let aggregator = aggregator.clone();
6952            async move { aggregator.list_identities().await }
6953        });
6954
6955        tokio::time::sleep(Duration::from_millis(20)).await;
6956        assert!(
6957            !waiter.is_finished(),
6958            "cold identity list should wait for the in-flight refresh to finish"
6959        );
6960
6961        let record = identity_record_for_test("agent-primed");
6962        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6963        aggregator
6964            .inner
6965            .identity_read_model
6966            .primed
6967            .store(true, Ordering::Release);
6968        drop(guard);
6969
6970        let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6971            .await
6972            .expect("cold identity list waiter should resume")
6973            .expect("waiter joins")
6974            .expect("identity list succeeds");
6975        assert_eq!(identities, vec![record]);
6976    }
6977
6978    #[tokio::test]
6979    async fn query_timeline_reads_from_aggregate_store() {
6980        let aggregator = MobKitConsoleAggregator::in_memory();
6981        let frame = NewConsoleFrame {
6982            id: None,
6983            dedupe_key: "event-1".to_string(),
6984            timestamp_ms: 1,
6985            runtime_key: "runtime-a".to_string(),
6986            identity: "agent-a".to_string(),
6987            conversation_id: Some("agent-a".to_string()),
6988            session_id: None,
6989            kind: "text_delta".to_string(),
6990            status: ConsoleFrameStatus::Delivered,
6991            payload: json!({ "delta": "hello" }),
6992            source: ConsoleFrameSource {
6993                kind: ConsoleFrameSourceKind::ConsoleEvent,
6994                source_cursor: None,
6995            },
6996            source_event_id: Some("event-1".to_string()),
6997            interaction_id: None,
6998            turn_id: None,
6999            run_id: None,
7000            parent_frame_id: None,
7001            caused_by_frame_id: None,
7002        };
7003        aggregator
7004            .store()
7005            .append_if_absent(frame)
7006            .await
7007            .expect("append frame");
7008
7009        let page = aggregator
7010            .query_timeline(ConsoleTimelineQuery {
7011                identity: Some("agent-a".to_string()),
7012                limit: 10,
7013                ..ConsoleTimelineQuery::default()
7014            })
7015            .await
7016            .expect("query timeline");
7017        assert_eq!(page.frames.len(), 1);
7018        assert_eq!(page.frames[0].kind, "text_delta");
7019    }
7020
7021    #[tokio::test]
7022    async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7023        let aggregator = MobKitConsoleAggregator::in_memory();
7024        let runtime = build_single_member_runtime().await;
7025        let entry = runtime_entry_for_test("runtime-a", &runtime);
7026        for event_type in ["reasoning_delta", "reasoning_complete"] {
7027            let frame = frame_from_console_event(
7028                &entry,
7029                console_envelope_for_test(
7030                    event_type,
7031                    Some("interaction-a"),
7032                    event_type,
7033                    json!({
7034                        "delta": "I should inspect the file first.",
7035                        "text": "I should inspect the file first.",
7036                        "turn_id": "turn-a",
7037                    }),
7038                ),
7039            );
7040            aggregator
7041                .store()
7042                .append_if_absent(frame)
7043                .await
7044                .expect("append reasoning frame");
7045        }
7046
7047        let page = aggregator
7048            .query_timeline(ConsoleTimelineQuery {
7049                identity: Some("test/agent-a".to_string()),
7050                limit: 10,
7051                ..ConsoleTimelineQuery::default()
7052            })
7053            .await
7054            .expect("query timeline");
7055
7056        assert_eq!(
7057            page.frames.len(),
7058            1,
7059            "identical reasoning re-emissions in one turn should collapse"
7060        );
7061        assert_eq!(page.frames[0].kind, "reasoning_delta");
7062        assert_eq!(
7063            page.frames[0].dedupe_key,
7064            format!(
7065                "console-reasoning:runtime-a:interaction-a:{}",
7066                hash_short("I should inspect the file first.")
7067            )
7068        );
7069    }
7070
7071    #[tokio::test]
7072    async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7073        let aggregator = MobKitConsoleAggregator::in_memory();
7074        let runtime = build_single_member_runtime().await;
7075        let entry = runtime_entry_for_test("runtime-a", &runtime);
7076        for (event_id, text) in [
7077            ("reasoning-1", "I should inspect the file first."),
7078            ("reasoning-2", "Now I should run the focused test."),
7079        ] {
7080            let frame = frame_from_console_event(
7081                &entry,
7082                console_envelope_for_test(
7083                    event_id,
7084                    Some("interaction-a"),
7085                    "reasoning_complete",
7086                    json!({
7087                        "text": text,
7088                        "turn_id": "turn-a",
7089                    }),
7090                ),
7091            );
7092            aggregator
7093                .store()
7094                .append_if_absent(frame)
7095                .await
7096                .expect("append reasoning frame");
7097        }
7098
7099        let page = aggregator
7100            .query_timeline(ConsoleTimelineQuery {
7101                identity: Some("test/agent-a".to_string()),
7102                limit: 10,
7103                ..ConsoleTimelineQuery::default()
7104            })
7105            .await
7106            .expect("query timeline");
7107
7108        assert_eq!(
7109            page.frames.len(),
7110            2,
7111            "distinct same-turn reasoning segments should not collapse"
7112        );
7113        assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7114    }
7115
7116    #[tokio::test]
7117    async fn identity_recent_anchor_respects_query_limit() {
7118        let aggregator = MobKitConsoleAggregator::in_memory();
7119        aggregator
7120            .store()
7121            .append_if_absent(NewConsoleFrame {
7122                id: None,
7123                dedupe_key: "anchored-user-input".to_string(),
7124                timestamp_ms: 1,
7125                runtime_key: "runtime-a".to_string(),
7126                identity: "agent-a".to_string(),
7127                conversation_id: Some("agent-a".to_string()),
7128                session_id: None,
7129                kind: "user_input".to_string(),
7130                status: ConsoleFrameStatus::Delivered,
7131                payload: json!({ "content": "anchor me" }),
7132                source: ConsoleFrameSource {
7133                    kind: ConsoleFrameSourceKind::Synthetic,
7134                    source_cursor: None,
7135                },
7136                source_event_id: Some("anchored-user-input".to_string()),
7137                interaction_id: Some("turn-a".to_string()),
7138                turn_id: None,
7139                run_id: None,
7140                parent_frame_id: None,
7141                caused_by_frame_id: None,
7142            })
7143            .await
7144            .expect("append anchor");
7145        for idx in 2..=40 {
7146            aggregator
7147                .store()
7148                .append_if_absent(NewConsoleFrame {
7149                    id: None,
7150                    dedupe_key: format!("noisy-tail-{idx}"),
7151                    timestamp_ms: idx,
7152                    runtime_key: "runtime-a".to_string(),
7153                    identity: "agent-a".to_string(),
7154                    conversation_id: Some("agent-a".to_string()),
7155                    session_id: None,
7156                    kind: "reasoning_delta".to_string(),
7157                    status: ConsoleFrameStatus::Delivered,
7158                    payload: json!({ "delta": idx }),
7159                    source: ConsoleFrameSource {
7160                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7161                        source_cursor: None,
7162                    },
7163                    source_event_id: Some(format!("noisy-tail-{idx}")),
7164                    interaction_id: Some("turn-a".to_string()),
7165                    turn_id: None,
7166                    run_id: None,
7167                    parent_frame_id: None,
7168                    caused_by_frame_id: None,
7169                })
7170                .await
7171                .expect("append noisy tail");
7172        }
7173
7174        let page = aggregator
7175            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7176                identity: Some("agent-a".to_string()),
7177                mode: ConsoleTimelineMode::Recent,
7178                limit: 5,
7179                ..ConsoleTimelineWindowQuery::default()
7180            })
7181            .await
7182            .expect("query recent identity timeline");
7183
7184        assert_eq!(
7185            page.frames.len(),
7186            5,
7187            "identity anchor merge must not exceed the requested limit"
7188        );
7189        assert!(
7190            page.frames
7191                .iter()
7192                .any(|frame| frame.dedupe_key == "anchored-user-input"),
7193            "the bounded result should still retain the useful turn anchor: {:#?}",
7194            page.frames
7195        );
7196        assert_eq!(
7197            page.frames.last().and_then(|frame| frame.cursor.seq()),
7198            Some(40)
7199        );
7200    }
7201
7202    #[tokio::test]
7203    async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7204        let aggregator = MobKitConsoleAggregator::in_memory();
7205        let runtime = build_single_member_runtime().await;
7206        let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7207        entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7208        aggregator
7209            .inner
7210            .runtimes
7211            .write()
7212            .expect("runtime registry")
7213            .insert("runtime-a".to_string(), entry);
7214
7215        for idx in 0..1_500 {
7216            aggregator
7217                .store()
7218                .append_if_absent(NewConsoleFrame {
7219                    id: None,
7220                    dedupe_key: format!("hidden-gap-{idx}"),
7221                    timestamp_ms: idx,
7222                    runtime_key: "runtime-a".to_string(),
7223                    identity: "agent-a".to_string(),
7224                    conversation_id: Some("agent-a".to_string()),
7225                    session_id: None,
7226                    kind: "hidden_noise".to_string(),
7227                    status: ConsoleFrameStatus::Delivered,
7228                    payload: json!({ "delta": idx }),
7229                    source: ConsoleFrameSource {
7230                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7231                        source_cursor: None,
7232                    },
7233                    source_event_id: Some(format!("hidden-gap-{idx}")),
7234                    interaction_id: None,
7235                    turn_id: None,
7236                    run_id: None,
7237                    parent_frame_id: None,
7238                    caused_by_frame_id: None,
7239                })
7240                .await
7241                .expect("append hidden frame");
7242        }
7243        aggregator
7244            .store()
7245            .append_if_absent(NewConsoleFrame {
7246                id: None,
7247                dedupe_key: "visible-after-gap".to_string(),
7248                timestamp_ms: 1_501,
7249                runtime_key: "runtime-a".to_string(),
7250                identity: "agent-a".to_string(),
7251                conversation_id: Some("agent-a".to_string()),
7252                session_id: None,
7253                kind: "interaction_complete".to_string(),
7254                status: ConsoleFrameStatus::Delivered,
7255                payload: json!({ "text": "visible after hidden gap" }),
7256                source: ConsoleFrameSource {
7257                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7258                    source_cursor: None,
7259                },
7260                source_event_id: Some("visible-after-gap".to_string()),
7261                interaction_id: Some("turn-visible".to_string()),
7262                turn_id: None,
7263                run_id: None,
7264                parent_frame_id: None,
7265                caused_by_frame_id: None,
7266            })
7267            .await
7268            .expect("append visible frame");
7269
7270        let page = aggregator
7271            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7272                identity: Some("agent-a".to_string()),
7273                mode: ConsoleTimelineMode::Since,
7274                limit: 1,
7275                ..ConsoleTimelineWindowQuery::default()
7276            })
7277            .await
7278            .expect("query timeline");
7279        assert_eq!(page.frames.len(), 1);
7280        assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
7281        assert_eq!(
7282            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7283            Some(1_501)
7284        );
7285        let _ = runtime.mob_handle().stop().await;
7286    }
7287
7288    #[tokio::test]
7289    async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
7290        let aggregator = MobKitConsoleAggregator::in_memory();
7291        for idx in 1..=300 {
7292            aggregator
7293                .store()
7294                .append_if_absent(NewConsoleFrame {
7295                    id: None,
7296                    dedupe_key: format!("visible-{idx}"),
7297                    timestamp_ms: idx,
7298                    runtime_key: "runtime-a".to_string(),
7299                    identity: "agent-a".to_string(),
7300                    conversation_id: Some("agent-a".to_string()),
7301                    session_id: None,
7302                    kind: "interaction_complete".to_string(),
7303                    status: ConsoleFrameStatus::Delivered,
7304                    payload: json!({ "text": format!("visible {idx}") }),
7305                    source: ConsoleFrameSource {
7306                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7307                        source_cursor: None,
7308                    },
7309                    source_event_id: Some(format!("visible-{idx}")),
7310                    interaction_id: Some(format!("turn-{idx}")),
7311                    turn_id: None,
7312                    run_id: None,
7313                    parent_frame_id: None,
7314                    caused_by_frame_id: None,
7315                })
7316                .await
7317                .expect("append visible frame");
7318        }
7319
7320        let first = aggregator
7321            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7322                identity: Some("agent-a".to_string()),
7323                mode: ConsoleTimelineMode::Since,
7324                limit: 10,
7325                ..ConsoleTimelineWindowQuery::default()
7326            })
7327            .await
7328            .expect("query first page");
7329        assert_eq!(first.frames.len(), 10);
7330        assert_eq!(
7331            first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7332            Some(10)
7333        );
7334
7335        let second = aggregator
7336            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7337                identity: Some("agent-a".to_string()),
7338                mode: ConsoleTimelineMode::Since,
7339                after: first.next_cursor,
7340                limit: 10,
7341                ..ConsoleTimelineWindowQuery::default()
7342            })
7343            .await
7344            .expect("query second page");
7345        assert_eq!(second.frames.len(), 10);
7346        assert_eq!(second.frames[0].dedupe_key, "visible-11");
7347        assert_eq!(
7348            second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7349            Some(20)
7350        );
7351    }
7352
7353    #[tokio::test]
7354    async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
7355        let store = Arc::new(CountingConsoleLogStore::new());
7356        let aggregator = MobKitConsoleAggregator::new(store.clone());
7357        let runtime = build_single_member_runtime().await;
7358        aggregator
7359            .inner
7360            .runtimes
7361            .write()
7362            .expect("runtime registry")
7363            .insert(
7364                "runtime-a".to_string(),
7365                runtime_entry_for_test("runtime-a", &runtime),
7366            );
7367        let inserted = store
7368            .append_if_absent(NewConsoleFrame {
7369                id: None,
7370                dedupe_key: "event-1".to_string(),
7371                timestamp_ms: 1,
7372                runtime_key: "runtime-a".to_string(),
7373                identity: "agent-a".to_string(),
7374                conversation_id: Some("agent-a".to_string()),
7375                session_id: None,
7376                kind: "text_delta".to_string(),
7377                status: ConsoleFrameStatus::Delivered,
7378                payload: json!({ "delta": "hello" }),
7379                source: ConsoleFrameSource {
7380                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7381                    source_cursor: None,
7382                },
7383                source_event_id: Some("event-1".to_string()),
7384                interaction_id: None,
7385                turn_id: None,
7386                run_id: None,
7387                parent_frame_id: None,
7388                caused_by_frame_id: None,
7389            })
7390            .await
7391            .expect("append frame");
7392
7393        let page = aggregator
7394            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7395                identity: Some("agent-a".to_string()),
7396                mode: ConsoleTimelineMode::Since,
7397                after: Some(inserted.frame.cursor),
7398                limit: 10,
7399                ..ConsoleTimelineWindowQuery::default()
7400            })
7401            .await
7402            .expect("query continuation");
7403
7404        assert!(page.frames.is_empty());
7405        assert_eq!(
7406            store.source_watermark_calls(),
7407            0,
7408            "empty since continuation must not synchronously force session-history backfill"
7409        );
7410        let _ = runtime.mob_handle().stop().await;
7411    }
7412
7413    #[test]
7414    fn legacy_timeline_struct_literals_remain_source_compatible() {
7415        let query = ConsoleTimelineQuery {
7416            identity: Some("agent-a".to_string()),
7417            conversation_id: None,
7418            after: Some(ConsoleCursor::from("console:1")),
7419            limit: 10,
7420        };
7421        let page = ConsoleTimelinePage {
7422            frames: Vec::new(),
7423            next_cursor: query.after.clone(),
7424        };
7425
7426        assert_eq!(query.limit, 10);
7427        assert_eq!(
7428            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7429            Some(1)
7430        );
7431    }
7432
7433    #[tokio::test]
7434    async fn query_timeline_is_store_local_for_registered_runtimes() {
7435        let store = Arc::new(CountingConsoleLogStore::new());
7436        let aggregator = MobKitConsoleAggregator::new(store.clone());
7437        let runtime = build_single_member_runtime().await;
7438        aggregator
7439            .inner
7440            .runtimes
7441            .write()
7442            .expect("runtime registry")
7443            .insert(
7444                "runtime-a".to_string(),
7445                runtime_entry_for_test("runtime-a", &runtime),
7446            );
7447        store
7448            .append_if_absent(NewConsoleFrame {
7449                id: None,
7450                dedupe_key: "event-1".to_string(),
7451                timestamp_ms: 1,
7452                runtime_key: "runtime-a".to_string(),
7453                identity: "agent-a".to_string(),
7454                conversation_id: Some("agent-a".to_string()),
7455                session_id: None,
7456                kind: "text_delta".to_string(),
7457                status: ConsoleFrameStatus::Delivered,
7458                payload: json!({ "delta": "hello" }),
7459                source: ConsoleFrameSource {
7460                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7461                    source_cursor: None,
7462                },
7463                source_event_id: Some("event-1".to_string()),
7464                interaction_id: None,
7465                turn_id: None,
7466                run_id: None,
7467                parent_frame_id: None,
7468                caused_by_frame_id: None,
7469            })
7470            .await
7471            .expect("append frame");
7472
7473        let page = tokio::time::timeout(
7474            Duration::from_millis(250),
7475            aggregator.query_timeline(ConsoleTimelineQuery {
7476                identity: Some("agent-a".to_string()),
7477                limit: 10,
7478                ..ConsoleTimelineQuery::default()
7479            }),
7480        )
7481        .await
7482        .expect("timeline query should not wait for session history")
7483        .expect("timeline query succeeds");
7484
7485        assert_eq!(page.frames.len(), 1);
7486        assert_eq!(
7487            store.source_watermark_calls(),
7488            0,
7489            "query_timeline must not synchronously touch session-history watermarks"
7490        );
7491        let _ = runtime.mob_handle().stop().await;
7492    }
7493
7494    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7495    async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
7496        let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
7497            delay: Duration::from_secs(2),
7498        }))
7499        .await;
7500        let aggregator = MobKitConsoleAggregator::in_memory();
7501        aggregator
7502            .inner
7503            .runtimes
7504            .write()
7505            .expect("runtime registry")
7506            .insert(
7507                "runtime-a".to_string(),
7508                runtime_entry_for_test("runtime-a", &runtime),
7509            );
7510
7511        let start = Instant::now();
7512        let accepted = tokio::time::timeout(
7513            Duration::from_millis(300),
7514            aggregator.send(ConsoleSendRequest {
7515                identity: "test/agent-a".to_string(),
7516                content: json!("hello slow agent"),
7517                origin: "console:test".to_string(),
7518                idempotency_key: "nonblocking-send".to_string(),
7519                handling_mode: Some("queue".to_string()),
7520            }),
7521        )
7522        .await
7523        .expect("console send should return once the input is accepted")
7524        .expect("send succeeds");
7525
7526        assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
7527        assert!(
7528            start.elapsed() < Duration::from_secs(1),
7529            "console send should not wait for the delayed LLM turn"
7530        );
7531
7532        wait_for_session_history_text(
7533            &aggregator,
7534            "test/agent-a",
7535            "slow ok",
7536            Duration::from_secs(5),
7537        )
7538        .await
7539        .expect("background dispatch should still complete and project history");
7540        let _ = runtime.mob_handle().stop().await;
7541    }
7542
7543    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7544    async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
7545    {
7546        let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
7547        let aggregator = MobKitConsoleAggregator::in_memory();
7548        aggregator.register_runtime(ConsoleRuntimeRegistration {
7549            runtime_key: "runtime-late".to_string(),
7550            runtime: runtime.clone(),
7551            identity_namespace: "late".to_string(),
7552            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7553        });
7554
7555        runtime
7556            .spawn(SpawnMemberSpec::from_wire(
7557                "worker".to_string(),
7558                "agent-late".to_string(),
7559                Some("You are agent-late.".into()),
7560                None,
7561                None,
7562            ))
7563            .await
7564            .expect("late member spawns");
7565        let session_id = send_message_on_mob_with_mode(
7566            &runtime.mob_handle(),
7567            "agent-late",
7568            ContentInput::Text("hello after registration".to_string()),
7569            meerkat_core::types::HandlingMode::Queue,
7570        )
7571        .await
7572        .expect("direct member send succeeds");
7573        wait_for_identity_record(
7574            &aggregator,
7575            "late/agent-late",
7576            Some(session_id.as_str()),
7577            Duration::from_secs(5),
7578        )
7579        .await?;
7580
7581        wait_for_session_history_text(
7582            &aggregator,
7583            "late/agent-late",
7584            "You are agent-late.",
7585            Duration::from_secs(5),
7586        )
7587        .await?;
7588        let _ = runtime.mob_handle().stop().await;
7589        Ok(())
7590    }
7591
7592    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7593    async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
7594    -> Result<(), String> {
7595        let runtime = build_single_member_runtime().await;
7596        let entry = runtime_entry_for_test("runtime-a", &runtime);
7597        let resolved = member_sources_for_entry(&entry)
7598            .await
7599            .into_iter()
7600            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7601            .expect("agent-a member exists");
7602        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7603            .await
7604            .expect("identity record exists");
7605        let session_id = record.session_id.expect("agent-a has a session");
7606        wait_for_runtime_session_history_text(
7607            &runtime,
7608            &session_id,
7609            "You are agent-a.",
7610            Duration::from_secs(5),
7611        )
7612        .await?;
7613
7614        let aggregator = MobKitConsoleAggregator::in_memory();
7615        aggregator
7616            .inner
7617            .runtimes
7618            .write()
7619            .expect("runtime registry")
7620            .insert("runtime-a".to_string(), entry);
7621        let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
7622        aggregator
7623            .store()
7624            .record_source_watermark(
7625                &watermark_key,
7626                ConsoleFrameSourceKind::SessionHistory,
7627                &format_session_history_watermark(&session_id, 0, current_time_ms()),
7628            )
7629            .await
7630            .expect("record fresh empty watermark");
7631
7632        // Hang/stall backstop only — generous so CPU starvation under full
7633        // parallel CI load can't false-trip it (matches the sibling timeline
7634        // query test). The real assertions are on the returned frames below.
7635        let page = tokio::time::timeout(
7636            Duration::from_mins(1),
7637            aggregator.query_timeline(ConsoleTimelineQuery {
7638                identity: Some("test/agent-a".to_string()),
7639                limit: 20,
7640                ..ConsoleTimelineQuery::default()
7641            }),
7642        )
7643        .await
7644        .expect("explicit identity query should not stall")
7645        .expect("query succeeds");
7646
7647        if !page.frames.is_empty() {
7648            assert!(
7649                page.frames.iter().any(|frame| {
7650                    frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7651                        && session_history_content_text(frame).as_deref()
7652                            == Some("You are agent-a.")
7653                }),
7654                "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
7655                page.frames
7656            );
7657        }
7658        wait_for_session_history_text(
7659            &aggregator,
7660            "test/agent-a",
7661            "You are agent-a.",
7662            Duration::from_secs(5),
7663        )
7664        .await?;
7665        let _ = runtime.mob_handle().stop().await;
7666        Ok(())
7667    }
7668
7669    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7670    async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
7671    {
7672        let store = Arc::new(CountingConsoleLogStore::new());
7673        let aggregator = MobKitConsoleAggregator::new(store.clone());
7674        let runtime = build_single_member_runtime().await;
7675        let entry = runtime_entry_for_test("runtime-a", &runtime);
7676        let resolved = member_sources_for_entry(&entry)
7677            .await
7678            .into_iter()
7679            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7680            .expect("agent-a member exists");
7681        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7682            .await
7683            .expect("identity record exists");
7684        let session_id = record.session_id.expect("agent-a has a session");
7685        wait_for_runtime_session_history_text(
7686            &runtime,
7687            &session_id,
7688            "You are agent-a.",
7689            Duration::from_secs(5),
7690        )
7691        .await?;
7692        aggregator
7693            .inner
7694            .runtimes
7695            .write()
7696            .expect("runtime registry")
7697            .insert("runtime-a".to_string(), entry);
7698
7699        store
7700            .append_if_absent(NewConsoleFrame {
7701                id: None,
7702                dedupe_key: "stale-session-history-agent-a".to_string(),
7703                timestamp_ms: 10,
7704                runtime_key: "runtime-a".to_string(),
7705                identity: "test/agent-a".to_string(),
7706                conversation_id: Some("test/agent-a".to_string()),
7707                session_id: Some(session_id.clone()),
7708                kind: "user_input".to_string(),
7709                status: ConsoleFrameStatus::Delivered,
7710                payload: json!({
7711                    "text": "stale projected history",
7712                    "type": "user_input",
7713                }),
7714                source: ConsoleFrameSource {
7715                    kind: ConsoleFrameSourceKind::SessionHistory,
7716                    source_cursor: Some("stale-session-history-agent-a".to_string()),
7717                },
7718                source_event_id: Some("stale-session-history-agent-a".to_string()),
7719                interaction_id: None,
7720                turn_id: None,
7721                run_id: None,
7722                parent_frame_id: None,
7723                caused_by_frame_id: None,
7724            })
7725            .await
7726            .expect("append stale history frame");
7727
7728        let fresh_prompt = "fresh prompt after stale history";
7729        let sent_session_id = send_message_on_mob_with_mode(
7730            &runtime.mob_handle(),
7731            "agent-a",
7732            ContentInput::Text(fresh_prompt.to_string()),
7733            meerkat_core::types::HandlingMode::Queue,
7734        )
7735        .await
7736        .expect("direct member send succeeds");
7737        assert_eq!(sent_session_id, session_id);
7738        wait_for_runtime_session_history_text(
7739            &runtime,
7740            &session_id,
7741            fresh_prompt,
7742            Duration::from_secs(5),
7743        )
7744        .await?;
7745
7746        let page = aggregator
7747            .query_timeline(ConsoleTimelineQuery {
7748                identity: Some("test/agent-a".to_string()),
7749                limit: 50,
7750                ..ConsoleTimelineQuery::default()
7751            })
7752            .await
7753            .expect("query child timeline");
7754
7755        assert!(
7756            page.frames.iter().any(|frame| {
7757                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7758                    && session_history_content_text(frame).as_deref()
7759                        == Some("stale projected history")
7760            }),
7761            "explicit identity query should return existing history before async refresh; frames: {:#?}",
7762            page.frames
7763        );
7764        wait_for_session_history_text(
7765            &aggregator,
7766            "test/agent-a",
7767            fresh_prompt,
7768            Duration::from_secs(5),
7769        )
7770        .await?;
7771        assert!(
7772            store.source_watermark_calls() > 0,
7773            "stale existing session history should force a targeted source refresh"
7774        );
7775        let _ = runtime.mob_handle().stop().await;
7776        Ok(())
7777    }
7778
7779    async fn wait_for_session_history_text(
7780        aggregator: &MobKitConsoleAggregator,
7781        identity: &str,
7782        expected: &str,
7783        timeout: Duration,
7784    ) -> Result<(), String> {
7785        let deadline = Instant::now() + timeout;
7786        let mut observed = Vec::new();
7787        while Instant::now() < deadline {
7788            let _ = aggregator.list_identities().await;
7789            let page = aggregator
7790                .query_timeline(ConsoleTimelineQuery {
7791                    identity: Some(identity.to_string()),
7792                    limit: 20,
7793                    ..ConsoleTimelineQuery::default()
7794                })
7795                .await
7796                .expect("query timeline");
7797            observed = page.frames;
7798            if observed.iter().any(|frame| {
7799                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7800                    && session_history_content_text(frame).as_deref() == Some(expected)
7801            }) {
7802                return Ok(());
7803            }
7804            tokio::time::sleep(Duration::from_millis(25)).await;
7805        }
7806
7807        Err(format!(
7808            "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7809        ))
7810    }
7811
7812    async fn wait_for_session_backfill_target(
7813        aggregator: &MobKitConsoleAggregator,
7814        identity: &str,
7815        timeout: Duration,
7816    ) -> Result<String, String> {
7817        let deadline = Instant::now() + timeout;
7818        while Instant::now() < deadline {
7819            if let Some(target) =
7820                session_backfill_target_for_identity(&aggregator.inner, identity).await
7821            {
7822                return Ok(target.session_id);
7823            }
7824            tokio::time::sleep(Duration::from_millis(25)).await;
7825        }
7826        Err(format!(
7827            "session backfill target for {identity:?} was not resolvable"
7828        ))
7829    }
7830
7831    async fn wait_for_runtime_session_history_text(
7832        runtime: &UnifiedRuntime,
7833        session_id: &str,
7834        expected: &str,
7835        timeout: Duration,
7836    ) -> Result<(), String> {
7837        let deadline = Instant::now() + timeout;
7838        let mut observed = Vec::new();
7839        while Instant::now() < deadline {
7840            let page = runtime
7841                .mob_runtime()
7842                .read_session_history(session_id, 0, Some(20))
7843                .await
7844                .map_err(|err| err.to_string())?;
7845            observed = page.messages;
7846            if observed.iter().enumerate().any(|(idx, message)| {
7847                let Some(message) = serde_json::to_value(message).ok() else {
7848                    return false;
7849                };
7850                frames_from_session_history_message(
7851                    "runtime-a",
7852                    "test/agent-a",
7853                    session_id,
7854                    idx,
7855                    message,
7856                )
7857                .iter()
7858                .any(|frame| {
7859                    matches!(
7860                        frame.kind.as_str(),
7861                        "user_input" | "system_notice" | "interaction_complete"
7862                    ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7863                })
7864            }) {
7865                return Ok(());
7866            }
7867            tokio::time::sleep(Duration::from_millis(25)).await;
7868        }
7869
7870        Err(format!(
7871            "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7872        ))
7873    }
7874
7875    async fn wait_for_identity_record(
7876        aggregator: &MobKitConsoleAggregator,
7877        identity: &str,
7878        session_id: Option<&str>,
7879        timeout: Duration,
7880    ) -> Result<(), String> {
7881        let deadline = Instant::now() + timeout;
7882        let mut observed = Vec::new();
7883        while Instant::now() < deadline {
7884            observed = aggregator
7885                .list_identities()
7886                .await
7887                .map_err(|err| err.to_string())?;
7888            if observed.iter().any(|record| {
7889                record.identity == identity && record.session_id.as_deref() == session_id
7890            }) {
7891                return Ok(());
7892            }
7893            tokio::time::sleep(Duration::from_millis(25)).await;
7894        }
7895
7896        Err(format!(
7897            "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7898        ))
7899    }
7900
7901    fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7902        session_history_payload_text(&frame.payload)
7903    }
7904
7905    fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7906        session_history_payload_text(&frame.payload)
7907    }
7908
7909    fn session_history_payload_text(payload: &Value) -> Option<String> {
7910        if let Some(text) = payload.get("text").and_then(Value::as_str) {
7911            return Some(text.to_string());
7912        }
7913        if let Some(text) = payload.get("result").and_then(Value::as_str) {
7914            return Some(text.to_string());
7915        }
7916        if let Some(text) = payload.get("body").and_then(Value::as_str) {
7917            return Some(text.to_string());
7918        }
7919        match payload.get("content")? {
7920            Value::String(text) => Some(text.clone()),
7921            Value::Array(blocks) => Some(
7922                blocks
7923                    .iter()
7924                    .filter_map(|block| block.get("text").and_then(Value::as_str))
7925                    .collect::<Vec<_>>()
7926                    .join(""),
7927            ),
7928            _ => None,
7929        }
7930    }
7931
7932    #[tokio::test]
7933    async fn query_timeline_handles_large_store_without_backfill_calls() {
7934        let store = Arc::new(CountingConsoleLogStore::new());
7935        let aggregator = MobKitConsoleAggregator::new(store.clone());
7936        for idx in 0..5_000 {
7937            store
7938                .append_if_absent(NewConsoleFrame {
7939                    id: None,
7940                    dedupe_key: format!("event-{idx}"),
7941                    timestamp_ms: idx,
7942                    runtime_key: "runtime-a".to_string(),
7943                    identity: "agent-a".to_string(),
7944                    conversation_id: Some("agent-a".to_string()),
7945                    session_id: Some("session-a".to_string()),
7946                    kind: "text_delta".to_string(),
7947                    status: ConsoleFrameStatus::Delivered,
7948                    payload: json!({ "delta": idx }),
7949                    source: ConsoleFrameSource {
7950                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7951                        source_cursor: None,
7952                    },
7953                    source_event_id: Some(format!("event-{idx}")),
7954                    interaction_id: None,
7955                    turn_id: None,
7956                    run_id: None,
7957                    parent_frame_id: None,
7958                    caused_by_frame_id: None,
7959                })
7960                .await
7961                .expect("append frame");
7962        }
7963
7964        let page = aggregator
7965            .query_timeline(ConsoleTimelineQuery {
7966                identity: Some("agent-a".to_string()),
7967                limit: 1_000,
7968                ..ConsoleTimelineQuery::default()
7969            })
7970            .await
7971            .expect("large query");
7972
7973        assert_eq!(page.frames.len(), 1_000);
7974        assert_eq!(store.source_watermark_calls(), 0);
7975    }
7976
7977    #[tokio::test]
7978    async fn query_timeline_rejects_future_cursor_after_store_reset() {
7979        let aggregator = MobKitConsoleAggregator::in_memory();
7980        let err = aggregator
7981            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7982                after: Some(ConsoleCursor::from("console:99")),
7983                limit: 10,
7984                ..ConsoleTimelineWindowQuery::default()
7985            })
7986            .await
7987            .expect_err("future cursor on empty/reset store must be replay-unavailable");
7988
7989        assert!(
7990            err.to_string()
7991                .contains("beyond the current store frontier"),
7992            "unexpected error: {err}"
7993        );
7994    }
7995
7996    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7997    async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7998        let store = Arc::new(CountingConsoleLogStore::new());
7999        let aggregator = MobKitConsoleAggregator::new(store.clone());
8000        let runtime = build_single_member_runtime().await;
8001        aggregator
8002            .inner
8003            .runtimes
8004            .write()
8005            .expect("runtime registry")
8006            .insert(
8007                "runtime-a".to_string(),
8008                runtime_entry_for_test("runtime-a", &runtime),
8009            );
8010        let session_id =
8011            wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8012                .await
8013                .expect("spawned member is resolvable for targeted backfill");
8014        wait_for_runtime_session_history_text(
8015            &runtime,
8016            &session_id,
8017            "You are agent-a.",
8018            Duration::from_secs(5),
8019        )
8020        .await
8021        .expect("spawned member kickoff is readable before live-frame race");
8022        store
8023            .append_if_absent(NewConsoleFrame {
8024                id: None,
8025                dedupe_key: "live-tool-before-history".to_string(),
8026                timestamp_ms: 10,
8027                runtime_key: "runtime-a".to_string(),
8028                identity: "test/agent-a".to_string(),
8029                conversation_id: Some("test/agent-a".to_string()),
8030                session_id: None,
8031                kind: "tool_execution_started".to_string(),
8032                status: ConsoleFrameStatus::Delivered,
8033                payload: json!({
8034                    "id": "call-live",
8035                    "name": "king_search",
8036                    "source_event_type": "tool_execution_started",
8037                    "type": "tool_execution_started",
8038                }),
8039                source: ConsoleFrameSource {
8040                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8041                    source_cursor: Some("live-tool-before-history".to_string()),
8042                },
8043                source_event_id: Some("live-tool-before-history".to_string()),
8044                interaction_id: None,
8045                turn_id: None,
8046                run_id: None,
8047                parent_frame_id: None,
8048                caused_by_frame_id: None,
8049            })
8050            .await
8051            .expect("append live tool frame");
8052
8053        let page = aggregator
8054            .query_timeline(ConsoleTimelineQuery {
8055                identity: Some("test/agent-a".to_string()),
8056                limit: 20,
8057                ..ConsoleTimelineQuery::default()
8058            })
8059            .await
8060            .expect("query child timeline");
8061
8062        assert!(
8063            page.frames
8064                .iter()
8065                .any(|frame| frame.kind == "tool_execution_started"),
8066            "initial explicit child timeline should return existing store frames without waiting for session history"
8067        );
8068
8069        let mut observed_backfill = false;
8070        for _ in 0..80 {
8071            let page = aggregator
8072                .query_timeline(ConsoleTimelineQuery {
8073                    identity: Some("test/agent-a".to_string()),
8074                    limit: 20,
8075                    ..ConsoleTimelineQuery::default()
8076                })
8077                .await
8078                .expect("query child timeline after scheduled backfill");
8079            if page.frames.iter().any(|frame| {
8080                frame.kind == "user_input"
8081                    && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8082                    && frame.payload.to_string().contains("You are agent-a.")
8083            }) {
8084                observed_backfill = true;
8085                break;
8086            }
8087            tokio::time::sleep(Duration::from_millis(25)).await;
8088        }
8089        assert!(
8090            observed_backfill,
8091            "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8092        );
8093        let _ = runtime.mob_handle().stop().await;
8094    }
8095
8096    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8097    async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8098        let store = Arc::new(CountingConsoleLogStore::new());
8099        let aggregator = MobKitConsoleAggregator::new(store.clone());
8100        let (_temp, runtime, _delayed_service) =
8101            build_stress_runtime(64, Duration::from_millis(0)).await;
8102        aggregator
8103            .inner
8104            .runtimes
8105            .write()
8106            .expect("runtime registry")
8107            .insert(
8108                "runtime-a".to_string(),
8109                runtime_entry_for_test("runtime-a", runtime.as_ref()),
8110            );
8111        for idx in 0..1_000 {
8112            store
8113                .append_if_absent(NewConsoleFrame {
8114                    id: None,
8115                    dedupe_key: format!("event-{idx}"),
8116                    timestamp_ms: idx,
8117                    runtime_key: "runtime-a".to_string(),
8118                    identity: "test/agent-0".to_string(),
8119                    conversation_id: Some("test/agent-0".to_string()),
8120                    session_id: Some("session-a".to_string()),
8121                    kind: "text_delta".to_string(),
8122                    status: ConsoleFrameStatus::Delivered,
8123                    payload: json!({ "delta": idx }),
8124                    source: ConsoleFrameSource {
8125                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8126                        source_cursor: None,
8127                    },
8128                    source_event_id: Some(format!("event-{idx}")),
8129                    interaction_id: None,
8130                    turn_id: None,
8131                    run_id: None,
8132                    parent_frame_id: None,
8133                    caused_by_frame_id: None,
8134                })
8135                .await
8136                .expect("append frame");
8137        }
8138
8139        // The per-frame roster-rediscovery regression is asserted exactly by
8140        // `source_watermark_calls() == 0` below (and would also blow this query
8141        // up to O(frames) async roster resolutions). The timeout is only a
8142        // hang/O(n^2)-blowup backstop — generous so it never false-trips on CPU
8143        // starvation under full parallel CI load (the old 2s ceiling flaked the
8144        // suite); a real regression takes minutes, not seconds, on 1000 frames.
8145        let page = tokio::time::timeout(
8146            Duration::from_mins(1),
8147            aggregator.query_timeline(ConsoleTimelineQuery {
8148                identity: Some("test/agent-0".to_string()),
8149                limit: 1_000,
8150                ..ConsoleTimelineQuery::default()
8151            }),
8152        )
8153        .await
8154        .expect("identity timeline query should not hang or rediscover the roster per frame")
8155        .expect("timeline query succeeds");
8156
8157        assert_eq!(page.frames.len(), 1_000);
8158        assert_eq!(store.source_watermark_calls(), 0);
8159        let _ = runtime.mob_handle().stop().await;
8160    }
8161
8162    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8163    async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8164        const MEMBER_COUNT: usize = 32;
8165        let (_temp, runtime, delayed_service) =
8166            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8167        let aggregator = MobKitConsoleAggregator::in_memory();
8168        aggregator
8169            .inner
8170            .runtimes
8171            .write()
8172            .expect("runtime registry")
8173            .insert(
8174                "runtime-stress".to_string(),
8175                runtime_entry_for_test("runtime-stress", &runtime),
8176            );
8177
8178        let started = Instant::now();
8179        aggregator
8180            .refresh_session_history()
8181            .await
8182            .expect("stress refresh");
8183        let elapsed = started.elapsed();
8184
8185        assert!(
8186            delayed_service.read_calls() >= MEMBER_COUNT,
8187            "expected at least one history read per member, saw {}",
8188            delayed_service.read_calls()
8189        );
8190        assert!(
8191            delayed_service.max_active_reads() > 1,
8192            "session history backfill should fan out instead of reading members serially"
8193        );
8194        assert!(
8195            delayed_service.max_active_reads()
8196                <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8197            "session history backfill should respect the default concurrency limit"
8198        );
8199        assert!(
8200            elapsed < Duration::from_millis(600),
8201            "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8202            MEMBER_COUNT * 40
8203        );
8204        let _ = runtime.mob_handle().stop().await;
8205    }
8206
8207    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8208    async fn session_history_backfill_respects_configured_concurrency_limit() {
8209        const MEMBER_COUNT: usize = 16;
8210        let (_temp, runtime, delayed_service) =
8211            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8212        let aggregator =
8213            MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8214                max_concurrent_session_backfills: 4,
8215                ..ConsoleAggregatorOptions::default()
8216            });
8217        aggregator
8218            .inner
8219            .runtimes
8220            .write()
8221            .expect("runtime registry")
8222            .insert(
8223                "runtime-stress".to_string(),
8224                runtime_entry_for_test("runtime-stress", &runtime),
8225            );
8226
8227        aggregator
8228            .refresh_session_history()
8229            .await
8230            .expect("stress refresh");
8231
8232        assert!(
8233            delayed_service.max_active_reads() <= 4,
8234            "configured concurrency limit should bound session history reads, saw {}",
8235            delayed_service.max_active_reads()
8236        );
8237        let _ = runtime.mob_handle().stop().await;
8238    }
8239
8240    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8241    async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8242        const MEMBER_COUNT: usize = 24;
8243        const LIVE_EVENT_COUNT: usize = 2_048;
8244        let (_temp, runtime, _delayed_service) =
8245            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
8246        let aggregator = MobKitConsoleAggregator::in_memory();
8247        aggregator.register_runtime(ConsoleRuntimeRegistration {
8248            runtime_key: "runtime-burst".to_string(),
8249            runtime: runtime.clone(),
8250            identity_namespace: "stress".to_string(),
8251            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8252        });
8253        let console_events = runtime.console_events();
8254        for idx in 0..LIVE_EVENT_COUNT {
8255            console_events
8256                .append(
8257                    "agent-0",
8258                    Some("burst-turn".to_string()),
8259                    "text_delta",
8260                    json!({ "delta": format!("frame-{idx}") }),
8261                )
8262                .await;
8263        }
8264
8265        // Generous deadline: the loop breaks the instant all live frames land,
8266        // so a large ceiling is free in the normal case and only prevents a
8267        // false drop-detection when the live pump is CPU-starved under full
8268        // parallel CI load (the old 5s deadline flaked the suite).
8269        let deadline = Instant::now() + Duration::from_secs(30);
8270        let mut observed = 0;
8271        while Instant::now() < deadline {
8272            observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
8273            if observed >= LIVE_EVENT_COUNT {
8274                break;
8275            }
8276            tokio::time::sleep(Duration::from_millis(25)).await;
8277        }
8278
8279        assert_eq!(
8280            observed, LIVE_EVENT_COUNT,
8281            "live pump should not drop frames while slow background backfill is running"
8282        );
8283        let _ = runtime.mob_handle().stop().await;
8284    }
8285
8286    async fn count_console_event_frames(
8287        aggregator: &MobKitConsoleAggregator,
8288        identity: &str,
8289    ) -> usize {
8290        let mut after = None;
8291        let mut count = 0;
8292        loop {
8293            let page = aggregator
8294                .query_timeline(ConsoleTimelineQuery {
8295                    identity: Some(identity.to_string()),
8296                    after,
8297                    limit: 1_000,
8298                    ..ConsoleTimelineQuery::default()
8299                })
8300                .await
8301                .expect("query burst timeline");
8302            if page.frames.is_empty() {
8303                break;
8304            }
8305            count += page
8306                .frames
8307                .iter()
8308                .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
8309                .count();
8310            after = page.next_cursor;
8311            if after.is_none() {
8312                break;
8313            }
8314        }
8315        count
8316    }
8317
8318    #[tokio::test]
8319    async fn status_updates_get_replayable_aggregate_cursors() {
8320        let aggregator = MobKitConsoleAggregator::in_memory();
8321        let frame = NewConsoleFrame {
8322            id: None,
8323            dedupe_key: "send-1".to_string(),
8324            timestamp_ms: 1,
8325            runtime_key: "runtime-a".to_string(),
8326            identity: "agent-a".to_string(),
8327            conversation_id: Some("agent-a".to_string()),
8328            session_id: Some("session-1".to_string()),
8329            kind: "user_input".to_string(),
8330            status: ConsoleFrameStatus::Accepted,
8331            payload: json!({ "content": "hello" }),
8332            source: ConsoleFrameSource {
8333                kind: ConsoleFrameSourceKind::Send,
8334                source_cursor: None,
8335            },
8336            source_event_id: None,
8337            interaction_id: Some("interaction-1".to_string()),
8338            turn_id: None,
8339            run_id: None,
8340            parent_frame_id: None,
8341            caused_by_frame_id: None,
8342        };
8343        let inserted = aggregator
8344            .store()
8345            .append_if_absent(frame)
8346            .await
8347            .expect("append frame");
8348
8349        update_frame_status_and_emit(
8350            &aggregator.inner,
8351            &inserted.frame.id,
8352            ConsoleFrameStatus::Delivered,
8353        )
8354        .await
8355        .expect("update status");
8356
8357        let page = aggregator
8358            .query_timeline(ConsoleTimelineQuery {
8359                identity: Some("agent-a".to_string()),
8360                after: Some(inserted.frame.cursor.clone()),
8361                limit: 10,
8362                ..ConsoleTimelineQuery::default()
8363            })
8364            .await
8365            .expect("query timeline");
8366        assert_eq!(page.frames.len(), 1);
8367        assert_eq!(page.frames[0].kind, "frame_updated");
8368        assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
8369        assert_eq!(
8370            page.frames[0]
8371                .payload
8372                .get("frame")
8373                .and_then(|frame| frame.get("status"))
8374                .and_then(Value::as_str),
8375            Some("delivered")
8376        );
8377    }
8378
8379    #[tokio::test]
8380    async fn steer_delivery_appends_terminal_control_frame() {
8381        let aggregator = MobKitConsoleAggregator::in_memory();
8382        let frame = NewConsoleFrame {
8383            id: None,
8384            dedupe_key: "send-steer-1".to_string(),
8385            timestamp_ms: 1,
8386            runtime_key: "runtime-a".to_string(),
8387            identity: "agent-a".to_string(),
8388            conversation_id: Some("agent-a".to_string()),
8389            session_id: Some("session-1".to_string()),
8390            kind: "user_input".to_string(),
8391            status: ConsoleFrameStatus::Delivered,
8392            payload: json!({
8393                "content": "operator steer",
8394                "handling_mode": "steer",
8395            }),
8396            source: ConsoleFrameSource {
8397                kind: ConsoleFrameSourceKind::Send,
8398                source_cursor: None,
8399            },
8400            source_event_id: None,
8401            interaction_id: Some("interaction-steer-1".to_string()),
8402            turn_id: None,
8403            run_id: None,
8404            parent_frame_id: None,
8405            caused_by_frame_id: None,
8406        };
8407        let inserted = aggregator
8408            .store()
8409            .append_if_absent(frame)
8410            .await
8411            .expect("append steer input");
8412
8413        append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
8414            .await
8415            .expect("append steer terminal");
8416
8417        let page = aggregator
8418            .query_timeline(ConsoleTimelineQuery {
8419                identity: Some("agent-a".to_string()),
8420                after: Some(inserted.frame.cursor.clone()),
8421                limit: 10,
8422                ..ConsoleTimelineQuery::default()
8423            })
8424            .await
8425            .expect("query timeline");
8426
8427        assert_eq!(page.frames.len(), 1);
8428        let terminal = &page.frames[0];
8429        assert_eq!(terminal.kind, "interaction_complete");
8430        assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
8431        assert_eq!(
8432            terminal.interaction_id.as_deref(),
8433            Some("interaction-steer-1")
8434        );
8435        assert_eq!(
8436            terminal.parent_frame_id.as_deref(),
8437            Some(inserted.frame.id.as_str())
8438        );
8439        assert_eq!(
8440            terminal.payload.get("reason").and_then(Value::as_str),
8441            Some("steer_delivered")
8442        );
8443    }
8444
8445    #[test]
8446    fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
8447        let live = transcript_fingerprint(
8448            "reasoning_complete",
8449            &json!({ "text": "I should inspect the file first." }),
8450        );
8451        let replay = transcript_fingerprint(
8452            "reasoning_complete",
8453            &json!({ "summary": "I should inspect the file first." }),
8454        );
8455        let delta = transcript_fingerprint(
8456            "reasoning_delta",
8457            &json!({ "delta": "I should inspect the file first." }),
8458        );
8459
8460        assert_eq!(live.as_deref(), Some("I should inspect the file first."));
8461        assert_eq!(live, replay);
8462        assert_eq!(live, delta);
8463    }
8464
8465    #[tokio::test]
8466    async fn history_counterpart_scan_is_not_capped_to_one_page() {
8467        let aggregator = MobKitConsoleAggregator::in_memory();
8468        for idx in 0..1_005 {
8469            aggregator
8470                .store()
8471                .append_if_absent(NewConsoleFrame {
8472                    id: None,
8473                    dedupe_key: format!("filler-{idx}"),
8474                    timestamp_ms: idx,
8475                    runtime_key: "runtime-a".to_string(),
8476                    identity: "agent-a".to_string(),
8477                    conversation_id: Some("agent-a".to_string()),
8478                    session_id: Some("session-a".to_string()),
8479                    kind: "text_delta".to_string(),
8480                    status: ConsoleFrameStatus::Completed,
8481                    payload: json!({ "delta": idx }),
8482                    source: ConsoleFrameSource {
8483                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8484                        source_cursor: None,
8485                    },
8486                    source_event_id: Some(format!("filler-{idx}")),
8487                    interaction_id: None,
8488                    turn_id: None,
8489                    run_id: None,
8490                    parent_frame_id: None,
8491                    caused_by_frame_id: None,
8492                })
8493                .await
8494                .expect("append filler");
8495        }
8496        aggregator
8497            .store()
8498            .append_if_absent(NewConsoleFrame {
8499                id: None,
8500                dedupe_key: "live-user-input".to_string(),
8501                timestamp_ms: 2_000,
8502                runtime_key: "runtime-a".to_string(),
8503                identity: "agent-a".to_string(),
8504                conversation_id: Some("agent-a".to_string()),
8505                session_id: Some("session-a".to_string()),
8506                kind: "user_input".to_string(),
8507                status: ConsoleFrameStatus::Delivered,
8508                payload: json!({ "content": "already here" }),
8509                source: ConsoleFrameSource {
8510                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8511                    source_cursor: None,
8512                },
8513                source_event_id: Some("live-user-input".to_string()),
8514                interaction_id: None,
8515                turn_id: None,
8516                run_id: None,
8517                parent_frame_id: None,
8518                caused_by_frame_id: None,
8519            })
8520            .await
8521            .expect("append live input");
8522
8523        let history = NewConsoleFrame {
8524            id: None,
8525            dedupe_key: "history-user-input".to_string(),
8526            timestamp_ms: 3_000,
8527            runtime_key: "runtime-a".to_string(),
8528            identity: "agent-a".to_string(),
8529            conversation_id: Some("agent-a".to_string()),
8530            session_id: Some("session-a".to_string()),
8531            kind: "user_input".to_string(),
8532            status: ConsoleFrameStatus::Completed,
8533            payload: json!({ "content": "already here" }),
8534            source: ConsoleFrameSource {
8535                kind: ConsoleFrameSourceKind::SessionHistory,
8536                source_cursor: Some("session-a:1006".to_string()),
8537            },
8538            source_event_id: None,
8539            interaction_id: None,
8540            turn_id: None,
8541            run_id: None,
8542            parent_frame_id: None,
8543            caused_by_frame_id: None,
8544        };
8545
8546        assert!(
8547            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8548                .await
8549                .expect("counterpart scan")
8550        );
8551    }
8552
8553    #[tokio::test]
8554    async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
8555        let aggregator = MobKitConsoleAggregator::in_memory();
8556        aggregator
8557            .store()
8558            .append_if_absent(NewConsoleFrame {
8559                id: None,
8560                dedupe_key: "live-user-input".to_string(),
8561                timestamp_ms: 2_000,
8562                runtime_key: "runtime-a".to_string(),
8563                identity: "agent-a".to_string(),
8564                conversation_id: Some("agent-a".to_string()),
8565                session_id: Some("session-a".to_string()),
8566                kind: "user_input".to_string(),
8567                status: ConsoleFrameStatus::Delivered,
8568                payload: json!({ "content": "hello from operator" }),
8569                source: ConsoleFrameSource {
8570                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8571                    source_cursor: None,
8572                },
8573                source_event_id: Some("live-user-input".to_string()),
8574                interaction_id: None,
8575                turn_id: None,
8576                run_id: None,
8577                parent_frame_id: None,
8578                caused_by_frame_id: None,
8579            })
8580            .await
8581            .expect("append live input");
8582
8583        let history = NewConsoleFrame {
8584            id: None,
8585            dedupe_key: "history-user-input".to_string(),
8586            timestamp_ms: 3_000,
8587            runtime_key: "runtime-a".to_string(),
8588            identity: "agent-a".to_string(),
8589            conversation_id: Some("agent-a".to_string()),
8590            session_id: Some("session-a".to_string()),
8591            kind: "user_input".to_string(),
8592            status: ConsoleFrameStatus::Completed,
8593            payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
8594            source: ConsoleFrameSource {
8595                kind: ConsoleFrameSourceKind::SessionHistory,
8596                source_cursor: Some("session-a:2".to_string()),
8597            },
8598            source_event_id: None,
8599            interaction_id: None,
8600            turn_id: None,
8601            run_id: None,
8602            parent_frame_id: None,
8603            caused_by_frame_id: None,
8604        };
8605
8606        assert!(
8607            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8608                .await
8609                .expect("counterpart scan")
8610        );
8611    }
8612
8613    #[tokio::test]
8614    async fn history_counterpart_scan_matches_content_block_user_prompts() {
8615        let aggregator = MobKitConsoleAggregator::in_memory();
8616        aggregator
8617            .store()
8618            .append_if_absent(NewConsoleFrame {
8619                id: None,
8620                dedupe_key: "live-user-input".to_string(),
8621                timestamp_ms: 2_000,
8622                runtime_key: "runtime-a".to_string(),
8623                identity: "agent-a".to_string(),
8624                conversation_id: Some("agent-a".to_string()),
8625                session_id: Some("session-a".to_string()),
8626                kind: "user_input".to_string(),
8627                status: ConsoleFrameStatus::Delivered,
8628                payload: json!({ "content": "hello from operator" }),
8629                source: ConsoleFrameSource {
8630                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8631                    source_cursor: None,
8632                },
8633                source_event_id: Some("live-user-input".to_string()),
8634                interaction_id: None,
8635                turn_id: None,
8636                run_id: None,
8637                parent_frame_id: None,
8638                caused_by_frame_id: None,
8639            })
8640            .await
8641            .expect("append live input");
8642
8643        let history = NewConsoleFrame {
8644            id: None,
8645            dedupe_key: "history-user-input".to_string(),
8646            timestamp_ms: 3_000,
8647            runtime_key: "runtime-a".to_string(),
8648            identity: "agent-a".to_string(),
8649            conversation_id: Some("agent-a".to_string()),
8650            session_id: Some("session-a".to_string()),
8651            kind: "user_input".to_string(),
8652            status: ConsoleFrameStatus::Completed,
8653            payload: json!({
8654                "content": [{ "type": "text", "text": "hello from operator" }]
8655            }),
8656            source: ConsoleFrameSource {
8657                kind: ConsoleFrameSourceKind::SessionHistory,
8658                source_cursor: Some("session-a:2".to_string()),
8659            },
8660            source_event_id: None,
8661            interaction_id: None,
8662            turn_id: None,
8663            run_id: None,
8664            parent_frame_id: None,
8665            caused_by_frame_id: None,
8666        };
8667
8668        assert!(
8669            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8670                .await
8671                .expect("counterpart scan")
8672        );
8673    }
8674
8675    #[tokio::test]
8676    async fn history_counterpart_scan_matches_live_tool_results() {
8677        let aggregator = MobKitConsoleAggregator::in_memory();
8678        aggregator
8679            .store()
8680            .append_if_absent(NewConsoleFrame {
8681                id: None,
8682                dedupe_key: "live-tool-result".to_string(),
8683                timestamp_ms: 2_000,
8684                runtime_key: "runtime-a".to_string(),
8685                identity: "agent-a".to_string(),
8686                conversation_id: Some("agent-a".to_string()),
8687                session_id: Some("session-a".to_string()),
8688                kind: "tool_execution_completed".to_string(),
8689                status: ConsoleFrameStatus::Delivered,
8690                payload: json!({
8691                    "id": "call-1",
8692                    "tool_call_id": "call-1",
8693                    "result": "{ \"count\": 70 }"
8694                }),
8695                source: ConsoleFrameSource {
8696                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8697                    source_cursor: None,
8698                },
8699                source_event_id: Some("live-tool-result".to_string()),
8700                interaction_id: None,
8701                turn_id: None,
8702                run_id: None,
8703                parent_frame_id: None,
8704                caused_by_frame_id: None,
8705            })
8706            .await
8707            .expect("append live tool result");
8708
8709        let history = NewConsoleFrame {
8710            id: None,
8711            dedupe_key: "history-tool-result".to_string(),
8712            timestamp_ms: 3_000,
8713            runtime_key: "runtime-a".to_string(),
8714            identity: "agent-a".to_string(),
8715            conversation_id: Some("agent-a".to_string()),
8716            session_id: Some("session-a".to_string()),
8717            kind: "tool_execution_completed".to_string(),
8718            status: ConsoleFrameStatus::Completed,
8719            payload: json!({
8720                "id": "call-1",
8721                "tool_call_id": "call-1",
8722                "result": "{ \"count\": 70 }",
8723                "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
8724            }),
8725            source: ConsoleFrameSource {
8726                kind: ConsoleFrameSourceKind::SessionHistory,
8727                source_cursor: Some("session-a:3:0".to_string()),
8728            },
8729            source_event_id: None,
8730            interaction_id: None,
8731            turn_id: None,
8732            run_id: None,
8733            parent_frame_id: None,
8734            caused_by_frame_id: None,
8735        };
8736
8737        assert!(
8738            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8739                .await
8740                .expect("counterpart scan")
8741        );
8742    }
8743
8744    #[tokio::test]
8745    async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
8746        let aggregator = MobKitConsoleAggregator::in_memory();
8747        for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
8748            aggregator
8749                .store()
8750                .append_if_absent(NewConsoleFrame {
8751                    id: None,
8752                    dedupe_key: format!("live-delta-{idx}"),
8753                    timestamp_ms: 2_000 + idx as u64,
8754                    runtime_key: "runtime-a".to_string(),
8755                    identity: "agent-a".to_string(),
8756                    conversation_id: Some("agent-a".to_string()),
8757                    session_id: Some("session-a".to_string()),
8758                    kind: "text_delta".to_string(),
8759                    status: ConsoleFrameStatus::Delivered,
8760                    payload: json!({ "delta": delta }),
8761                    source: ConsoleFrameSource {
8762                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8763                        source_cursor: None,
8764                    },
8765                    source_event_id: Some(format!("live-delta-{idx}")),
8766                    interaction_id: Some("turn-a".to_string()),
8767                    turn_id: None,
8768                    run_id: None,
8769                    parent_frame_id: None,
8770                    caused_by_frame_id: None,
8771                })
8772                .await
8773                .expect("append live delta");
8774        }
8775
8776        let history = NewConsoleFrame {
8777            id: None,
8778            dedupe_key: "history-assistant-complete".to_string(),
8779            timestamp_ms: 3_000,
8780            runtime_key: "runtime-a".to_string(),
8781            identity: "agent-a".to_string(),
8782            conversation_id: Some("agent-a".to_string()),
8783            session_id: Some("session-a".to_string()),
8784            kind: "interaction_complete".to_string(),
8785            status: ConsoleFrameStatus::Completed,
8786            payload: json!({ "result": "Ready and standing by." }),
8787            source: ConsoleFrameSource {
8788                kind: ConsoleFrameSourceKind::SessionHistory,
8789                source_cursor: Some("session-a:3".to_string()),
8790            },
8791            source_event_id: None,
8792            interaction_id: None,
8793            turn_id: None,
8794            run_id: None,
8795            parent_frame_id: None,
8796            caused_by_frame_id: None,
8797        };
8798
8799        assert!(
8800            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8801                .await
8802                .expect("counterpart scan")
8803        );
8804    }
8805
8806    #[test]
8807    fn session_history_watermark_key_is_session_scoped() {
8808        assert_ne!(
8809            session_history_watermark_runtime_key("runtime-a", "session-1"),
8810            session_history_watermark_runtime_key("runtime-a", "session-2")
8811        );
8812    }
8813
8814    #[test]
8815    fn backfill_offset_advances_only_on_strict_forward_progress() {
8816        // Normal full-page advance from the start.
8817        assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
8818        // Subsequent full page from a correct cursor advances.
8819        assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
8820        // A partial page that still moves forward advances.
8821        assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
8822        // One element past the requested offset still counts as progress.
8823        assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
8824
8825        // Livelock shapes: the store re-serves a page whose (base + len) does
8826        // NOT advance past what we asked for. These must stop the loop.
8827        assert_eq!(advance_backfill_offset(500, 0, 500), None); // cursor reset to 0, full page
8828        assert_eq!(advance_backfill_offset(500, 0, 400), None); // strictly behind
8829        assert_eq!(advance_backfill_offset(1000, 500, 500), None); // lands exactly on offset
8830        assert_eq!(advance_backfill_offset(10, 10, 0), None); // empty page at same offset
8831
8832        // Saturating add: a pathological base near usize::MAX cannot panic.
8833        assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
8834    }
8835
8836    #[test]
8837    fn session_history_watermarks_are_cursor_and_ttl_aware() {
8838        let legacy = "session:with:colon:42";
8839        let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8840        let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8841
8842        assert_eq!(
8843            parse_session_history_watermark(legacy, "session:with:colon"),
8844            Some(42)
8845        );
8846        assert_eq!(
8847            parse_session_history_watermark(&checked, "session:with:colon"),
8848            Some(43)
8849        );
8850        assert!(session_history_watermark_is_fresh(
8851            &checked,
8852            "session:with:colon",
8853            1_500
8854        ));
8855        assert!(!session_history_watermark_is_fresh(
8856            &checked,
8857            "session:with:colon",
8858            1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8859        ));
8860        assert!(session_history_watermark_is_fresh(
8861            &empty_checked,
8862            "session:with:colon",
8863            1_500
8864        ));
8865        assert!(!session_history_watermark_is_fresh(
8866            &empty_checked,
8867            "session:with:colon",
8868            1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8869        ));
8870    }
8871
8872    #[test]
8873    fn session_history_messages_project_to_renderable_frames() {
8874        let user = frame_from_session_history_message(
8875            "runtime-a",
8876            "agent-a",
8877            "session-a",
8878            0,
8879            json!({
8880                "role": "user",
8881                "content": "hello",
8882                "timestamp_ms": 10
8883            }),
8884        )
8885        .expect("user history frame");
8886        // meerkat 0.7 removed the legacy flat-text `Message::Assistant`
8887        // variant: session stores serialize assistant turns only as
8888        // `block_assistant` (MessageWire), so the history projection sees the
8889        // block shape.
8890        let assistant = frame_from_session_history_message(
8891            "runtime-a",
8892            "agent-a",
8893            "session-a",
8894            1,
8895            json!({
8896                "role": "block_assistant",
8897                "blocks": [
8898                    { "block_type": "text", "data": { "text": "hi there" } }
8899                ],
8900                "stop_reason": "end_turn",
8901                "timestamp_ms": 11
8902            }),
8903        )
8904        .expect("assistant history frame");
8905
8906        assert_eq!(user.kind, "user_input");
8907        assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8908        assert_eq!(
8909            user.payload["content"],
8910            json!([{ "type": "text", "text": "hello" }])
8911        );
8912        assert_eq!(assistant.kind, "interaction_complete");
8913        assert_eq!(assistant.payload["text"], json!("hi there"));
8914        assert!(
8915            assistant
8916                .dedupe_key
8917                .starts_with("session-history:runtime-a:session-a:1:")
8918        );
8919    }
8920
8921    #[test]
8922    fn session_history_projection_filters_scaffold_user_messages() {
8923        let spawn_notice = frame_from_session_history_message(
8924            "runtime-a",
8925            "agent-a",
8926            "session-a",
8927            0,
8928            json!({
8929                "role": "user",
8930                "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8931                "timestamp_ms": 10
8932            }),
8933        );
8934        let peer_update = frame_from_session_history_message(
8935            "runtime-a",
8936            "agent-a",
8937            "session-a",
8938            1,
8939            json!({
8940                "role": "user",
8941                "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8942                "timestamp_ms": 11
8943            }),
8944        );
8945        let real_user = frame_from_session_history_message(
8946            "runtime-a",
8947            "agent-a",
8948            "session-a",
8949            2,
8950            json!({
8951                "role": "user",
8952                "content": "Please review the incident notes.",
8953                "timestamp_ms": 12
8954            }),
8955        );
8956
8957        assert!(spawn_notice.is_none());
8958        assert!(peer_update.is_none());
8959        assert!(real_user.is_some());
8960    }
8961
8962    #[test]
8963    fn session_history_projection_skips_non_transcript_messages() {
8964        let skipped = frame_from_session_history_message(
8965            "runtime-a",
8966            "agent-a",
8967            "session-a",
8968            0,
8969            json!({
8970                "content": "internal system prompt"
8971            }),
8972        );
8973        assert!(skipped.is_none());
8974    }
8975
8976    #[test]
8977    fn session_history_projection_extracts_assistant_blocks() {
8978        let frame = frame_from_session_history_message(
8979            "runtime-a",
8980            "agent-a",
8981            "session-a",
8982            0,
8983            json!({
8984                "role": "block_assistant",
8985                "blocks": [
8986                    { "block_type": "text", "data": { "text": "hello " } },
8987                    { "block_type": "text", "data": { "text": "there" } }
8988                ],
8989                "stop_reason": "end_turn"
8990            }),
8991        )
8992        .expect("assistant block history frame");
8993        assert_eq!(frame.payload["text"], json!("hello there"));
8994    }
8995
8996    #[test]
8997    fn session_history_projection_extracts_nested_text_block_data() {
8998        let frame = frame_from_session_history_message(
8999            "runtime-a",
9000            "agent-a",
9001            "session-a",
9002            0,
9003            json!({
9004                "role": "block_assistant",
9005                "blocks": [
9006                    {
9007                        "block_type": "text",
9008                        "data": { "text": "Ready and standing by." }
9009                    }
9010                ],
9011                "stop_reason": "end_turn",
9012                "created_at": "1970-01-01T00:00:00.010Z"
9013            }),
9014        )
9015        .expect("assistant block history frame");
9016
9017        assert_eq!(frame.kind, "interaction_complete");
9018        assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9019    }
9020
9021    #[test]
9022    fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9023        let frame = frame_from_session_history_message(
9024            "runtime-a",
9025            "agent-a",
9026            "session-a",
9027            0,
9028            json!({
9029                "role": "block_assistant",
9030                "blocks": [
9031                    {
9032                        "block_type": "reasoning",
9033                        "data": { "text": "**Planning**\nI should not be rendered." }
9034                    },
9035                    {
9036                        "block_type": "text",
9037                        "data": { "text": "Visible answer." }
9038                    }
9039                ],
9040                "stop_reason": "end_turn",
9041                "created_at": "1970-01-01T00:00:00.010Z"
9042            }),
9043        )
9044        .expect("assistant block history frame");
9045
9046        assert_eq!(frame.kind, "interaction_complete");
9047        assert_eq!(frame.payload["result"], json!("Visible answer."));
9048        assert_eq!(frame.payload["text"], json!("Visible answer."));
9049    }
9050
9051    #[test]
9052    fn session_history_projection_leaves_reasoning_only_result_empty() {
9053        let frame = frame_from_session_history_message(
9054            "runtime-a",
9055            "agent-a",
9056            "session-a",
9057            0,
9058            json!({
9059                "role": "block_assistant",
9060                "blocks": [
9061                    {
9062                        "block_type": "reasoning",
9063                        "data": { "text": "Private planning text." }
9064                    },
9065                    {
9066                        "block_type": "tool_use",
9067                        "data": { "id": "toolu-1", "name": "peers", "args": {} }
9068                    }
9069                ],
9070                "stop_reason": "end_turn",
9071                "created_at": "1970-01-01T00:00:00.010Z"
9072            }),
9073        )
9074        .expect("assistant block history frame");
9075
9076        assert_eq!(frame.kind, "interaction_complete");
9077        assert_eq!(frame.payload["result"], json!(""));
9078        assert_eq!(frame.payload["text"], json!(""));
9079    }
9080
9081    #[test]
9082    fn session_history_projection_preserves_tool_results() {
9083        let frames = frames_from_session_history_message(
9084            "runtime-a",
9085            "agent-a",
9086            "session-a",
9087            5,
9088            json!({
9089                "role": "tool_results",
9090                "results": [
9091                    {
9092                        "tool_use_id": "call-peers",
9093                        "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9094                        "is_error": false
9095                    }
9096                ],
9097                "created_at": "1970-01-01T00:00:00.050Z"
9098            }),
9099        );
9100
9101        assert_eq!(frames.len(), 1);
9102        let frame = &frames[0];
9103        assert_eq!(frame.kind, "tool_execution_completed");
9104        assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9105        assert_eq!(
9106            frame.payload["result"],
9107            json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9108        );
9109        assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9110        assert_eq!(frame.timestamp_ms, 50);
9111    }
9112
9113    #[test]
9114    fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9115        // Regression: a generate_image tool result rebuilt from session-history
9116        // backfill must derive an `assistant_image` frame (image_id/blob_id/
9117        // media_type/width/height) just like the live ingest edge does. Without
9118        // it the image renders as raw tool JSON after a reload / fresh store.
9119        // The serialized ImageGenerationToolResult carries NO tool name, so the
9120        // backfill parses it directly.
9121        let image_result = json!({
9122            "operation_id": "00000000-0000-0000-0000-0000000000aa",
9123            "terminal": { "terminal": "generated" },
9124            "images": [
9125                {
9126                    "image_id": "00000000-0000-0000-0000-000000000051",
9127                    "blob_ref": {
9128                        "blob_id": "sha256:test-generated-image",
9129                        "media_type": "image/png"
9130                    },
9131                    "media_type": "image/png",
9132                    "width": 1024,
9133                    "height": 768
9134                }
9135            ],
9136            "provider_text": { "disposition": "not_emitted" },
9137            "revised_prompt": { "disposition": "not_requested" },
9138            "native_metadata": { "provider": "not_emitted" }
9139        });
9140        // Sanity: the fixture must be a valid ImageGenerationToolResult so the
9141        // test pins the real wire contract, not an arbitrary blob.
9142        serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9143            image_result.clone(),
9144        )
9145        .expect("fixture must be a valid ImageGenerationToolResult");
9146        let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9147
9148        let frames = frames_from_session_history_message(
9149            "runtime-a",
9150            "artist:singleton",
9151            "session-a",
9152            6,
9153            json!({
9154                "role": "tool_results",
9155                "results": [
9156                    {
9157                        "tool_use_id": "call-image",
9158                        "content": result_text,
9159                        "is_error": false
9160                    }
9161                ],
9162                "created_at": "1970-01-01T00:00:00.060Z"
9163            }),
9164        );
9165
9166        // The generic tool_execution_completed frame is still emitted...
9167        assert!(
9168            frames
9169                .iter()
9170                .any(|frame| frame.kind == "tool_execution_completed"),
9171            "tool_execution_completed frame must still be present: {frames:#?}"
9172        );
9173        // ...plus a derived assistant_image frame so the picture renders.
9174        let image_frame = frames
9175            .iter()
9176            .find(|frame| frame.kind == "assistant_image")
9177            .expect("backfill must derive an assistant_image frame");
9178        assert_eq!(
9179            image_frame.payload["blob_id"],
9180            json!("sha256:test-generated-image")
9181        );
9182        assert_eq!(
9183            image_frame.payload["image_id"],
9184            json!("00000000-0000-0000-0000-000000000051")
9185        );
9186        assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9187        assert_eq!(image_frame.payload["width"], json!(1024));
9188        assert_eq!(image_frame.payload["height"], json!(768));
9189        assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9190        assert_eq!(
9191            image_frame.source.kind,
9192            ConsoleFrameSourceKind::SessionHistory
9193        );
9194        assert_eq!(image_frame.timestamp_ms, 60);
9195    }
9196
9197    #[test]
9198    fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9199        let frames = frames_from_session_history_message(
9200            "runtime-a",
9201            "review:singleton",
9202            "session-a",
9203            7,
9204            json!({
9205                "role": "block_assistant",
9206                "blocks": [
9207                    {
9208                        "block_type": "tool_use",
9209                        "data": {
9210                            "id": "call-spawn",
9211                            "name": "mob_spawn_member",
9212                            "args": {
9213                                "member_id": "review-worker-alpha",
9214                                "profile": "review-worker",
9215                                "initial_message": "Review Initiative Alpha and save the result."
9216                            }
9217                        }
9218                    }
9219                ],
9220                "stop_reason": "tool_use",
9221                "created_at": "1970-01-01T00:00:00.070Z"
9222            }),
9223        );
9224
9225        assert_eq!(frames.len(), 2);
9226        let child = frames
9227            .iter()
9228            .find(|frame| frame.identity == "review-worker-alpha")
9229            .expect("spawned member initial message frame");
9230        assert_eq!(child.kind, "user_input");
9231        assert_eq!(child.timestamp_ms, 70);
9232        assert_eq!(
9233            child.payload["message"]["content"],
9234            "Review Initiative Alpha and save the result."
9235        );
9236        assert_eq!(child.payload["parent_identity"], "review:singleton");
9237        assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9238        assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9239    }
9240
9241    #[test]
9242    fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9243        let frames = frames_from_session_history_message(
9244            "runtime-a",
9245            "review:singleton",
9246            "session-a",
9247            8,
9248            json!({
9249                "role": "block_assistant",
9250                "blocks": [
9251                    {
9252                        "block_type": "tool_use",
9253                        "data": {
9254                            "id": "call-spawn-many",
9255                            "name": "mob_spawn_member",
9256                            "args": {
9257                                "specs": [
9258                                    {
9259                                        "agent_identity": "review-worker-alpha",
9260                                        "profile": "review-worker",
9261                                        "initial_message": "Review Initiative Alpha."
9262                                    },
9263                                    {
9264                                        "agent_identity": "review-worker-beta",
9265                                        "profile": "review-worker",
9266                                        "initial_message": "Review Initiative Beta."
9267                                    }
9268                                ]
9269                            }
9270                        }
9271                    }
9272                ],
9273                "stop_reason": "tool_use",
9274                "created_at": "1970-01-01T00:00:00.080Z"
9275            }),
9276        );
9277
9278        assert_eq!(frames.len(), 3);
9279        let child_messages = frames
9280            .iter()
9281            .filter(|frame| frame.kind == "user_input")
9282            .map(|frame| {
9283                (
9284                    frame.identity.as_str(),
9285                    frame.payload["message"]["content"]
9286                        .as_str()
9287                        .unwrap_or_default(),
9288                )
9289            })
9290            .collect::<Vec<_>>();
9291        assert_eq!(
9292            child_messages,
9293            vec![
9294                ("review-worker-alpha", "Review Initiative Alpha."),
9295                ("review-worker-beta", "Review Initiative Beta."),
9296            ]
9297        );
9298    }
9299
9300    #[tokio::test]
9301    async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
9302        let aggregator = MobKitConsoleAggregator::in_memory();
9303        let frames = frames_from_session_history_message(
9304            "runtime-a",
9305            "review:singleton",
9306            "session-a",
9307            9,
9308            json!({
9309                "role": "block_assistant",
9310                "blocks": [
9311                    {
9312                        "block_type": "tool_use",
9313                        "data": {
9314                            "id": "call-spawn-many",
9315                            "name": "mob_spawn_member",
9316                            "args": {
9317                                "specs": [
9318                                    {
9319                                        "agent_identity": "review-worker-alpha",
9320                                        "profile": "review-worker",
9321                                        "initial_message": "Review Initiative Alpha."
9322                                    },
9323                                    {
9324                                        "agent_identity": "review-worker-beta",
9325                                        "profile": "review-worker",
9326                                        "initial_message": "Review Initiative Beta."
9327                                    }
9328                                ]
9329                            }
9330                        }
9331                    }
9332                ],
9333                "stop_reason": "tool_use",
9334                "created_at": "1970-01-01T00:00:00.090Z"
9335            }),
9336        );
9337        for frame in frames {
9338            aggregator
9339                .store()
9340                .append_if_absent(frame)
9341                .await
9342                .expect("append projected session-history frame");
9343        }
9344
9345        let child_page = aggregator
9346            .query_timeline(ConsoleTimelineQuery {
9347                identity: Some("review-worker-alpha".to_string()),
9348                limit: 20,
9349                ..ConsoleTimelineQuery::default()
9350            })
9351            .await
9352            .expect("query child timeline");
9353        assert_eq!(child_page.frames.len(), 1);
9354        assert_eq!(child_page.frames[0].kind, "user_input");
9355        assert_eq!(
9356            child_page.frames[0].payload["message"]["content"],
9357            "Review Initiative Alpha."
9358        );
9359        assert_eq!(
9360            child_page.frames[0].payload["parent_identity"],
9361            "review:singleton"
9362        );
9363
9364        let parent_page = aggregator
9365            .query_timeline(ConsoleTimelineQuery {
9366                identity: Some("review:singleton".to_string()),
9367                limit: 20,
9368                ..ConsoleTimelineQuery::default()
9369            })
9370            .await
9371            .expect("query parent timeline");
9372        assert!(
9373            parent_page
9374                .frames
9375                .iter()
9376                .all(|frame| frame.identity == "review:singleton"),
9377            "child initial messages must not leak into parent timeline: {:#?}",
9378            parent_page.frames
9379        );
9380    }
9381
9382    #[test]
9383    fn session_history_projection_applies_namespace_to_spawned_child_identity() {
9384        let frames = frames_from_session_history_message_with_namespace(
9385            "runtime-a",
9386            "test/review:singleton",
9387            "test",
9388            "session-a",
9389            10,
9390            json!({
9391                "role": "block_assistant",
9392                "blocks": [
9393                    {
9394                        "block_type": "tool_use",
9395                        "data": {
9396                            "id": "call-spawn-many",
9397                            "name": "mob_spawn_member",
9398                            "args": {
9399                                "specs": [
9400                                    {
9401                                        "agent_identity": "review-worker-alpha",
9402                                        "profile": "review-worker",
9403                                        "initial_message": "Review Initiative Alpha."
9404                                    },
9405                                    {
9406                                        "agent_identity": "test/review-worker-beta",
9407                                        "profile": "review-worker",
9408                                        "initial_message": "Review Initiative Beta."
9409                                    }
9410                                ]
9411                            }
9412                        }
9413                    }
9414                ],
9415                "stop_reason": "tool_use",
9416                "created_at": "1970-01-01T00:00:00.100Z"
9417            }),
9418        );
9419
9420        let child_identities = frames
9421            .iter()
9422            .filter(|frame| frame.kind == "user_input")
9423            .map(|frame| frame.identity.as_str())
9424            .collect::<Vec<_>>();
9425        assert_eq!(
9426            child_identities,
9427            vec!["test/review-worker-alpha", "test/review-worker-beta"]
9428        );
9429    }
9430
9431    #[tokio::test]
9432    async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
9433        let aggregator = MobKitConsoleAggregator::in_memory();
9434        aggregator
9435            .inner
9436            .identity_read_model
9437            .replace(vec![
9438                identity_record_for_test("test/review:singleton"),
9439                identity_record_for_test("test/review-worker-alpha"),
9440            ])
9441            .await;
9442        let frames = frames_from_session_history_message_with_namespace(
9443            "runtime-a",
9444            "test/review:singleton",
9445            "test",
9446            "session-a",
9447            11,
9448            json!({
9449                "role": "block_assistant",
9450                "blocks": [
9451                    {
9452                        "block_type": "tool_use",
9453                        "data": {
9454                            "id": "call-spawn",
9455                            "name": "mob_spawn_member",
9456                            "args": {
9457                                "agent_identity": "review-worker-alpha",
9458                                "profile": "review-worker",
9459                                "initial_message": "Review Initiative Alpha."
9460                            }
9461                        }
9462                    }
9463                ],
9464                "stop_reason": "tool_use",
9465                "created_at": "1970-01-01T00:00:00.110Z"
9466            }),
9467        );
9468        for frame in frames {
9469            aggregator
9470                .store()
9471                .append_if_absent(frame)
9472                .await
9473                .expect("append projected namespaced session-history frame");
9474        }
9475
9476        let namespaced_child_page = aggregator
9477            .query_timeline(ConsoleTimelineQuery {
9478                identity: Some("test/review-worker-alpha".to_string()),
9479                limit: 20,
9480                ..ConsoleTimelineQuery::default()
9481            })
9482            .await
9483            .expect("query namespaced child timeline");
9484        assert_eq!(namespaced_child_page.frames.len(), 1);
9485        assert_eq!(
9486            namespaced_child_page.frames[0].identity,
9487            "test/review-worker-alpha"
9488        );
9489        assert_eq!(
9490            namespaced_child_page.frames[0].payload["message"]["content"],
9491            "Review Initiative Alpha."
9492        );
9493
9494        let raw_child_page = aggregator
9495            .query_timeline(ConsoleTimelineQuery {
9496                identity: Some("review-worker-alpha".to_string()),
9497                limit: 20,
9498                ..ConsoleTimelineQuery::default()
9499            })
9500            .await
9501            .expect("query raw child timeline");
9502        assert!(
9503            raw_child_page.frames.is_empty(),
9504            "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
9505            raw_child_page.frames
9506        );
9507    }
9508
9509    #[test]
9510    fn namespace_helpers_preserve_namespace_named_member_identity() {
9511        assert_eq!(apply_namespace("test", "test"), "test/test");
9512        assert_eq!(
9513            strip_namespace("test/test", "test").as_deref(),
9514            Some("test")
9515        );
9516        assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
9517        assert_eq!(
9518            strip_namespace("test/worker", "test").as_deref(),
9519            Some("worker")
9520        );
9521        assert_eq!(strip_namespace("test", "test"), None);
9522    }
9523
9524    #[test]
9525    fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
9526        assert!(!member_id_matches_durable_identity(
9527            "rt:review:singleton:0",
9528            "review:singleton"
9529        ));
9530        assert!(!member_id_matches_durable_identity(
9531            "review:singleton:gen1",
9532            "review:singleton"
9533        ));
9534        assert!(!member_id_matches_durable_identity(
9535            "review:singleton:1",
9536            "review:singleton"
9537        ));
9538        assert!(!member_id_matches_durable_identity(
9539            "rt:review:singleton:qa:0",
9540            "review:singleton"
9541        ));
9542        assert!(!member_id_matches_durable_identity(
9543            "review:singleton:qa",
9544            "review:singleton"
9545        ));
9546    }
9547
9548    #[test]
9549    fn session_history_projection_uses_rfc3339_created_at_timestamp() {
9550        let frame = frame_from_session_history_message(
9551            "runtime-a",
9552            "agent-a",
9553            "session-a",
9554            0,
9555            json!({
9556                "role": "user",
9557                "content": "hello",
9558                "created_at": "2026-05-12T05:00:06.564227Z"
9559            }),
9560        )
9561        .expect("user history frame");
9562
9563        assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
9564    }
9565
9566    async fn wait_for_kickoff_frames(
9567        aggregator: &MobKitConsoleAggregator,
9568        identity: &str,
9569        timeout: Duration,
9570    ) -> Vec<ConsoleFrame> {
9571        let deadline = Instant::now() + timeout;
9572        let mut observed = Vec::new();
9573        while Instant::now() < deadline {
9574            let page = aggregator
9575                .query_timeline(ConsoleTimelineQuery {
9576                    identity: Some(identity.to_string()),
9577                    limit: 50,
9578                    ..ConsoleTimelineQuery::default()
9579                })
9580                .await
9581                .expect("query timeline");
9582            observed = page.frames;
9583            if observed.iter().any(|frame| frame.kind == "user_input") {
9584                break;
9585            }
9586            tokio::time::sleep(Duration::from_millis(25)).await;
9587        }
9588        observed
9589    }
9590
9591    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9592    async fn spawn_registry_primes_access_lineage() {
9593        let runtime = build_empty_runtime("spawn-access-lineage-test").await;
9594        let sink = runtime
9595            .mob_runtime()
9596            .console_spawn_sink()
9597            .expect("unified runtime installs the console spawn sink at bootstrap");
9598
9599        runtime
9600            .spawn(SpawnMemberSpec::from_wire(
9601                "worker".to_string(),
9602                "worker-3".to_string(),
9603                None,
9604                None,
9605                None,
9606            ))
9607            .await
9608            .expect("member spawns");
9609        sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
9610            mob_id: Some("spawn-access-lineage-test".to_string()),
9611            member_id: "worker-3".to_string(),
9612            identity: "worker-3".to_string(),
9613            initial_message: None,
9614            labels: BTreeMap::new(),
9615            spawned_by: Some("ops-lead".to_string()),
9616            via_tool: "mob_spawn_member".to_string(),
9617        })
9618        .await;
9619
9620        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9621            enabled: true,
9622            admins: vec!["root@example.test".to_string()],
9623            groups: BTreeMap::new(),
9624            rules: vec![crate::access::AccessRule {
9625                id: "bob-ops-lead".to_string(),
9626                subjects: vec!["bob@example.test".to_string()],
9627                actions: vec!["agent.view".to_string()],
9628                agents: vec!["ops-lead".to_string()],
9629                ..crate::access::AccessRule::default()
9630            }],
9631        })
9632        .expect("controller");
9633
9634        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9635            .await;
9636
9637        let bob = controller.view_for_subject(Some("bob@example.test"));
9638        assert!(
9639            bob.can_view_agent("worker-3"),
9640            "spawn registry lineage must let bob see the member ops-lead spawned"
9641        );
9642        assert!(
9643            !bob.can_view_agent("unrelated-agent"),
9644            "agents outside the granted lineage stay denied"
9645        );
9646
9647        let _ = runtime.mob_handle().stop().await;
9648    }
9649
9650    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9651    async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
9652        let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
9653
9654        // Server-side spawns with caller-controlled labels and NO spawn
9655        // registry entry: lineage claims in roster labels are untrusted.
9656        let mut imposter = SpawnMemberSpec::from_wire(
9657            "worker".to_string(),
9658            "imposter".to_string(),
9659            None,
9660            None,
9661            None,
9662        );
9663        imposter.labels = Some(BTreeMap::from([(
9664            "spawned_by".to_string(),
9665            "ops-lead".to_string(),
9666        )]));
9667        runtime.spawn(imposter).await.expect("imposter spawns");
9668
9669        let mut evader = SpawnMemberSpec::from_wire(
9670            "worker".to_string(),
9671            "evader".to_string(),
9672            None,
9673            None,
9674            None,
9675        );
9676        evader.labels = Some(BTreeMap::from([(
9677            "spawned_by".to_string(),
9678            "secret-lead".to_string(),
9679        )]));
9680        runtime.spawn(evader).await.expect("evader spawns");
9681
9682        // The console identity list must not present unverified lineage.
9683        let aggregator = MobKitConsoleAggregator::in_memory();
9684        aggregator.register_runtime_handles_with_policy(
9685            "runtime-a",
9686            "",
9687            runtime.mob_runtime().clone(),
9688            None,
9689            runtime.console_events(),
9690            Arc::new(AllowAllConsoleVisibilityPolicy),
9691        );
9692        let identities = aggregator
9693            .list_identities_fresh()
9694            .await
9695            .expect("list identities");
9696        let imposter_record = identities
9697            .iter()
9698            .find(|record| record.identity == "imposter")
9699            .expect("imposter listed");
9700        assert!(
9701            !imposter_record.labels.contains_key("spawned_by"),
9702            "identity records must drop roster-claimed lineage when the spawn registry has none"
9703        );
9704
9705        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9706            enabled: true,
9707            admins: vec!["root@example.test".to_string()],
9708            groups: BTreeMap::new(),
9709            rules: vec![
9710                crate::access::AccessRule {
9711                    id: "bob-ops-lead".to_string(),
9712                    subjects: vec!["bob@example.test".to_string()],
9713                    actions: vec!["agent.view".to_string()],
9714                    agents: vec!["ops-lead".to_string()],
9715                    ..crate::access::AccessRule::default()
9716                },
9717                crate::access::AccessRule {
9718                    id: "carol-view-all".to_string(),
9719                    subjects: vec!["carol@example.test".to_string()],
9720                    actions: vec!["agent.view".to_string()],
9721                    agents: vec!["*".to_string()],
9722                    ..crate::access::AccessRule::default()
9723                },
9724                crate::access::AccessRule {
9725                    id: "hide-secret-lead".to_string(),
9726                    effect: crate::access::AccessEffect::Deny,
9727                    actions: vec!["agent.*".to_string()],
9728                    agents: vec!["secret-lead".to_string()],
9729                    ..crate::access::AccessRule::default()
9730                },
9731            ],
9732        })
9733        .expect("controller");
9734
9735        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9736            .await;
9737
9738        let bob = controller.view_for_subject(Some("bob@example.test"));
9739        assert!(
9740            !bob.can_view_agent("imposter"),
9741            "a roster-claimed spawned_by must not mint inherited visibility"
9742        );
9743        let carol = controller.view_for_subject(Some("carol@example.test"));
9744        assert!(
9745            carol.can_view_agent("evader"),
9746            "a roster-claimed spawned_by must not let a member hide behind a denied parent"
9747        );
9748
9749        let _ = runtime.mob_handle().stop().await;
9750    }
9751
9752    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9753    async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
9754        let runtime = build_empty_runtime("agent-spawn-console-test").await;
9755
9756        // The unified runtime must hand the agent-tool spawn path a console
9757        // sink at bootstrap — without it, agent-spawned members stay
9758        // invisible until their first runtime activity.
9759        let sink = runtime
9760            .mob_runtime()
9761            .console_spawn_sink()
9762            .expect("unified runtime installs the console spawn sink at bootstrap");
9763
9764        // The member exists in the roster exactly as an agent-tool spawn
9765        // would leave it; the server-side spawn carries no initial message
9766        // so the projected kickoff below is the only chat seed.
9767        runtime
9768            .spawn(SpawnMemberSpec::from_wire(
9769                "worker".to_string(),
9770                "worker-3".to_string(),
9771                None,
9772                None,
9773                None,
9774            ))
9775            .await
9776            .expect("member spawns");
9777
9778        let seed = crate::console_spawn::ConsoleSpawnSeed {
9779            mob_id: Some("agent-spawn-console-test".to_string()),
9780            member_id: "worker-3".to_string(),
9781            identity: "worker-3".to_string(),
9782            initial_message: Some(json!("Kick off the person hunt")),
9783            labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
9784            spawned_by: Some("ops-lead".to_string()),
9785            via_tool: "mob_spawn_member".to_string(),
9786        };
9787        sink.project_spawned_member(&seed).await;
9788        // Spawn retries must not fork or duplicate the chat.
9789        sink.project_spawned_member(&seed).await;
9790
9791        let aggregator = MobKitConsoleAggregator::in_memory();
9792        aggregator.register_runtime_handles_with_policy(
9793            "runtime-a",
9794            "test",
9795            runtime.mob_runtime().clone(),
9796            None,
9797            runtime.console_events(),
9798            Arc::new(AllowAllConsoleVisibilityPolicy),
9799        );
9800
9801        let frames =
9802            wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
9803        let kickoffs = frames
9804            .iter()
9805            .filter(|frame| frame.kind == "user_input")
9806            .collect::<Vec<_>>();
9807        assert_eq!(
9808            kickoffs.len(),
9809            1,
9810            "member timeline must contain exactly one kickoff frame; got {frames:#?}"
9811        );
9812        assert_eq!(
9813            kickoffs[0].payload["content"][0]["text"],
9814            "Kick off the person hunt"
9815        );
9816
9817        // A live runtime event for the member must aggregate into the same
9818        // identity/chat as the kickoff, not fork into a second conversation.
9819        runtime
9820            .project_console_event_from_unified(&crate::types::EventEnvelope {
9821                event_id: "evt-live-worker-3".to_string(),
9822                source: "test".to_string(),
9823                timestamp_ms: current_time_ms(),
9824                event: crate::types::UnifiedEvent::Agent {
9825                    agent_id: "worker-3:0".to_string(),
9826                    event_type: "text_delta".to_string(),
9827                    payload: Some(json!({ "delta": "on it" })),
9828                },
9829            })
9830            .await;
9831
9832        let deadline = Instant::now() + Duration::from_secs(5);
9833        let mut live_frame = None;
9834        while Instant::now() < deadline && live_frame.is_none() {
9835            let page = aggregator
9836                .query_timeline(ConsoleTimelineQuery {
9837                    identity: Some("test/worker-3".to_string()),
9838                    limit: 50,
9839                    ..ConsoleTimelineQuery::default()
9840                })
9841                .await
9842                .expect("query timeline");
9843            live_frame = page
9844                .frames
9845                .into_iter()
9846                .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
9847            if live_frame.is_none() {
9848                tokio::time::sleep(Duration::from_millis(25)).await;
9849            }
9850        }
9851        let live_frame = live_frame.expect("live runtime event lands in the member chat");
9852        assert_eq!(live_frame.identity, "test/worker-3");
9853
9854        // The identity list must carry the registered spawn labels so the
9855        // sidebar can group the member and ABAC can see its lineage.
9856        let identities = aggregator
9857            .list_identities_fresh()
9858            .await
9859            .expect("list identities");
9860        let record = identities
9861            .iter()
9862            .find(|record| record.identity == "test/worker-3")
9863            .unwrap_or_else(|| {
9864                panic!("spawned member missing from identity list: {identities:#?}")
9865            });
9866        assert_eq!(
9867            record.labels.get("group").map(String::as_str),
9868            Some("workers")
9869        );
9870        assert_eq!(
9871            record.labels.get("spawned_by").map(String::as_str),
9872            Some("ops-lead")
9873        );
9874
9875        let _ = runtime.mob_handle().stop().await;
9876    }
9877}