Skip to main content

meerkat_mobkit/console_aggregator/
mod.rs

1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23    MobRuntime, MobRuntimeError, assert_member_accepts_images,
24    is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30    ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31    SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32    SourceIngestionTransition,
33};
34pub use store::{
35    ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36    SqliteConsoleLogStore,
37};
38pub use types::{
39    AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40    ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41    ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42    ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43    ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59    CachedOnly,
60    IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65    inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70    pub session_history_backfill_enabled: bool,
71    pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75    fn default() -> Self {
76        Self {
77            session_history_backfill_enabled: true,
78            max_concurrent_session_backfills: 16,
79        }
80    }
81}
82
83struct AggregatorInner {
84    store: Arc<dyn ConsoleLogStore>,
85    runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86    event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87    active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88    opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89    session_backfill_permits: Arc<Semaphore>,
90    identity_read_model: ConsoleIdentityReadModel,
91    options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96    inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97    refresh_lock: Arc<tokio::sync::Mutex<()>>,
98    primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102    fn default() -> Self {
103        Self {
104            inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106            primed: Arc::new(AtomicBool::new(false)),
107        }
108    }
109}
110
111impl ConsoleIdentityReadModel {
112    async fn snapshot(
113        &self,
114        inner: Arc<AggregatorInner>,
115    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116        if !self.primed.load(Ordering::Acquire) {
117            Box::pin(self.prime_now(inner)).await?;
118        }
119        Ok(self.inner.read().await.clone())
120    }
121
122    async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123        self.inner.read().await.clone()
124    }
125
126    async fn refresh_now_if_idle(
127        &self,
128        inner: Arc<AggregatorInner>,
129        mode: IdentityCollectionMode,
130    ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132            return None;
133        };
134        let result = Box::pin(collect_identity_records(&inner, mode)).await;
135        drop(guard);
136        match result {
137            Ok(identities) => {
138                self.replace(identities.clone()).await;
139                Some(Ok(identities))
140            }
141            Err(err) => Some(Err(err)),
142        }
143    }
144
145    async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146        if self.primed.load(Ordering::Acquire) {
147            return Ok(());
148        }
149        let _guard = self.refresh_lock.clone().lock_owned().await;
150        if self.primed.load(Ordering::Acquire) {
151            return Ok(());
152        }
153        let identities = Box::pin(collect_identity_records(
154            &inner,
155            IdentityCollectionMode::CachedOnly,
156        ))
157        .await?;
158        *self.inner.write().await = identities;
159        self.primed.store(true, Ordering::Release);
160        Ok(())
161    }
162
163    fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
164        let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
165            return;
166        };
167        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
168            return;
169        };
170        let read_model = self.clone();
171        runtime_handle.spawn(async move {
172            let _guard = guard;
173            match Box::pin(collect_identity_records(
174                &inner,
175                IdentityCollectionMode::IncludeLiveMembers,
176            ))
177            .await
178            {
179                Ok(identities) => {
180                    *read_model.inner.write().await = identities;
181                    read_model.primed.store(true, Ordering::Release);
182                }
183                Err(err) => {
184                    tracing::warn!(error = %err, "console identity read-model refresh failed");
185                }
186            }
187        });
188    }
189
190    async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
191        *self.inner.write().await = identities;
192        self.primed.store(true, Ordering::Release);
193    }
194}
195
196#[derive(Clone)]
197struct RuntimeEntry {
198    runtime_key: String,
199    identity_namespace: String,
200    runtime: MobRuntime,
201    identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
202    console_events: ConsoleEventStore,
203    visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
204}
205
206#[derive(Clone)]
207struct ResolvedConsoleMember {
208    entry: RuntimeEntry,
209    handle: MobHandle,
210    member: MobMemberListEntry,
211    source_mob_id: String,
212    runtime_identity: String,
213}
214
215fn console_member_for_resolved_member(
216    resolved: &ResolvedConsoleMember,
217    record: &ConsoleIdentityRecord,
218) -> ConsoleMember {
219    ConsoleMember {
220        agent_identity: crate::member_comms_id::runtime_alias_str(
221            resolved.member.agent_identity.as_str(),
222        )
223        .into_owned(),
224        role: resolved.member.role.to_string(),
225        state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
226        model_capabilities: Default::default(),
227        runtime_mode: Some(resolved.member.runtime_mode.to_string()),
228        session_id: record.session_id.clone(),
229        wired_to: resolved
230            .member
231            .wired_to
232            .iter()
233            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
234            .collect(),
235        labels: record.labels.clone(),
236    }
237}
238
239async fn resolved_member_visible(
240    resolved: &ResolvedConsoleMember,
241    record: &ConsoleIdentityRecord,
242) -> bool {
243    if !raw_resolved_member_visible(resolved, record) {
244        return false;
245    }
246    !Box::pin(live_record_shadowed_by_hidden_durable_binding(
247        resolved, record,
248    ))
249    .await
250}
251
252fn raw_resolved_member_visible(
253    resolved: &ResolvedConsoleMember,
254    record: &ConsoleIdentityRecord,
255) -> bool {
256    resolved.entry.visibility_policy.identity_visible(record)
257        && resolved
258            .entry
259            .visibility_policy
260            .member_visible(&console_member_for_resolved_member(resolved, record))
261}
262
263pub trait ConsoleVisibilityPolicy: Send + Sync {
264    fn include_implicit_delegate_members(&self) -> bool {
265        true
266    }
267
268    fn member_visible(&self, _member: &ConsoleMember) -> bool {
269        true
270    }
271
272    fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
273        true
274    }
275
276    fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
277        true
278    }
279
280    fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
281        None
282    }
283}
284
285#[derive(Debug, Default)]
286pub struct AllowAllConsoleVisibilityPolicy;
287
288impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
289
290#[derive(Debug, Default)]
291pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
292
293impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
294    fn include_implicit_delegate_members(&self) -> bool {
295        false
296    }
297
298    fn member_visible(&self, member: &ConsoleMember) -> bool {
299        !is_implicit_delegate_member(member.role.as_str(), &member.labels)
300    }
301
302    fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
303        !is_implicit_delegate_member(
304            record
305                .labels
306                .get("role")
307                .map(String::as_str)
308                .unwrap_or_default(),
309            &record.labels,
310        )
311    }
312}
313
314#[derive(Clone)]
315pub struct ConsoleRuntimeRegistration {
316    pub runtime_key: String,
317    pub runtime: Arc<UnifiedRuntime>,
318    pub identity_namespace: String,
319    pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
320}
321
322impl MobKitConsoleAggregator {
323    pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
324        Self::new_with_options(store, ConsoleAggregatorOptions::default())
325    }
326
327    pub fn new_with_options(
328        store: Arc<dyn ConsoleLogStore>,
329        mut options: ConsoleAggregatorOptions,
330    ) -> Self {
331        options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
332        let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
333        Self {
334            inner: Arc::new(AggregatorInner {
335                store,
336                runtimes: RwLock::new(BTreeMap::new()),
337                event_tx,
338                active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
339                opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
340                session_backfill_permits: Arc::new(Semaphore::new(
341                    options.max_concurrent_session_backfills,
342                )),
343                identity_read_model: ConsoleIdentityReadModel::default(),
344                options,
345            }),
346        }
347    }
348
349    pub fn in_memory() -> Self {
350        Self::new(Arc::new(InMemoryConsoleLogStore::new()))
351    }
352
353    pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
354        Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
355    }
356
357    pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
358        self.inner.event_tx.subscribe()
359    }
360
361    pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
362        self.inner.store.clone()
363    }
364
365    pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
366        let identity_runtime = registration.runtime.identity_runtime().cloned();
367        self.register_runtime_handles_with_policy(
368            registration.runtime_key,
369            registration.identity_namespace,
370            registration.runtime.mob_runtime().clone(),
371            identity_runtime,
372            registration.runtime.console_events(),
373            registration.visibility_policy,
374        );
375    }
376
377    pub(crate) fn register_runtime_handles_with_policy(
378        &self,
379        runtime_key: impl Into<String>,
380        identity_namespace: impl Into<String>,
381        runtime: MobRuntime,
382        identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
383        console_events: ConsoleEventStore,
384        visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
385    ) {
386        let runtime_key = runtime_key.into();
387        let identity_namespace = identity_namespace.into();
388        let entry = RuntimeEntry {
389            runtime_key: runtime_key.clone(),
390            identity_namespace,
391            runtime,
392            identity_runtime,
393            console_events: console_events.clone(),
394            visibility_policy,
395        };
396        if let Ok(mut runtimes) = self.inner.runtimes.write() {
397            runtimes.insert(runtime_key.clone(), entry);
398        }
399        self.inner
400            .identity_read_model
401            .refresh_soon(self.inner.clone());
402        let inner = self.inner.clone();
403        let events_for_live = console_events.clone();
404        let events_for_live_recovery = console_events.clone();
405        let runtime_key_for_live = runtime_key.clone();
406        tokio::spawn(async move {
407            let mut rx = events_for_live.subscribe();
408            loop {
409                match rx.recv().await {
410                    Ok(envelope) => {
411                        let _ =
412                            project_console_event(inner.clone(), &runtime_key_for_live, envelope)
413                                .await;
414                    }
415                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
416                        let _ = recover_lagged_source_events(
417                            inner.clone(),
418                            &runtime_key_for_live,
419                            &events_for_live_recovery,
420                        )
421                        .await;
422                    }
423                    Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
424                }
425            }
426        });
427
428        let inner = self.inner.clone();
429        let events_for_replay = console_events;
430        let runtime_key_for_replay = runtime_key;
431        tokio::spawn(async move {
432            let mut ingestion_state = SourceIngestionState::Registered;
433            if let Ok((next, _effects)) =
434                ingestion_state.apply(SourceIngestionTransition::StartBackfill)
435            {
436                ingestion_state = next;
437            }
438            if let Ok(events) = events_for_replay.replay_all(None).await {
439                for envelope in events {
440                    let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
441                        .await;
442                }
443            }
444            spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
445            spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
446            if let Ok((next, _effects)) =
447                ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
448            {
449                ingestion_state = next;
450            }
451            let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
452        });
453    }
454
455    pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
456        if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
457            self.inner.clone(),
458            IdentityCollectionMode::IncludeLiveMembers,
459        ))
460        .await
461        {
462            let identities = identities?;
463            spawn_identity_backfills_for_records(self.inner.clone(), &identities);
464            return Ok(identities);
465        }
466        self.inner
467            .identity_read_model
468            .refresh_soon(self.inner.clone());
469        let identities =
470            Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
471        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
472        Ok(identities)
473    }
474
475    #[cfg(test)]
476    pub(crate) async fn list_identities_fresh(
477        &self,
478    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
479        let _guard = self
480            .inner
481            .identity_read_model
482            .refresh_lock
483            .clone()
484            .lock_owned()
485            .await;
486        let identities =
487            collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
488                .await?;
489        self.inner
490            .identity_read_model
491            .replace(identities.clone())
492            .await;
493        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
494        Ok(identities)
495    }
496
497    pub async fn inspect_identity(
498        &self,
499        identity: &str,
500    ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
501        let entries = self
502            .inner
503            .runtimes
504            .read()
505            .map_err(|_| runtime_registry_lock_error())?
506            .values()
507            .cloned()
508            .collect::<Vec<_>>();
509        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
510        let mut durable_matches = Vec::new();
511        if !requested_identity_has_runtime_member_alias(identity, &entries) {
512            for entry in &entries {
513                let Some(identity_runtime) = entry.identity_runtime.clone() else {
514                    continue;
515                };
516                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
517                {
518                    let Ok(parsed_identity) =
519                        crate::identity_first::AgentIdentity::parse(&raw_identity)
520                    else {
521                        continue;
522                    };
523                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
524                        continue;
525                    };
526                    let mut record = identity_record_for_status(entry, &status);
527                    let topology_peers =
528                        identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
529                    record.topology_peers = topology_peers
530                        .get(&record.identity)
531                        .cloned()
532                        .unwrap_or_default();
533                    if Box::pin(console_identity_record_visible(entry, &record)).await {
534                        durable_matches.push((
535                            entry.clone(),
536                            identity_runtime.clone(),
537                            parsed_identity,
538                            record,
539                        ));
540                    }
541                }
542            }
543        }
544        if durable_matches.len() > 1 {
545            let candidates = durable_matches
546                .iter()
547                .map(|(_, _, _, record)| record.runtime_member_id.clone())
548                .collect::<Vec<_>>()
549                .join(", ");
550            return Err(format!(
551                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
552            )
553            .into());
554        }
555        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
556            durable_matches.into_iter().next()
557        {
558            let durable_live_records = Box::pin(self.live_records_for_durable_record(
559                &entry,
560                identity,
561                &record,
562                &live_records,
563            ))
564            .await;
565            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
566                &entry,
567                &durable_live_records,
568            ))
569            .await;
570            if let Some(ambiguous_error) =
571                ambiguous_live_alias_error(identity, &visible_durable_live_records)
572            {
573                return Err(ambiguous_error.into());
574            }
575            if let Some(rebound_record) = self_heal_durable_session_mismatch(
576                &entry,
577                identity_runtime.as_ref(),
578                &parsed_identity,
579                identity,
580                &record,
581                &durable_live_records,
582            )
583            .await
584            .map_err(ConsoleLogError::from)?
585            {
586                record = rebound_record;
587            }
588            if let Some(stale_error) =
589                stale_durable_record_error(identity, &record, &durable_live_records)
590            {
591                return Err(stale_error.into());
592            }
593            if let Some(session_id) = record.session_id.clone() {
594                spawn_session_history_backfill_target(
595                    self.inner.clone(),
596                    SessionBackfillTarget {
597                        entry,
598                        record: record.clone(),
599                        session_id,
600                    },
601                    false,
602                );
603            }
604            return Ok(Some(ConsoleIdentityInspection {
605                peers: record.topology_peers.clone(),
606                identity: record,
607            }));
608        }
609        let mut runtime_id_matches = Vec::new();
610        for entry in &entries {
611            let Some(identity_runtime) = entry.identity_runtime.clone() else {
612                continue;
613            };
614            let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
615                continue;
616            };
617            let Some(status) = identity_runtime
618                .statuses()
619                .await
620                .into_iter()
621                .find(|status| {
622                    status
623                        .agent_runtime_id
624                        .as_ref()
625                        .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
626                })
627            else {
628                continue;
629            };
630            let mut record = identity_record_for_status(entry, &status);
631            let topology_peers =
632                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
633            record.topology_peers = topology_peers
634                .get(&record.identity)
635                .cloned()
636                .unwrap_or_default();
637            if Box::pin(console_identity_record_visible(entry, &record)).await {
638                runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
639            }
640        }
641        if runtime_id_matches.len() > 1 {
642            let candidates = runtime_id_matches
643                .iter()
644                .map(|(_, _, _, record)| record.identity.clone())
645                .collect::<Vec<_>>()
646                .join(", ");
647            return Err(format!(
648                "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
649            )
650            .into());
651        }
652        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
653            runtime_id_matches.into_iter().next()
654        {
655            let durable_live_records = Box::pin(self.live_records_for_durable_record(
656                &entry,
657                identity,
658                &record,
659                &live_records,
660            ))
661            .await;
662            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
663                &entry,
664                &durable_live_records,
665            ))
666            .await;
667            if let Some(ambiguous_error) =
668                ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
669            {
670                return Err(ambiguous_error.into());
671            }
672            if let Some(rebound_record) = self_heal_durable_session_mismatch(
673                &entry,
674                identity_runtime.as_ref(),
675                &parsed_identity,
676                identity,
677                &record,
678                &durable_live_records,
679            )
680            .await
681            .map_err(ConsoleLogError::from)?
682            {
683                record = rebound_record;
684            }
685            if let Some(stale_error) =
686                stale_durable_record_error(identity, &record, &durable_live_records)
687            {
688                return Err(stale_error.into());
689            }
690            if let Some(session_id) = record.session_id.clone() {
691                spawn_session_history_backfill_target(
692                    self.inner.clone(),
693                    SessionBackfillTarget {
694                        entry: entry.clone(),
695                        record: record.clone(),
696                        session_id,
697                    },
698                    false,
699                );
700            }
701            return Ok(Some(ConsoleIdentityInspection {
702                peers: record.topology_peers.clone(),
703                identity: record,
704            }));
705        }
706
707        let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
708        if live_matches.len() > 1 {
709            let candidates = live_matches
710                .iter()
711                .map(|resolved| resolved.runtime_identity.clone())
712                .collect::<Vec<_>>()
713                .join(", ");
714            return Err(format!(
715                "ambiguous live identity alias {identity}: candidates [{candidates}]"
716            )
717            .into());
718        }
719        if let Some(resolved) = live_matches.into_iter().next() {
720            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
721                return Ok(None);
722            };
723            if let Some(stale_error) =
724                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
725            {
726                return Err(stale_error.into());
727            }
728            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
729                && let Ok(parsed_identity) =
730                    crate::identity_first::AgentIdentity::parse(&record.identity)
731                && let Ok(status) = identity_runtime.status(&parsed_identity).await
732            {
733                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
734                let durable_live_records = Box::pin(self.live_records_for_durable_record(
735                    &resolved.entry,
736                    identity,
737                    &durable_record,
738                    &live_records,
739                ))
740                .await;
741                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
742                    &resolved.entry,
743                    &durable_live_records,
744                ))
745                .await;
746                if let Some(ambiguous_error) =
747                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
748                {
749                    return Err(ambiguous_error.into());
750                }
751                if let Some(rebound_record) = self_heal_durable_session_mismatch(
752                    &resolved.entry,
753                    identity_runtime.as_ref(),
754                    &parsed_identity,
755                    identity,
756                    &durable_record,
757                    &durable_live_records,
758                )
759                .await
760                .map_err(ConsoleLogError::from)?
761                {
762                    durable_record = rebound_record;
763                }
764                if let Some(stale_error) =
765                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
766                {
767                    return Err(stale_error.into());
768                }
769                if status
770                    .agent_runtime_id
771                    .as_ref()
772                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
773                {
774                    return Ok(None);
775                }
776            }
777            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
778                return Ok(None);
779            }
780            if let Some(session_id) = record.session_id.clone() {
781                spawn_session_history_backfill_target(
782                    self.inner.clone(),
783                    SessionBackfillTarget {
784                        entry: resolved.entry.clone(),
785                        record: record.clone(),
786                        session_id,
787                    },
788                    false,
789                );
790            }
791            let peers = resolved
792                .member
793                .wired_to
794                .iter()
795                .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
796                .collect();
797            return Ok(Some(ConsoleIdentityInspection {
798                identity: record,
799                peers,
800            }));
801        }
802
803        Ok(None)
804    }
805
806    pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
807        let entries = self
808            .inner
809            .runtimes
810            .read()
811            .map_err(|_| runtime_registry_lock_error())?
812            .clone();
813        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
814        let entries_vec = entries.values().cloned().collect::<Vec<_>>();
815        let mut durable_matches = Vec::new();
816        if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
817            for entry in entries.values() {
818                let Some(identity_runtime) = entry.identity_runtime.clone() else {
819                    continue;
820                };
821                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
822                {
823                    let Ok(parsed_identity) =
824                        crate::identity_first::AgentIdentity::parse(&raw_identity)
825                    else {
826                        continue;
827                    };
828                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
829                        continue;
830                    };
831                    let record = identity_record_for_status(entry, &status);
832                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
833                        continue;
834                    }
835                    durable_matches.push((
836                        entry.clone(),
837                        identity_runtime.clone(),
838                        parsed_identity,
839                        record,
840                    ));
841                }
842            }
843        }
844        if durable_matches.len() > 1 {
845            let candidates = durable_matches
846                .iter()
847                .map(|(_, _, identity, _)| identity.as_str().to_string())
848                .collect::<Vec<_>>()
849                .join(", ");
850            return Err(format!(
851                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
852            )
853            .into());
854        }
855        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
856            durable_matches.into_iter().next()
857        {
858            let durable_live_records = Box::pin(self.live_records_for_durable_record(
859                &entry,
860                identity,
861                &record,
862                &live_records,
863            ))
864            .await;
865            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
866                &entry,
867                &durable_live_records,
868            ))
869            .await;
870            if let Some(ambiguous_error) =
871                ambiguous_live_alias_error(identity, &visible_durable_live_records)
872            {
873                return Err(ambiguous_error.into());
874            }
875            if let Some(rebound_record) = self_heal_durable_session_mismatch(
876                &entry,
877                identity_runtime.as_ref(),
878                &parsed_identity,
879                identity,
880                &record,
881                &durable_live_records,
882            )
883            .await
884            .map_err(ConsoleLogError::from)?
885            {
886                record = rebound_record;
887            }
888            if let Some(stale_error) =
889                stale_durable_record_error(identity, &record, &durable_live_records)
890            {
891                return Err(stale_error.into());
892            }
893            identity_runtime
894                .retire(&parsed_identity)
895                .await
896                .map_err(|err| -> ConsoleLogError {
897                    format!("retire failed for {identity}: {err}").into()
898                })?;
899            {
900                let mut durable_identities =
901                    namespace_match_candidates(identity, &entry.identity_namespace);
902                durable_identities.push(parsed_identity.as_str().to_string());
903                durable_identities.sort();
904                durable_identities.dedup();
905                if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
906                    &entry,
907                    &durable_identities,
908                ))
909                .await
910                {
911                    tracing::warn!(
912                        identity,
913                        error = %err,
914                        "stale console member cleanup failed after durable identity retire"
915                    );
916                }
917            }
918            return Ok(true);
919        }
920        let matches = Box::pin(self.resolve_visible_members(identity)).await;
921        if matches.len() > 1 {
922            let candidates = matches
923                .iter()
924                .map(|resolved| resolved.runtime_identity.clone())
925                .collect::<Vec<_>>()
926                .join(", ");
927            return Err(format!(
928                "ambiguous live identity alias {identity}: candidates [{candidates}]"
929            )
930            .into());
931        }
932        let mut live_retired_any = false;
933        for resolved in matches {
934            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
935                continue;
936            };
937            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
938                continue;
939            }
940            if let Some(stale_error) =
941                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
942            {
943                return Err(stale_error.into());
944            }
945            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
946                && let Ok(parsed_identity) =
947                    crate::identity_first::AgentIdentity::parse(&record.identity)
948                && let Ok(status) = identity_runtime.status(&parsed_identity).await
949            {
950                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
951                let durable_live_records = Box::pin(self.live_records_for_durable_record(
952                    &resolved.entry,
953                    identity,
954                    &durable_record,
955                    &live_records,
956                ))
957                .await;
958                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
959                    &resolved.entry,
960                    &durable_live_records,
961                ))
962                .await;
963                if let Some(ambiguous_error) =
964                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
965                {
966                    return Err(ambiguous_error.into());
967                }
968                if let Some(rebound_record) = self_heal_durable_session_mismatch(
969                    &resolved.entry,
970                    identity_runtime.as_ref(),
971                    &parsed_identity,
972                    identity,
973                    &durable_record,
974                    &durable_live_records,
975                )
976                .await
977                .map_err(ConsoleLogError::from)?
978                {
979                    durable_record = rebound_record;
980                }
981                if let Some(stale_error) =
982                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
983                {
984                    return Err(stale_error.into());
985                }
986                if status
987                    .agent_runtime_id
988                    .as_ref()
989                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
990                {
991                    continue;
992                }
993            }
994            resolved
995                .handle
996                .retire(crate::member_comms_id::mob_member_id(
997                    resolved.runtime_identity.as_str(),
998                ))
999                .await
1000                .map_err(|err| -> ConsoleLogError {
1001                    format!("retire failed for {identity}: {err}").into()
1002                })
1003                .or_else(|err| {
1004                    if lifecycle_archive_cleanup_completed(&err.to_string()) {
1005                        Ok(())
1006                    } else {
1007                        Err(err)
1008                    }
1009                })?;
1010            live_retired_any = true;
1011        }
1012        Ok(live_retired_any)
1013    }
1014
1015    pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1016        self.inner.store.clear_frames().await
1017    }
1018
1019    pub async fn query_timeline(
1020        &self,
1021        query: ConsoleTimelineQuery,
1022    ) -> ConsoleLogResult<ConsoleTimelinePage> {
1023        let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1024        Ok(ConsoleTimelinePage {
1025            frames: page.frames,
1026            next_cursor: page.next_cursor,
1027        })
1028    }
1029
1030    pub async fn query_timeline_windowed(
1031        &self,
1032        query: ConsoleTimelineWindowQuery,
1033    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1034        self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1035            .await?;
1036        if query.after.is_none()
1037            && query.before.is_none()
1038            && let Some(identity) = query.identity.clone()
1039        {
1040            let mut probe_query = query.clone();
1041            probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1042            let page = self.inner.store.query_windowed_frames(probe_query).await?;
1043            if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1044                spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1045            }
1046        }
1047        Box::pin(self.query_timeline_visible(query)).await
1048    }
1049
1050    async fn reject_after_cursor_beyond_store_frontier(
1051        &self,
1052        after: Option<&ConsoleCursor>,
1053    ) -> ConsoleLogResult<()> {
1054        let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1055            return Ok(());
1056        };
1057        let latest_seq = self
1058            .inner
1059            .store
1060            .latest_cursor()
1061            .await?
1062            .and_then(|cursor| cursor.seq())
1063            .unwrap_or(0);
1064        if after_seq > latest_seq {
1065            return Err(std::io::Error::other(
1066                "timeline replay cursor is beyond the current store frontier",
1067            )
1068            .into());
1069        }
1070        Ok(())
1071    }
1072
1073    async fn query_timeline_visible(
1074        &self,
1075        query: ConsoleTimelineWindowQuery,
1076    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1077        let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1078        let mut scan_query = query.clone();
1079        scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1080        let mut visible_frames = Vec::with_capacity(requested_limit);
1081        let mut anchor_frames = Vec::new();
1082        let mut identity_visibility_cache = HashMap::new();
1083        let identity_records = self.inner.identity_read_model.current().await;
1084        let mut next_cursor = query.after.clone();
1085        let mut since_last_scanned_cursor = query.after.clone();
1086        let mut since_last_delivered_cursor = None;
1087        let mut since_stopped_at_visible_limit = false;
1088        let mut latest_cursor = None;
1089        let mut exhausted = false;
1090        let mut scanned = 0usize;
1091
1092        loop {
1093            let page = self
1094                .inner
1095                .store
1096                .query_windowed_frames(scan_query.clone())
1097                .await?;
1098            latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1099            if page.frames.is_empty() {
1100                exhausted = true;
1101                break;
1102            }
1103            let raw_len = page.frames.len();
1104            scanned = scanned.saturating_add(raw_len);
1105            match query.mode {
1106                ConsoleTimelineMode::Since => {
1107                    if let Some(cursor) = page.next_cursor.clone() {
1108                        since_last_scanned_cursor = Some(cursor.clone());
1109                        scan_query.after = Some(cursor);
1110                    }
1111                }
1112                ConsoleTimelineMode::Recent => {
1113                    if let Some(first) = page.frames.first() {
1114                        scan_query.before = Some(first.cursor.clone());
1115                    }
1116                    next_cursor = page.next_cursor.clone().or(next_cursor);
1117                }
1118            }
1119
1120            for frame in page.frames {
1121                let allow_historical_identity =
1122                    query.identity.as_deref() == Some(frame.identity.as_str());
1123                if Box::pin(frame_is_visible_cached(
1124                    &self.inner,
1125                    &frame,
1126                    allow_historical_identity,
1127                    &mut identity_visibility_cache,
1128                    &identity_records,
1129                ))
1130                .await
1131                .unwrap_or(false)
1132                {
1133                    if query.mode == ConsoleTimelineMode::Recent
1134                        && query.identity.is_some()
1135                        && is_identity_timeline_anchor_frame(&frame)
1136                        && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1137                    {
1138                        anchor_frames.push(frame.clone());
1139                    }
1140                    match query.mode {
1141                        ConsoleTimelineMode::Since => {
1142                            if visible_frames.len() >= requested_limit {
1143                                since_stopped_at_visible_limit = true;
1144                                break;
1145                            }
1146                            since_last_delivered_cursor = Some(frame.cursor.clone());
1147                            visible_frames.push(frame);
1148                            if visible_frames.len() >= requested_limit {
1149                                since_stopped_at_visible_limit = true;
1150                                exhausted = false;
1151                                break;
1152                            }
1153                        }
1154                        ConsoleTimelineMode::Recent => {
1155                            visible_frames.push(frame);
1156                        }
1157                    }
1158                }
1159            }
1160            let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1161                && query.identity.is_some()
1162                && anchor_frames.is_empty()
1163                && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1164            if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1165                break;
1166            }
1167            if since_stopped_at_visible_limit {
1168                break;
1169            }
1170            if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1171                exhausted = true;
1172                break;
1173            }
1174            if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1175                break;
1176            }
1177        }
1178
1179        if query.mode == ConsoleTimelineMode::Recent {
1180            visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1181            if visible_frames.len() > requested_limit {
1182                visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1183            }
1184            if !anchor_frames.is_empty() {
1185                let anchor_cursors = anchor_frames
1186                    .iter()
1187                    .map(|frame| frame.cursor.clone())
1188                    .collect::<Vec<_>>();
1189                let mut merged = anchor_frames;
1190                for frame in visible_frames {
1191                    if !merged
1192                        .iter()
1193                        .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1194                    {
1195                        merged.push(frame);
1196                    }
1197                }
1198                merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1199                visible_frames = merged;
1200                if visible_frames.len() > requested_limit {
1201                    let mut anchors = Vec::new();
1202                    let mut tail = Vec::new();
1203                    for frame in visible_frames {
1204                        if anchor_cursors.contains(&frame.cursor) {
1205                            anchors.push(frame);
1206                        } else {
1207                            tail.push(frame);
1208                        }
1209                    }
1210                    visible_frames = if anchors.len() >= requested_limit {
1211                        anchors.split_off(anchors.len() - requested_limit)
1212                    } else {
1213                        let keep_tail = requested_limit.saturating_sub(anchors.len());
1214                        if tail.len() > keep_tail {
1215                            tail = tail.split_off(tail.len() - keep_tail);
1216                        }
1217                        anchors.extend(tail);
1218                        anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1219                        anchors
1220                    };
1221                }
1222            }
1223        }
1224
1225        Ok(ConsoleTimelineWindowPage {
1226            frames: visible_frames,
1227            next_cursor: match query.mode {
1228                ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1229                    since_last_delivered_cursor.or(since_last_scanned_cursor)
1230                }
1231                ConsoleTimelineMode::Since => since_last_scanned_cursor,
1232                ConsoleTimelineMode::Recent => next_cursor,
1233            },
1234            latest_cursor,
1235            exhausted,
1236        })
1237    }
1238
1239    pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1240        let runtime_keys = self
1241            .inner
1242            .runtimes
1243            .read()
1244            .map_err(|_| runtime_registry_lock_error())?
1245            .keys()
1246            .cloned()
1247            .collect::<Vec<_>>();
1248        let results =
1249            join_all(runtime_keys.into_iter().map(|runtime_key| {
1250                backfill_session_history(self.inner.clone(), runtime_key, true)
1251            }))
1252            .await;
1253        for result in results {
1254            result?;
1255        }
1256        Ok(())
1257    }
1258
1259    pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1260        self.inner.store.latest_cursor().await
1261    }
1262
1263    pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1264        match event {
1265            ConsoleTimelineEvent::ConsoleFrame { frame }
1266            | ConsoleTimelineEvent::FrameUpdated { frame } => {
1267                let identity_records = self.inner.identity_read_model.current().await;
1268                Box::pin(frame_is_visible(
1269                    &self.inner,
1270                    frame,
1271                    false,
1272                    &identity_records,
1273                ))
1274                .await
1275                .unwrap_or(false)
1276            }
1277            ConsoleTimelineEvent::SnapshotStarted { .. }
1278            | ConsoleTimelineEvent::SnapshotComplete { .. }
1279            | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1280        }
1281    }
1282
1283    pub async fn timeline_frame_visible_for_query(
1284        &self,
1285        frame: &ConsoleFrame,
1286        identity: Option<&str>,
1287    ) -> bool {
1288        let allow_historical_identity = identity == Some(frame.identity.as_str());
1289        let identity_records = self.inner.identity_read_model.current().await;
1290        Box::pin(frame_is_visible(
1291            &self.inner,
1292            frame,
1293            allow_historical_identity,
1294            &identity_records,
1295        ))
1296        .await
1297        .unwrap_or(false)
1298    }
1299
1300    pub async fn send(
1301        &self,
1302        request: ConsoleSendRequest,
1303    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1304        validate_send_request(&request)?;
1305        let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1306            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1307        };
1308        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1309            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1310        };
1311        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1312            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1313        }
1314        if !member_is_addressable(&resolved.member) {
1315            return Err(ConsoleSendError::NotAddressable(request.identity));
1316        }
1317        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1318            return Err(ConsoleSendError::Retired(request.identity));
1319        }
1320
1321        let content = content_input_from_value(&request.content)?;
1322        let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1323        assert_member_accepts_images(
1324            &resolved.handle,
1325            resolved.entry.runtime.session_service(),
1326            &resolved.runtime_identity,
1327            &content,
1328        )
1329        .await
1330        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1331
1332        let dedupe_key = send_dedupe_key(
1333            &resolved.entry.runtime_key,
1334            &request.identity,
1335            &request.origin,
1336            &request.idempotency_key,
1337        );
1338        let handling_mode_value = request
1339            .handling_mode
1340            .as_deref()
1341            .unwrap_or("queue")
1342            .to_string();
1343        let request_fingerprint =
1344            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1345        if let Some(existing) = self
1346            .inner
1347            .store
1348            .frame_by_dedupe_key(&dedupe_key)
1349            .await
1350            .map_err(ConsoleSendError::Log)?
1351        {
1352            let same_request = existing.source.source_cursor.as_deref()
1353                == Some(request_fingerprint.as_str())
1354                || existing.source.source_cursor.is_none()
1355                    && existing.payload.get("origin").and_then(Value::as_str)
1356                        == Some(request.origin.as_str())
1357                    && existing.payload.get("content") == Some(&request.content)
1358                    && existing
1359                        .payload
1360                        .get("handling_mode")
1361                        .and_then(Value::as_str)
1362                        == Some(handling_mode_value.as_str());
1363            if !same_request {
1364                return Err(ConsoleSendError::IdempotencyConflict(
1365                    request.idempotency_key,
1366                ));
1367            }
1368            return Ok(accepted_from_frame(&existing));
1369        }
1370
1371        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1372        resolved
1373            .entry
1374            .console_events
1375            .reserve_interaction_value(
1376                &resolved.runtime_identity,
1377                Some(resolved.runtime_identity.as_str()),
1378                &interaction_id,
1379                &request.origin,
1380                request.content.clone(),
1381            )
1382            .await
1383            .map_err(ConsoleSendError::State)?;
1384        let session_id = resolved
1385            .handle
1386            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1387                resolved.runtime_identity.as_str(),
1388            ))
1389            .await
1390            .map(|sid| sid.to_string());
1391        let mut new_frame = NewConsoleFrame {
1392            id: None,
1393            dedupe_key,
1394            timestamp_ms: current_time_ms(),
1395            runtime_key: resolved.entry.runtime_key.clone(),
1396            identity: request.identity.clone(),
1397            conversation_id: Some(request.identity.clone()),
1398            session_id: session_id.clone(),
1399            kind: "user_input".to_string(),
1400            status: ConsoleFrameStatus::Accepted,
1401            payload: json!({
1402                "content": request.content,
1403                "origin": request.origin,
1404                "idempotency_key": request.idempotency_key,
1405                "handling_mode": handling_mode_value,
1406            }),
1407            source: ConsoleFrameSource {
1408                kind: ConsoleFrameSourceKind::Send,
1409                source_cursor: Some(request_fingerprint),
1410            },
1411            source_event_id: None,
1412            interaction_id: Some(interaction_id.clone()),
1413            turn_id: None,
1414            run_id: None,
1415            parent_frame_id: None,
1416            caused_by_frame_id: None,
1417        };
1418        if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1419            new_frame.payload = redacted;
1420            new_frame.status = ConsoleFrameStatus::Redacted;
1421        }
1422
1423        let _ = SendState::Requested
1424            .apply(SendTransition::PersistAccepted)
1425            .map_err(ConsoleSendError::State)?;
1426        let outcome = self
1427            .inner
1428            .store
1429            .append_if_absent(new_frame)
1430            .await
1431            .map_err(ConsoleSendError::Log)?;
1432        if outcome.disposition == AppendDisposition::Existing {
1433            return Ok(accepted_from_frame(&outcome.frame));
1434        }
1435        let _ = self
1436            .inner
1437            .event_tx
1438            .send(ConsoleTimelineEvent::ConsoleFrame {
1439                frame: outcome.frame.clone(),
1440            });
1441        let accepted = accepted_from_frame(&outcome.frame);
1442
1443        let (dispatching, _effects) = SendState::AcceptedPersisted
1444            .apply(SendTransition::StartDispatch)
1445            .map_err(ConsoleSendError::State)?;
1446        update_frame_status_and_emit(
1447            &self.inner,
1448            &outcome.frame.id,
1449            ConsoleFrameStatus::Dispatching,
1450        )
1451        .await
1452        .map_err(ConsoleSendError::Log)?;
1453
1454        spawn_console_send_dispatch(
1455            self.inner.clone(),
1456            resolved,
1457            content,
1458            handling_mode,
1459            dispatching,
1460            outcome.frame,
1461            interaction_id,
1462        );
1463        Ok(accepted)
1464    }
1465
1466    pub async fn reserve_identity_first_interaction(
1467        &self,
1468        request: ConsoleSendRequest,
1469        session_id: Option<&str>,
1470    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1471        validate_send_request(&request)?;
1472        let _content = content_input_from_value(&request.content)?;
1473        let runtime_key =
1474            Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1475        let handling_mode_value = request
1476            .handling_mode
1477            .as_deref()
1478            .unwrap_or("queue")
1479            .to_string();
1480        let dedupe_key = send_dedupe_key(
1481            &runtime_key,
1482            &request.identity,
1483            &request.origin,
1484            &request.idempotency_key,
1485        );
1486        let request_fingerprint =
1487            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1488        if let Some(existing) = self
1489            .inner
1490            .store
1491            .frame_by_dedupe_key(&dedupe_key)
1492            .await
1493            .map_err(ConsoleSendError::Log)?
1494        {
1495            let same_request = existing.source.source_cursor.as_deref()
1496                == Some(request_fingerprint.as_str())
1497                || existing.source.source_cursor.is_none()
1498                    && existing.payload.get("origin").and_then(Value::as_str)
1499                        == Some(request.origin.as_str())
1500                    && existing.payload.get("content") == Some(&request.content)
1501                    && existing
1502                        .payload
1503                        .get("handling_mode")
1504                        .and_then(Value::as_str)
1505                        == Some(handling_mode_value.as_str());
1506            if !same_request {
1507                return Err(ConsoleSendError::IdempotencyConflict(
1508                    request.idempotency_key,
1509                ));
1510            }
1511            return Ok(accepted_from_frame(&existing));
1512        }
1513
1514        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1515        let new_frame = NewConsoleFrame {
1516            id: None,
1517            dedupe_key,
1518            timestamp_ms: current_time_ms(),
1519            runtime_key,
1520            identity: request.identity.clone(),
1521            conversation_id: Some(request.identity.clone()),
1522            session_id: session_id.map(ToString::to_string),
1523            kind: "user_input".to_string(),
1524            status: ConsoleFrameStatus::Accepted,
1525            payload: json!({
1526                "content": request.content,
1527                "origin": request.origin,
1528                "idempotency_key": request.idempotency_key,
1529                "handling_mode": handling_mode_value,
1530            }),
1531            source: ConsoleFrameSource {
1532                kind: ConsoleFrameSourceKind::Send,
1533                source_cursor: Some(request_fingerprint),
1534            },
1535            source_event_id: None,
1536            interaction_id: Some(interaction_id),
1537            turn_id: None,
1538            run_id: None,
1539            parent_frame_id: None,
1540            caused_by_frame_id: None,
1541        };
1542        let outcome = self
1543            .inner
1544            .store
1545            .append_if_absent(new_frame)
1546            .await
1547            .map_err(ConsoleSendError::Log)?;
1548        let _ = self
1549            .inner
1550            .event_tx
1551            .send(ConsoleTimelineEvent::ConsoleFrame {
1552                frame: outcome.frame.clone(),
1553            });
1554        Ok(accepted_from_frame(&outcome.frame))
1555    }
1556
1557    async fn runtime_key_for_identity_first_send(
1558        &self,
1559        identity: &str,
1560    ) -> Result<String, ConsoleSendError> {
1561        let entries = self
1562            .inner
1563            .runtimes
1564            .read()
1565            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1566            .clone();
1567        if entries.is_empty() {
1568            return Ok("identity-first".to_string());
1569        }
1570
1571        let mut identity_runtime_keys = Vec::new();
1572        let mut hidden_durable_match = false;
1573        let mut durable_matches = Vec::new();
1574        for entry in entries.values() {
1575            let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1576                continue;
1577            };
1578            identity_runtime_keys.push(entry.runtime_key.clone());
1579            if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1580                continue;
1581            }
1582            for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1583            {
1584                let Ok(parsed_identity) =
1585                    crate::identity_first::AgentIdentity::parse(&runtime_identity)
1586                else {
1587                    continue;
1588                };
1589                if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1590                    let record = identity_record_for_status(entry, &status);
1591                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
1592                        hidden_durable_match = true;
1593                        continue;
1594                    }
1595                    durable_matches.push((
1596                        entry.clone(),
1597                        identity_runtime.clone(),
1598                        parsed_identity,
1599                        record,
1600                    ));
1601                }
1602            }
1603        }
1604        if durable_matches.len() > 1 {
1605            let candidates = durable_matches
1606                .iter()
1607                .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1608                .collect::<Vec<_>>()
1609                .join(", ");
1610            return Err(ConsoleSendError::InvalidRequest(format!(
1611                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1612            )));
1613        }
1614        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1615            durable_matches.into_iter().next()
1616        {
1617            let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1618            let durable_live_records = Box::pin(self.live_records_for_durable_record(
1619                &entry,
1620                identity,
1621                &record,
1622                &live_records,
1623            ))
1624            .await;
1625            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1626                &entry,
1627                &durable_live_records,
1628            ))
1629            .await;
1630            if let Some(ambiguous_error) =
1631                ambiguous_live_alias_error(identity, &visible_durable_live_records)
1632            {
1633                return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1634            }
1635            if let Some(rebound_record) = self_heal_durable_session_mismatch(
1636                &entry,
1637                identity_runtime.as_ref(),
1638                &parsed_identity,
1639                identity,
1640                &record,
1641                &durable_live_records,
1642            )
1643            .await
1644            .map_err(ConsoleSendError::InvalidRequest)?
1645            {
1646                record = rebound_record;
1647            }
1648            if let Some(stale_error) =
1649                stale_durable_record_error(identity, &record, &durable_live_records)
1650            {
1651                return Err(ConsoleSendError::InvalidRequest(stale_error));
1652            }
1653            return Ok(entry.runtime_key);
1654        }
1655        if entries.values().any(|entry| {
1656            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1657        }) {
1658            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1659        }
1660        if hidden_durable_match {
1661            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1662        }
1663
1664        match identity_runtime_keys.as_slice() {
1665            [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1666            [runtime_key] => Ok(runtime_key.clone()),
1667            _ => Err(ConsoleSendError::InvalidRequest(format!(
1668                "identity-first send for '{identity}' did not match exactly one registered runtime"
1669            ))),
1670        }
1671    }
1672
1673    pub async fn mark_interaction_delivery_failed(
1674        &self,
1675        input_frame_id: &str,
1676    ) -> Result<(), ConsoleSendError> {
1677        update_frame_status_and_emit(
1678            &self.inner,
1679            input_frame_id,
1680            ConsoleFrameStatus::DeliveryFailed,
1681        )
1682        .await
1683        .map_err(ConsoleSendError::Log)?;
1684        Ok(())
1685    }
1686
1687    pub async fn mark_interaction_delivered(
1688        &self,
1689        input_frame_id: &str,
1690    ) -> Result<(), ConsoleSendError> {
1691        update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1692            .await
1693            .map_err(ConsoleSendError::Log)?;
1694        Ok(())
1695    }
1696
1697    pub async fn mark_steer_interaction_delivered(
1698        &self,
1699        input_frame_id: &str,
1700        interaction_id: &str,
1701    ) -> Result<(), ConsoleSendError> {
1702        let Some(updated) = update_frame_status_and_emit(
1703            &self.inner,
1704            input_frame_id,
1705            ConsoleFrameStatus::Delivered,
1706        )
1707        .await
1708        .map_err(ConsoleSendError::Log)?
1709        else {
1710            return Ok(());
1711        };
1712        append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1713            .await
1714            .map_err(ConsoleSendError::Log)?;
1715        Ok(())
1716    }
1717
1718    pub async fn binary_blob_store_for_identity(
1719        &self,
1720        identity: &str,
1721    ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1722        if identity.trim().is_empty() {
1723            return Err(ConsoleSendError::InvalidRequest(
1724                "identity must be non-empty".to_string(),
1725            ));
1726        }
1727        let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1728            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1729        };
1730        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1731            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1732        };
1733        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1734            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1735        }
1736        if !member_is_addressable(&resolved.member) {
1737            return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1738        }
1739        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1740            return Err(ConsoleSendError::Retired(identity.to_string()));
1741        }
1742        Ok(resolved.entry.runtime.binary_blob_store())
1743    }
1744
1745    pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1746        self.inner
1747            .runtimes
1748            .read()
1749            .map(|entries| {
1750                entries
1751                    .values()
1752                    .filter_map(|entry| entry.runtime.binary_blob_store())
1753                    .collect()
1754            })
1755            .unwrap_or_default()
1756    }
1757
1758    async fn resolve_send_member(
1759        &self,
1760        identity: &str,
1761    ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1762        let matches = Box::pin(self.resolve_visible_members(identity)).await;
1763        if matches.len() > 1 {
1764            let candidates = matches
1765                .iter()
1766                .map(|resolved| resolved.runtime_identity.clone())
1767                .collect::<Vec<_>>()
1768                .join(", ");
1769            return Err(ConsoleSendError::AmbiguousIdentity {
1770                identity: identity.to_string(),
1771                candidates,
1772            });
1773        }
1774        let Some(resolved) = matches.into_iter().next() else {
1775            if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1776                return Err(ConsoleSendError::InvalidRequest(stale_error));
1777            }
1778            return Ok(None);
1779        };
1780        if let Some(record) = identity_record_for_resolved_member(&resolved).await
1781            && let Some(stale_error) =
1782                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1783        {
1784            return Err(ConsoleSendError::InvalidRequest(stale_error));
1785        }
1786        if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1787            return Err(ConsoleSendError::InvalidRequest(stale_error));
1788        }
1789        Ok(Some(resolved))
1790    }
1791
1792    async fn durable_send_binding_error(
1793        &self,
1794        identity: &str,
1795    ) -> Result<Option<String>, ConsoleSendError> {
1796        let entries = self
1797            .inner
1798            .runtimes
1799            .read()
1800            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1801            .clone();
1802        if entries.values().any(|entry| {
1803            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1804        }) {
1805            return Ok(None);
1806        }
1807        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1808        let mut durable_matches = Vec::new();
1809        for entry in entries.values() {
1810            let Some(identity_runtime) = entry.identity_runtime.clone() else {
1811                continue;
1812            };
1813            for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1814                let Ok(parsed_identity) =
1815                    crate::identity_first::AgentIdentity::parse(&raw_identity)
1816                else {
1817                    continue;
1818                };
1819                let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1820                    continue;
1821                };
1822                let record = identity_record_for_status(entry, &status);
1823                if !Box::pin(console_identity_record_visible(entry, &record)).await {
1824                    continue;
1825                }
1826                durable_matches.push((
1827                    entry.clone(),
1828                    identity_runtime.clone(),
1829                    parsed_identity,
1830                    record,
1831                ));
1832            }
1833        }
1834        if durable_matches.len() > 1 {
1835            let candidates = durable_matches
1836                .iter()
1837                .map(|(_, _, _, record)| record.runtime_member_id.clone())
1838                .collect::<Vec<_>>()
1839                .join(", ");
1840            return Err(ConsoleSendError::AmbiguousIdentity {
1841                identity: identity.to_string(),
1842                candidates,
1843            });
1844        }
1845        let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1846            durable_matches.into_iter().next()
1847        else {
1848            return Ok(None);
1849        };
1850        let durable_live_records = Box::pin(self.live_records_for_durable_record(
1851            &entry,
1852            identity,
1853            &durable_record,
1854            &live_records,
1855        ))
1856        .await;
1857        let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1858            &entry,
1859            &durable_live_records,
1860        ))
1861        .await;
1862        if let Some(ambiguous_error) =
1863            ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1864        {
1865            return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1866        }
1867        if let Some(rebound_record) = self_heal_durable_session_mismatch(
1868            &entry,
1869            identity_runtime.as_ref(),
1870            &parsed_identity,
1871            identity,
1872            &durable_record,
1873            &durable_live_records,
1874        )
1875        .await
1876        .map_err(ConsoleSendError::InvalidRequest)?
1877        {
1878            durable_record = rebound_record;
1879        }
1880        Ok(stale_durable_record_error(
1881            identity,
1882            &durable_record,
1883            &durable_live_records,
1884        ))
1885    }
1886
1887    async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1888        let entries = self
1889            .inner
1890            .runtimes
1891            .read()
1892            .ok()
1893            .map(|entries| entries.clone())
1894            .unwrap_or_default();
1895        let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1896        let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1897        for entry in entries.values() {
1898            let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1899            if raw_identities.is_empty() {
1900                continue;
1901            }
1902            let mids = raw_identities
1903                .iter()
1904                .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
1905                .collect::<Vec<_>>();
1906            for resolved in Box::pin(member_sources_for_entry(entry)).await {
1907                let session_id = resolved
1908                    .handle
1909                    .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1910                    .await
1911                    .map(|sid| sid.to_string())
1912                    .unwrap_or_default();
1913                if mids.contains(&resolved.member.agent_identity) {
1914                    exact_matches.push((session_id, resolved));
1915                } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1916                {
1917                    label_matches.push((session_id, resolved));
1918                }
1919            }
1920        }
1921        let mut matches = exact_matches;
1922        matches.extend(label_matches);
1923        let mut seen_members = BTreeSet::new();
1924        matches.retain(|(session_id, resolved)| {
1925            seen_members.insert((
1926                resolved.entry.runtime_key.clone(),
1927                resolved.source_mob_id.clone(),
1928                session_id.clone(),
1929                resolved.runtime_identity.clone(),
1930            ))
1931        });
1932        matches.sort_by(|left, right| right.0.cmp(&left.0));
1933        matches.into_iter().map(|(_, resolved)| resolved).collect()
1934    }
1935
1936    async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1937        let mut visible = Vec::new();
1938        for resolved in Box::pin(self.resolve_members(identity)).await {
1939            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1940                continue;
1941            };
1942            if Box::pin(resolved_member_visible(&resolved, &record)).await {
1943                visible.push(resolved);
1944            }
1945        }
1946        visible
1947    }
1948
1949    async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1950        let mut records = Vec::new();
1951        for resolved in Box::pin(self.resolve_members(identity)).await {
1952            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1953                continue;
1954            };
1955            if Box::pin(resolved_member_visible(&resolved, &record)).await {
1956                records.push(record);
1957            }
1958        }
1959        records
1960    }
1961
1962    async fn live_records_for_durable_record(
1963        &self,
1964        entry: &RuntimeEntry,
1965        requested_identity: &str,
1966        durable: &ConsoleIdentityRecord,
1967        requested_live_records: &[ConsoleIdentityRecord],
1968    ) -> Vec<ConsoleIdentityRecord> {
1969        let mut records = if durable.identity == requested_identity {
1970            requested_live_records.to_vec()
1971        } else {
1972            Box::pin(self.live_records_for_identity(&durable.identity)).await
1973        };
1974        for runtime_record in Box::pin(live_records_for_runtime_member(
1975            entry,
1976            &durable.runtime_member_id,
1977        ))
1978        .await
1979        {
1980            if records.iter().any(|record| {
1981                record.runtime_key == runtime_record.runtime_key
1982                    && record.runtime_member_id == runtime_record.runtime_member_id
1983                    && record.session_id == runtime_record.session_id
1984                    && record.identity == runtime_record.identity
1985            }) {
1986                continue;
1987            }
1988            records.push(runtime_record);
1989        }
1990        records
1991    }
1992}
1993
1994async fn console_identity_record_visible(
1995    entry: &RuntimeEntry,
1996    record: &ConsoleIdentityRecord,
1997) -> bool {
1998    if !entry.visibility_policy.identity_visible(record) {
1999        return false;
2000    }
2001    let mut bound_live_member_seen = false;
2002    let mut wrong_live_projection_seen = false;
2003    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2004        if resolved.runtime_identity != record.runtime_member_id {
2005            continue;
2006        }
2007        bound_live_member_seen = true;
2008        let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2009            continue;
2010        };
2011        if live_record.identity != record.identity {
2012            wrong_live_projection_seen = true;
2013            continue;
2014        }
2015        if raw_resolved_member_visible(&resolved, &live_record) {
2016            return true;
2017        }
2018    }
2019    if wrong_live_projection_seen {
2020        return true;
2021    }
2022    !bound_live_member_seen
2023}
2024
2025async fn live_record_shadowed_by_hidden_durable_binding(
2026    resolved: &ResolvedConsoleMember,
2027    record: &ConsoleIdentityRecord,
2028) -> bool {
2029    let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2030        return false;
2031    };
2032    let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2033        return false;
2034    };
2035    let Ok(status) = identity_runtime.status(&identity).await else {
2036        return false;
2037    };
2038    let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2039        return false;
2040    };
2041    if bound_runtime_id.as_str() == resolved.runtime_identity {
2042        return false;
2043    }
2044    for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2045        if candidate.runtime_identity != bound_runtime_id.as_str() {
2046            continue;
2047        }
2048        let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2049            continue;
2050        };
2051        if bound_record.identity == record.identity
2052            && !raw_resolved_member_visible(&candidate, &bound_record)
2053        {
2054            return true;
2055        }
2056    }
2057    false
2058}
2059
2060async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2061    let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2062        .unwrap_or_else(|| frame.identity.clone());
2063    let frame_session_id = frame.session_id.as_deref();
2064    let mut saw_hidden = false;
2065    let mut saw_visible = false;
2066    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2067        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2068            continue;
2069        };
2070        let session_matches =
2071            frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2072        let runtime_matches = record.runtime_member_id == runtime_member_id;
2073        let identity_matches = record.identity == frame.identity;
2074        if !session_matches && !runtime_matches && !identity_matches {
2075            continue;
2076        }
2077        if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2078            return true;
2079        }
2080        if Box::pin(resolved_member_visible(&resolved, &record)).await {
2081            saw_visible = true;
2082        } else {
2083            saw_hidden = true;
2084        }
2085    }
2086    saw_hidden && !saw_visible
2087}
2088
2089async fn live_records_for_runtime_member(
2090    entry: &RuntimeEntry,
2091    runtime_member_id: &str,
2092) -> Vec<ConsoleIdentityRecord> {
2093    let mut records = Vec::new();
2094    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2095        if resolved.runtime_identity != runtime_member_id {
2096            continue;
2097        }
2098        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2099            continue;
2100        };
2101        records.push(record);
2102    }
2103    records
2104}
2105
2106async fn visible_live_records_for_entry(
2107    entry: &RuntimeEntry,
2108    records: &[ConsoleIdentityRecord],
2109) -> Vec<ConsoleIdentityRecord> {
2110    let mut visible = Vec::new();
2111    for record in records {
2112        if record.runtime_key != entry.runtime_key
2113            || Box::pin(console_identity_record_visible(entry, record)).await
2114        {
2115            visible.push(record.clone());
2116        }
2117    }
2118    visible
2119}
2120
2121async fn reconcile_stale_live_record_binding(
2122    entry: &RuntimeEntry,
2123    requested_identity: &str,
2124    live_record: &ConsoleIdentityRecord,
2125) -> Option<String> {
2126    let identity_runtime = entry.identity_runtime.as_ref()?;
2127    for status in identity_runtime.statuses().await {
2128        let matches_runtime = status
2129            .agent_runtime_id
2130            .as_ref()
2131            .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2132        if !matches_runtime {
2133            continue;
2134        }
2135        let durable_record = identity_record_for_status(entry, &status);
2136        if let Some(session_mismatch) =
2137            stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2138        {
2139            let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2140                continue;
2141            };
2142            let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2143                return Some(format!(
2144                    "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2145                ));
2146            };
2147            return match identity_runtime
2148                .rebind_session_after_live_respawn(&status.identity, session_id)
2149                .await
2150            {
2151                Ok(_) => {
2152                    tracing::warn!(
2153                        requested_identity,
2154                        identity = %durable_record.identity,
2155                        runtime_member_id = %durable_record.runtime_member_id,
2156                        old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2157                        new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2158                        "self-healed stale durable identity alias session mismatch"
2159                    );
2160                    None
2161                }
2162                Err(err) => Some(format!(
2163                    "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2164                    durable_record.identity,
2165                    session_mismatch.session_id.as_deref().unwrap_or("<none>")
2166                )),
2167            };
2168        }
2169        if let Some(stale_error) = stale_durable_record_error(
2170            requested_identity,
2171            &durable_record,
2172            std::slice::from_ref(live_record),
2173        ) {
2174            return Some(stale_error);
2175        }
2176    }
2177    None
2178}
2179
2180async fn self_heal_durable_session_mismatch(
2181    entry: &RuntimeEntry,
2182    identity_runtime: &crate::identity_first::IdentityRuntime,
2183    parsed_identity: &crate::identity_first::AgentIdentity,
2184    requested_identity: &str,
2185    durable: &ConsoleIdentityRecord,
2186    live_records: &[ConsoleIdentityRecord],
2187) -> Result<Option<ConsoleIdentityRecord>, String> {
2188    let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2189        return Ok(None);
2190    };
2191    let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2192        return Ok(None);
2193    };
2194    let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2195        format!(
2196            "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2197        )
2198    })?;
2199    identity_runtime
2200        .rebind_session_after_live_respawn(parsed_identity, session_id)
2201        .await
2202        .map_err(|err| {
2203            format!(
2204                "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2205                durable.identity, live_session_id
2206            )
2207        })?;
2208    tracing::warn!(
2209        requested_identity,
2210        identity = %durable.identity,
2211        runtime_member_id = %durable.runtime_member_id,
2212        old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2213        new_session_id = live_session_id,
2214        "self-healed stale durable identity alias session mismatch"
2215    );
2216    let status = identity_runtime
2217        .status(parsed_identity)
2218        .await
2219        .map_err(|err| {
2220            format!(
2221                "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2222                durable.identity
2223            )
2224        })?;
2225    Ok(Some(identity_record_for_status(entry, &status)))
2226}
2227
2228fn stale_durable_session_mismatch<'a>(
2229    durable: &ConsoleIdentityRecord,
2230    live_records: &'a [ConsoleIdentityRecord],
2231) -> Option<&'a ConsoleIdentityRecord> {
2232    live_records.iter().find(|record| {
2233        record.identity == durable.identity
2234            && record.runtime_member_id == durable.runtime_member_id
2235            && durable.session_id.is_some()
2236            && record.session_id.is_some()
2237            && durable.session_id != record.session_id
2238    })
2239}
2240
2241fn stale_durable_record_error(
2242    requested_identity: &str,
2243    durable: &ConsoleIdentityRecord,
2244    live_records: &[ConsoleIdentityRecord],
2245) -> Option<String> {
2246    let matching_live = live_records
2247        .iter()
2248        .filter(|record| record.identity == durable.identity)
2249        .collect::<Vec<_>>();
2250    if let Some(wrong_projection) = live_records.iter().find(|record| {
2251        record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2252    }) {
2253        return Some(format!(
2254            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2255            durable.identity, durable.runtime_member_id, wrong_projection.identity
2256        ));
2257    }
2258    if matching_live.is_empty() {
2259        return None;
2260    }
2261    if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2262        return Some(format!(
2263            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2264            durable.identity,
2265            durable.runtime_member_id,
2266            durable.session_id.as_deref().unwrap_or("<none>"),
2267            session_mismatch.session_id.as_deref().unwrap_or("<none>")
2268        ));
2269    }
2270    if matching_live
2271        .iter()
2272        .any(|record| record.runtime_member_id == durable.runtime_member_id)
2273    {
2274        return None;
2275    }
2276    let live_candidates = matching_live
2277        .iter()
2278        .map(|record| record.runtime_member_id.as_str())
2279        .collect::<Vec<_>>()
2280        .join(", ");
2281    Some(format!(
2282        "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2283        durable.identity, durable.runtime_member_id, live_candidates
2284    ))
2285}
2286
2287fn ambiguous_live_alias_error(
2288    requested_identity: &str,
2289    live_records: &[ConsoleIdentityRecord],
2290) -> Option<String> {
2291    if live_records.len() <= 1 {
2292        return None;
2293    }
2294    let candidates = live_records
2295        .iter()
2296        .map(|record| {
2297            format!(
2298                "{}@{}",
2299                record.runtime_member_id,
2300                record
2301                    .labels
2302                    .get("source_mob_id")
2303                    .map(String::as_str)
2304                    .unwrap_or(record.runtime_key.as_str())
2305            )
2306        })
2307        .collect::<Vec<_>>()
2308        .join(", ");
2309    Some(format!(
2310        "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2311    ))
2312}
2313
2314fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2315    // Roster ids are comms-safe encodings of public aliases (meerkat 0.7
2316    // MemberCommsName); compare in the public alias space.
2317    crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2318}
2319
2320async fn retire_stale_console_members_for_identity(
2321    handle: &MobHandle,
2322    durable_identities: &[String],
2323    keep_runtime_member_id: Option<&str>,
2324) -> Result<(), String> {
2325    let stale_members = handle
2326        .list_members_including_retiring()
2327        .await
2328        .into_iter()
2329        .filter(|member| {
2330            (durable_identities.iter().any(|durable_identity| {
2331                member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2332            }) || member.labels.get("agent_identity").is_some_and(|identity| {
2333                durable_identities
2334                    .iter()
2335                    .any(|durable_identity| identity == durable_identity)
2336            })) && keep_runtime_member_id
2337                .map(|keep| {
2338                    // `keep` is a public alias; compare decoded.
2339                    crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2340                        != keep
2341                })
2342                .unwrap_or(true)
2343        })
2344        .map(|member| member.agent_identity)
2345        .collect::<Vec<_>>();
2346    for member_id in stale_members {
2347        match handle.retire(member_id).await {
2348            Ok(()) => {}
2349            Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2350            Err(err) => return Err(err.to_string()),
2351        }
2352    }
2353    Ok(())
2354}
2355
2356async fn retire_stale_console_members_for_runtime_entry(
2357    entry: &RuntimeEntry,
2358    durable_identities: &[String],
2359) -> Result<(), String> {
2360    let primary_handle = entry.runtime.handle();
2361    let primary_mob_id = primary_handle.mob_id().to_string();
2362    let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2363    if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2364        for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2365            .await
2366            .unwrap_or_default()
2367        {
2368            if mob_id.as_str() != primary_mob_id {
2369                handles.push((mob_id.to_string(), handle));
2370            }
2371        }
2372    }
2373    let mut seen_handles = BTreeSet::new();
2374    for (mob_id, handle) in handles {
2375        if !seen_handles.insert(mob_id) {
2376            continue;
2377        }
2378        retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2379    }
2380    Ok(())
2381}
2382
2383fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2384    is_recoverable_lifecycle_cleanup_error(error)
2385}
2386
2387fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2388    if frames.is_empty() {
2389        return true;
2390    }
2391    let latest_session_history_timestamp_ms = frames
2392        .iter()
2393        .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2394        .map(|frame| frame.timestamp_ms)
2395        .max();
2396    if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2397        return current_time_ms().saturating_sub(latest_timestamp_ms)
2398            >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2399    }
2400    frames.iter().any(|frame| {
2401        matches!(
2402            frame.kind.as_str(),
2403            "turn_started"
2404                | "run_started"
2405                | "reasoning_delta"
2406                | "reasoning_complete"
2407                | "tool_call_requested"
2408                | "tool_call"
2409                | "tool_execution_started"
2410                | "tool_execution_completed"
2411                | "tool_result_received"
2412        )
2413    })
2414}
2415
2416fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2417    match frame.kind.as_str() {
2418        "user_input" | "run_started" => true,
2419        "interaction_started" => frame
2420            .payload
2421            .get("content")
2422            .or_else(|| frame.payload.get("prompt"))
2423            .is_some(),
2424        _ => false,
2425    }
2426}
2427
2428fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2429    let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2430    for record in records {
2431        by_identity
2432            .entry(identity_record_dedupe_key(&record))
2433            .and_modify(|current| {
2434                if identity_record_prefer(&record, current) {
2435                    *current = record.clone();
2436                }
2437            })
2438            .or_insert(record);
2439    }
2440    by_identity.into_values().collect()
2441}
2442
2443fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2444    format!(
2445        "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2446        record.identity,
2447        record.runtime_key,
2448        record
2449            .labels
2450            .get("source_mob_id")
2451            .map(String::as_str)
2452            .unwrap_or(""),
2453        record.runtime_member_id,
2454        record.session_id.as_deref().unwrap_or("")
2455    )
2456}
2457
2458fn identity_record_prefer(
2459    candidate: &ConsoleIdentityRecord,
2460    current: &ConsoleIdentityRecord,
2461) -> bool {
2462    let candidate_is_live_label_projection =
2463        candidate
2464            .labels
2465            .get("agent_identity")
2466            .is_some_and(|identity| {
2467                identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2468                    || candidate.runtime_member_id != candidate.identity
2469            });
2470    let current_is_live_label_projection =
2471        current
2472            .labels
2473            .get("agent_identity")
2474            .is_some_and(|identity| {
2475                identity == &strip_namespace(&current.identity, "").unwrap_or_default()
2476                    || current.runtime_member_id != current.identity
2477            });
2478    if candidate_is_live_label_projection != current_is_live_label_projection {
2479        return candidate_is_live_label_projection;
2480    }
2481    let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2482    let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2483    if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2484        return candidate_has_distinct_runtime_binding;
2485    }
2486    let candidate_live = candidate.addressable && candidate.health != "retired";
2487    let current_live = current.addressable && current.health != "retired";
2488    if candidate_live != current_live {
2489        return candidate_live;
2490    }
2491    candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2492}
2493
2494async fn collect_identity_records(
2495    inner: &Arc<AggregatorInner>,
2496    mode: IdentityCollectionMode,
2497) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2498    let entries = inner
2499        .runtimes
2500        .read()
2501        .map_err(|_| runtime_registry_lock_error())?
2502        .clone();
2503    let mut identities = Vec::new();
2504    for entry in entries.values() {
2505        if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2506            let topology_peers =
2507                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2508            for status in identity_runtime.statuses().await {
2509                let mut record = identity_record_for_status(entry, &status);
2510                record.topology_peers = topology_peers
2511                    .get(&record.identity)
2512                    .cloned()
2513                    .unwrap_or_default();
2514                if Box::pin(console_identity_record_visible(entry, &record)).await {
2515                    identities.push(record);
2516                }
2517            }
2518            if mode == IdentityCollectionMode::CachedOnly {
2519                continue;
2520            }
2521        }
2522        let members = if entry.identity_runtime.is_some() {
2523            match Box::pin(tokio::time::timeout(
2524                IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2525                member_sources_for_entry(entry),
2526            ))
2527            .await
2528            {
2529                Ok(members) => members,
2530                Err(_) => {
2531                    tracing::warn!(
2532                        runtime_key = %entry.runtime_key,
2533                        "identity-first live member refresh timed out; keeping cached identity read model"
2534                    );
2535                    Vec::new()
2536                }
2537            }
2538        } else {
2539            Box::pin(member_sources_for_entry(entry)).await
2540        };
2541        for resolved in members {
2542            if let Some(record) = identity_record_for_resolved_member(&resolved).await
2543                && Box::pin(resolved_member_visible(&resolved, &record)).await
2544            {
2545                identities.push(record);
2546            }
2547        }
2548    }
2549    Ok(dedupe_identity_records(identities))
2550}
2551
2552async fn identity_runtime_topology_peers(
2553    entry: &RuntimeEntry,
2554    identity_runtime: &crate::identity_first::IdentityRuntime,
2555) -> BTreeMap<String, Vec<String>> {
2556    let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2557    for edge in identity_runtime.desired_peer_edges().await {
2558        let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2559        let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2560        peers.entry(a.clone()).or_default().insert(b.clone());
2561        peers.entry(b).or_default().insert(a);
2562    }
2563    peers
2564        .into_iter()
2565        .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2566        .collect()
2567}
2568
2569fn spawn_identity_backfills_for_records(
2570    inner: Arc<AggregatorInner>,
2571    records: &[ConsoleIdentityRecord],
2572) {
2573    if !inner.options.session_history_backfill_enabled {
2574        return;
2575    }
2576    let entries = match inner.runtimes.read() {
2577        Ok(entries) => entries.clone(),
2578        Err(_) => return,
2579    };
2580    for record in records {
2581        if record.health == "dormant" || record.health == "uninitialized" {
2582            continue;
2583        }
2584        let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2585            continue;
2586        };
2587        let Some(session_id) = record.session_id.clone() else {
2588            continue;
2589        };
2590        spawn_session_history_backfill_target(
2591            inner.clone(),
2592            SessionBackfillTarget {
2593                entry,
2594                record: record.clone(),
2595                session_id,
2596            },
2597            false,
2598        );
2599    }
2600}
2601
2602async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2603    let mut resolved = Vec::new();
2604    let primary_handle = entry.runtime.handle();
2605    let primary_mob_id = primary_handle.mob_id().to_string();
2606    for member in primary_handle.list_members_observation_snapshot().await {
2607        resolved.push(ResolvedConsoleMember {
2608            entry: entry.clone(),
2609            handle: primary_handle.clone(),
2610            // Roster ids are comms-safe encodings (meerkat 0.7); the
2611            // aggregator's runtime identity is the public alias.
2612            runtime_identity: crate::member_comms_id::runtime_alias_str(
2613                member.agent_identity.as_str(),
2614            )
2615            .into_owned(),
2616            source_mob_id: primary_mob_id.clone(),
2617            member,
2618        });
2619    }
2620
2621    let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2622        return resolved;
2623    };
2624    if !entry.visibility_policy.include_implicit_delegate_members() {
2625        return resolved;
2626    }
2627    for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2628        .await
2629        .unwrap_or_default()
2630    {
2631        if mob_id.as_str() == primary_mob_id {
2632            continue;
2633        }
2634        for member in handle.list_members_observation_snapshot().await {
2635            resolved.push(ResolvedConsoleMember {
2636                entry: entry.clone(),
2637                handle: handle.clone(),
2638                // Roster ids are comms-safe encodings (meerkat 0.7); the
2639                // aggregator's runtime identity is the public alias.
2640                runtime_identity: crate::member_comms_id::runtime_alias_str(
2641                    member.agent_identity.as_str(),
2642                )
2643                .into_owned(),
2644                source_mob_id: mob_id.to_string(),
2645                member,
2646            });
2647        }
2648    }
2649    resolved
2650}
2651
2652async fn dispatch_message_to_resolved_member(
2653    resolved: &ResolvedConsoleMember,
2654    content: ContentInput,
2655    handling_mode: meerkat_core::types::HandlingMode,
2656) -> Result<String, String> {
2657    let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2658    match send_message_on_mob_with_mode(
2659        &resolved.handle,
2660        &resolved.runtime_identity,
2661        content.clone(),
2662        handling_mode,
2663    )
2664    .await
2665    {
2666        Ok(session_id) => Ok(session_id),
2667        Err(err) if is_not_externally_addressable(&err) => {
2668            let member = resolved
2669                .handle
2670                .member(&mid)
2671                .await
2672                .map_err(|err| err.to_string())?;
2673            let _receipt = member
2674                .internal_turn(content)
2675                .await
2676                .map_err(|err| err.to_string())?;
2677            resolved
2678                .handle
2679                .resolve_bridge_session_id_observation(&mid)
2680                .await
2681                .map(|sid| sid.to_string())
2682                .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2683        }
2684        Err(err) => Err(err.to_string()),
2685    }
2686}
2687
2688fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2689    matches!(
2690        err,
2691        MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2692    )
2693}
2694
2695fn spawn_console_send_dispatch(
2696    inner: Arc<AggregatorInner>,
2697    resolved: ResolvedConsoleMember,
2698    content: ContentInput,
2699    handling_mode: meerkat_core::types::HandlingMode,
2700    dispatching: SendState,
2701    user_frame: ConsoleFrame,
2702    interaction_id: String,
2703) {
2704    tokio::spawn(async move {
2705        match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2706            Ok(_) => {
2707                let _ = dispatching.apply(SendTransition::MarkDelivered);
2708                if let Err(err) = update_frame_status_and_emit(
2709                    &inner,
2710                    &user_frame.id,
2711                    ConsoleFrameStatus::Delivered,
2712                )
2713                .await
2714                {
2715                    tracing::warn!(
2716                        frame_id = %user_frame.id,
2717                        error = %err,
2718                        "failed to update console send delivery status"
2719                    );
2720                }
2721                if handling_mode == HandlingMode::Steer
2722                    && let Err(err) =
2723                        append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2724                {
2725                    tracing::warn!(
2726                        frame_id = %user_frame.id,
2727                        interaction_id = %interaction_id,
2728                        error = %err,
2729                        "failed to append console steer terminal frame"
2730                    );
2731                }
2732            }
2733            Err(err) => {
2734                let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2735                if let Err(update_err) = update_frame_status_and_emit(
2736                    &inner,
2737                    &user_frame.id,
2738                    ConsoleFrameStatus::DeliveryFailed,
2739                )
2740                .await
2741                {
2742                    tracing::warn!(
2743                        frame_id = %user_frame.id,
2744                        error = %update_err,
2745                        "failed to update console send failure status"
2746                    );
2747                }
2748                let failure_frame = NewConsoleFrame {
2749                    id: None,
2750                    dedupe_key: format!("delivery-failed:{}", user_frame.id),
2751                    timestamp_ms: current_time_ms(),
2752                    runtime_key: user_frame.runtime_key,
2753                    identity: user_frame.identity,
2754                    conversation_id: user_frame.conversation_id,
2755                    session_id: user_frame.session_id,
2756                    kind: "message_delivery_failed".to_string(),
2757                    status: ConsoleFrameStatus::DeliveryFailed,
2758                    payload: json!({ "reason": err }),
2759                    source: ConsoleFrameSource {
2760                        kind: ConsoleFrameSourceKind::Synthetic,
2761                        source_cursor: None,
2762                    },
2763                    source_event_id: None,
2764                    interaction_id: Some(interaction_id),
2765                    turn_id: None,
2766                    run_id: None,
2767                    parent_frame_id: Some(user_frame.id.clone()),
2768                    caused_by_frame_id: Some(user_frame.id),
2769                };
2770                if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2771                    tracing::warn!(
2772                        error = %append_err,
2773                        "failed to append console send failure frame"
2774                    );
2775                }
2776            }
2777        }
2778    });
2779}
2780
2781async fn append_steer_delivery_terminal(
2782    inner: &AggregatorInner,
2783    user_frame: &ConsoleFrame,
2784    interaction_id: &str,
2785) -> ConsoleLogResult<AppendOutcome> {
2786    append_and_emit(
2787        inner,
2788        NewConsoleFrame {
2789            id: None,
2790            dedupe_key: format!("steer-delivered:{}", user_frame.id),
2791            timestamp_ms: current_time_ms(),
2792            runtime_key: user_frame.runtime_key.clone(),
2793            identity: user_frame.identity.clone(),
2794            conversation_id: user_frame.conversation_id.clone(),
2795            session_id: user_frame.session_id.clone(),
2796            kind: "interaction_complete".to_string(),
2797            status: ConsoleFrameStatus::Completed,
2798            payload: json!({
2799                "reason": "steer_delivered",
2800                "handling_mode": "steer",
2801            }),
2802            source: ConsoleFrameSource {
2803                kind: ConsoleFrameSourceKind::Synthetic,
2804                source_cursor: None,
2805            },
2806            source_event_id: None,
2807            interaction_id: Some(interaction_id.to_string()),
2808            turn_id: None,
2809            run_id: None,
2810            parent_frame_id: Some(user_frame.id.clone()),
2811            caused_by_frame_id: Some(user_frame.id.clone()),
2812        },
2813    )
2814    .await
2815}
2816
2817#[derive(Debug)]
2818pub enum ConsoleSendError {
2819    UnknownIdentity(String),
2820    AmbiguousIdentity {
2821        identity: String,
2822        candidates: String,
2823    },
2824    NotAddressable(String),
2825    Retired(String),
2826    InvalidContent(String),
2827    InvalidHandlingMode(String),
2828    InvalidRequest(String),
2829    IdempotencyConflict(String),
2830    State(&'static str),
2831    Dispatch(String),
2832    Log(ConsoleLogError),
2833}
2834
2835impl std::fmt::Display for ConsoleSendError {
2836    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2837        match self {
2838            Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2839            Self::AmbiguousIdentity {
2840                identity,
2841                candidates,
2842            } => write!(
2843                f,
2844                "ambiguous live identity alias {identity}: candidates [{candidates}]"
2845            ),
2846            Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2847            Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2848            Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2849            Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2850            Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2851            Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2852            Self::State(message) => write!(f, "console send state error: {message}"),
2853            Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2854            Self::Log(err) => write!(f, "console log error: {err}"),
2855        }
2856    }
2857}
2858
2859impl std::error::Error for ConsoleSendError {}
2860
2861async fn backfill_session_history(
2862    inner: Arc<AggregatorInner>,
2863    runtime_key: String,
2864    force_refresh: bool,
2865) -> ConsoleLogResult<()> {
2866    if !inner.options.session_history_backfill_enabled {
2867        return Ok(());
2868    }
2869    let Some(entry) = inner
2870        .runtimes
2871        .read()
2872        .ok()
2873        .and_then(|entries| entries.get(&runtime_key).cloned())
2874    else {
2875        return Ok(());
2876    };
2877    let members = Box::pin(member_sources_for_entry(&entry)).await;
2878    let mut targets = Vec::new();
2879    for resolved in members {
2880        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2881            continue;
2882        };
2883        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
2884            continue;
2885        }
2886        let Some(session_id) = record.session_id.clone() else {
2887            continue;
2888        };
2889        targets.push(SessionBackfillTarget {
2890            entry: entry.clone(),
2891            record,
2892            session_id,
2893        });
2894    }
2895    backfill_session_history_targets(inner, targets, force_refresh).await
2896}
2897
2898#[derive(Clone)]
2899struct SessionBackfillTarget {
2900    entry: RuntimeEntry,
2901    record: ConsoleIdentityRecord,
2902    session_id: String,
2903}
2904
2905async fn backfill_session_history_targets(
2906    inner: Arc<AggregatorInner>,
2907    targets: Vec<SessionBackfillTarget>,
2908    force_refresh: bool,
2909) -> ConsoleLogResult<()> {
2910    let mut tasks = tokio::task::JoinSet::new();
2911    for target in targets {
2912        tasks.spawn(backfill_one_session_history(
2913            inner.clone(),
2914            target,
2915            force_refresh,
2916        ));
2917    }
2918    let mut first_error = None;
2919    while let Some(result) = tasks.join_next().await {
2920        match result {
2921            Ok(Ok(())) => {}
2922            Ok(Err(err)) => {
2923                if first_error.is_none() {
2924                    first_error = Some(err);
2925                }
2926            }
2927            Err(err) => {
2928                if first_error.is_none() {
2929                    first_error = Some(Box::new(std::io::Error::other(format!(
2930                        "session backfill task failed: {err}"
2931                    ))) as ConsoleLogError);
2932                }
2933            }
2934        }
2935    }
2936    if let Some(err) = first_error {
2937        Err(err)
2938    } else {
2939        Ok(())
2940    }
2941}
2942
2943async fn backfill_one_session_history(
2944    inner: Arc<AggregatorInner>,
2945    target: SessionBackfillTarget,
2946    force_refresh: bool,
2947) -> ConsoleLogResult<()> {
2948    let _permit = inner
2949        .session_backfill_permits
2950        .clone()
2951        .acquire_owned()
2952        .await
2953        .map_err(|err| -> ConsoleLogError {
2954            Box::new(std::io::Error::other(format!(
2955                "session backfill limiter closed: {err}"
2956            )))
2957        })?;
2958    let SessionBackfillTarget {
2959        entry,
2960        record,
2961        session_id,
2962    } = target;
2963    let watermark_runtime_key =
2964        session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2965    let watermark = inner
2966        .store
2967        .source_watermark(
2968            &watermark_runtime_key,
2969            ConsoleFrameSourceKind::SessionHistory,
2970        )
2971        .await?;
2972    let now_ms = current_time_ms();
2973    let mut offset = watermark
2974        .as_deref()
2975        .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2976        .unwrap_or(0);
2977    if !force_refresh
2978        && watermark.as_deref().is_some_and(|watermark| {
2979            session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2980        })
2981    {
2982        return Ok(());
2983    }
2984    loop {
2985        let page = match entry
2986            .runtime
2987            .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2988            .await
2989        {
2990            Ok(page) => page,
2991            Err(err) => {
2992                record_session_history_backfill_failure(
2993                    &inner,
2994                    &watermark_runtime_key,
2995                    &entry.runtime_key,
2996                    &record.identity,
2997                    &session_id,
2998                    offset,
2999                    err.to_string(),
3000                )
3001                .await?;
3002                break;
3003            }
3004        };
3005        let page_value = match serde_json::to_value(page) {
3006            Ok(value) => value,
3007            Err(err) => {
3008                record_session_history_backfill_failure(
3009                    &inner,
3010                    &watermark_runtime_key,
3011                    &entry.runtime_key,
3012                    &record.identity,
3013                    &session_id,
3014                    offset,
3015                    err.to_string(),
3016                )
3017                .await?;
3018                break;
3019            }
3020        };
3021        let base_offset = page_value
3022            .get("offset")
3023            .and_then(Value::as_u64)
3024            .unwrap_or(offset as u64) as usize;
3025        let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3026            record_session_history_backfill_failure(
3027                &inner,
3028                &watermark_runtime_key,
3029                &entry.runtime_key,
3030                &record.identity,
3031                &session_id,
3032                offset,
3033                "session history page missing messages".to_string(),
3034            )
3035            .await?;
3036            break;
3037        };
3038        if messages.is_empty() {
3039            if offset > 0 {
3040                record_session_history_watermark(
3041                    &inner,
3042                    &watermark_runtime_key,
3043                    &session_id,
3044                    offset,
3045                )
3046                .await?;
3047            }
3048            break;
3049        }
3050        for (idx, message) in messages.iter().enumerate() {
3051            let absolute_offset = base_offset + idx;
3052            let frames = frames_from_session_history_message_with_namespace(
3053                &entry.runtime_key,
3054                &record.identity,
3055                &entry.identity_namespace,
3056                &session_id,
3057                absolute_offset,
3058                message.clone(),
3059            );
3060            for mut frame in frames {
3061                if history_frame_has_existing_counterpart(&inner, &frame).await? {
3062                    continue;
3063                }
3064                if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3065                    frame.payload = redacted;
3066                    frame.status = ConsoleFrameStatus::Redacted;
3067                }
3068                append_and_emit(&inner, frame).await?;
3069            }
3070        }
3071        let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3072            // Forward-progress guard: a session store that returns a page whose
3073            // (base_offset + len) does not advance past the offset we requested
3074            // — e.g. a cursor that resets to 0 while still reporting has_more —
3075            // would make this loop re-read the same page forever, re-recording
3076            // the same watermark (a livelock observed on an orphaned gateway).
3077            // Stop instead of spinning; the last good watermark stands.
3078            tracing::warn!(
3079                session_id = %session_id,
3080                requested_offset = offset,
3081                page_base_offset = base_offset,
3082                page_len = messages.len(),
3083                "session history backfill made no forward progress; stopping to avoid a livelock"
3084            );
3085            break;
3086        };
3087        offset = next_offset;
3088        record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3089            .await?;
3090        let has_more = page_value
3091            .get("has_more")
3092            .and_then(Value::as_bool)
3093            .unwrap_or(false);
3094        if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3095            break;
3096        }
3097    }
3098    Ok(())
3099}
3100
3101/// Compute the next session-history backfill offset, enforcing strict forward
3102/// progress. Returns `None` when the page (`base_offset + page_len`) does not
3103/// advance past `requested_offset` — a non-advancing/looping cursor — which
3104/// signals the caller to stop rather than re-read the same page forever.
3105fn advance_backfill_offset(
3106    requested_offset: usize,
3107    base_offset: usize,
3108    page_len: usize,
3109) -> Option<usize> {
3110    let next = base_offset.saturating_add(page_len);
3111    (next > requested_offset).then_some(next)
3112}
3113
3114async fn record_session_history_watermark(
3115    inner: &AggregatorInner,
3116    watermark_runtime_key: &str,
3117    session_id: &str,
3118    offset: usize,
3119) -> ConsoleLogResult<()> {
3120    inner
3121        .store
3122        .record_source_watermark(
3123            watermark_runtime_key,
3124            ConsoleFrameSourceKind::SessionHistory,
3125            &format_session_history_watermark(session_id, offset, current_time_ms()),
3126        )
3127        .await
3128}
3129
3130fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3131    if !inner.options.session_history_backfill_enabled {
3132        return;
3133    }
3134    tokio::spawn(async move {
3135        {
3136            let mut active = inner.active_session_backfills.lock().await;
3137            if !active.insert(runtime_key.clone()) {
3138                return;
3139            }
3140        }
3141        let result = Box::pin(backfill_session_history(
3142            inner.clone(),
3143            runtime_key.clone(),
3144            false,
3145        ))
3146        .await;
3147        let mut active = inner.active_session_backfills.lock().await;
3148        active.remove(&runtime_key);
3149        drop(active);
3150        if let Err(err) = result {
3151            tracing::warn!(
3152                runtime_key = %runtime_key,
3153                error = %err,
3154                "console session-history backfill failed"
3155            );
3156        }
3157    });
3158}
3159
3160fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3161    if !inner.options.session_history_backfill_enabled {
3162        return;
3163    }
3164    tokio::spawn(async move {
3165        let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3166        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3167        loop {
3168            interval.tick().await;
3169            let runtime_still_registered = inner
3170                .runtimes
3171                .read()
3172                .ok()
3173                .is_some_and(|entries| entries.contains_key(&runtime_key));
3174            if !runtime_still_registered {
3175                break;
3176            }
3177            spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3178        }
3179    });
3180}
3181
3182fn spawn_session_history_backfill_target(
3183    inner: Arc<AggregatorInner>,
3184    target: SessionBackfillTarget,
3185    force_refresh: bool,
3186) {
3187    if !inner.options.session_history_backfill_enabled {
3188        return;
3189    }
3190    tokio::spawn(async move {
3191        let active_key = targeted_session_history_active_key(&target, force_refresh);
3192        let result =
3193            run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3194        if let Err(err) = result {
3195            tracing::warn!(
3196                active_key = %active_key,
3197                error = %err,
3198                "console targeted session-history backfill failed"
3199            );
3200        }
3201    });
3202}
3203
3204async fn run_targeted_session_history_backfill(
3205    inner: Arc<AggregatorInner>,
3206    target: SessionBackfillTarget,
3207    force_refresh: bool,
3208) -> ConsoleLogResult<()> {
3209    let active_key = targeted_session_history_active_key(&target, force_refresh);
3210    {
3211        let mut active = inner.active_session_backfills.lock().await;
3212        if !active.insert(active_key.clone()) {
3213            return Ok(());
3214        }
3215    }
3216    let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3217    let mut active = inner.active_session_backfills.lock().await;
3218    active.remove(&active_key);
3219    result
3220}
3221
3222fn targeted_session_history_active_key(
3223    target: &SessionBackfillTarget,
3224    force_refresh: bool,
3225) -> String {
3226    let mode = if force_refresh { "force" } else { "refresh" };
3227    format!(
3228        "{}:session-history:{}:{mode}",
3229        target.entry.runtime_key, target.session_id
3230    )
3231}
3232
3233fn spawn_session_history_backfill_for_identity(
3234    inner: Arc<AggregatorInner>,
3235    identity: String,
3236    force_refresh: bool,
3237) {
3238    if !inner.options.session_history_backfill_enabled {
3239        return;
3240    }
3241    tokio::spawn(async move {
3242        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3243            spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3244        }
3245    });
3246}
3247
3248fn spawn_opportunistic_session_history_backfill_for_identity(
3249    inner: Arc<AggregatorInner>,
3250    identity: String,
3251) {
3252    if !inner.options.session_history_backfill_enabled {
3253        return;
3254    }
3255    tokio::spawn(async move {
3256        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3257            let active_key = format!(
3258                "{}:session-history:{}",
3259                target.entry.runtime_key, target.session_id
3260            );
3261            {
3262                let mut seen = inner.opportunistic_session_backfills.lock().await;
3263                if !seen.insert(active_key) {
3264                    continue;
3265                }
3266            }
3267            spawn_session_history_backfill_target(inner.clone(), target, false);
3268        }
3269    });
3270}
3271
3272#[cfg(test)]
3273async fn session_backfill_target_for_identity(
3274    inner: &AggregatorInner,
3275    identity: &str,
3276) -> Option<SessionBackfillTarget> {
3277    session_backfill_targets_for_identity(inner, identity)
3278        .await
3279        .into_iter()
3280        .next()
3281}
3282
3283async fn session_backfill_targets_for_identity(
3284    inner: &AggregatorInner,
3285    identity: &str,
3286) -> Vec<SessionBackfillTarget> {
3287    let entries = inner
3288        .runtimes
3289        .read()
3290        .ok()
3291        .map(|entries| entries.clone())
3292        .unwrap_or_default();
3293    let mut targets = Vec::new();
3294    for entry in entries.values() {
3295        let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3296        if raw_identities.is_empty() {
3297            continue;
3298        }
3299        let mids = raw_identities
3300            .iter()
3301            .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3302            .collect::<Vec<_>>();
3303        for resolved in Box::pin(member_sources_for_entry(entry))
3304            .await
3305            .into_iter()
3306            .filter(|candidate| {
3307                resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3308            })
3309        {
3310            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3311                continue;
3312            };
3313            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3314                continue;
3315            }
3316            let Some(session_id) = record.session_id.clone() else {
3317                continue;
3318            };
3319            targets.push(SessionBackfillTarget {
3320                entry: entry.clone(),
3321                record,
3322                session_id,
3323            });
3324        }
3325    }
3326    targets
3327}
3328
3329fn resolved_member_matches_raw_identities(
3330    resolved: &ResolvedConsoleMember,
3331    raw_identities: &[String],
3332    mids: &[AgentIdentity],
3333) -> bool {
3334    mids.contains(&resolved.member.agent_identity)
3335        || resolved
3336            .member
3337            .labels
3338            .get("agent_identity")
3339            .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3340}
3341
3342async fn recover_lagged_source_events(
3343    inner: Arc<AggregatorInner>,
3344    runtime_key: &str,
3345    console_events: &ConsoleEventStore,
3346) -> ConsoleLogResult<()> {
3347    let watermark = inner
3348        .store
3349        .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3350        .await?;
3351    match console_events.replay_all(watermark.as_deref()).await {
3352        Ok(events) => {
3353            for envelope in events {
3354                project_console_event(inner.clone(), runtime_key, envelope).await?;
3355            }
3356        }
3357        Err(err) => {
3358            append_source_gap(
3359                &inner,
3360                runtime_key,
3361                format!(
3362                    "{}:{}:{}",
3363                    err.error, err.stream, err.requested_last_event_id
3364                ),
3365            )
3366            .await?;
3367        }
3368    }
3369    Ok(())
3370}
3371
3372async fn append_source_gap(
3373    inner: &AggregatorInner,
3374    runtime_key: &str,
3375    reason: String,
3376) -> ConsoleLogResult<()> {
3377    append_and_emit(
3378        inner,
3379        NewConsoleFrame {
3380            id: None,
3381            dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3382            timestamp_ms: current_time_ms(),
3383            runtime_key: runtime_key.to_string(),
3384            identity: "__console__".to_string(),
3385            conversation_id: None,
3386            session_id: None,
3387            kind: "replay_unavailable".to_string(),
3388            status: ConsoleFrameStatus::DeliveryFailed,
3389            payload: json!({
3390                "reason": reason,
3391                "source_kind": "console_event",
3392            }),
3393            source: ConsoleFrameSource {
3394                kind: ConsoleFrameSourceKind::Synthetic,
3395                source_cursor: None,
3396            },
3397            source_event_id: None,
3398            interaction_id: None,
3399            turn_id: None,
3400            run_id: None,
3401            parent_frame_id: None,
3402            caused_by_frame_id: None,
3403        },
3404    )
3405    .await?;
3406    let _ = inner
3407        .event_tx
3408        .send(ConsoleTimelineEvent::ReplayUnavailable {
3409            requested_cursor: format!("source-gap:{runtime_key}"),
3410            latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3411        });
3412    Ok(())
3413}
3414
3415/// Record a session-history backfill failure: stamp a watermark at the current
3416/// offset so the freshness gate throttles retries to once per
3417/// `SESSION_HISTORY_REFRESH_TTL_MS` (without it the 5s discovery loop re-reads
3418/// a persistently-failing session forever), and append a single gap frame.
3419async fn record_session_history_backfill_failure(
3420    inner: &AggregatorInner,
3421    watermark_runtime_key: &str,
3422    runtime_key: &str,
3423    identity: &str,
3424    session_id: &str,
3425    offset: usize,
3426    reason: String,
3427) -> ConsoleLogResult<()> {
3428    record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3429    append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3430}
3431
3432/// Stable dedupe key for a session-history backfill gap frame: a persistently
3433/// failing session collapses to a single gap frame across retries.
3434fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3435    format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3436}
3437
3438async fn append_backfill_gap(
3439    inner: &AggregatorInner,
3440    runtime_key: &str,
3441    identity: &str,
3442    session_id: &str,
3443    reason: String,
3444) -> ConsoleLogResult<()> {
3445    append_and_emit(
3446        inner,
3447        NewConsoleFrame {
3448            id: None,
3449            // Stable per (runtime, identity, session): repeated failures for
3450            // the same session collapse to one gap frame instead of minting a
3451            // fresh one (timestamped) on every retry — which grew the timeline
3452            // store unbounded.
3453            dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3454            timestamp_ms: current_time_ms(),
3455            runtime_key: runtime_key.to_string(),
3456            identity: identity.to_string(),
3457            conversation_id: Some(identity.to_string()),
3458            session_id: Some(session_id.to_string()),
3459            kind: "replay_unavailable".to_string(),
3460            status: ConsoleFrameStatus::DeliveryFailed,
3461            payload: json!({
3462                "reason": reason,
3463                "source_kind": "session_history",
3464            }),
3465            source: ConsoleFrameSource {
3466                kind: ConsoleFrameSourceKind::Synthetic,
3467                source_cursor: None,
3468            },
3469            source_event_id: None,
3470            interaction_id: None,
3471            turn_id: None,
3472            run_id: None,
3473            parent_frame_id: None,
3474            caused_by_frame_id: None,
3475        },
3476    )
3477    .await?;
3478    Ok(())
3479}
3480
3481async fn project_console_event(
3482    inner: Arc<AggregatorInner>,
3483    runtime_key: &str,
3484    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3485) -> ConsoleLogResult<()> {
3486    let Some(entry) = inner
3487        .runtimes
3488        .read()
3489        .ok()
3490        .and_then(|entries| entries.get(runtime_key).cloned())
3491    else {
3492        return Ok(());
3493    };
3494    let mut frame = frame_from_console_event(&entry, envelope);
3495    if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3496        frame.payload = redacted;
3497        frame.status = ConsoleFrameStatus::Redacted;
3498    }
3499    let source_cursor = frame
3500        .source_event_id
3501        .clone()
3502        .unwrap_or_else(|| frame.dedupe_key.clone());
3503    let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3504        Some(frame.identity.clone())
3505    } else {
3506        None
3507    };
3508    let opportunistic_refresh_identity = if refresh_identity.is_none()
3509        && console_event_should_start_session_history_backfill(&frame)
3510    {
3511        Some(frame.identity.clone())
3512    } else {
3513        None
3514    };
3515    append_and_emit(&inner, frame).await?;
3516    inner
3517        .store
3518        .record_source_watermark(
3519            &entry.runtime_key,
3520            ConsoleFrameSourceKind::ConsoleEvent,
3521            &source_cursor,
3522        )
3523        .await?;
3524    if let Some(identity) = refresh_identity {
3525        spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3526    } else if let Some(identity) = opportunistic_refresh_identity {
3527        spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3528    }
3529    Ok(())
3530}
3531
3532fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3533    matches!(
3534        frame.kind.as_str(),
3535        "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3536    ) || frame.session_id.is_some()
3537}
3538
3539fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3540    if frame.identity == SYSTEM_EVENT_IDENTITY {
3541        return false;
3542    }
3543    matches!(
3544        frame.kind.as_str(),
3545        "turn_started"
3546            | "run_started"
3547            | "reasoning_complete"
3548            | "tool_call_requested"
3549            | "tool_call"
3550            | "tool_execution_started"
3551            | "text_delta"
3552            | "system_notice"
3553    )
3554}
3555
3556async fn append_and_emit(
3557    inner: &AggregatorInner,
3558    frame: NewConsoleFrame,
3559) -> ConsoleLogResult<AppendOutcome> {
3560    let outcome = inner.store.append_if_absent(frame).await?;
3561    if outcome.disposition == AppendDisposition::Inserted {
3562        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3563            frame: outcome.frame.clone(),
3564        });
3565    }
3566    Ok(outcome)
3567}
3568
3569async fn update_frame_status_and_emit(
3570    inner: &AggregatorInner,
3571    frame_id: &str,
3572    status: ConsoleFrameStatus,
3573) -> ConsoleLogResult<Option<ConsoleFrame>> {
3574    let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3575        return Ok(None);
3576    };
3577    let update_marker = NewConsoleFrame {
3578        id: None,
3579        dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3580        timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3581        runtime_key: updated.runtime_key.clone(),
3582        identity: updated.identity.clone(),
3583        conversation_id: updated.conversation_id.clone(),
3584        session_id: updated.session_id.clone(),
3585        kind: "frame_updated".to_string(),
3586        status: updated.status,
3587        payload: json!({ "frame": updated.clone() }),
3588        source: ConsoleFrameSource {
3589            kind: ConsoleFrameSourceKind::Synthetic,
3590            source_cursor: None,
3591        },
3592        source_event_id: None,
3593        interaction_id: updated.interaction_id.clone(),
3594        turn_id: updated.turn_id.clone(),
3595        run_id: updated.run_id.clone(),
3596        parent_frame_id: Some(updated.id.clone()),
3597        caused_by_frame_id: Some(updated.id.clone()),
3598    };
3599    let outcome = inner.store.append_if_absent(update_marker).await?;
3600    if outcome.disposition == AppendDisposition::Inserted {
3601        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3602            frame: outcome.frame,
3603        });
3604    }
3605    Ok(Some(updated))
3606}
3607
3608fn frame_from_console_event(
3609    entry: &RuntimeEntry,
3610    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3611) -> NewConsoleFrame {
3612    let event_id = envelope.event_id;
3613    let interaction_id = envelope.interaction_id;
3614    let event_type = envelope.event_type;
3615    let payload = envelope.data;
3616    let turn_id = payload
3617        .get("turn_id")
3618        .and_then(Value::as_str)
3619        .map(ToString::to_string);
3620    let run_id = payload
3621        .get("run_id")
3622        .and_then(Value::as_str)
3623        .map(ToString::to_string);
3624    let status = match event_type.as_str() {
3625        "interaction_started" => ConsoleFrameStatus::Accepted,
3626        "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3627        "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3628        _ => ConsoleFrameStatus::Delivered,
3629    };
3630    // The system identity is runtime-plane, not agent-plane: namespacing it
3631    // would defeat the global-timeline visibility exemption in
3632    // `frame_is_visible_cached` on namespaced runtimes (memory.* events
3633    // attributed to `_system` would become `ns/_system` and vanish again).
3634    let identity = if envelope.identity == SYSTEM_EVENT_IDENTITY {
3635        envelope.identity
3636    } else {
3637        apply_namespace(&envelope.identity, &entry.identity_namespace)
3638    };
3639    let dedupe_key = reasoning_payload_text(&event_type, &payload)
3640        .map(|reasoning_text| {
3641            // Reasoning hashes assume console events carry full/cumulative text; if deltas become
3642            // fragmentary, key by a stable reasoning segment id or emit only the complete frame.
3643            let turn_scope = interaction_id
3644                .as_deref()
3645                .or(turn_id.as_deref())
3646                .or(run_id.as_deref())
3647                .unwrap_or(event_id.as_str());
3648            format!(
3649                "console-reasoning:{}:{}:{}",
3650                entry.runtime_key,
3651                turn_scope,
3652                hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3653            )
3654        })
3655        .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3656    NewConsoleFrame {
3657        id: Some(event_id.clone()),
3658        dedupe_key,
3659        timestamp_ms: envelope.timestamp_ms,
3660        runtime_key: entry.runtime_key.clone(),
3661        identity: identity.clone(),
3662        conversation_id: Some(identity),
3663        session_id: payload
3664            .get("session_id")
3665            .and_then(Value::as_str)
3666            .map(ToString::to_string),
3667        kind: event_type,
3668        status,
3669        payload,
3670        source: ConsoleFrameSource {
3671            kind: ConsoleFrameSourceKind::ConsoleEvent,
3672            source_cursor: None,
3673        },
3674        source_event_id: Some(event_id),
3675        interaction_id,
3676        turn_id,
3677        run_id,
3678        parent_frame_id: None,
3679        caused_by_frame_id: None,
3680    }
3681}
3682
3683#[cfg(test)]
3684fn frame_from_session_history_message(
3685    runtime_key: &str,
3686    identity: &str,
3687    session_id: &str,
3688    offset: usize,
3689    message: Value,
3690) -> Option<NewConsoleFrame> {
3691    frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3692        .into_iter()
3693        .next()
3694}
3695
3696#[cfg(test)]
3697fn frames_from_session_history_message(
3698    runtime_key: &str,
3699    identity: &str,
3700    session_id: &str,
3701    offset: usize,
3702    message: Value,
3703) -> Vec<NewConsoleFrame> {
3704    frames_from_session_history_message_with_namespace(
3705        runtime_key,
3706        identity,
3707        "",
3708        session_id,
3709        offset,
3710        message,
3711    )
3712}
3713
3714/// Derive `assistant_image` frames from a session-history tool result's text,
3715/// mirroring the live ingest edge (`parse_generate_image_tool_result` ->
3716/// per-image `assistant_image` envelopes). The serialized
3717/// `ImageGenerationToolResult` is self-describing, so — unlike the live path —
3718/// no tool name is needed: we parse the result text directly. Returns an empty
3719/// vec for non-image tool results. The console dedupes images by blob/reference,
3720/// so an overlap with a live `assistant_image` frame collapses to one image.
3721#[allow(clippy::too_many_arguments)]
3722fn session_history_assistant_image_frames(
3723    runtime_key: &str,
3724    identity: &str,
3725    session_id: &str,
3726    offset: usize,
3727    result_idx: usize,
3728    timestamp_ms: u64,
3729    tool_use_id: &str,
3730    result_text: &str,
3731) -> Vec<NewConsoleFrame> {
3732    let Ok(image_result) = serde_json::from_str::<
3733        meerkat_core::image_generation::ImageGenerationToolResult,
3734    >(result_text) else {
3735        return Vec::new();
3736    };
3737    image_result
3738        .images
3739        .iter()
3740        .enumerate()
3741        .map(|(image_idx, image)| NewConsoleFrame {
3742            id: None,
3743            // Key on the stable image identity so a backfill frame and any live
3744            // `assistant_image` frame for the same image collapse on dedupe.
3745            dedupe_key: format!(
3746                "console-assistant-image:{runtime_key}:{}",
3747                image.blob_ref.blob_id
3748            ),
3749            timestamp_ms,
3750            runtime_key: runtime_key.to_string(),
3751            identity: identity.to_string(),
3752            conversation_id: Some(identity.to_string()),
3753            session_id: Some(session_id.to_string()),
3754            kind: "assistant_image".to_string(),
3755            status: ConsoleFrameStatus::Completed,
3756            payload: json!({
3757                "source_event_type": "session_history",
3758                "tool_call_id": tool_use_id,
3759                "image_id": image.image_id.0.to_string(),
3760                "blob_id": image.blob_ref.blob_id,
3761                "media_type": image.media_type.as_str(),
3762                "width": image.width,
3763                "height": image.height,
3764                "revised_prompt": image_result.revised_prompt.clone(),
3765                "type": "session_history",
3766            }),
3767            source: ConsoleFrameSource {
3768                kind: ConsoleFrameSourceKind::SessionHistory,
3769                source_cursor: Some(format!(
3770                    "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3771                )),
3772            },
3773            source_event_id: None,
3774            interaction_id: None,
3775            turn_id: None,
3776            run_id: None,
3777            parent_frame_id: None,
3778            caused_by_frame_id: None,
3779        })
3780        .collect()
3781}
3782
3783fn frames_from_session_history_message_with_namespace(
3784    runtime_key: &str,
3785    identity: &str,
3786    identity_namespace: &str,
3787    session_id: &str,
3788    offset: usize,
3789    message: Value,
3790) -> Vec<NewConsoleFrame> {
3791    let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3792    let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3793        return Vec::new();
3794    };
3795    if let Message::ToolResults {
3796        results,
3797        created_at,
3798    } = parsed
3799    {
3800        return results
3801            .into_iter()
3802            .enumerate()
3803            .flat_map(|(idx, result)| {
3804                let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3805                let result_text = result.text_content();
3806                let tool_use_id = result.tool_use_id.clone();
3807                let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3808                let mut frames = vec![NewConsoleFrame {
3809                    id: None,
3810                    dedupe_key: format!(
3811                        "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3812                    ),
3813                    timestamp_ms,
3814                    runtime_key: runtime_key.to_string(),
3815                    identity: identity.to_string(),
3816                    conversation_id: Some(identity.to_string()),
3817                    session_id: Some(session_id.to_string()),
3818                    kind: "tool_execution_completed".to_string(),
3819                    status: ConsoleFrameStatus::Completed,
3820                    payload: json!({
3821                        "id": tool_use_id,
3822                        "tool_call_id": tool_use_id,
3823                        "result": result_text,
3824                        "content": content,
3825                        "is_error": result.is_error,
3826                        "source_event_type": "session_history",
3827                        "type": "session_history",
3828                    }),
3829                    source: ConsoleFrameSource {
3830                        kind: ConsoleFrameSourceKind::SessionHistory,
3831                        source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3832                    },
3833                    source_event_id: None,
3834                    interaction_id: None,
3835                    turn_id: None,
3836                    run_id: None,
3837                    parent_frame_id: None,
3838                    caused_by_frame_id: None,
3839                }];
3840                // Backfill parity with the live ingest edge: a generate_image
3841                // tool result serializes an `ImageGenerationToolResult` into the
3842                // result text (which carries NO tool name on meerkat 0.7.x). The
3843                // live path emits per-image `assistant_image` frames; without
3844                // this the image renders as raw tool JSON after a reload /
3845                // history backfill. Parse the result directly and mirror those
3846                // frames (the console dedupes images by blob/reference).
3847                frames.extend(session_history_assistant_image_frames(
3848                    runtime_key,
3849                    identity,
3850                    session_id,
3851                    offset,
3852                    idx,
3853                    timestamp_ms,
3854                    &tool_use_id,
3855                    &result_text,
3856                ));
3857                frames
3858            })
3859            .collect();
3860    }
3861    let (kind, timestamp_ms, payload) = match &parsed {
3862        Message::User(user) => {
3863            if session_history_user_message_is_scaffold(&message) {
3864                return Vec::new();
3865            }
3866            (
3867                "user_input",
3868                user.created_at.timestamp_millis().max(0) as u64,
3869                json!({
3870                    "content": user.content,
3871                    "message": message,
3872                }),
3873            )
3874        }
3875        // Meerkat 0.7 removed the legacy flat-text `Message::Assistant`
3876        // variant; all assistant history is block-shaped now.
3877        Message::BlockAssistant(assistant) => {
3878            let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3879            (
3880                "interaction_complete",
3881                assistant.created_at.timestamp_millis().max(0) as u64,
3882                json!({
3883                    "result": text,
3884                    "text": text,
3885                    "message": message,
3886                    "source_event_type": "session_history",
3887                    "type": "session_history",
3888                }),
3889            )
3890        }
3891        Message::SystemNotice(notice) => (
3892            "system_notice",
3893            notice.created_at.timestamp_millis().max(0) as u64,
3894            json!({
3895                "message": message,
3896                "kind": notice.kind,
3897                "render_class": notice.kind.render_class(),
3898                "body": notice.body,
3899                "blocks": notice.blocks,
3900                "source_event_type": "session_history",
3901                "type": "session_history",
3902            }),
3903        ),
3904        Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3905    };
3906    let mut frames = vec![NewConsoleFrame {
3907        id: None,
3908        dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3909        timestamp_ms,
3910        runtime_key: runtime_key.to_string(),
3911        identity: identity.to_string(),
3912        conversation_id: Some(identity.to_string()),
3913        session_id: Some(session_id.to_string()),
3914        kind: kind.to_string(),
3915        status: ConsoleFrameStatus::Completed,
3916        payload,
3917        source: ConsoleFrameSource {
3918            kind: ConsoleFrameSourceKind::SessionHistory,
3919            source_cursor: Some(format!("{session_id}:{offset}")),
3920        },
3921        source_event_id: None,
3922        interaction_id: None,
3923        turn_id: None,
3924        run_id: None,
3925        parent_frame_id: None,
3926        caused_by_frame_id: None,
3927    }];
3928    if let Message::BlockAssistant(assistant) = &parsed {
3929        frames.extend(spawn_initial_message_frames_from_assistant(
3930            runtime_key,
3931            identity,
3932            identity_namespace,
3933            session_id,
3934            offset,
3935            assistant,
3936            &payload_hash,
3937        ));
3938    }
3939    frames
3940}
3941
3942fn spawn_initial_message_frames_from_assistant(
3943    runtime_key: &str,
3944    parent_identity: &str,
3945    identity_namespace: &str,
3946    session_id: &str,
3947    offset: usize,
3948    assistant: &meerkat_core::types::BlockAssistantMessage,
3949    payload_hash: &str,
3950) -> Vec<NewConsoleFrame> {
3951    let mut frames = Vec::new();
3952    for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3953        if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3954            continue;
3955        }
3956        let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3957            continue;
3958        };
3959        for (spawn_idx, (target_identity, initial_message)) in
3960            spawn_initial_messages_from_tool_args(&args)
3961                .into_iter()
3962                .enumerate()
3963        {
3964            let target_identity = apply_namespace(&target_identity, identity_namespace);
3965            let message_content = match &initial_message {
3966                Value::String(text) => json!([
3967                    {
3968                        "type": "text",
3969                        "text": text,
3970                    }
3971                ]),
3972                other => other.clone(),
3973            };
3974            let message = match &initial_message {
3975                Value::String(text) => json!({
3976                    "role": "user",
3977                    "content": text,
3978                    "created_at": assistant.created_at,
3979                }),
3980                other => json!({
3981                    "role": "user",
3982                    "content": other,
3983                    "created_at": assistant.created_at,
3984                }),
3985            };
3986            frames.push(NewConsoleFrame {
3987                id: None,
3988                dedupe_key: format!(
3989                    "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3990                ),
3991                timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3992                runtime_key: runtime_key.to_string(),
3993                identity: target_identity.clone(),
3994                conversation_id: Some(target_identity),
3995                session_id: None,
3996                kind: "user_input".to_string(),
3997                status: ConsoleFrameStatus::Completed,
3998                payload: json!({
3999                    "content": message_content,
4000                    "message": message,
4001                    "source_event_type": "session_history_spawn_initial_message",
4002                    "type": "session_history",
4003                    "tool_call_id": tool.id,
4004                    "parent_identity": parent_identity,
4005                    "via_tool": tool.name,
4006                }),
4007                source: ConsoleFrameSource {
4008                    kind: ConsoleFrameSourceKind::SessionHistory,
4009                    source_cursor: Some(format!(
4010                        "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4011                    )),
4012                },
4013                source_event_id: None,
4014                interaction_id: None,
4015                turn_id: None,
4016                run_id: None,
4017                parent_frame_id: None,
4018                caused_by_frame_id: None,
4019            });
4020        }
4021    }
4022    frames
4023}
4024
4025fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4026    let mut messages = Vec::new();
4027    if let Some(message) = spawn_initial_message_from_record(args) {
4028        messages.push(message);
4029    }
4030    if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4031        for spec in specs {
4032            if let Some(message) = spawn_initial_message_from_record(spec) {
4033                messages.push(message);
4034            }
4035        }
4036    }
4037    messages
4038}
4039
4040fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4041    let target_identity = record
4042        .get("member_id")
4043        .or_else(|| record.get("agent_identity"))
4044        .and_then(Value::as_str)?
4045        .trim();
4046    if target_identity.is_empty() {
4047        return None;
4048    }
4049    let initial_message = record
4050        .get("initial_message")
4051        .or_else(|| record.get("task"))?
4052        .clone();
4053    Some((target_identity.to_string(), initial_message))
4054}
4055
4056fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4057    message
4058        .get("content")
4059        .is_some_and(scaffold_message_content_is_noise)
4060}
4061
4062fn scaffold_message_content_is_noise(value: &Value) -> bool {
4063    match value {
4064        Value::String(text) => scaffold_message_text_is_noise(text),
4065        Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4066        Value::Object(map) => ["text", "content", "message"]
4067            .iter()
4068            .filter_map(|key| map.get(*key))
4069            .any(scaffold_message_content_is_noise),
4070        _ => false,
4071    }
4072}
4073
4074fn scaffold_message_text_is_noise(text: &str) -> bool {
4075    let trimmed = text.trim_start();
4076    trimmed.starts_with("[PEER UPDATE]")
4077        || trimmed
4078            .to_ascii_lowercase()
4079            .starts_with("you have been spawned")
4080}
4081
4082fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4083    let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4084    rest.split(':').next()?.parse().ok()
4085}
4086
4087fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4088    format!("{session_id}:{offset}:{checked_at_ms}")
4089}
4090
4091fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4092    let Some(checked_at_ms) = watermark
4093        .rsplit_once(':')
4094        .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4095    else {
4096        return false;
4097    };
4098    let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4099    let ttl_ms = if offset > 0 {
4100        SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4101    } else {
4102        SESSION_HISTORY_REFRESH_TTL_MS
4103    };
4104    now_ms.saturating_sub(checked_at_ms) < ttl_ms
4105}
4106
4107async fn history_frame_has_existing_counterpart(
4108    inner: &AggregatorInner,
4109    frame: &NewConsoleFrame,
4110) -> ConsoleLogResult<bool> {
4111    let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4112    let Some(fingerprint) = fingerprint else {
4113        return Ok(false);
4114    };
4115    let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4116    let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4117    let mut after = None;
4118    loop {
4119        let page = inner
4120            .store
4121            .query_frames(ConsoleTimelineQuery {
4122                identity: Some(frame.identity.clone()),
4123                conversation_id: frame.conversation_id.clone(),
4124                after,
4125                limit: 1_000,
4126            })
4127            .await?;
4128        for existing in &page.frames {
4129            let same_session = existing.session_id == frame.session_id
4130                || existing.session_id.is_none()
4131                || frame.session_id.is_none();
4132            if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4133                continue;
4134            }
4135            if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4136                == Some(&fingerprint)
4137            {
4138                return Ok(true);
4139            }
4140            if assistant_terminal
4141                && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4142            {
4143                let turn_key = existing
4144                    .interaction_id
4145                    .as_deref()
4146                    .or(existing.turn_id.as_deref())
4147                    .or(existing.run_id.as_deref())
4148                    .unwrap_or("session");
4149                let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4150                aggregated.push_str(delta);
4151                if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4152                    return Ok(true);
4153                }
4154            }
4155        }
4156        if page.frames.is_empty() || page.next_cursor.is_none() {
4157            return Ok(false);
4158        }
4159        after = page.next_cursor;
4160    }
4161}
4162
4163fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4164    format!("{runtime_key}:session-history:{session_id}")
4165}
4166
4167fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4168    match kind {
4169        "user_input" | "interaction_started" => payload
4170            .get("content")
4171            .map(stable_value_fingerprint)
4172            .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4173        "tool_execution_completed" => tool_result_fingerprint(payload),
4174        "text_delta" => {
4175            text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4176        }
4177        kind if is_reasoning_event_kind(kind) => {
4178            reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4179        }
4180        "text_complete" | "interaction_complete" | "run_completed" => {
4181            assistant_terminal_fingerprint(kind, payload)
4182        }
4183        _ => None,
4184    }
4185}
4186
4187fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4188    let id = payload
4189        .get("tool_call_id")
4190        .or_else(|| payload.get("id"))
4191        .and_then(Value::as_str)
4192        .unwrap_or("");
4193    let result = payload
4194        .get("result")
4195        .or_else(|| payload.get("content"))
4196        .map(stable_value_fingerprint)?;
4197    Some(format!("{id}:{result}"))
4198}
4199
4200fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4201    match kind {
4202        "text_complete" | "interaction_complete" | "run_completed" => payload
4203            .get("text")
4204            .or_else(|| payload.get("result"))
4205            .or_else(|| payload.get("content"))
4206            .map(stable_value_fingerprint),
4207        _ => None,
4208    }
4209}
4210
4211fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4212    if kind != "text_delta" {
4213        return None;
4214    }
4215    payload
4216        .get("delta")
4217        .or_else(|| payload.get("text"))
4218        .or_else(|| payload.get("content"))
4219        .and_then(Value::as_str)
4220        .or_else(|| payload.as_str())
4221}
4222
4223fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4224    if !is_reasoning_event_kind(kind) {
4225        return None;
4226    }
4227    if kind == "reasoning_delta" {
4228        return payload
4229            .get("delta")
4230            .or_else(|| payload.get("text"))
4231            .or_else(|| payload.get("content"))
4232            .or_else(|| payload.get("summary"))
4233            .or_else(|| payload.get("reasoning"))
4234            .and_then(Value::as_str)
4235            .or_else(|| payload.as_str());
4236    }
4237    payload
4238        .get("summary")
4239        .or_else(|| payload.get("text"))
4240        .or_else(|| payload.get("content"))
4241        .or_else(|| payload.get("delta"))
4242        .or_else(|| payload.get("reasoning"))
4243        .and_then(Value::as_str)
4244        .or_else(|| payload.as_str())
4245}
4246
4247fn is_reasoning_event_kind(kind: &str) -> bool {
4248    matches!(
4249        kind,
4250        "reasoning_delta"
4251            | "reasoning_complete"
4252            | "reasoning_summary"
4253            | "reasoning_summary_delta"
4254            | "reasoning_summary_complete"
4255    )
4256}
4257
4258fn stable_value_fingerprint(value: &Value) -> String {
4259    if let Some(text) = content_value_text(value) {
4260        return normalize_transcript_fingerprint_text(&text);
4261    }
4262    match value {
4263        Value::String(text) => normalize_transcript_fingerprint_text(text),
4264        other => serde_json::to_string(other).unwrap_or_default(),
4265    }
4266}
4267
4268fn content_value_text(value: &Value) -> Option<String> {
4269    match value {
4270        Value::String(text) => Some(text.clone()),
4271        Value::Array(items) => {
4272            let text = items
4273                .iter()
4274                .filter_map(content_value_text)
4275                .collect::<String>();
4276            (!text.is_empty()).then_some(text)
4277        }
4278        Value::Object(map) => ["text", "content", "message", "blocks"]
4279            .iter()
4280            .filter_map(|key| map.get(*key))
4281            .find_map(content_value_text),
4282        _ => None,
4283    }
4284}
4285
4286fn normalize_transcript_fingerprint_text(text: &str) -> String {
4287    let trimmed = text.trim();
4288    trimmed
4289        .strip_prefix("[EVENT via rpc] ")
4290        .unwrap_or(trimmed)
4291        .trim()
4292        .to_string()
4293}
4294
4295#[derive(Debug, Clone, Copy)]
4296enum CachedIdentityVisibility {
4297    Visible,
4298    Hidden,
4299    Missing,
4300}
4301
4302async fn frame_is_visible(
4303    inner: &AggregatorInner,
4304    frame: &ConsoleFrame,
4305    allow_historical_identity: bool,
4306    identity_records: &[ConsoleIdentityRecord],
4307) -> ConsoleLogResult<bool> {
4308    let mut identity_visibility_cache = HashMap::new();
4309    Box::pin(frame_is_visible_cached(
4310        inner,
4311        frame,
4312        allow_historical_identity,
4313        &mut identity_visibility_cache,
4314        identity_records,
4315    ))
4316    .await
4317}
4318
4319async fn frame_is_visible_cached(
4320    inner: &AggregatorInner,
4321    frame: &ConsoleFrame,
4322    allow_historical_identity: bool,
4323    identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4324    identity_records: &[ConsoleIdentityRecord],
4325) -> ConsoleLogResult<bool> {
4326    let entry = {
4327        let entries = inner
4328            .runtimes
4329            .read()
4330            .map_err(|_| runtime_registry_lock_error())?;
4331        if entries.is_empty() {
4332            return Ok(true);
4333        }
4334        let Some(entry) = entries.get(&frame.runtime_key) else {
4335            return Ok(false);
4336        };
4337        entry.clone()
4338    };
4339    // Runtime-plane frames aren't agent frames: `__console__` (synthetic
4340    // gap markers) and `_system` (ConsoleMemoryEventSink attributes
4341    // identity-less §9.3 memory.* events here) have no roster record to
4342    // resolve, so the per-identity gate below would hide them from every
4343    // global timeline query. They skip straight to the visibility policy;
4344    // ABAC still filters them per-caller in the RPC layer
4345    // (`retain_visible_timeline_frames`).
4346    if frame.identity != "__console__" && frame.identity != SYSTEM_EVENT_IDENTITY {
4347        let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4348        let identity_visibility =
4349            if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4350                cached
4351            } else {
4352                let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4353                    .unwrap_or_else(|| frame.identity.clone());
4354                let visibility = match identity_records.iter().find(|record| {
4355                    record.runtime_key == frame.runtime_key
4356                        && (record.identity == frame.identity
4357                            || record.runtime_member_id == runtime_member_id)
4358                }) {
4359                    Some(record) => {
4360                        if Box::pin(console_identity_record_visible(&entry, record)).await {
4361                            CachedIdentityVisibility::Visible
4362                        } else {
4363                            CachedIdentityVisibility::Hidden
4364                        }
4365                    }
4366                    None => CachedIdentityVisibility::Missing,
4367                };
4368                identity_visibility_cache.insert(cache_key, visibility);
4369                visibility
4370            };
4371        match identity_visibility {
4372            CachedIdentityVisibility::Visible => {
4373                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4374                    return Ok(false);
4375                }
4376            }
4377            CachedIdentityVisibility::Hidden => return Ok(false),
4378            CachedIdentityVisibility::Missing => {
4379                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4380                    return Ok(false);
4381                }
4382                return Ok(
4383                    allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4384                );
4385            }
4386        }
4387    }
4388    Ok(entry.visibility_policy.frame_visible(frame))
4389}
4390
4391async fn identity_record_for_member(
4392    entry: &RuntimeEntry,
4393    handle: &MobHandle,
4394    member: &MobMemberListEntry,
4395) -> Option<ConsoleIdentityRecord> {
4396    // Roster ids are comms-safe encodings (meerkat 0.7 MemberCommsName);
4397    // identity records carry the public alias.
4398    let runtime_member_id =
4399        crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4400    let durable_identity = member
4401        .labels
4402        .get("agent_identity")
4403        .filter(|value| !value.trim().is_empty())
4404        .map_or(runtime_member_id.as_str(), String::as_str);
4405    let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4406    let addressable = member_is_addressable(member);
4407    let visibility = match member.status {
4408        meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4409            ConsoleVisibility::RetiredReadable
4410        }
4411        // Broken/unknown members have no live runtime binding; mirror the
4412        // identity-first projection, which marks them unreachable.
4413        meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4414            ConsoleVisibility::Unreachable
4415        }
4416        _ if addressable => ConsoleVisibility::Addressable,
4417        _ => ConsoleVisibility::Hidden,
4418    };
4419    let session_id = handle
4420        .resolve_bridge_session_id_observation(&member.agent_identity)
4421        .await
4422        .map(|sid| sid.to_string());
4423    let mut labels = member.labels.clone();
4424    labels
4425        .entry("role".to_string())
4426        .or_insert_with(|| member.role.to_string());
4427    // Spawn-time console metadata (group/display labels, spawned_by lineage)
4428    // registered by the agent-tool spawn path fills gaps the roster does not
4429    // carry; roster labels win on conflict — except lineage, which only the
4430    // spawn registry may assert (these records feed aggregate-mode ABAC
4431    // attribute priming).
4432    crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4433    if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4434        crate::console_spawn::merge_registered_labels(&mut labels, &registered);
4435    }
4436    let display_name = labels
4437        .get("display_name")
4438        .cloned()
4439        .unwrap_or_else(|| runtime_member_id.clone());
4440    Some(ConsoleIdentityRecord {
4441        identity,
4442        display_name,
4443        runtime_key: entry.runtime_key.clone(),
4444        runtime_member_id,
4445        session_id,
4446        visibility,
4447        addressable,
4448        health: if addressable {
4449            "ready"
4450        } else {
4451            "hidden_by_policy"
4452        }
4453        .to_string(),
4454        topology_peers: member
4455            .wired_to
4456            .iter()
4457            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4458            .collect(),
4459        labels,
4460    })
4461}
4462
4463async fn identity_record_for_resolved_member(
4464    resolved: &ResolvedConsoleMember,
4465) -> Option<ConsoleIdentityRecord> {
4466    let mut record =
4467        identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4468    if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4469        record
4470            .labels
4471            .entry("source_mob_id".to_string())
4472            .or_insert_with(|| resolved.source_mob_id.clone());
4473    }
4474    Some(record)
4475}
4476
4477fn identity_record_for_status(
4478    entry: &RuntimeEntry,
4479    status: &crate::identity_first::IdentityStatus,
4480) -> ConsoleIdentityRecord {
4481    let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4482    let runtime_member_id = status
4483        .agent_runtime_id
4484        .as_ref()
4485        .map(crate::identity_first::AgentRuntimeId::as_str)
4486        .unwrap_or_else(|| status.identity.as_str())
4487        .to_string();
4488    let addressable = status.addressability
4489        == crate::identity_first::AgentAddressability::Addressable
4490        && matches!(
4491            status.state,
4492            crate::identity_first::IdentityLifecycleState::Active
4493                | crate::identity_first::IdentityLifecycleState::Dormant
4494                | crate::identity_first::IdentityLifecycleState::Uninitialized
4495        );
4496    let visibility = match status.state {
4497        crate::identity_first::IdentityLifecycleState::Retiring => {
4498            ConsoleVisibility::RetiredReadable
4499        }
4500        crate::identity_first::IdentityLifecycleState::Broken
4501        | crate::identity_first::IdentityLifecycleState::Suspended => {
4502            ConsoleVisibility::Unreachable
4503        }
4504        _ if addressable => ConsoleVisibility::Addressable,
4505        _ => ConsoleVisibility::Hidden,
4506    };
4507    let health = match status.state {
4508        crate::identity_first::IdentityLifecycleState::Active => "ready",
4509        crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4510        crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4511        crate::identity_first::IdentityLifecycleState::Broken => "broken",
4512        crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4513        crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4514    }
4515    .to_string();
4516    let display_name = status
4517        .display_name
4518        .as_ref()
4519        .map(crate::identity_first::DisplayName::as_str)
4520        .unwrap_or_else(|| status.identity.as_str())
4521        .to_string();
4522    ConsoleIdentityRecord {
4523        identity,
4524        display_name,
4525        runtime_key: entry.runtime_key.clone(),
4526        runtime_member_id,
4527        session_id: status.session_id.as_ref().map(ToString::to_string),
4528        visibility,
4529        addressable,
4530        health,
4531        topology_peers: Vec::new(),
4532        labels: status.labels.clone(),
4533    }
4534}
4535
4536pub(crate) fn is_implicit_delegate_member(
4537    role: &str,
4538    labels: &std::collections::BTreeMap<String, String>,
4539) -> bool {
4540    role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4541}
4542
4543fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4544    member
4545        .labels
4546        .get("addressable")
4547        .map(|value| !value.eq_ignore_ascii_case("false"))
4548        .unwrap_or(true)
4549}
4550
4551fn apply_namespace(identity: &str, namespace: &str) -> String {
4552    let namespace = namespace.trim().trim_matches('/');
4553    if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4554        identity.to_string()
4555    } else {
4556        format!("{namespace}/{identity}")
4557    }
4558}
4559
4560fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4561    let namespace = namespace.trim().trim_matches('/');
4562    if namespace.is_empty() {
4563        return Some(identity.to_string());
4564    }
4565    identity
4566        .strip_prefix(namespace)
4567        .and_then(|rest| rest.strip_prefix('/'))
4568        .map(ToString::to_string)
4569}
4570
4571fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4572    let namespace = namespace.trim().trim_matches('/');
4573    if namespace.is_empty() {
4574        return vec![identity.to_string()];
4575    }
4576    let mut candidates = Vec::new();
4577    if let Some(stripped) = strip_namespace(identity, namespace) {
4578        candidates.push(stripped);
4579        candidates.push(identity.to_string());
4580    }
4581    candidates.sort();
4582    candidates.dedup();
4583    candidates
4584}
4585
4586fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4587    namespace_match_candidates(identity, namespace)
4588        .iter()
4589        .any(|candidate| candidate.starts_with("rt:"))
4590}
4591
4592fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4593    entries.iter().any(|entry| {
4594        requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4595    })
4596}
4597
4598fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4599    if request.identity.trim().is_empty() {
4600        return Err(ConsoleSendError::InvalidRequest(
4601            "identity must be non-empty".to_string(),
4602        ));
4603    }
4604    if request.origin.trim().is_empty() {
4605        return Err(ConsoleSendError::InvalidRequest(
4606            "origin must be non-empty".to_string(),
4607        ));
4608    }
4609    if request.idempotency_key.trim().is_empty() {
4610        return Err(ConsoleSendError::InvalidRequest(
4611            "idempotency_key must be non-empty".to_string(),
4612        ));
4613    }
4614    Ok(())
4615}
4616
4617fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4618    let content: ContentInput = serde_json::from_value(value.clone())
4619        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4620    match &content {
4621        ContentInput::Text(text) if text.trim().is_empty() => Err(
4622            ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4623        ),
4624        ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4625            "content blocks must be non-empty".to_string(),
4626        )),
4627        _ => Ok(content),
4628    }
4629}
4630
4631fn parse_handling_mode(
4632    value: Option<&str>,
4633) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4634    match value.unwrap_or("queue") {
4635        "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4636        "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4637        other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4638    }
4639}
4640
4641fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4642    ConsoleInteractionAccepted {
4643        interaction_id: frame
4644            .interaction_id
4645            .clone()
4646            .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4647        identity: frame.identity.clone(),
4648        conversation_id: frame.conversation_id.clone(),
4649        session_id: frame.session_id.clone(),
4650        input_frame_id: frame.id.clone(),
4651        cursor: frame.cursor.clone(),
4652        status: frame.status,
4653    }
4654}
4655
4656fn send_dedupe_key(
4657    runtime_key: &str,
4658    identity: &str,
4659    origin: &str,
4660    idempotency_key: &str,
4661) -> String {
4662    format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4663}
4664
4665fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4666    let content_json = serde_json::to_string(content).unwrap_or_default();
4667    hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4668}
4669
4670fn hash_short(value: &str) -> String {
4671    let mut hasher = Sha256::new();
4672    hasher.update(value.as_bytes());
4673    let digest = hasher.finalize();
4674    to_hex(&digest[..8])
4675}
4676
4677fn to_hex(bytes: &[u8]) -> String {
4678    const HEX: &[u8; 16] = b"0123456789abcdef";
4679    let mut out = String::with_capacity(bytes.len() * 2);
4680    for byte in bytes {
4681        out.push(HEX[(byte >> 4) as usize] as char);
4682        out.push(HEX[(byte & 0x0f) as usize] as char);
4683    }
4684    out
4685}
4686
4687fn current_time_ms() -> u64 {
4688    match SystemTime::now().duration_since(UNIX_EPOCH) {
4689        Ok(duration) => duration.as_millis() as u64,
4690        Err(_) => 0,
4691    }
4692}
4693
4694fn runtime_registry_lock_error() -> ConsoleLogError {
4695    Box::new(std::io::Error::other(
4696        "console runtime registry lock poisoned",
4697    ))
4698}
4699
4700#[cfg(test)]
4701#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4702mod tests {
4703    /// Regression: a session that keeps failing backfill must collapse to ONE
4704    /// gap frame. The dedupe key is stable per (runtime, identity, session) —
4705    /// independent of wall-clock — so repeated failures don't mint a fresh
4706    /// timestamped frame each retry and grow the timeline store unbounded.
4707    #[test]
4708    fn backfill_gap_dedupe_key_is_stable_per_session() {
4709        use super::backfill_gap_dedupe_key;
4710        let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4711        let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4712        assert_eq!(a, b, "same target must reuse one gap frame");
4713        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4714        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4715        // The key carries no wall-clock component (the pre-fix bug appended
4716        // current_time_ms, so every retry was a unique frame).
4717        assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4718    }
4719
4720    use std::sync::Arc;
4721    use std::sync::atomic::{AtomicUsize, Ordering};
4722    use std::time::Duration;
4723    use std::time::Instant;
4724
4725    use futures::StreamExt;
4726    use meerkat::{AgentFactory, Config, build_ephemeral_service};
4727    use meerkat_client::types::LlmStream;
4728    use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4729    use meerkat_core::{
4730        AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4731        RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4732        SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4733        SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4734        StreamError,
4735    };
4736    use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4737    use serde_json::json;
4738
4739    use super::*;
4740    use crate::mob_handle_runtime::MobBootstrapSpec;
4741
4742    #[derive(Debug)]
4743    struct HideHiddenNoiseFrames;
4744
4745    impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4746        fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4747            frame.kind != "hidden_noise"
4748        }
4749    }
4750
4751    #[derive(Debug)]
4752    struct HideRuntimeMemberIdentity(&'static str);
4753
4754    impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4755        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4756            record.runtime_member_id != self.0
4757        }
4758    }
4759
4760    #[derive(Debug)]
4761    struct HideRuntimeMemberOnly(&'static str);
4762
4763    impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4764        fn member_visible(&self, member: &ConsoleMember) -> bool {
4765            member.agent_identity != self.0
4766        }
4767    }
4768
4769    #[derive(Debug)]
4770    struct HideConsoleIdentity(&'static str);
4771
4772    impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4773        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4774            record.identity != self.0
4775        }
4776    }
4777
4778    struct CountingConsoleLogStore {
4779        inner: InMemoryConsoleLogStore,
4780        source_watermark_calls: AtomicUsize,
4781        record_watermark_calls: AtomicUsize,
4782    }
4783
4784    impl CountingConsoleLogStore {
4785        fn new() -> Self {
4786            Self {
4787                inner: InMemoryConsoleLogStore::new(),
4788                source_watermark_calls: AtomicUsize::new(0),
4789                record_watermark_calls: AtomicUsize::new(0),
4790            }
4791        }
4792
4793        fn source_watermark_calls(&self) -> usize {
4794            self.source_watermark_calls.load(Ordering::SeqCst)
4795        }
4796    }
4797
4798    struct SlowTestClient {
4799        delay: Duration,
4800    }
4801
4802    #[async_trait::async_trait]
4803    impl LlmClient for SlowTestClient {
4804        fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4805            Ok(messages.to_vec())
4806        }
4807
4808        fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4809            let delay = self.delay;
4810            let delayed_text = futures::stream::once(async move {
4811                tokio::time::sleep(delay).await;
4812                Ok(LlmEvent::TextDelta {
4813                    delta: "slow ok".to_string(),
4814                    meta: None,
4815                })
4816            });
4817            let done = futures::stream::once(async {
4818                Ok(LlmEvent::Done {
4819                    outcome: LlmDoneOutcome::Success {
4820                        stop_reason: StopReason::EndTurn,
4821                    },
4822                })
4823            });
4824            Box::pin(delayed_text.chain(done))
4825        }
4826
4827        fn provider(&self) -> meerkat_core::Provider {
4828            meerkat_core::Provider::Other
4829        }
4830
4831        async fn health_check(&self) -> Result<(), LlmError> {
4832            Ok(())
4833        }
4834    }
4835
4836    #[derive(Clone)]
4837    struct DelayedHistorySessionService {
4838        inner: Arc<dyn MobSessionService>,
4839        delay: Duration,
4840        read_calls: Arc<AtomicUsize>,
4841        active_reads: Arc<AtomicUsize>,
4842        max_active_reads: Arc<AtomicUsize>,
4843    }
4844
4845    impl DelayedHistorySessionService {
4846        fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4847            Self {
4848                inner,
4849                delay,
4850                read_calls: Arc::new(AtomicUsize::new(0)),
4851                active_reads: Arc::new(AtomicUsize::new(0)),
4852                max_active_reads: Arc::new(AtomicUsize::new(0)),
4853            }
4854        }
4855
4856        fn read_calls(&self) -> usize {
4857            self.read_calls.load(Ordering::SeqCst)
4858        }
4859
4860        fn max_active_reads(&self) -> usize {
4861            self.max_active_reads.load(Ordering::SeqCst)
4862        }
4863    }
4864
4865    #[async_trait::async_trait]
4866    impl SessionService for DelayedHistorySessionService {
4867        async fn create_session(
4868            &self,
4869            req: meerkat_core::CreateSessionRequest,
4870        ) -> Result<RunResult, SessionError> {
4871            self.inner.create_session(req).await
4872        }
4873
4874        async fn start_turn(
4875            &self,
4876            id: &SessionId,
4877            req: StartTurnRequest,
4878        ) -> Result<RunResult, SessionError> {
4879            self.inner.start_turn(id, req).await
4880        }
4881
4882        async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4883            self.inner.interrupt(id).await
4884        }
4885
4886        async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4887            self.inner.read(id).await
4888        }
4889
4890        async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4891            self.inner.list(query).await
4892        }
4893
4894        async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4895            self.inner.archive(id).await
4896        }
4897
4898        async fn subscribe_session_events(
4899            &self,
4900            id: &SessionId,
4901        ) -> Result<EventStream, StreamError> {
4902            SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4903        }
4904    }
4905
4906    #[async_trait::async_trait]
4907    impl SessionServiceCommsExt for DelayedHistorySessionService {
4908        async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4909            self.inner.comms_runtime(session_id).await
4910        }
4911    }
4912
4913    #[async_trait::async_trait]
4914    impl SessionServiceControlExt for DelayedHistorySessionService {
4915        async fn append_system_context(
4916            &self,
4917            id: &SessionId,
4918            req: AppendSystemContextRequest,
4919        ) -> Result<AppendSystemContextResult, SessionControlError> {
4920            self.inner.append_system_context(id, req).await
4921        }
4922    }
4923
4924    #[async_trait::async_trait]
4925    impl SessionServiceHistoryExt for DelayedHistorySessionService {
4926        async fn read_history(
4927            &self,
4928            id: &SessionId,
4929            query: SessionHistoryQuery,
4930        ) -> Result<SessionHistoryPage, SessionError> {
4931            self.read_calls.fetch_add(1, Ordering::SeqCst);
4932            let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4933            self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4934            tokio::time::sleep(self.delay).await;
4935            let result = self.inner.read_history(id, query).await;
4936            self.active_reads.fetch_sub(1, Ordering::SeqCst);
4937            result
4938        }
4939    }
4940
4941    #[async_trait::async_trait]
4942    impl MobSessionService for DelayedHistorySessionService {
4943        fn supports_persistent_sessions(&self) -> bool {
4944            self.inner.supports_persistent_sessions()
4945        }
4946
4947        fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4948            self.inner.runtime_adapter()
4949        }
4950
4951        async fn session_belongs_to_mob(
4952            &self,
4953            session_id: &SessionId,
4954            mob_id: &meerkat_mob::MobId,
4955        ) -> bool {
4956            self.inner.session_belongs_to_mob(session_id, mob_id).await
4957        }
4958
4959        async fn cancel_all_checkpointers(&self) {
4960            self.inner.cancel_all_checkpointers().await;
4961        }
4962
4963        async fn rearm_all_checkpointers(&self) {
4964            self.inner.rearm_all_checkpointers().await;
4965        }
4966    }
4967
4968    #[async_trait::async_trait]
4969    impl ConsoleLogStore for CountingConsoleLogStore {
4970        async fn append_if_absent(
4971            &self,
4972            frame: NewConsoleFrame,
4973        ) -> ConsoleLogResult<AppendOutcome> {
4974            self.inner.append_if_absent(frame).await
4975        }
4976
4977        async fn update_frame_status(
4978            &self,
4979            frame_id: &str,
4980            status: ConsoleFrameStatus,
4981        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4982            self.inner.update_frame_status(frame_id, status).await
4983        }
4984
4985        async fn query_frames(
4986            &self,
4987            query: ConsoleTimelineQuery,
4988        ) -> ConsoleLogResult<ConsoleTimelinePage> {
4989            self.inner.query_frames(query).await
4990        }
4991
4992        async fn query_windowed_frames(
4993            &self,
4994            query: ConsoleTimelineWindowQuery,
4995        ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4996            self.inner.query_windowed_frames(query).await
4997        }
4998
4999        async fn frame_by_dedupe_key(
5000            &self,
5001            dedupe_key: &str,
5002        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5003            self.inner.frame_by_dedupe_key(dedupe_key).await
5004        }
5005
5006        async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
5007            self.inner.latest_cursor().await
5008        }
5009
5010        async fn clear_frames(&self) -> ConsoleLogResult<()> {
5011            self.inner.clear_frames().await
5012        }
5013
5014        async fn record_source_watermark(
5015            &self,
5016            runtime_key: &str,
5017            source_kind: ConsoleFrameSourceKind,
5018            source_cursor: &str,
5019        ) -> ConsoleLogResult<()> {
5020            self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5021            self.inner
5022                .record_source_watermark(runtime_key, source_kind, source_cursor)
5023                .await
5024        }
5025
5026        async fn source_watermark(
5027            &self,
5028            runtime_key: &str,
5029            source_kind: ConsoleFrameSourceKind,
5030        ) -> ConsoleLogResult<Option<String>> {
5031            self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5032            self.inner.source_watermark(runtime_key, source_kind).await
5033        }
5034    }
5035
5036    async fn build_single_member_runtime() -> UnifiedRuntime {
5037        build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5038    }
5039
5040    async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5041        let definition = MobDefinition::from_toml(
5042            r#"
5043[mob]
5044id = "console-aggregator-perf-test"
5045
5046[profiles.worker]
5047model = "gpt-5.5"
5048external_addressable = true
5049
5050[profiles.worker.tools]
5051comms = true
5052"#,
5053        )
5054        .expect("definition parses");
5055        let runtime = UnifiedRuntime::builder()
5056            .definition(definition)
5057            .default_llm_client(client)
5058            .build()
5059            .await
5060            .expect("runtime builds");
5061        runtime
5062            .spawn(SpawnMemberSpec::from_wire(
5063                "worker".to_string(),
5064                "agent-a".to_string(),
5065                Some("You are agent-a.".into()),
5066                None,
5067                None,
5068            ))
5069            .await
5070            .expect("member spawns");
5071        runtime
5072    }
5073
5074    async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5075        let definition = MobDefinition::from_toml(&format!(
5076            r#"
5077[mob]
5078id = "{mob_id}"
5079
5080[profiles.worker]
5081model = "gpt-5.5"
5082external_addressable = true
5083
5084[profiles.worker.tools]
5085comms = true
5086"#
5087        ))
5088        .expect("definition parses");
5089        UnifiedRuntime::builder()
5090            .definition(definition)
5091            .default_llm_client(Arc::new(TestClient::default()))
5092            .build()
5093            .await
5094            .expect("runtime builds")
5095    }
5096
5097    async fn build_stress_runtime(
5098        member_count: usize,
5099        history_delay: Duration,
5100    ) -> (
5101        tempfile::TempDir,
5102        Arc<UnifiedRuntime>,
5103        DelayedHistorySessionService,
5104    ) {
5105        let temp_dir = tempfile::tempdir().expect("temp dir");
5106        let session_path = temp_dir.path().join("sessions");
5107        std::fs::create_dir_all(&session_path).expect("session path");
5108        let factory = AgentFactory::new(&session_path).comms(true);
5109        let base_service = Arc::new(build_ephemeral_service(
5110            factory,
5111            Config::default(),
5112            member_count + 8,
5113        ));
5114        let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5115        let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5116        let definition = MobDefinition::from_toml(
5117            r#"
5118[mob]
5119id = "console-aggregator-stress-test"
5120
5121[profiles.worker]
5122model = "gpt-5.5"
5123external_addressable = true
5124
5125[profiles.worker.tools]
5126comms = true
5127"#,
5128        )
5129        .expect("definition parses");
5130        let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5131            .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5132                allow_ephemeral_sessions: true,
5133                notify_orchestrator_on_resume: true,
5134                default_llm_client: Some(Arc::new(TestClient::default())),
5135            });
5136        let runtime = Arc::new(
5137            UnifiedRuntime::bootstrap(
5138                spec,
5139                crate::types::MobKitConfig {
5140                    modules: Vec::new(),
5141                    discovery: crate::types::DiscoverySpec {
5142                        namespace: "stress".to_string(),
5143                        modules: Vec::new(),
5144                    },
5145                    pre_spawn: Vec::new(),
5146                },
5147                Duration::from_secs(5),
5148            )
5149            .await
5150            .expect("runtime boots"),
5151        );
5152        for idx in 0..member_count {
5153            runtime
5154                .spawn(SpawnMemberSpec::from_wire(
5155                    "worker".to_string(),
5156                    format!("agent-{idx}"),
5157                    Some(format!("You are agent-{idx}.").into()),
5158                    None,
5159                    None,
5160                ))
5161                .await
5162                .expect("member spawns");
5163        }
5164        (temp_dir, runtime, delayed_service)
5165    }
5166
5167    fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5168        RuntimeEntry {
5169            runtime_key: runtime_key.to_string(),
5170            identity_namespace: "test".to_string(),
5171            runtime: runtime.mob_runtime().clone(),
5172            identity_runtime: runtime.identity_runtime().cloned(),
5173            console_events: runtime.console_events(),
5174            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5175        }
5176    }
5177
5178    fn console_envelope_for_test(
5179        event_id: &str,
5180        interaction_id: Option<&str>,
5181        event_type: &str,
5182        data: Value,
5183    ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5184        crate::console_contracts::ConsoleIdentityEventEnvelope {
5185            event_id: event_id.to_string(),
5186            interaction_id: interaction_id.map(ToString::to_string),
5187            identity: "agent-a".to_string(),
5188            event_type: event_type.to_string(),
5189            timestamp_ms: 1_000,
5190            data,
5191        }
5192    }
5193
5194    fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5195        ConsoleIdentityRecord {
5196            identity: identity.to_string(),
5197            display_name: identity.to_string(),
5198            runtime_key: "runtime-cache".to_string(),
5199            runtime_member_id: identity.to_string(),
5200            session_id: Some(format!("session-{identity}")),
5201            visibility: ConsoleVisibility::Addressable,
5202            addressable: true,
5203            health: "ready".to_string(),
5204            topology_peers: Vec::new(),
5205            labels: BTreeMap::new(),
5206        }
5207    }
5208
5209    #[test]
5210    fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5211        let mut durable = identity_record_for_test("test/review:singleton");
5212        durable.runtime_member_id = "rt:review:singleton:1".to_string();
5213        durable.session_id = Some("session-stale".to_string());
5214        durable
5215            .labels
5216            .insert("agent_identity".to_string(), "review:singleton".to_string());
5217
5218        let mut live = identity_record_for_test("test/review:singleton");
5219        live.runtime_member_id = "rt:review:singleton:0".to_string();
5220        live.session_id = Some("session-live".to_string());
5221        live.labels
5222            .insert("agent_identity".to_string(), "review:singleton".to_string());
5223
5224        let records = dedupe_identity_records(vec![durable, live]);
5225
5226        assert_eq!(
5227            records.len(),
5228            2,
5229            "list projection must expose stale durable/live alias split-brain instead of picking one record"
5230        );
5231        assert!(
5232            records
5233                .iter()
5234                .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5235        );
5236        assert!(
5237            records
5238                .iter()
5239                .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5240        );
5241    }
5242
5243    #[test]
5244    fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5245        let mut first = identity_record_for_test("test/review:singleton");
5246        first.runtime_member_id = "rt:review:singleton:0".to_string();
5247        first
5248            .labels
5249            .insert("agent_identity".to_string(), "review:singleton".to_string());
5250
5251        let mut second = identity_record_for_test("test/review:singleton");
5252        second.runtime_member_id = "rt:review:singleton:1".to_string();
5253        second
5254            .labels
5255            .insert("agent_identity".to_string(), "review:singleton".to_string());
5256
5257        let records = dedupe_identity_records(vec![first, second]);
5258
5259        assert_eq!(
5260            records.len(),
5261            2,
5262            "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5263        );
5264    }
5265
5266    #[test]
5267    fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5268        let mut first = identity_record_for_test("test/review:singleton");
5269        first.runtime_member_id = "rt:review:singleton:0".to_string();
5270        first.session_id = None;
5271        first
5272            .labels
5273            .insert("agent_identity".to_string(), "review:singleton".to_string());
5274        first
5275            .labels
5276            .insert("source_mob_id".to_string(), "mob-a".to_string());
5277
5278        let mut second = first.clone();
5279        second
5280            .labels
5281            .insert("source_mob_id".to_string(), "mob-b".to_string());
5282
5283        let records = dedupe_identity_records(vec![first, second]);
5284
5285        assert_eq!(
5286            records.len(),
5287            2,
5288            "same member/session aliases from different source mobs must remain visible"
5289        );
5290    }
5291
5292    #[test]
5293    fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5294        let durable = identity_record_for_test("test/review:singleton");
5295        let live = durable.clone();
5296
5297        let records = dedupe_identity_records(vec![durable, live]);
5298
5299        assert_eq!(
5300            records.len(),
5301            1,
5302            "healthy durable and primary live projections should not duplicate the roster"
5303        );
5304    }
5305
5306    #[test]
5307    fn stale_durable_record_error_detects_session_split_brain() {
5308        let mut durable = identity_record_for_test("test/review:singleton");
5309        durable.runtime_member_id = "rt:review:singleton:0".to_string();
5310        durable.session_id = Some("session-durable".to_string());
5311
5312        let mut live = durable.clone();
5313        live.session_id = Some("session-live".to_string());
5314
5315        let err = stale_durable_record_error("review:singleton", &durable, &[live])
5316            .expect("session mismatch must be stale split-brain");
5317
5318        assert!(
5319            err.contains("stale durable identity alias"),
5320            "unexpected error: {err}"
5321        );
5322        assert!(
5323            err.contains("session-durable") && err.contains("session-live"),
5324            "error should name both sessions: {err}"
5325        );
5326    }
5327
5328    async fn identity_runtime_for_test(
5329        identities: &[&str],
5330    ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5331    {
5332        let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5333            crate::identity_first::IdentityRuntimeConfig {
5334                continuity_store: Arc::new(
5335                    crate::identity_first::LocalContinuityStore::in_memory()?
5336                ),
5337                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5338                runtime_instance_id: "console-aggregator-identity-test".to_string(),
5339                has_runtime_store: true,
5340                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5341                bridge: None,
5342                default_timeout: None,
5343            },
5344        ));
5345        for identity in identities {
5346            let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5347            let record = crate::identity_first::ContinuityRecord {
5348                identity: identity.clone(),
5349                agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5350                    "rt:{}:0",
5351                    identity.as_str()
5352                ))?,
5353                session_id: SessionId::new(),
5354                generation: crate::identity_first::ContinuityGeneration::new(0),
5355                checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5356            };
5357            runtime
5358                .register(
5359                    crate::identity_first::DurableAgentSpec {
5360                        identity: identity.clone(),
5361                        profile: meerkat_mob::ProfileName::from("worker"),
5362                        addressability: crate::identity_first::AgentAddressability::Addressable,
5363                        display_name: None,
5364                        labels: BTreeMap::new(),
5365                        context: None,
5366                        additional_instructions: Vec::new(),
5367                        initial_message: None,
5368                        runtime_mode_override: None,
5369                        backend: None,
5370                        binding: None,
5371                    },
5372                    crate::identity_first::IdentityLifecycleState::Active,
5373                    Some(record),
5374                    Some(crate::identity_first::LeaseGrant {
5375                        identity,
5376                        fencing_token: crate::identity_first::FencingToken::new(1),
5377                        ttl: Duration::from_mins(5),
5378                    }),
5379                )
5380                .await;
5381        }
5382        Ok(runtime)
5383    }
5384
5385    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5386    async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5387    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5388        let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5389        let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5390        let aggregator = MobKitConsoleAggregator::in_memory();
5391        aggregator.register_runtime_handles_with_policy(
5392            "runtime-a",
5393            "",
5394            runtime.mob_runtime().clone(),
5395            Some(identity_runtime),
5396            runtime.console_events(),
5397            Arc::new(AllowAllConsoleVisibilityPolicy),
5398        );
5399
5400        assert!(
5401            aggregator
5402                .inspect_identity("rt:review:singleton:0")
5403                .await?
5404                .is_none(),
5405            "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5406        );
5407        assert!(
5408            !aggregator.retire_identity("rt:review:singleton:0").await?,
5409            "runtime-member-id shaped requests must not retire a durable rt:* identity"
5410        );
5411        let reserve_err = aggregator
5412            .reserve_identity_first_interaction(
5413                ConsoleSendRequest {
5414                    identity: "rt:review:singleton:0".to_string(),
5415                    content: json!("hello"),
5416                    origin: "console:test".to_string(),
5417                    idempotency_key: "rt-shadow-reserve".to_string(),
5418                    handling_mode: Some("queue".to_string()),
5419                },
5420                None,
5421            )
5422            .await
5423            .expect_err(
5424                "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5425            );
5426        assert!(
5427            reserve_err.to_string().contains("unknown identity"),
5428            "unexpected reserve error: {reserve_err}"
5429        );
5430        let send_err = aggregator
5431            .send(ConsoleSendRequest {
5432                identity: "rt:review:singleton:0".to_string(),
5433                content: json!("hello"),
5434                origin: "console:test".to_string(),
5435                idempotency_key: "rt-shadow-send".to_string(),
5436                handling_mode: Some("queue".to_string()),
5437            })
5438            .await
5439            .expect_err(
5440                "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5441            );
5442        assert!(
5443            send_err.to_string().contains("unknown identity"),
5444            "unexpected send error: {send_err}"
5445        );
5446        let blob_err = match aggregator
5447            .binary_blob_store_for_identity("rt:review:singleton:0")
5448            .await
5449        {
5450            Ok(_) => panic!(
5451                "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5452            ),
5453            Err(err) => err,
5454        };
5455        assert!(
5456            blob_err.to_string().contains("unknown identity"),
5457            "unexpected blob error: {blob_err}"
5458        );
5459
5460        let _ = runtime.mob_handle().stop().await;
5461        Ok(())
5462    }
5463
5464    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5465    async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5466    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5467        let runtime = build_empty_runtime("identity-label-test").await;
5468        let mut labels = BTreeMap::new();
5469        labels.insert(
5470            "agent_identity".to_string(),
5471            "channel:C0SMOKEOB3".to_string(),
5472        );
5473        labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5474        runtime
5475            .spawn(
5476                SpawnMemberSpec::from_wire(
5477                    "worker".to_string(),
5478                    "rt:channel:C0SMOKEOB3:0".to_string(),
5479                    Some("You are C0SMOKEOB3.".into()),
5480                    None,
5481                    None,
5482                )
5483                .with_labels(labels),
5484            )
5485            .await
5486            .expect("member spawns");
5487
5488        let entry = RuntimeEntry {
5489            runtime_key: "runtime-a".to_string(),
5490            identity_namespace: String::new(),
5491            runtime: runtime.mob_runtime().clone(),
5492            identity_runtime: runtime.identity_runtime().cloned(),
5493            console_events: runtime.console_events(),
5494            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5495        };
5496        let aggregator = MobKitConsoleAggregator::in_memory();
5497        aggregator
5498            .inner
5499            .runtimes
5500            .write()
5501            .expect("runtime registry")
5502            .insert("runtime-a".to_string(), entry);
5503
5504        let records = aggregator.list_identities_fresh().await?;
5505        let record = records
5506            .iter()
5507            .find(|record| record.identity == "channel:C0SMOKEOB3")
5508            .expect("durable identity is exposed");
5509        assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5510
5511        let inspection = aggregator
5512            .inspect_identity("channel:C0SMOKEOB3")
5513            .await?
5514            .expect("durable identity resolves back to runtime member");
5515        assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5516        assert_eq!(
5517            inspection.identity.runtime_member_id,
5518            "rt:channel:C0SMOKEOB3:0"
5519        );
5520
5521        let _ = runtime.mob_handle().stop().await;
5522        Ok(())
5523    }
5524
5525    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5526    async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
5527    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5528        let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
5529        let mut labels = BTreeMap::new();
5530        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5531        runtime
5532            .spawn(
5533                SpawnMemberSpec::from_wire(
5534                    "worker".to_string(),
5535                    "rt:review:singleton:0".to_string(),
5536                    Some("You are the Review Agent.".into()),
5537                    None,
5538                    None,
5539                )
5540                .with_labels(labels),
5541            )
5542            .await?;
5543        let hidden_session_id = runtime
5544            .mob_handle()
5545            .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
5546                "rt:review:singleton:0",
5547            ))
5548            .await
5549            .map(|sid| sid.to_string());
5550        let mut duplicate_labels = BTreeMap::new();
5551        duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5552        runtime
5553            .spawn(
5554                SpawnMemberSpec::from_wire(
5555                    "worker".to_string(),
5556                    "rt:review:singleton:1".to_string(),
5557                    Some("You are a visible duplicate Review Agent.".into()),
5558                    None,
5559                    None,
5560                )
5561                .with_labels(duplicate_labels),
5562            )
5563            .await?;
5564
5565        let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5566        let aggregator = MobKitConsoleAggregator::in_memory();
5567        aggregator.register_runtime_handles_with_policy(
5568            "runtime-a",
5569            "",
5570            runtime.mob_runtime().clone(),
5571            Some(identity_runtime),
5572            runtime.console_events(),
5573            Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5574        );
5575
5576        assert!(
5577            aggregator
5578                .inspect_identity("review:singleton")
5579                .await?
5580                .is_none(),
5581            "member-hidden live alias must not inspect through aggregator"
5582        );
5583        let records = aggregator.list_identities_fresh().await?;
5584        assert!(
5585            records
5586                .iter()
5587                .all(|record| record.identity != "review:singleton"),
5588            "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5589        );
5590        let appended = aggregator
5591            .store()
5592            .append_if_absent(NewConsoleFrame {
5593                id: None,
5594                dedupe_key: "member-hidden-frame".to_string(),
5595                timestamp_ms: 1,
5596                runtime_key: "runtime-a".to_string(),
5597                identity: "review:singleton".to_string(),
5598                conversation_id: Some("review:singleton".to_string()),
5599                session_id: hidden_session_id,
5600                kind: "text_delta".to_string(),
5601                status: ConsoleFrameStatus::Delivered,
5602                payload: json!({ "delta": "hidden" }),
5603                source: ConsoleFrameSource {
5604                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5605                    source_cursor: None,
5606                },
5607                source_event_id: Some("member-hidden-frame".to_string()),
5608                interaction_id: None,
5609                turn_id: None,
5610                run_id: None,
5611                parent_frame_id: None,
5612                caused_by_frame_id: None,
5613            })
5614            .await?;
5615        assert!(
5616            !aggregator
5617                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5618                    frame: appended.frame.clone()
5619                })
5620                .await,
5621            "member-hidden live alias must not pass timeline event visibility"
5622        );
5623        let identity_only = aggregator
5624            .store()
5625            .append_if_absent(NewConsoleFrame {
5626                id: None,
5627                dedupe_key: "member-hidden-identity-only-frame".to_string(),
5628                timestamp_ms: 2,
5629                runtime_key: "runtime-a".to_string(),
5630                identity: "review:singleton".to_string(),
5631                conversation_id: Some("review:singleton".to_string()),
5632                session_id: None,
5633                kind: "text_delta".to_string(),
5634                status: ConsoleFrameStatus::Delivered,
5635                payload: json!({ "delta": "identity-only-hidden" }),
5636                source: ConsoleFrameSource {
5637                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5638                    source_cursor: None,
5639                },
5640                source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5641                interaction_id: None,
5642                turn_id: None,
5643                run_id: None,
5644                parent_frame_id: None,
5645                caused_by_frame_id: None,
5646            })
5647            .await?;
5648        assert!(
5649            !aggregator
5650                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5651                    frame: identity_only.frame.clone()
5652                })
5653                .await,
5654            "member-hidden durable alias must not leak identity-only timeline frames"
5655        );
5656        let page = aggregator
5657            .query_timeline(ConsoleTimelineQuery {
5658                identity: Some("review:singleton".to_string()),
5659                limit: 10,
5660                ..ConsoleTimelineQuery::default()
5661            })
5662            .await?;
5663        assert!(
5664            page.frames.is_empty(),
5665            "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5666        );
5667        assert!(
5668            !aggregator.retire_identity("review:singleton").await?,
5669            "member-hidden live alias must not retire through aggregator"
5670        );
5671        let reserve_err = aggregator
5672            .reserve_identity_first_interaction(
5673                ConsoleSendRequest {
5674                    identity: "review:singleton".to_string(),
5675                    content: json!("hello"),
5676                    origin: "console:test".to_string(),
5677                    idempotency_key: "member-hidden-reserve".to_string(),
5678                    handling_mode: Some("queue".to_string()),
5679                },
5680                None,
5681            )
5682            .await
5683            .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5684        assert!(
5685            reserve_err.to_string().contains("unknown identity"),
5686            "unexpected reserve error: {reserve_err}"
5687        );
5688        let send_err = aggregator
5689            .send(ConsoleSendRequest {
5690                identity: "review:singleton".to_string(),
5691                content: json!("hello"),
5692                origin: "console:test".to_string(),
5693                idempotency_key: "member-hidden-send".to_string(),
5694                handling_mode: Some("queue".to_string()),
5695            })
5696            .await
5697            .expect_err("member-hidden live alias must not receive sends");
5698        assert!(
5699            send_err.to_string().contains("unknown identity"),
5700            "unexpected send error: {send_err}"
5701        );
5702        let blob_result = aggregator
5703            .binary_blob_store_for_identity("review:singleton")
5704            .await;
5705        let Err(blob_err) = blob_result else {
5706            panic!("member-hidden live alias must not expose blob store");
5707        };
5708        assert!(
5709            blob_err.to_string().contains("unknown identity"),
5710            "unexpected blob error: {blob_err}"
5711        );
5712
5713        let _ = runtime.mob_handle().stop().await;
5714        Ok(())
5715    }
5716
5717    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5718    async fn explicit_session_backfill_target_resolves_durable_identity_label()
5719    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5720        let runtime = build_empty_runtime("identity-label-backfill-test").await;
5721        let mut labels = BTreeMap::new();
5722        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5723        runtime
5724            .spawn(
5725                SpawnMemberSpec::from_wire(
5726                    "worker".to_string(),
5727                    "rt:review:singleton:0".to_string(),
5728                    Some("You are the Review Agent.".into()),
5729                    None,
5730                    None,
5731                )
5732                .with_labels(labels),
5733            )
5734            .await
5735            .expect("member spawns");
5736
5737        let entry = RuntimeEntry {
5738            runtime_key: "runtime-a".to_string(),
5739            identity_namespace: String::new(),
5740            runtime: runtime.mob_runtime().clone(),
5741            identity_runtime: runtime.identity_runtime().cloned(),
5742            console_events: runtime.console_events(),
5743            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5744        };
5745        let aggregator = MobKitConsoleAggregator::in_memory();
5746        aggregator
5747            .inner
5748            .runtimes
5749            .write()
5750            .expect("runtime registry")
5751            .insert("runtime-a".to_string(), entry);
5752
5753        let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5754            .await
5755            .expect("durable label should resolve targeted session backfill");
5756        assert_eq!(target.record.identity, "review:singleton");
5757        assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5758
5759        let _ = runtime.mob_handle().stop().await;
5760        Ok(())
5761    }
5762
5763    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5764    async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5765    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5766        let runtime = build_empty_runtime("identity-stale-durable-test").await;
5767        let mut labels = BTreeMap::new();
5768        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5769        runtime
5770            .spawn(
5771                SpawnMemberSpec::from_wire(
5772                    "worker".to_string(),
5773                    "rt:review:singleton:0".to_string(),
5774                    Some("You are the live Review Agent.".into()),
5775                    None,
5776                    None,
5777                )
5778                .with_labels(labels),
5779            )
5780            .await
5781            .expect("member spawns");
5782
5783        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5784            crate::identity_first::IdentityRuntimeConfig {
5785                continuity_store: Arc::new(
5786                    crate::identity_first::LocalContinuityStore::in_memory()?
5787                ),
5788                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5789                runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5790                has_runtime_store: true,
5791                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5792                bridge: None,
5793                default_timeout: None,
5794            },
5795        ));
5796        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5797        let record = crate::identity_first::ContinuityRecord {
5798            identity: identity.clone(),
5799            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5800                "rt:review:singleton:1",
5801            )?,
5802            session_id: SessionId::new(),
5803            generation: crate::identity_first::ContinuityGeneration::new(1),
5804            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5805        };
5806        identity_runtime
5807            .register(
5808                crate::identity_first::DurableAgentSpec {
5809                    identity: identity.clone(),
5810                    profile: meerkat_mob::ProfileName::from("worker"),
5811                    addressability: crate::identity_first::AgentAddressability::Addressable,
5812                    display_name: None,
5813                    labels: BTreeMap::new(),
5814                    context: None,
5815                    additional_instructions: Vec::new(),
5816                    initial_message: None,
5817                    runtime_mode_override: None,
5818                    backend: None,
5819                    binding: None,
5820                },
5821                crate::identity_first::IdentityLifecycleState::Active,
5822                Some(record),
5823                Some(crate::identity_first::LeaseGrant {
5824                    identity,
5825                    fencing_token: crate::identity_first::FencingToken::new(7),
5826                    ttl: Duration::from_mins(5),
5827                }),
5828            )
5829            .await;
5830
5831        let aggregator = MobKitConsoleAggregator::in_memory();
5832        aggregator
5833            .inner
5834            .runtimes
5835            .write()
5836            .expect("runtime registry")
5837            .insert(
5838                "runtime-a".to_string(),
5839                RuntimeEntry {
5840                    runtime_key: "runtime-a".to_string(),
5841                    identity_namespace: String::new(),
5842                    runtime: runtime.mob_runtime().clone(),
5843                    identity_runtime: Some(identity_runtime),
5844                    console_events: runtime.console_events(),
5845                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5846                },
5847            );
5848
5849        let err = aggregator
5850            .inspect_identity("review:singleton")
5851            .await
5852            .expect_err("stale durable binding must not inspect as healthy");
5853        assert!(
5854            err.to_string().contains("stale durable identity alias"),
5855            "unexpected error: {err}"
5856        );
5857        let retire_err = aggregator
5858            .retire_identity("review:singleton")
5859            .await
5860            .expect_err("stale durable binding must not retire as healthy");
5861        assert!(
5862            retire_err
5863                .to_string()
5864                .contains("stale durable identity alias"),
5865            "unexpected retire error: {retire_err}"
5866        );
5867        let send_err = aggregator
5868            .send(ConsoleSendRequest {
5869                identity: "review:singleton".to_string(),
5870                content: json!("hello"),
5871                origin: "console:test".to_string(),
5872                idempotency_key: "stale-durable-send".to_string(),
5873                handling_mode: Some("queue".to_string()),
5874            })
5875            .await
5876            .expect_err("stale durable binding must not send to the wrong live member");
5877        assert!(
5878            send_err
5879                .to_string()
5880                .contains("stale durable identity alias"),
5881            "unexpected send error: {send_err}"
5882        );
5883        let reserve_err = aggregator
5884            .reserve_identity_first_interaction(
5885                ConsoleSendRequest {
5886                    identity: "review:singleton".to_string(),
5887                    content: json!("hello"),
5888                    origin: "console:test".to_string(),
5889                    idempotency_key: "stale-durable-reserve".to_string(),
5890                    handling_mode: Some("queue".to_string()),
5891                },
5892                None,
5893            )
5894            .await
5895            .expect_err("identity-first reserve must not bypass stale durable binding");
5896        assert!(
5897            reserve_err
5898                .to_string()
5899                .contains("stale durable identity alias"),
5900            "unexpected reserve error: {reserve_err}"
5901        );
5902        let blob_err = match aggregator
5903            .binary_blob_store_for_identity("review:singleton")
5904            .await
5905        {
5906            Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5907            Err(err) => err,
5908        };
5909        assert!(
5910            blob_err
5911                .to_string()
5912                .contains("stale durable identity alias"),
5913            "unexpected blob error: {blob_err}"
5914        );
5915        let records = aggregator.list_identities_fresh().await?;
5916        assert!(
5917            records
5918                .iter()
5919                .any(|record| record.identity == "review:singleton"
5920                    && record.runtime_member_id == "rt:review:singleton:0"),
5921            "live alias should still be listed; records: {records:#?}"
5922        );
5923        assert!(
5924            records
5925                .iter()
5926                .any(|record| record.identity == "review:singleton"
5927                    && record.runtime_member_id == "rt:review:singleton:1"),
5928            "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5929        );
5930
5931        let _ = runtime.mob_handle().stop().await;
5932        Ok(())
5933    }
5934
5935    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5936    async fn identity_first_alias_self_heals_session_only_split_brain()
5937    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5938        let runtime = build_empty_runtime("identity-session-rebind-test").await;
5939        let mut labels = BTreeMap::new();
5940        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5941        runtime
5942            .spawn(
5943                SpawnMemberSpec::from_wire(
5944                    "worker".to_string(),
5945                    "rt:review:singleton:0".to_string(),
5946                    Some("You are the live Review Agent.".into()),
5947                    None,
5948                    None,
5949                )
5950                .with_labels(labels),
5951            )
5952            .await
5953            .expect("member spawns");
5954        let live_session_id = runtime
5955            .mob_handle()
5956            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
5957                "rt:review:singleton:0",
5958            ))
5959            .await
5960            .expect("live member has bridge session");
5961
5962        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5963            crate::identity_first::IdentityRuntimeConfig {
5964                continuity_store: Arc::new(
5965                    crate::identity_first::LocalContinuityStore::in_memory()?
5966                ),
5967                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5968                runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
5969                has_runtime_store: true,
5970                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5971                bridge: None,
5972                default_timeout: None,
5973            },
5974        ));
5975        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5976        let stale_session_id = SessionId::new();
5977        assert_ne!(stale_session_id, live_session_id);
5978        let record = crate::identity_first::ContinuityRecord {
5979            identity: identity.clone(),
5980            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5981                "rt:review:singleton:0",
5982            )?,
5983            session_id: stale_session_id,
5984            generation: crate::identity_first::ContinuityGeneration::new(1),
5985            checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
5986        };
5987        identity_runtime
5988            .register(
5989                crate::identity_first::DurableAgentSpec {
5990                    identity: identity.clone(),
5991                    profile: meerkat_mob::ProfileName::from("worker"),
5992                    addressability: crate::identity_first::AgentAddressability::Addressable,
5993                    display_name: None,
5994                    labels: BTreeMap::new(),
5995                    context: None,
5996                    additional_instructions: Vec::new(),
5997                    initial_message: None,
5998                    runtime_mode_override: None,
5999                    backend: None,
6000                    binding: None,
6001                },
6002                crate::identity_first::IdentityLifecycleState::Active,
6003                Some(record),
6004                Some(crate::identity_first::LeaseGrant {
6005                    identity: identity.clone(),
6006                    fencing_token: crate::identity_first::FencingToken::new(7),
6007                    ttl: Duration::from_mins(5),
6008                }),
6009            )
6010            .await;
6011
6012        let aggregator = MobKitConsoleAggregator::in_memory();
6013        aggregator
6014            .inner
6015            .runtimes
6016            .write()
6017            .expect("runtime registry")
6018            .insert(
6019                "runtime-a".to_string(),
6020                RuntimeEntry {
6021                    runtime_key: "runtime-a".to_string(),
6022                    identity_namespace: String::new(),
6023                    runtime: runtime.mob_runtime().clone(),
6024                    identity_runtime: Some(identity_runtime.clone()),
6025                    console_events: runtime.console_events(),
6026                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6027                },
6028            );
6029
6030        aggregator
6031            .reserve_identity_first_interaction(
6032                ConsoleSendRequest {
6033                    identity: "review:singleton".to_string(),
6034                    content: json!("hello"),
6035                    origin: "console:test".to_string(),
6036                    idempotency_key: "session-rebind-reserve".to_string(),
6037                    handling_mode: Some("queue".to_string()),
6038                },
6039                None,
6040            )
6041            .await
6042            .expect("session-only split-brain should self-heal before reserve");
6043
6044        let rebound_status = identity_runtime.status(&identity).await?;
6045        assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6046
6047        let inspection = aggregator
6048            .inspect_identity("review:singleton")
6049            .await?
6050            .expect("rebound identity should inspect");
6051        let live_session_string = live_session_id.to_string();
6052        assert_eq!(
6053            inspection.identity.session_id.as_deref(),
6054            Some(live_session_string.as_str())
6055        );
6056
6057        let _ = runtime.mob_handle().stop().await;
6058        Ok(())
6059    }
6060
6061    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6062    async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6063    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6064        let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6065        let mut labels = BTreeMap::new();
6066        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6067        runtime
6068            .spawn(
6069                SpawnMemberSpec::from_wire(
6070                    "worker".to_string(),
6071                    "rt:review:singleton:0".to_string(),
6072                    Some("You are the mislabeled Review Agent.".into()),
6073                    None,
6074                    None,
6075                )
6076                .with_labels(labels),
6077            )
6078            .await
6079            .expect("member spawns");
6080
6081        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6082            crate::identity_first::IdentityRuntimeConfig {
6083                continuity_store: Arc::new(
6084                    crate::identity_first::LocalContinuityStore::in_memory()?
6085                ),
6086                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6087                runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6088                has_runtime_store: true,
6089                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6090                bridge: None,
6091                default_timeout: None,
6092            },
6093        ));
6094        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6095        let record = crate::identity_first::ContinuityRecord {
6096            identity: identity.clone(),
6097            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6098                "rt:review:singleton:0",
6099            )?,
6100            session_id: SessionId::new(),
6101            generation: crate::identity_first::ContinuityGeneration::new(1),
6102            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6103        };
6104        identity_runtime
6105            .register(
6106                crate::identity_first::DurableAgentSpec {
6107                    identity: identity.clone(),
6108                    profile: meerkat_mob::ProfileName::from("worker"),
6109                    addressability: crate::identity_first::AgentAddressability::Addressable,
6110                    display_name: None,
6111                    labels: BTreeMap::new(),
6112                    context: None,
6113                    additional_instructions: Vec::new(),
6114                    initial_message: None,
6115                    runtime_mode_override: None,
6116                    backend: None,
6117                    binding: None,
6118                },
6119                crate::identity_first::IdentityLifecycleState::Active,
6120                Some(record),
6121                Some(crate::identity_first::LeaseGrant {
6122                    identity,
6123                    fencing_token: crate::identity_first::FencingToken::new(7),
6124                    ttl: Duration::from_mins(5),
6125                }),
6126            )
6127            .await;
6128
6129        let aggregator = MobKitConsoleAggregator::in_memory();
6130        aggregator
6131            .inner
6132            .runtimes
6133            .write()
6134            .expect("runtime registry")
6135            .insert(
6136                "runtime-a".to_string(),
6137                RuntimeEntry {
6138                    runtime_key: "runtime-a".to_string(),
6139                    identity_namespace: String::new(),
6140                    runtime: runtime.mob_runtime().clone(),
6141                    identity_runtime: Some(identity_runtime),
6142                    console_events: runtime.console_events(),
6143                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6144                },
6145            );
6146
6147        let err = aggregator
6148            .inspect_identity("review:singleton")
6149            .await
6150            .expect_err("wrong projected live identity must not inspect as healthy");
6151        assert!(
6152            err.to_string()
6153                .contains("projects identity other:singleton"),
6154            "unexpected error: {err}"
6155        );
6156        let retire_err = aggregator
6157            .retire_identity("review:singleton")
6158            .await
6159            .expect_err("wrong projected live identity must not retire as healthy");
6160        assert!(
6161            retire_err
6162                .to_string()
6163                .contains("projects identity other:singleton"),
6164            "unexpected retire error: {retire_err}"
6165        );
6166        let send_err = aggregator
6167            .send(ConsoleSendRequest {
6168                identity: "review:singleton".to_string(),
6169                content: json!("hello"),
6170                origin: "console:test".to_string(),
6171                idempotency_key: "wrong-projection-send".to_string(),
6172                handling_mode: Some("queue".to_string()),
6173            })
6174            .await
6175            .expect_err("wrong projected live identity must not send as unknown");
6176        assert!(
6177            send_err
6178                .to_string()
6179                .contains("projects identity other:singleton"),
6180            "unexpected send error: {send_err}"
6181        );
6182        let reserve_err = aggregator
6183            .reserve_identity_first_interaction(
6184                ConsoleSendRequest {
6185                    identity: "review:singleton".to_string(),
6186                    content: json!("hello"),
6187                    origin: "console:test".to_string(),
6188                    idempotency_key: "wrong-projection-reserve".to_string(),
6189                    handling_mode: Some("queue".to_string()),
6190                },
6191                None,
6192            )
6193            .await
6194            .expect_err("identity-first reserve must not bypass wrong live projection");
6195        assert!(
6196            reserve_err
6197                .to_string()
6198                .contains("projects identity other:singleton"),
6199            "unexpected reserve error: {reserve_err}"
6200        );
6201        let blob_err = match aggregator
6202            .binary_blob_store_for_identity("review:singleton")
6203            .await
6204        {
6205            Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6206            Err(err) => err,
6207        };
6208        assert!(
6209            blob_err
6210                .to_string()
6211                .contains("projects identity other:singleton"),
6212            "unexpected blob error: {blob_err}"
6213        );
6214        let projected_err = aggregator
6215            .inspect_identity("other:singleton")
6216            .await
6217            .expect_err("wrong projected alias must not inspect through live-only fallback");
6218        assert!(
6219            projected_err
6220                .to_string()
6221                .contains("identity runtime binding for review:singleton"),
6222            "unexpected projected-alias error: {projected_err}"
6223        );
6224        let projected_retire_err = aggregator
6225            .retire_identity("other:singleton")
6226            .await
6227            .expect_err("wrong projected alias must not retire through live-only fallback");
6228        assert!(
6229            projected_retire_err
6230                .to_string()
6231                .contains("identity runtime binding for review:singleton"),
6232            "unexpected projected-alias retire error: {projected_retire_err}"
6233        );
6234
6235        let _ = runtime.mob_handle().stop().await;
6236        Ok(())
6237    }
6238
6239    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6240    async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6241    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6242        let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6243        let mut labels = BTreeMap::new();
6244        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6245        runtime
6246            .spawn(
6247                SpawnMemberSpec::from_wire(
6248                    "worker".to_string(),
6249                    "rt:review:singleton:0".to_string(),
6250                    Some("You are a hidden wrong projection.".into()),
6251                    None,
6252                    None,
6253                )
6254                .with_labels(labels),
6255            )
6256            .await
6257            .expect("member spawns");
6258
6259        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6260            crate::identity_first::IdentityRuntimeConfig {
6261                continuity_store: Arc::new(
6262                    crate::identity_first::LocalContinuityStore::in_memory()?
6263                ),
6264                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6265                runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6266                has_runtime_store: true,
6267                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6268                bridge: None,
6269                default_timeout: None,
6270            },
6271        ));
6272        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6273        identity_runtime
6274            .register(
6275                crate::identity_first::DurableAgentSpec {
6276                    identity: identity.clone(),
6277                    profile: meerkat_mob::ProfileName::from("worker"),
6278                    addressability: crate::identity_first::AgentAddressability::Addressable,
6279                    display_name: None,
6280                    labels: BTreeMap::new(),
6281                    context: None,
6282                    additional_instructions: Vec::new(),
6283                    initial_message: None,
6284                    runtime_mode_override: None,
6285                    backend: None,
6286                    binding: None,
6287                },
6288                crate::identity_first::IdentityLifecycleState::Active,
6289                Some(crate::identity_first::ContinuityRecord {
6290                    identity: identity.clone(),
6291                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6292                        "rt:review:singleton:0",
6293                    )?,
6294                    session_id: SessionId::new(),
6295                    generation: crate::identity_first::ContinuityGeneration::new(1),
6296                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6297                }),
6298                Some(crate::identity_first::LeaseGrant {
6299                    identity,
6300                    fencing_token: crate::identity_first::FencingToken::new(7),
6301                    ttl: Duration::from_mins(5),
6302                }),
6303            )
6304            .await;
6305
6306        let aggregator = MobKitConsoleAggregator::in_memory();
6307        aggregator
6308            .inner
6309            .runtimes
6310            .write()
6311            .expect("runtime registry")
6312            .insert(
6313                "runtime-a".to_string(),
6314                RuntimeEntry {
6315                    runtime_key: "runtime-a".to_string(),
6316                    identity_namespace: String::new(),
6317                    runtime: runtime.mob_runtime().clone(),
6318                    identity_runtime: Some(identity_runtime),
6319                    console_events: runtime.console_events(),
6320                    visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6321                },
6322            );
6323
6324        let err = aggregator
6325            .inspect_identity("review:singleton")
6326            .await
6327            .expect_err("hidden wrong projection must still fail durable binding validation");
6328        assert!(
6329            err.to_string()
6330                .contains("projects identity other:singleton"),
6331            "unexpected hidden wrong-projection error: {err}"
6332        );
6333        let reserve_err = aggregator
6334            .reserve_identity_first_interaction(
6335                ConsoleSendRequest {
6336                    identity: "review:singleton".to_string(),
6337                    content: json!("hello"),
6338                    origin: "console:test".to_string(),
6339                    idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6340                    handling_mode: Some("queue".to_string()),
6341                },
6342                None,
6343            )
6344            .await
6345            .expect_err("reserve must not hide wrong projection from durable validation");
6346        assert!(
6347            reserve_err
6348                .to_string()
6349                .contains("projects identity other:singleton"),
6350            "unexpected reserve error: {reserve_err}"
6351        );
6352
6353        let _ = runtime.mob_handle().stop().await;
6354        Ok(())
6355    }
6356
6357    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6358    async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6359    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6360        let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6361        let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6362        let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6363            crate::identity_first::IdentityRuntimeConfig {
6364                continuity_store: Arc::new(
6365                    crate::identity_first::LocalContinuityStore::in_memory()?
6366                ),
6367                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6368                runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6369                has_runtime_store: true,
6370                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6371                bridge: None,
6372                default_timeout: None,
6373            },
6374        ));
6375        let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6376            crate::identity_first::IdentityRuntimeConfig {
6377                continuity_store: Arc::new(
6378                    crate::identity_first::LocalContinuityStore::in_memory()?
6379                ),
6380                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6381                runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6382                has_runtime_store: true,
6383                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6384                bridge: None,
6385                default_timeout: None,
6386            },
6387        ));
6388        for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6389            let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6390            identity_runtime
6391                .register(
6392                    crate::identity_first::DurableAgentSpec {
6393                        identity: identity.clone(),
6394                        profile: meerkat_mob::ProfileName::from("worker"),
6395                        addressability: crate::identity_first::AgentAddressability::Addressable,
6396                        display_name: None,
6397                        labels: BTreeMap::new(),
6398                        context: None,
6399                        additional_instructions: Vec::new(),
6400                        initial_message: None,
6401                        runtime_mode_override: None,
6402                        backend: None,
6403                        binding: None,
6404                    },
6405                    crate::identity_first::IdentityLifecycleState::Active,
6406                    Some(crate::identity_first::ContinuityRecord {
6407                        identity: identity.clone(),
6408                        agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6409                            "rt:review:singleton:0",
6410                        )?,
6411                        session_id: SessionId::new(),
6412                        generation: crate::identity_first::ContinuityGeneration::new(1),
6413                        checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6414                    }),
6415                    Some(crate::identity_first::LeaseGrant {
6416                        identity,
6417                        fencing_token: crate::identity_first::FencingToken::new(7),
6418                        ttl: Duration::from_mins(5),
6419                    }),
6420                )
6421                .await;
6422        }
6423
6424        let aggregator = MobKitConsoleAggregator::in_memory();
6425        {
6426            let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6427            runtimes.insert(
6428                "a-hidden".to_string(),
6429                RuntimeEntry {
6430                    runtime_key: "a-hidden".to_string(),
6431                    identity_namespace: String::new(),
6432                    runtime: hidden_runtime.mob_runtime().clone(),
6433                    identity_runtime: Some(hidden_identity_runtime),
6434                    console_events: hidden_runtime.console_events(),
6435                    visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6436                },
6437            );
6438            runtimes.insert(
6439                "b-visible".to_string(),
6440                RuntimeEntry {
6441                    runtime_key: "b-visible".to_string(),
6442                    identity_namespace: String::new(),
6443                    runtime: visible_runtime.mob_runtime().clone(),
6444                    identity_runtime: Some(visible_identity_runtime),
6445                    console_events: visible_runtime.console_events(),
6446                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6447                },
6448            );
6449        }
6450
6451        let inspection = aggregator
6452            .inspect_identity("rt:review:singleton:0")
6453            .await?
6454            .expect("visible runtime-id match should be returned");
6455        assert_eq!(inspection.identity.runtime_key, "b-visible");
6456
6457        let _ = hidden_runtime.mob_handle().stop().await;
6458        let _ = visible_runtime.mob_handle().stop().await;
6459        Ok(())
6460    }
6461
6462    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6463    async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6464    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6465        let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6466        let mut primary_labels = BTreeMap::new();
6467        primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6468        runtime
6469            .spawn(
6470                SpawnMemberSpec::from_wire(
6471                    "worker".to_string(),
6472                    "rt:review:singleton:0".to_string(),
6473                    Some("You are the primary Review Agent.".into()),
6474                    None,
6475                    None,
6476                )
6477                .with_labels(primary_labels),
6478            )
6479            .await
6480            .expect("primary member spawns");
6481
6482        let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6483        let mut delegate_labels = BTreeMap::new();
6484        delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6485        delegate_runtime
6486            .spawn(
6487                SpawnMemberSpec::from_wire(
6488                    "worker".to_string(),
6489                    "rt:review:singleton:0".to_string(),
6490                    Some("You are the wrong projected Review Agent.".into()),
6491                    None,
6492                    None,
6493                )
6494                .with_labels(delegate_labels),
6495            )
6496            .await
6497            .expect("delegate member spawns");
6498        runtime
6499            .mob_runtime()
6500            .agent_mob_mcp_state()
6501            .expect("runtime should expose mob MCP state")
6502            .mob_insert_handle(
6503                meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6504                delegate_runtime.mob_handle().clone(),
6505            )
6506            .await;
6507
6508        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6509            crate::identity_first::IdentityRuntimeConfig {
6510                continuity_store: Arc::new(
6511                    crate::identity_first::LocalContinuityStore::in_memory()?
6512                ),
6513                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6514                runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
6515                has_runtime_store: true,
6516                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6517                bridge: None,
6518                default_timeout: None,
6519            },
6520        ));
6521        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6522        let record = crate::identity_first::ContinuityRecord {
6523            identity: identity.clone(),
6524            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6525                "rt:review:singleton:0",
6526            )?,
6527            session_id: SessionId::new(),
6528            generation: crate::identity_first::ContinuityGeneration::new(1),
6529            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6530        };
6531        identity_runtime
6532            .register(
6533                crate::identity_first::DurableAgentSpec {
6534                    identity: identity.clone(),
6535                    profile: meerkat_mob::ProfileName::from("worker"),
6536                    addressability: crate::identity_first::AgentAddressability::Addressable,
6537                    display_name: None,
6538                    labels: BTreeMap::new(),
6539                    context: None,
6540                    additional_instructions: Vec::new(),
6541                    initial_message: None,
6542                    runtime_mode_override: None,
6543                    backend: None,
6544                    binding: None,
6545                },
6546                crate::identity_first::IdentityLifecycleState::Active,
6547                Some(record),
6548                Some(crate::identity_first::LeaseGrant {
6549                    identity,
6550                    fencing_token: crate::identity_first::FencingToken::new(7),
6551                    ttl: Duration::from_mins(5),
6552                }),
6553            )
6554            .await;
6555
6556        let aggregator = MobKitConsoleAggregator::in_memory();
6557        aggregator
6558            .inner
6559            .runtimes
6560            .write()
6561            .expect("runtime registry")
6562            .insert(
6563                "runtime-a".to_string(),
6564                RuntimeEntry {
6565                    runtime_key: "runtime-a".to_string(),
6566                    identity_namespace: String::new(),
6567                    runtime: runtime.mob_runtime().clone(),
6568                    identity_runtime: Some(identity_runtime),
6569                    console_events: runtime.console_events(),
6570                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6571                },
6572            );
6573
6574        let err = aggregator
6575            .inspect_identity("review:singleton")
6576            .await
6577            .expect_err("cross-source wrong projection must not be hidden by primary source");
6578        assert!(
6579            err.to_string()
6580                .contains("projects identity other:singleton")
6581                || err.to_string().contains("ambiguous live identity alias"),
6582            "unexpected error: {err}"
6583        );
6584
6585        let _ = delegate_runtime.mob_handle().stop().await;
6586        let _ = runtime.mob_handle().stop().await;
6587        Ok(())
6588    }
6589
6590    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6591    async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
6592    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6593        let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
6594        let mut labels = BTreeMap::new();
6595        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6596        runtime
6597            .spawn(
6598                SpawnMemberSpec::from_wire(
6599                    "worker".to_string(),
6600                    "rt:review:singleton:0".to_string(),
6601                    Some("You are the first Review Agent.".into()),
6602                    None,
6603                    None,
6604                )
6605                .with_labels(labels.clone()),
6606            )
6607            .await
6608            .expect("first member spawns");
6609        runtime
6610            .spawn(
6611                SpawnMemberSpec::from_wire(
6612                    "worker".to_string(),
6613                    "rt:review:singleton:1".to_string(),
6614                    Some("You are the second Review Agent.".into()),
6615                    None,
6616                    None,
6617                )
6618                .with_labels(labels),
6619            )
6620            .await
6621            .expect("second member spawns");
6622
6623        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6624            crate::identity_first::IdentityRuntimeConfig {
6625                continuity_store: Arc::new(
6626                    crate::identity_first::LocalContinuityStore::in_memory()?
6627                ),
6628                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6629                runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
6630                has_runtime_store: true,
6631                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6632                bridge: None,
6633                default_timeout: None,
6634            },
6635        ));
6636        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6637        identity_runtime
6638            .register(
6639                crate::identity_first::DurableAgentSpec {
6640                    identity: identity.clone(),
6641                    profile: meerkat_mob::ProfileName::from("worker"),
6642                    addressability: crate::identity_first::AgentAddressability::Addressable,
6643                    display_name: None,
6644                    labels: BTreeMap::new(),
6645                    context: None,
6646                    additional_instructions: Vec::new(),
6647                    initial_message: None,
6648                    runtime_mode_override: None,
6649                    backend: None,
6650                    binding: None,
6651                },
6652                crate::identity_first::IdentityLifecycleState::Active,
6653                Some(crate::identity_first::ContinuityRecord {
6654                    identity: identity.clone(),
6655                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6656                        "rt:review:singleton:0",
6657                    )?,
6658                    session_id: SessionId::new(),
6659                    generation: crate::identity_first::ContinuityGeneration::new(1),
6660                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6661                }),
6662                Some(crate::identity_first::LeaseGrant {
6663                    identity,
6664                    fencing_token: crate::identity_first::FencingToken::new(8),
6665                    ttl: Duration::from_mins(5),
6666                }),
6667            )
6668            .await;
6669
6670        let aggregator = MobKitConsoleAggregator::in_memory();
6671        aggregator
6672            .inner
6673            .runtimes
6674            .write()
6675            .expect("runtime registry")
6676            .insert(
6677                "runtime-a".to_string(),
6678                RuntimeEntry {
6679                    runtime_key: "runtime-a".to_string(),
6680                    identity_namespace: String::new(),
6681                    runtime: runtime.mob_runtime().clone(),
6682                    identity_runtime: Some(identity_runtime),
6683                    console_events: runtime.console_events(),
6684                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6685                },
6686            );
6687
6688        let err = aggregator
6689            .inspect_identity("review:singleton")
6690            .await
6691            .expect_err("duplicate live alias must not inspect via durable fast path");
6692        assert!(
6693            err.to_string().contains("ambiguous live identity alias"),
6694            "unexpected error: {err}"
6695        );
6696        let retire_err = aggregator
6697            .retire_identity("review:singleton")
6698            .await
6699            .expect_err("duplicate live alias must not retire via durable fast path");
6700        assert!(
6701            retire_err
6702                .to_string()
6703                .contains("ambiguous live identity alias"),
6704            "unexpected retire error: {retire_err}"
6705        );
6706        let runtime_id_err = aggregator
6707            .inspect_identity("rt:review:singleton:0")
6708            .await
6709            .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6710        assert!(
6711            runtime_id_err
6712                .to_string()
6713                .contains("ambiguous live identity alias"),
6714            "unexpected runtime-id inspect error: {runtime_id_err}"
6715        );
6716        let runtime_id_retire_err = aggregator
6717            .retire_identity("rt:review:singleton:0")
6718            .await
6719            .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6720        assert!(
6721            runtime_id_retire_err
6722                .to_string()
6723                .contains("ambiguous live identity alias")
6724                || runtime_id_retire_err
6725                    .to_string()
6726                    .contains("stale durable identity alias"),
6727            "unexpected runtime-id retire error: {runtime_id_retire_err}"
6728        );
6729
6730        let _ = runtime.mob_handle().stop().await;
6731        Ok(())
6732    }
6733
6734    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6735    async fn inspect_identity_ignores_hidden_live_alias_candidates()
6736    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6737        let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6738        let mut labels = BTreeMap::new();
6739        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6740        runtime
6741            .spawn(
6742                SpawnMemberSpec::from_wire(
6743                    "worker".to_string(),
6744                    "rt:review:singleton:0".to_string(),
6745                    Some("You are the visible Review Agent.".into()),
6746                    None,
6747                    None,
6748                )
6749                .with_labels(labels.clone()),
6750            )
6751            .await
6752            .expect("visible member spawns");
6753        runtime
6754            .spawn(
6755                SpawnMemberSpec::from_wire(
6756                    "worker".to_string(),
6757                    "rt:review:singleton:1".to_string(),
6758                    Some("You are hidden maintenance noise.".into()),
6759                    None,
6760                    None,
6761                )
6762                .with_labels(labels),
6763            )
6764            .await
6765            .expect("hidden member spawns");
6766
6767        let aggregator = MobKitConsoleAggregator::in_memory();
6768        aggregator.register_runtime_handles_with_policy(
6769            "runtime-a",
6770            "",
6771            runtime.mob_runtime().clone(),
6772            None,
6773            runtime.console_events(),
6774            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6775        );
6776
6777        let inspection = aggregator
6778            .inspect_identity("review:singleton")
6779            .await?
6780            .expect("visible live alias should inspect without hidden ambiguity");
6781        assert_eq!(inspection.identity.identity, "review:singleton");
6782        assert_eq!(
6783            inspection.identity.runtime_member_id,
6784            "rt:review:singleton:0"
6785        );
6786
6787        let retired = aggregator.retire_identity("review:singleton").await?;
6788        assert!(
6789            retired,
6790            "visible live alias should retire without hidden ambiguity"
6791        );
6792
6793        let _ = runtime.mob_handle().stop().await;
6794        Ok(())
6795    }
6796
6797    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6798    async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6799    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6800        let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6801        let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6802        let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6803        let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6804
6805        let aggregator = MobKitConsoleAggregator::in_memory();
6806        aggregator.register_runtime_handles_with_policy(
6807            "a-hidden",
6808            "",
6809            hidden_runtime.mob_runtime().clone(),
6810            Some(hidden_identity_runtime),
6811            hidden_runtime.console_events(),
6812            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6813        );
6814        aggregator.register_runtime_handles_with_policy(
6815            "b-visible",
6816            "",
6817            visible_runtime.mob_runtime().clone(),
6818            Some(visible_identity_runtime),
6819            visible_runtime.console_events(),
6820            Arc::new(AllowAllConsoleVisibilityPolicy),
6821        );
6822
6823        let accepted = aggregator
6824            .reserve_identity_first_interaction(
6825                ConsoleSendRequest {
6826                    identity: "review:singleton".to_string(),
6827                    content: json!("hello"),
6828                    origin: "console:test".to_string(),
6829                    idempotency_key: "visible-after-hidden-reserve".to_string(),
6830                    handling_mode: Some("queue".to_string()),
6831                },
6832                None,
6833            )
6834            .await?;
6835        let page = aggregator
6836            .query_timeline(ConsoleTimelineQuery {
6837                identity: Some("review:singleton".to_string()),
6838                ..ConsoleTimelineQuery::default()
6839            })
6840            .await?;
6841        let frame = page
6842            .frames
6843            .iter()
6844            .find(|frame| frame.id == accepted.input_frame_id)
6845            .ok_or("accepted frame missing from timeline")?;
6846        assert_eq!(frame.runtime_key, "b-visible");
6847
6848        let _ = hidden_runtime.mob_handle().stop().await;
6849        let _ = visible_runtime.mob_handle().stop().await;
6850        Ok(())
6851    }
6852
6853    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6854    async fn identity_first_list_identities_projects_cached_topology_peers()
6855    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6856        let runtime = build_empty_runtime("identity-topology-cache-test").await;
6857        let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6858        identity_runtime
6859            .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6860                crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6861                crate::identity_first::AgentIdentity::parse("agent:beta")?,
6862            )?])
6863            .await;
6864
6865        let aggregator = MobKitConsoleAggregator::in_memory();
6866        aggregator.register_runtime_handles_with_policy(
6867            "identity-first",
6868            "",
6869            runtime.mob_runtime().clone(),
6870            Some(identity_runtime),
6871            runtime.console_events(),
6872            Arc::new(AllowAllConsoleVisibilityPolicy),
6873        );
6874
6875        let identities = aggregator.list_identities().await?;
6876        let alpha = identities
6877            .iter()
6878            .find(|record| record.identity == "agent:alpha")
6879            .ok_or("agent:alpha identity missing")?;
6880        assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6881        let inspection = aggregator
6882            .inspect_identity("agent:alpha")
6883            .await?
6884            .ok_or("agent:alpha inspection missing")?;
6885        assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6886
6887        let _ = runtime.mob_handle().stop().await;
6888        Ok(())
6889    }
6890
6891    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6892    async fn identity_first_list_identities_includes_member_only_spawned_workers()
6893    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6894        let runtime = build_empty_runtime("identity-member-only-test").await;
6895        let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6896        let aggregator = MobKitConsoleAggregator::in_memory();
6897        aggregator.register_runtime_handles_with_policy(
6898            "identity-first",
6899            "",
6900            runtime.mob_runtime().clone(),
6901            Some(identity_runtime),
6902            runtime.console_events(),
6903            Arc::new(AllowAllConsoleVisibilityPolicy),
6904        );
6905
6906        runtime
6907            .spawn(SpawnMemberSpec::from_wire(
6908                "worker".to_string(),
6909                "agent:beta".to_string(),
6910                Some("You are beta.".into()),
6911                None,
6912                None,
6913            ))
6914            .await?;
6915
6916        let identities = aggregator.list_identities().await?;
6917        assert!(
6918            identities
6919                .iter()
6920                .any(|record| record.identity == "agent:beta"),
6921            "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6922        );
6923
6924        let _ = runtime.mob_handle().stop().await;
6925        Ok(())
6926    }
6927
6928    #[tokio::test]
6929    async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6930        let aggregator = MobKitConsoleAggregator::in_memory();
6931        let record = identity_record_for_test("agent-cached");
6932        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6933        aggregator
6934            .inner
6935            .identity_read_model
6936            .primed
6937            .store(true, Ordering::Release);
6938        let _guard = aggregator
6939            .inner
6940            .identity_read_model
6941            .refresh_lock
6942            .clone()
6943            .lock_owned()
6944            .await;
6945
6946        let identities =
6947            tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6948                .await
6949                .expect("hot identity list should not wait for refresh lock")
6950                .expect("identity list succeeds");
6951
6952        assert_eq!(identities, vec![record]);
6953    }
6954
6955    #[tokio::test]
6956    async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6957        let aggregator = MobKitConsoleAggregator::in_memory();
6958        let guard = aggregator
6959            .inner
6960            .identity_read_model
6961            .refresh_lock
6962            .clone()
6963            .lock_owned()
6964            .await;
6965        let waiter = tokio::spawn({
6966            let aggregator = aggregator.clone();
6967            async move { aggregator.list_identities().await }
6968        });
6969
6970        tokio::time::sleep(Duration::from_millis(20)).await;
6971        assert!(
6972            !waiter.is_finished(),
6973            "cold identity list should wait for the in-flight refresh to finish"
6974        );
6975
6976        let record = identity_record_for_test("agent-primed");
6977        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6978        aggregator
6979            .inner
6980            .identity_read_model
6981            .primed
6982            .store(true, Ordering::Release);
6983        drop(guard);
6984
6985        let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6986            .await
6987            .expect("cold identity list waiter should resume")
6988            .expect("waiter joins")
6989            .expect("identity list succeeds");
6990        assert_eq!(identities, vec![record]);
6991    }
6992
6993    #[tokio::test]
6994    async fn query_timeline_reads_from_aggregate_store() {
6995        let aggregator = MobKitConsoleAggregator::in_memory();
6996        let frame = NewConsoleFrame {
6997            id: None,
6998            dedupe_key: "event-1".to_string(),
6999            timestamp_ms: 1,
7000            runtime_key: "runtime-a".to_string(),
7001            identity: "agent-a".to_string(),
7002            conversation_id: Some("agent-a".to_string()),
7003            session_id: None,
7004            kind: "text_delta".to_string(),
7005            status: ConsoleFrameStatus::Delivered,
7006            payload: json!({ "delta": "hello" }),
7007            source: ConsoleFrameSource {
7008                kind: ConsoleFrameSourceKind::ConsoleEvent,
7009                source_cursor: None,
7010            },
7011            source_event_id: Some("event-1".to_string()),
7012            interaction_id: None,
7013            turn_id: None,
7014            run_id: None,
7015            parent_frame_id: None,
7016            caused_by_frame_id: None,
7017        };
7018        aggregator
7019            .store()
7020            .append_if_absent(frame)
7021            .await
7022            .expect("append frame");
7023
7024        let page = aggregator
7025            .query_timeline(ConsoleTimelineQuery {
7026                identity: Some("agent-a".to_string()),
7027                limit: 10,
7028                ..ConsoleTimelineQuery::default()
7029            })
7030            .await
7031            .expect("query timeline");
7032        assert_eq!(page.frames.len(), 1);
7033        assert_eq!(page.frames[0].kind, "text_delta");
7034    }
7035
7036    /// §9.3: memory.* events with no affected identity are attributed to
7037    /// `_system` by the ConsoleMemoryEventSink. `_system` has no roster
7038    /// record, so without an explicit exemption the per-identity visibility
7039    /// gate hides them from every GLOBAL timeline query (the pre-fix bug:
7040    /// the rail and the Memory panel live strip never saw dream/quarantine/
7041    /// promotion events). The exemption must be exact — an arbitrary
7042    /// unknown identity stays hidden — and must survive identity
7043    /// namespacing (`frame_from_console_event` must not rewrite `_system`
7044    /// to `ns/_system`). ABAC is unaffected: under an enforced config the
7045    /// RPC layer still filters `_system` frames per caller via
7046    /// `retain_visible_timeline_frames` (covered in tests/access_control.rs).
7047    #[tokio::test]
7048    async fn system_identity_frames_are_visible_on_global_timeline_queries() {
7049        let aggregator = MobKitConsoleAggregator::in_memory();
7050        let runtime = build_single_member_runtime().await;
7051        // Register a runtime so the per-identity visibility gate engages
7052        // (an empty registry short-circuits every frame to visible). The
7053        // test entry carries the "test" identity namespace on purpose.
7054        aggregator
7055            .inner
7056            .runtimes
7057            .write()
7058            .expect("runtime registry")
7059            .insert(
7060                "runtime-a".to_string(),
7061                runtime_entry_for_test("runtime-a", &runtime),
7062            );
7063        let entry = runtime_entry_for_test("runtime-a", &runtime);
7064
7065        let envelope = |event_id: &str, identity: &str, event_type: &str| {
7066            crate::console_contracts::ConsoleIdentityEventEnvelope {
7067                event_id: event_id.to_string(),
7068                interaction_id: None,
7069                identity: identity.to_string(),
7070                event_type: event_type.to_string(),
7071                timestamp_ms: 1_000,
7072                data: json!({ "realm": "default" }),
7073            }
7074        };
7075
7076        let system_frame = frame_from_console_event(
7077            &entry,
7078            envelope("evt-mem-1", SYSTEM_EVENT_IDENTITY, "memory.dream.completed"),
7079        );
7080        assert_eq!(
7081            system_frame.identity, SYSTEM_EVENT_IDENTITY,
7082            "projection must not namespace the system identity"
7083        );
7084        let ghost_frame = frame_from_console_event(
7085            &entry,
7086            envelope("evt-ghost-1", "ghost-agent", "memory.dream.completed"),
7087        );
7088        assert_eq!(
7089            ghost_frame.identity, "test/ghost-agent",
7090            "ordinary identities keep the namespace"
7091        );
7092        for frame in [system_frame, ghost_frame] {
7093            aggregator
7094                .store()
7095                .append_if_absent(frame)
7096                .await
7097                .expect("append frame");
7098        }
7099
7100        // GLOBAL query: no identity requested, so allow_historical_identity
7101        // never applies — exactly the lane the pre-fix bug hid frames from.
7102        let page = aggregator
7103            .query_timeline(ConsoleTimelineQuery {
7104                identity: None,
7105                limit: 10,
7106                ..ConsoleTimelineQuery::default()
7107            })
7108            .await
7109            .expect("query timeline");
7110        let identities: Vec<&str> = page
7111            .frames
7112            .iter()
7113            .map(|frame| frame.identity.as_str())
7114            .collect();
7115        assert!(
7116            identities.contains(&SYSTEM_EVENT_IDENTITY),
7117            "system-attributed memory event must be globally visible: {identities:?}"
7118        );
7119        assert!(
7120            !identities.contains(&"test/ghost-agent"),
7121            "unknown identities must stay hidden — the exemption is exact: {identities:?}"
7122        );
7123
7124        let _ = runtime.mob_handle().stop().await;
7125    }
7126
7127    #[tokio::test]
7128    async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7129        let aggregator = MobKitConsoleAggregator::in_memory();
7130        let runtime = build_single_member_runtime().await;
7131        let entry = runtime_entry_for_test("runtime-a", &runtime);
7132        for event_type in ["reasoning_delta", "reasoning_complete"] {
7133            let frame = frame_from_console_event(
7134                &entry,
7135                console_envelope_for_test(
7136                    event_type,
7137                    Some("interaction-a"),
7138                    event_type,
7139                    json!({
7140                        "delta": "I should inspect the file first.",
7141                        "text": "I should inspect the file first.",
7142                        "turn_id": "turn-a",
7143                    }),
7144                ),
7145            );
7146            aggregator
7147                .store()
7148                .append_if_absent(frame)
7149                .await
7150                .expect("append reasoning frame");
7151        }
7152
7153        let page = aggregator
7154            .query_timeline(ConsoleTimelineQuery {
7155                identity: Some("test/agent-a".to_string()),
7156                limit: 10,
7157                ..ConsoleTimelineQuery::default()
7158            })
7159            .await
7160            .expect("query timeline");
7161
7162        assert_eq!(
7163            page.frames.len(),
7164            1,
7165            "identical reasoning re-emissions in one turn should collapse"
7166        );
7167        assert_eq!(page.frames[0].kind, "reasoning_delta");
7168        assert_eq!(
7169            page.frames[0].dedupe_key,
7170            format!(
7171                "console-reasoning:runtime-a:interaction-a:{}",
7172                hash_short("I should inspect the file first.")
7173            )
7174        );
7175    }
7176
7177    #[tokio::test]
7178    async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7179        let aggregator = MobKitConsoleAggregator::in_memory();
7180        let runtime = build_single_member_runtime().await;
7181        let entry = runtime_entry_for_test("runtime-a", &runtime);
7182        for (event_id, text) in [
7183            ("reasoning-1", "I should inspect the file first."),
7184            ("reasoning-2", "Now I should run the focused test."),
7185        ] {
7186            let frame = frame_from_console_event(
7187                &entry,
7188                console_envelope_for_test(
7189                    event_id,
7190                    Some("interaction-a"),
7191                    "reasoning_complete",
7192                    json!({
7193                        "text": text,
7194                        "turn_id": "turn-a",
7195                    }),
7196                ),
7197            );
7198            aggregator
7199                .store()
7200                .append_if_absent(frame)
7201                .await
7202                .expect("append reasoning frame");
7203        }
7204
7205        let page = aggregator
7206            .query_timeline(ConsoleTimelineQuery {
7207                identity: Some("test/agent-a".to_string()),
7208                limit: 10,
7209                ..ConsoleTimelineQuery::default()
7210            })
7211            .await
7212            .expect("query timeline");
7213
7214        assert_eq!(
7215            page.frames.len(),
7216            2,
7217            "distinct same-turn reasoning segments should not collapse"
7218        );
7219        assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7220    }
7221
7222    #[tokio::test]
7223    async fn identity_recent_anchor_respects_query_limit() {
7224        let aggregator = MobKitConsoleAggregator::in_memory();
7225        aggregator
7226            .store()
7227            .append_if_absent(NewConsoleFrame {
7228                id: None,
7229                dedupe_key: "anchored-user-input".to_string(),
7230                timestamp_ms: 1,
7231                runtime_key: "runtime-a".to_string(),
7232                identity: "agent-a".to_string(),
7233                conversation_id: Some("agent-a".to_string()),
7234                session_id: None,
7235                kind: "user_input".to_string(),
7236                status: ConsoleFrameStatus::Delivered,
7237                payload: json!({ "content": "anchor me" }),
7238                source: ConsoleFrameSource {
7239                    kind: ConsoleFrameSourceKind::Synthetic,
7240                    source_cursor: None,
7241                },
7242                source_event_id: Some("anchored-user-input".to_string()),
7243                interaction_id: Some("turn-a".to_string()),
7244                turn_id: None,
7245                run_id: None,
7246                parent_frame_id: None,
7247                caused_by_frame_id: None,
7248            })
7249            .await
7250            .expect("append anchor");
7251        for idx in 2..=40 {
7252            aggregator
7253                .store()
7254                .append_if_absent(NewConsoleFrame {
7255                    id: None,
7256                    dedupe_key: format!("noisy-tail-{idx}"),
7257                    timestamp_ms: idx,
7258                    runtime_key: "runtime-a".to_string(),
7259                    identity: "agent-a".to_string(),
7260                    conversation_id: Some("agent-a".to_string()),
7261                    session_id: None,
7262                    kind: "reasoning_delta".to_string(),
7263                    status: ConsoleFrameStatus::Delivered,
7264                    payload: json!({ "delta": idx }),
7265                    source: ConsoleFrameSource {
7266                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7267                        source_cursor: None,
7268                    },
7269                    source_event_id: Some(format!("noisy-tail-{idx}")),
7270                    interaction_id: Some("turn-a".to_string()),
7271                    turn_id: None,
7272                    run_id: None,
7273                    parent_frame_id: None,
7274                    caused_by_frame_id: None,
7275                })
7276                .await
7277                .expect("append noisy tail");
7278        }
7279
7280        let page = aggregator
7281            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7282                identity: Some("agent-a".to_string()),
7283                mode: ConsoleTimelineMode::Recent,
7284                limit: 5,
7285                ..ConsoleTimelineWindowQuery::default()
7286            })
7287            .await
7288            .expect("query recent identity timeline");
7289
7290        assert_eq!(
7291            page.frames.len(),
7292            5,
7293            "identity anchor merge must not exceed the requested limit"
7294        );
7295        assert!(
7296            page.frames
7297                .iter()
7298                .any(|frame| frame.dedupe_key == "anchored-user-input"),
7299            "the bounded result should still retain the useful turn anchor: {:#?}",
7300            page.frames
7301        );
7302        assert_eq!(
7303            page.frames.last().and_then(|frame| frame.cursor.seq()),
7304            Some(40)
7305        );
7306    }
7307
7308    #[tokio::test]
7309    async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7310        let aggregator = MobKitConsoleAggregator::in_memory();
7311        let runtime = build_single_member_runtime().await;
7312        let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7313        entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7314        aggregator
7315            .inner
7316            .runtimes
7317            .write()
7318            .expect("runtime registry")
7319            .insert("runtime-a".to_string(), entry);
7320
7321        for idx in 0..1_500 {
7322            aggregator
7323                .store()
7324                .append_if_absent(NewConsoleFrame {
7325                    id: None,
7326                    dedupe_key: format!("hidden-gap-{idx}"),
7327                    timestamp_ms: idx,
7328                    runtime_key: "runtime-a".to_string(),
7329                    identity: "agent-a".to_string(),
7330                    conversation_id: Some("agent-a".to_string()),
7331                    session_id: None,
7332                    kind: "hidden_noise".to_string(),
7333                    status: ConsoleFrameStatus::Delivered,
7334                    payload: json!({ "delta": idx }),
7335                    source: ConsoleFrameSource {
7336                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7337                        source_cursor: None,
7338                    },
7339                    source_event_id: Some(format!("hidden-gap-{idx}")),
7340                    interaction_id: None,
7341                    turn_id: None,
7342                    run_id: None,
7343                    parent_frame_id: None,
7344                    caused_by_frame_id: None,
7345                })
7346                .await
7347                .expect("append hidden frame");
7348        }
7349        aggregator
7350            .store()
7351            .append_if_absent(NewConsoleFrame {
7352                id: None,
7353                dedupe_key: "visible-after-gap".to_string(),
7354                timestamp_ms: 1_501,
7355                runtime_key: "runtime-a".to_string(),
7356                identity: "agent-a".to_string(),
7357                conversation_id: Some("agent-a".to_string()),
7358                session_id: None,
7359                kind: "interaction_complete".to_string(),
7360                status: ConsoleFrameStatus::Delivered,
7361                payload: json!({ "text": "visible after hidden gap" }),
7362                source: ConsoleFrameSource {
7363                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7364                    source_cursor: None,
7365                },
7366                source_event_id: Some("visible-after-gap".to_string()),
7367                interaction_id: Some("turn-visible".to_string()),
7368                turn_id: None,
7369                run_id: None,
7370                parent_frame_id: None,
7371                caused_by_frame_id: None,
7372            })
7373            .await
7374            .expect("append visible frame");
7375
7376        let page = aggregator
7377            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7378                identity: Some("agent-a".to_string()),
7379                mode: ConsoleTimelineMode::Since,
7380                limit: 1,
7381                ..ConsoleTimelineWindowQuery::default()
7382            })
7383            .await
7384            .expect("query timeline");
7385        assert_eq!(page.frames.len(), 1);
7386        assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
7387        assert_eq!(
7388            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7389            Some(1_501)
7390        );
7391        let _ = runtime.mob_handle().stop().await;
7392    }
7393
7394    #[tokio::test]
7395    async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
7396        let aggregator = MobKitConsoleAggregator::in_memory();
7397        for idx in 1..=300 {
7398            aggregator
7399                .store()
7400                .append_if_absent(NewConsoleFrame {
7401                    id: None,
7402                    dedupe_key: format!("visible-{idx}"),
7403                    timestamp_ms: idx,
7404                    runtime_key: "runtime-a".to_string(),
7405                    identity: "agent-a".to_string(),
7406                    conversation_id: Some("agent-a".to_string()),
7407                    session_id: None,
7408                    kind: "interaction_complete".to_string(),
7409                    status: ConsoleFrameStatus::Delivered,
7410                    payload: json!({ "text": format!("visible {idx}") }),
7411                    source: ConsoleFrameSource {
7412                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7413                        source_cursor: None,
7414                    },
7415                    source_event_id: Some(format!("visible-{idx}")),
7416                    interaction_id: Some(format!("turn-{idx}")),
7417                    turn_id: None,
7418                    run_id: None,
7419                    parent_frame_id: None,
7420                    caused_by_frame_id: None,
7421                })
7422                .await
7423                .expect("append visible frame");
7424        }
7425
7426        let first = aggregator
7427            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7428                identity: Some("agent-a".to_string()),
7429                mode: ConsoleTimelineMode::Since,
7430                limit: 10,
7431                ..ConsoleTimelineWindowQuery::default()
7432            })
7433            .await
7434            .expect("query first page");
7435        assert_eq!(first.frames.len(), 10);
7436        assert_eq!(
7437            first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7438            Some(10)
7439        );
7440
7441        let second = aggregator
7442            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7443                identity: Some("agent-a".to_string()),
7444                mode: ConsoleTimelineMode::Since,
7445                after: first.next_cursor,
7446                limit: 10,
7447                ..ConsoleTimelineWindowQuery::default()
7448            })
7449            .await
7450            .expect("query second page");
7451        assert_eq!(second.frames.len(), 10);
7452        assert_eq!(second.frames[0].dedupe_key, "visible-11");
7453        assert_eq!(
7454            second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7455            Some(20)
7456        );
7457    }
7458
7459    #[tokio::test]
7460    async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
7461        let store = Arc::new(CountingConsoleLogStore::new());
7462        let aggregator = MobKitConsoleAggregator::new(store.clone());
7463        let runtime = build_single_member_runtime().await;
7464        aggregator
7465            .inner
7466            .runtimes
7467            .write()
7468            .expect("runtime registry")
7469            .insert(
7470                "runtime-a".to_string(),
7471                runtime_entry_for_test("runtime-a", &runtime),
7472            );
7473        let inserted = store
7474            .append_if_absent(NewConsoleFrame {
7475                id: None,
7476                dedupe_key: "event-1".to_string(),
7477                timestamp_ms: 1,
7478                runtime_key: "runtime-a".to_string(),
7479                identity: "agent-a".to_string(),
7480                conversation_id: Some("agent-a".to_string()),
7481                session_id: None,
7482                kind: "text_delta".to_string(),
7483                status: ConsoleFrameStatus::Delivered,
7484                payload: json!({ "delta": "hello" }),
7485                source: ConsoleFrameSource {
7486                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7487                    source_cursor: None,
7488                },
7489                source_event_id: Some("event-1".to_string()),
7490                interaction_id: None,
7491                turn_id: None,
7492                run_id: None,
7493                parent_frame_id: None,
7494                caused_by_frame_id: None,
7495            })
7496            .await
7497            .expect("append frame");
7498
7499        let page = aggregator
7500            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7501                identity: Some("agent-a".to_string()),
7502                mode: ConsoleTimelineMode::Since,
7503                after: Some(inserted.frame.cursor),
7504                limit: 10,
7505                ..ConsoleTimelineWindowQuery::default()
7506            })
7507            .await
7508            .expect("query continuation");
7509
7510        assert!(page.frames.is_empty());
7511        assert_eq!(
7512            store.source_watermark_calls(),
7513            0,
7514            "empty since continuation must not synchronously force session-history backfill"
7515        );
7516        let _ = runtime.mob_handle().stop().await;
7517    }
7518
7519    #[test]
7520    fn legacy_timeline_struct_literals_remain_source_compatible() {
7521        let query = ConsoleTimelineQuery {
7522            identity: Some("agent-a".to_string()),
7523            conversation_id: None,
7524            after: Some(ConsoleCursor::from("console:1")),
7525            limit: 10,
7526        };
7527        let page = ConsoleTimelinePage {
7528            frames: Vec::new(),
7529            next_cursor: query.after.clone(),
7530        };
7531
7532        assert_eq!(query.limit, 10);
7533        assert_eq!(
7534            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
7535            Some(1)
7536        );
7537    }
7538
7539    #[tokio::test]
7540    async fn query_timeline_is_store_local_for_registered_runtimes() {
7541        let store = Arc::new(CountingConsoleLogStore::new());
7542        let aggregator = MobKitConsoleAggregator::new(store.clone());
7543        let runtime = build_single_member_runtime().await;
7544        aggregator
7545            .inner
7546            .runtimes
7547            .write()
7548            .expect("runtime registry")
7549            .insert(
7550                "runtime-a".to_string(),
7551                runtime_entry_for_test("runtime-a", &runtime),
7552            );
7553        store
7554            .append_if_absent(NewConsoleFrame {
7555                id: None,
7556                dedupe_key: "event-1".to_string(),
7557                timestamp_ms: 1,
7558                runtime_key: "runtime-a".to_string(),
7559                identity: "agent-a".to_string(),
7560                conversation_id: Some("agent-a".to_string()),
7561                session_id: None,
7562                kind: "text_delta".to_string(),
7563                status: ConsoleFrameStatus::Delivered,
7564                payload: json!({ "delta": "hello" }),
7565                source: ConsoleFrameSource {
7566                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7567                    source_cursor: None,
7568                },
7569                source_event_id: Some("event-1".to_string()),
7570                interaction_id: None,
7571                turn_id: None,
7572                run_id: None,
7573                parent_frame_id: None,
7574                caused_by_frame_id: None,
7575            })
7576            .await
7577            .expect("append frame");
7578
7579        let page = tokio::time::timeout(
7580            Duration::from_millis(250),
7581            aggregator.query_timeline(ConsoleTimelineQuery {
7582                identity: Some("agent-a".to_string()),
7583                limit: 10,
7584                ..ConsoleTimelineQuery::default()
7585            }),
7586        )
7587        .await
7588        .expect("timeline query should not wait for session history")
7589        .expect("timeline query succeeds");
7590
7591        assert_eq!(page.frames.len(), 1);
7592        assert_eq!(
7593            store.source_watermark_calls(),
7594            0,
7595            "query_timeline must not synchronously touch session-history watermarks"
7596        );
7597        let _ = runtime.mob_handle().stop().await;
7598    }
7599
7600    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7601    async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
7602        let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
7603            delay: Duration::from_secs(2),
7604        }))
7605        .await;
7606        let aggregator = MobKitConsoleAggregator::in_memory();
7607        aggregator
7608            .inner
7609            .runtimes
7610            .write()
7611            .expect("runtime registry")
7612            .insert(
7613                "runtime-a".to_string(),
7614                runtime_entry_for_test("runtime-a", &runtime),
7615            );
7616
7617        let start = Instant::now();
7618        let accepted = tokio::time::timeout(
7619            Duration::from_millis(300),
7620            aggregator.send(ConsoleSendRequest {
7621                identity: "test/agent-a".to_string(),
7622                content: json!("hello slow agent"),
7623                origin: "console:test".to_string(),
7624                idempotency_key: "nonblocking-send".to_string(),
7625                handling_mode: Some("queue".to_string()),
7626            }),
7627        )
7628        .await
7629        .expect("console send should return once the input is accepted")
7630        .expect("send succeeds");
7631
7632        assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
7633        assert!(
7634            start.elapsed() < Duration::from_secs(1),
7635            "console send should not wait for the delayed LLM turn"
7636        );
7637
7638        wait_for_session_history_text(
7639            &aggregator,
7640            "test/agent-a",
7641            "slow ok",
7642            Duration::from_secs(5),
7643        )
7644        .await
7645        .expect("background dispatch should still complete and project history");
7646        let _ = runtime.mob_handle().stop().await;
7647    }
7648
7649    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7650    async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
7651    {
7652        let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
7653        let aggregator = MobKitConsoleAggregator::in_memory();
7654        aggregator.register_runtime(ConsoleRuntimeRegistration {
7655            runtime_key: "runtime-late".to_string(),
7656            runtime: runtime.clone(),
7657            identity_namespace: "late".to_string(),
7658            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7659        });
7660
7661        runtime
7662            .spawn(SpawnMemberSpec::from_wire(
7663                "worker".to_string(),
7664                "agent-late".to_string(),
7665                Some("You are agent-late.".into()),
7666                None,
7667                None,
7668            ))
7669            .await
7670            .expect("late member spawns");
7671        let session_id = send_message_on_mob_with_mode(
7672            &runtime.mob_handle(),
7673            "agent-late",
7674            ContentInput::Text("hello after registration".to_string()),
7675            meerkat_core::types::HandlingMode::Queue,
7676        )
7677        .await
7678        .expect("direct member send succeeds");
7679        wait_for_identity_record(
7680            &aggregator,
7681            "late/agent-late",
7682            Some(session_id.as_str()),
7683            Duration::from_secs(5),
7684        )
7685        .await?;
7686
7687        wait_for_session_history_text(
7688            &aggregator,
7689            "late/agent-late",
7690            "You are agent-late.",
7691            Duration::from_secs(5),
7692        )
7693        .await?;
7694        let _ = runtime.mob_handle().stop().await;
7695        Ok(())
7696    }
7697
7698    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699    async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
7700    -> Result<(), String> {
7701        let runtime = build_single_member_runtime().await;
7702        let entry = runtime_entry_for_test("runtime-a", &runtime);
7703        let resolved = member_sources_for_entry(&entry)
7704            .await
7705            .into_iter()
7706            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7707            .expect("agent-a member exists");
7708        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7709            .await
7710            .expect("identity record exists");
7711        let session_id = record.session_id.expect("agent-a has a session");
7712        wait_for_runtime_session_history_text(
7713            &runtime,
7714            &session_id,
7715            "You are agent-a.",
7716            Duration::from_secs(5),
7717        )
7718        .await?;
7719
7720        let aggregator = MobKitConsoleAggregator::in_memory();
7721        aggregator
7722            .inner
7723            .runtimes
7724            .write()
7725            .expect("runtime registry")
7726            .insert("runtime-a".to_string(), entry);
7727        let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
7728        aggregator
7729            .store()
7730            .record_source_watermark(
7731                &watermark_key,
7732                ConsoleFrameSourceKind::SessionHistory,
7733                &format_session_history_watermark(&session_id, 0, current_time_ms()),
7734            )
7735            .await
7736            .expect("record fresh empty watermark");
7737
7738        // Hang/stall backstop only — generous so CPU starvation under full
7739        // parallel CI load can't false-trip it (matches the sibling timeline
7740        // query test). The real assertions are on the returned frames below.
7741        let page = tokio::time::timeout(
7742            Duration::from_mins(1),
7743            aggregator.query_timeline(ConsoleTimelineQuery {
7744                identity: Some("test/agent-a".to_string()),
7745                limit: 20,
7746                ..ConsoleTimelineQuery::default()
7747            }),
7748        )
7749        .await
7750        .expect("explicit identity query should not stall")
7751        .expect("query succeeds");
7752
7753        if !page.frames.is_empty() {
7754            assert!(
7755                page.frames.iter().any(|frame| {
7756                    frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7757                        && session_history_content_text(frame).as_deref()
7758                            == Some("You are agent-a.")
7759                }),
7760                "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
7761                page.frames
7762            );
7763        }
7764        wait_for_session_history_text(
7765            &aggregator,
7766            "test/agent-a",
7767            "You are agent-a.",
7768            Duration::from_secs(5),
7769        )
7770        .await?;
7771        let _ = runtime.mob_handle().stop().await;
7772        Ok(())
7773    }
7774
7775    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7776    async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
7777    {
7778        let store = Arc::new(CountingConsoleLogStore::new());
7779        let aggregator = MobKitConsoleAggregator::new(store.clone());
7780        let runtime = build_single_member_runtime().await;
7781        let entry = runtime_entry_for_test("runtime-a", &runtime);
7782        let resolved = member_sources_for_entry(&entry)
7783            .await
7784            .into_iter()
7785            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
7786            .expect("agent-a member exists");
7787        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
7788            .await
7789            .expect("identity record exists");
7790        let session_id = record.session_id.expect("agent-a has a session");
7791        wait_for_runtime_session_history_text(
7792            &runtime,
7793            &session_id,
7794            "You are agent-a.",
7795            Duration::from_secs(5),
7796        )
7797        .await?;
7798        aggregator
7799            .inner
7800            .runtimes
7801            .write()
7802            .expect("runtime registry")
7803            .insert("runtime-a".to_string(), entry);
7804
7805        store
7806            .append_if_absent(NewConsoleFrame {
7807                id: None,
7808                dedupe_key: "stale-session-history-agent-a".to_string(),
7809                timestamp_ms: 10,
7810                runtime_key: "runtime-a".to_string(),
7811                identity: "test/agent-a".to_string(),
7812                conversation_id: Some("test/agent-a".to_string()),
7813                session_id: Some(session_id.clone()),
7814                kind: "user_input".to_string(),
7815                status: ConsoleFrameStatus::Delivered,
7816                payload: json!({
7817                    "text": "stale projected history",
7818                    "type": "user_input",
7819                }),
7820                source: ConsoleFrameSource {
7821                    kind: ConsoleFrameSourceKind::SessionHistory,
7822                    source_cursor: Some("stale-session-history-agent-a".to_string()),
7823                },
7824                source_event_id: Some("stale-session-history-agent-a".to_string()),
7825                interaction_id: None,
7826                turn_id: None,
7827                run_id: None,
7828                parent_frame_id: None,
7829                caused_by_frame_id: None,
7830            })
7831            .await
7832            .expect("append stale history frame");
7833
7834        let fresh_prompt = "fresh prompt after stale history";
7835        let sent_session_id = send_message_on_mob_with_mode(
7836            &runtime.mob_handle(),
7837            "agent-a",
7838            ContentInput::Text(fresh_prompt.to_string()),
7839            meerkat_core::types::HandlingMode::Queue,
7840        )
7841        .await
7842        .expect("direct member send succeeds");
7843        assert_eq!(sent_session_id, session_id);
7844        wait_for_runtime_session_history_text(
7845            &runtime,
7846            &session_id,
7847            fresh_prompt,
7848            Duration::from_secs(5),
7849        )
7850        .await?;
7851
7852        let page = aggregator
7853            .query_timeline(ConsoleTimelineQuery {
7854                identity: Some("test/agent-a".to_string()),
7855                limit: 50,
7856                ..ConsoleTimelineQuery::default()
7857            })
7858            .await
7859            .expect("query child timeline");
7860
7861        assert!(
7862            page.frames.iter().any(|frame| {
7863                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7864                    && session_history_content_text(frame).as_deref()
7865                        == Some("stale projected history")
7866            }),
7867            "explicit identity query should return existing history before async refresh; frames: {:#?}",
7868            page.frames
7869        );
7870        wait_for_session_history_text(
7871            &aggregator,
7872            "test/agent-a",
7873            fresh_prompt,
7874            Duration::from_secs(5),
7875        )
7876        .await?;
7877        assert!(
7878            store.source_watermark_calls() > 0,
7879            "stale existing session history should force a targeted source refresh"
7880        );
7881        let _ = runtime.mob_handle().stop().await;
7882        Ok(())
7883    }
7884
7885    async fn wait_for_session_history_text(
7886        aggregator: &MobKitConsoleAggregator,
7887        identity: &str,
7888        expected: &str,
7889        timeout: Duration,
7890    ) -> Result<(), String> {
7891        let deadline = Instant::now() + timeout;
7892        let mut observed = Vec::new();
7893        while Instant::now() < deadline {
7894            let _ = aggregator.list_identities().await;
7895            let page = aggregator
7896                .query_timeline(ConsoleTimelineQuery {
7897                    identity: Some(identity.to_string()),
7898                    limit: 20,
7899                    ..ConsoleTimelineQuery::default()
7900                })
7901                .await
7902                .expect("query timeline");
7903            observed = page.frames;
7904            if observed.iter().any(|frame| {
7905                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7906                    && session_history_content_text(frame).as_deref() == Some(expected)
7907            }) {
7908                return Ok(());
7909            }
7910            tokio::time::sleep(Duration::from_millis(25)).await;
7911        }
7912
7913        Err(format!(
7914            "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7915        ))
7916    }
7917
7918    async fn wait_for_session_backfill_target(
7919        aggregator: &MobKitConsoleAggregator,
7920        identity: &str,
7921        timeout: Duration,
7922    ) -> Result<String, String> {
7923        let deadline = Instant::now() + timeout;
7924        while Instant::now() < deadline {
7925            if let Some(target) =
7926                session_backfill_target_for_identity(&aggregator.inner, identity).await
7927            {
7928                return Ok(target.session_id);
7929            }
7930            tokio::time::sleep(Duration::from_millis(25)).await;
7931        }
7932        Err(format!(
7933            "session backfill target for {identity:?} was not resolvable"
7934        ))
7935    }
7936
7937    async fn wait_for_runtime_session_history_text(
7938        runtime: &UnifiedRuntime,
7939        session_id: &str,
7940        expected: &str,
7941        timeout: Duration,
7942    ) -> Result<(), String> {
7943        let deadline = Instant::now() + timeout;
7944        let mut observed = Vec::new();
7945        while Instant::now() < deadline {
7946            let page = runtime
7947                .mob_runtime()
7948                .read_session_history(session_id, 0, Some(20))
7949                .await
7950                .map_err(|err| err.to_string())?;
7951            observed = page.messages;
7952            if observed.iter().enumerate().any(|(idx, message)| {
7953                let Some(message) = serde_json::to_value(message).ok() else {
7954                    return false;
7955                };
7956                frames_from_session_history_message(
7957                    "runtime-a",
7958                    "test/agent-a",
7959                    session_id,
7960                    idx,
7961                    message,
7962                )
7963                .iter()
7964                .any(|frame| {
7965                    matches!(
7966                        frame.kind.as_str(),
7967                        "user_input" | "system_notice" | "interaction_complete"
7968                    ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7969                })
7970            }) {
7971                return Ok(());
7972            }
7973            tokio::time::sleep(Duration::from_millis(25)).await;
7974        }
7975
7976        Err(format!(
7977            "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7978        ))
7979    }
7980
7981    async fn wait_for_identity_record(
7982        aggregator: &MobKitConsoleAggregator,
7983        identity: &str,
7984        session_id: Option<&str>,
7985        timeout: Duration,
7986    ) -> Result<(), String> {
7987        let deadline = Instant::now() + timeout;
7988        let mut observed = Vec::new();
7989        while Instant::now() < deadline {
7990            observed = aggregator
7991                .list_identities()
7992                .await
7993                .map_err(|err| err.to_string())?;
7994            if observed.iter().any(|record| {
7995                record.identity == identity && record.session_id.as_deref() == session_id
7996            }) {
7997                return Ok(());
7998            }
7999            tokio::time::sleep(Duration::from_millis(25)).await;
8000        }
8001
8002        Err(format!(
8003            "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
8004        ))
8005    }
8006
8007    fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
8008        session_history_payload_text(&frame.payload)
8009    }
8010
8011    fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
8012        session_history_payload_text(&frame.payload)
8013    }
8014
8015    fn session_history_payload_text(payload: &Value) -> Option<String> {
8016        if let Some(text) = payload.get("text").and_then(Value::as_str) {
8017            return Some(text.to_string());
8018        }
8019        if let Some(text) = payload.get("result").and_then(Value::as_str) {
8020            return Some(text.to_string());
8021        }
8022        if let Some(text) = payload.get("body").and_then(Value::as_str) {
8023            return Some(text.to_string());
8024        }
8025        match payload.get("content")? {
8026            Value::String(text) => Some(text.clone()),
8027            Value::Array(blocks) => Some(
8028                blocks
8029                    .iter()
8030                    .filter_map(|block| block.get("text").and_then(Value::as_str))
8031                    .collect::<Vec<_>>()
8032                    .join(""),
8033            ),
8034            _ => None,
8035        }
8036    }
8037
8038    #[tokio::test]
8039    async fn query_timeline_handles_large_store_without_backfill_calls() {
8040        let store = Arc::new(CountingConsoleLogStore::new());
8041        let aggregator = MobKitConsoleAggregator::new(store.clone());
8042        for idx in 0..5_000 {
8043            store
8044                .append_if_absent(NewConsoleFrame {
8045                    id: None,
8046                    dedupe_key: format!("event-{idx}"),
8047                    timestamp_ms: idx,
8048                    runtime_key: "runtime-a".to_string(),
8049                    identity: "agent-a".to_string(),
8050                    conversation_id: Some("agent-a".to_string()),
8051                    session_id: Some("session-a".to_string()),
8052                    kind: "text_delta".to_string(),
8053                    status: ConsoleFrameStatus::Delivered,
8054                    payload: json!({ "delta": idx }),
8055                    source: ConsoleFrameSource {
8056                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8057                        source_cursor: None,
8058                    },
8059                    source_event_id: Some(format!("event-{idx}")),
8060                    interaction_id: None,
8061                    turn_id: None,
8062                    run_id: None,
8063                    parent_frame_id: None,
8064                    caused_by_frame_id: None,
8065                })
8066                .await
8067                .expect("append frame");
8068        }
8069
8070        let page = aggregator
8071            .query_timeline(ConsoleTimelineQuery {
8072                identity: Some("agent-a".to_string()),
8073                limit: 1_000,
8074                ..ConsoleTimelineQuery::default()
8075            })
8076            .await
8077            .expect("large query");
8078
8079        assert_eq!(page.frames.len(), 1_000);
8080        assert_eq!(store.source_watermark_calls(), 0);
8081    }
8082
8083    #[tokio::test]
8084    async fn query_timeline_rejects_future_cursor_after_store_reset() {
8085        let aggregator = MobKitConsoleAggregator::in_memory();
8086        let err = aggregator
8087            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8088                after: Some(ConsoleCursor::from("console:99")),
8089                limit: 10,
8090                ..ConsoleTimelineWindowQuery::default()
8091            })
8092            .await
8093            .expect_err("future cursor on empty/reset store must be replay-unavailable");
8094
8095        assert!(
8096            err.to_string()
8097                .contains("beyond the current store frontier"),
8098            "unexpected error: {err}"
8099        );
8100    }
8101
8102    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8103    async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
8104        let store = Arc::new(CountingConsoleLogStore::new());
8105        let aggregator = MobKitConsoleAggregator::new(store.clone());
8106        let runtime = build_single_member_runtime().await;
8107        aggregator
8108            .inner
8109            .runtimes
8110            .write()
8111            .expect("runtime registry")
8112            .insert(
8113                "runtime-a".to_string(),
8114                runtime_entry_for_test("runtime-a", &runtime),
8115            );
8116        let session_id =
8117            wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8118                .await
8119                .expect("spawned member is resolvable for targeted backfill");
8120        wait_for_runtime_session_history_text(
8121            &runtime,
8122            &session_id,
8123            "You are agent-a.",
8124            Duration::from_secs(5),
8125        )
8126        .await
8127        .expect("spawned member kickoff is readable before live-frame race");
8128        store
8129            .append_if_absent(NewConsoleFrame {
8130                id: None,
8131                dedupe_key: "live-tool-before-history".to_string(),
8132                timestamp_ms: 10,
8133                runtime_key: "runtime-a".to_string(),
8134                identity: "test/agent-a".to_string(),
8135                conversation_id: Some("test/agent-a".to_string()),
8136                session_id: None,
8137                kind: "tool_execution_started".to_string(),
8138                status: ConsoleFrameStatus::Delivered,
8139                payload: json!({
8140                    "id": "call-live",
8141                    "name": "king_search",
8142                    "source_event_type": "tool_execution_started",
8143                    "type": "tool_execution_started",
8144                }),
8145                source: ConsoleFrameSource {
8146                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8147                    source_cursor: Some("live-tool-before-history".to_string()),
8148                },
8149                source_event_id: Some("live-tool-before-history".to_string()),
8150                interaction_id: None,
8151                turn_id: None,
8152                run_id: None,
8153                parent_frame_id: None,
8154                caused_by_frame_id: None,
8155            })
8156            .await
8157            .expect("append live tool frame");
8158
8159        let page = aggregator
8160            .query_timeline(ConsoleTimelineQuery {
8161                identity: Some("test/agent-a".to_string()),
8162                limit: 20,
8163                ..ConsoleTimelineQuery::default()
8164            })
8165            .await
8166            .expect("query child timeline");
8167
8168        assert!(
8169            page.frames
8170                .iter()
8171                .any(|frame| frame.kind == "tool_execution_started"),
8172            "initial explicit child timeline should return existing store frames without waiting for session history"
8173        );
8174
8175        let mut observed_backfill = false;
8176        for _ in 0..80 {
8177            let page = aggregator
8178                .query_timeline(ConsoleTimelineQuery {
8179                    identity: Some("test/agent-a".to_string()),
8180                    limit: 20,
8181                    ..ConsoleTimelineQuery::default()
8182                })
8183                .await
8184                .expect("query child timeline after scheduled backfill");
8185            if page.frames.iter().any(|frame| {
8186                frame.kind == "user_input"
8187                    && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8188                    && frame.payload.to_string().contains("You are agent-a.")
8189            }) {
8190                observed_backfill = true;
8191                break;
8192            }
8193            tokio::time::sleep(Duration::from_millis(25)).await;
8194        }
8195        assert!(
8196            observed_backfill,
8197            "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8198        );
8199        let _ = runtime.mob_handle().stop().await;
8200    }
8201
8202    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8203    async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8204        let store = Arc::new(CountingConsoleLogStore::new());
8205        let aggregator = MobKitConsoleAggregator::new(store.clone());
8206        let (_temp, runtime, _delayed_service) =
8207            build_stress_runtime(64, Duration::from_millis(0)).await;
8208        aggregator
8209            .inner
8210            .runtimes
8211            .write()
8212            .expect("runtime registry")
8213            .insert(
8214                "runtime-a".to_string(),
8215                runtime_entry_for_test("runtime-a", runtime.as_ref()),
8216            );
8217        for idx in 0..1_000 {
8218            store
8219                .append_if_absent(NewConsoleFrame {
8220                    id: None,
8221                    dedupe_key: format!("event-{idx}"),
8222                    timestamp_ms: idx,
8223                    runtime_key: "runtime-a".to_string(),
8224                    identity: "test/agent-0".to_string(),
8225                    conversation_id: Some("test/agent-0".to_string()),
8226                    session_id: Some("session-a".to_string()),
8227                    kind: "text_delta".to_string(),
8228                    status: ConsoleFrameStatus::Delivered,
8229                    payload: json!({ "delta": idx }),
8230                    source: ConsoleFrameSource {
8231                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8232                        source_cursor: None,
8233                    },
8234                    source_event_id: Some(format!("event-{idx}")),
8235                    interaction_id: None,
8236                    turn_id: None,
8237                    run_id: None,
8238                    parent_frame_id: None,
8239                    caused_by_frame_id: None,
8240                })
8241                .await
8242                .expect("append frame");
8243        }
8244
8245        // The per-frame roster-rediscovery regression is asserted exactly by
8246        // `source_watermark_calls() == 0` below (and would also blow this query
8247        // up to O(frames) async roster resolutions). The timeout is only a
8248        // hang/O(n^2)-blowup backstop — generous so it never false-trips on CPU
8249        // starvation under full parallel CI load (the old 2s ceiling flaked the
8250        // suite); a real regression takes minutes, not seconds, on 1000 frames.
8251        let page = tokio::time::timeout(
8252            Duration::from_mins(1),
8253            aggregator.query_timeline(ConsoleTimelineQuery {
8254                identity: Some("test/agent-0".to_string()),
8255                limit: 1_000,
8256                ..ConsoleTimelineQuery::default()
8257            }),
8258        )
8259        .await
8260        .expect("identity timeline query should not hang or rediscover the roster per frame")
8261        .expect("timeline query succeeds");
8262
8263        assert_eq!(page.frames.len(), 1_000);
8264        assert_eq!(store.source_watermark_calls(), 0);
8265        let _ = runtime.mob_handle().stop().await;
8266    }
8267
8268    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8269    async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8270        const MEMBER_COUNT: usize = 32;
8271        let (_temp, runtime, delayed_service) =
8272            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8273        let aggregator = MobKitConsoleAggregator::in_memory();
8274        aggregator
8275            .inner
8276            .runtimes
8277            .write()
8278            .expect("runtime registry")
8279            .insert(
8280                "runtime-stress".to_string(),
8281                runtime_entry_for_test("runtime-stress", &runtime),
8282            );
8283
8284        let started = Instant::now();
8285        aggregator
8286            .refresh_session_history()
8287            .await
8288            .expect("stress refresh");
8289        let elapsed = started.elapsed();
8290
8291        assert!(
8292            delayed_service.read_calls() >= MEMBER_COUNT,
8293            "expected at least one history read per member, saw {}",
8294            delayed_service.read_calls()
8295        );
8296        assert!(
8297            delayed_service.max_active_reads() > 1,
8298            "session history backfill should fan out instead of reading members serially"
8299        );
8300        assert!(
8301            delayed_service.max_active_reads()
8302                <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8303            "session history backfill should respect the default concurrency limit"
8304        );
8305        assert!(
8306            elapsed < Duration::from_millis(600),
8307            "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8308            MEMBER_COUNT * 40
8309        );
8310        let _ = runtime.mob_handle().stop().await;
8311    }
8312
8313    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8314    async fn session_history_backfill_respects_configured_concurrency_limit() {
8315        const MEMBER_COUNT: usize = 16;
8316        let (_temp, runtime, delayed_service) =
8317            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8318        let aggregator =
8319            MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8320                max_concurrent_session_backfills: 4,
8321                ..ConsoleAggregatorOptions::default()
8322            });
8323        aggregator
8324            .inner
8325            .runtimes
8326            .write()
8327            .expect("runtime registry")
8328            .insert(
8329                "runtime-stress".to_string(),
8330                runtime_entry_for_test("runtime-stress", &runtime),
8331            );
8332
8333        aggregator
8334            .refresh_session_history()
8335            .await
8336            .expect("stress refresh");
8337
8338        assert!(
8339            delayed_service.max_active_reads() <= 4,
8340            "configured concurrency limit should bound session history reads, saw {}",
8341            delayed_service.max_active_reads()
8342        );
8343        let _ = runtime.mob_handle().stop().await;
8344    }
8345
8346    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8347    async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8348        const MEMBER_COUNT: usize = 24;
8349        const LIVE_EVENT_COUNT: usize = 2_048;
8350        let (_temp, runtime, _delayed_service) =
8351            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
8352        let aggregator = MobKitConsoleAggregator::in_memory();
8353        aggregator.register_runtime(ConsoleRuntimeRegistration {
8354            runtime_key: "runtime-burst".to_string(),
8355            runtime: runtime.clone(),
8356            identity_namespace: "stress".to_string(),
8357            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8358        });
8359        let console_events = runtime.console_events();
8360        for idx in 0..LIVE_EVENT_COUNT {
8361            console_events
8362                .append(
8363                    "agent-0",
8364                    Some("burst-turn".to_string()),
8365                    "text_delta",
8366                    json!({ "delta": format!("frame-{idx}") }),
8367                )
8368                .await;
8369        }
8370
8371        // Generous deadline: the loop breaks the instant all live frames land,
8372        // so a large ceiling is free in the normal case and only prevents a
8373        // false drop-detection when the live pump is CPU-starved under full
8374        // parallel CI load (the old 5s deadline flaked the suite).
8375        let deadline = Instant::now() + Duration::from_secs(30);
8376        let mut observed = 0;
8377        while Instant::now() < deadline {
8378            observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
8379            if observed >= LIVE_EVENT_COUNT {
8380                break;
8381            }
8382            tokio::time::sleep(Duration::from_millis(25)).await;
8383        }
8384
8385        assert_eq!(
8386            observed, LIVE_EVENT_COUNT,
8387            "live pump should not drop frames while slow background backfill is running"
8388        );
8389        let _ = runtime.mob_handle().stop().await;
8390    }
8391
8392    async fn count_console_event_frames(
8393        aggregator: &MobKitConsoleAggregator,
8394        identity: &str,
8395    ) -> usize {
8396        let mut after = None;
8397        let mut count = 0;
8398        loop {
8399            let page = aggregator
8400                .query_timeline(ConsoleTimelineQuery {
8401                    identity: Some(identity.to_string()),
8402                    after,
8403                    limit: 1_000,
8404                    ..ConsoleTimelineQuery::default()
8405                })
8406                .await
8407                .expect("query burst timeline");
8408            if page.frames.is_empty() {
8409                break;
8410            }
8411            count += page
8412                .frames
8413                .iter()
8414                .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
8415                .count();
8416            after = page.next_cursor;
8417            if after.is_none() {
8418                break;
8419            }
8420        }
8421        count
8422    }
8423
8424    #[tokio::test]
8425    async fn status_updates_get_replayable_aggregate_cursors() {
8426        let aggregator = MobKitConsoleAggregator::in_memory();
8427        let frame = NewConsoleFrame {
8428            id: None,
8429            dedupe_key: "send-1".to_string(),
8430            timestamp_ms: 1,
8431            runtime_key: "runtime-a".to_string(),
8432            identity: "agent-a".to_string(),
8433            conversation_id: Some("agent-a".to_string()),
8434            session_id: Some("session-1".to_string()),
8435            kind: "user_input".to_string(),
8436            status: ConsoleFrameStatus::Accepted,
8437            payload: json!({ "content": "hello" }),
8438            source: ConsoleFrameSource {
8439                kind: ConsoleFrameSourceKind::Send,
8440                source_cursor: None,
8441            },
8442            source_event_id: None,
8443            interaction_id: Some("interaction-1".to_string()),
8444            turn_id: None,
8445            run_id: None,
8446            parent_frame_id: None,
8447            caused_by_frame_id: None,
8448        };
8449        let inserted = aggregator
8450            .store()
8451            .append_if_absent(frame)
8452            .await
8453            .expect("append frame");
8454
8455        update_frame_status_and_emit(
8456            &aggregator.inner,
8457            &inserted.frame.id,
8458            ConsoleFrameStatus::Delivered,
8459        )
8460        .await
8461        .expect("update status");
8462
8463        let page = aggregator
8464            .query_timeline(ConsoleTimelineQuery {
8465                identity: Some("agent-a".to_string()),
8466                after: Some(inserted.frame.cursor.clone()),
8467                limit: 10,
8468                ..ConsoleTimelineQuery::default()
8469            })
8470            .await
8471            .expect("query timeline");
8472        assert_eq!(page.frames.len(), 1);
8473        assert_eq!(page.frames[0].kind, "frame_updated");
8474        assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
8475        assert_eq!(
8476            page.frames[0]
8477                .payload
8478                .get("frame")
8479                .and_then(|frame| frame.get("status"))
8480                .and_then(Value::as_str),
8481            Some("delivered")
8482        );
8483    }
8484
8485    #[tokio::test]
8486    async fn steer_delivery_appends_terminal_control_frame() {
8487        let aggregator = MobKitConsoleAggregator::in_memory();
8488        let frame = NewConsoleFrame {
8489            id: None,
8490            dedupe_key: "send-steer-1".to_string(),
8491            timestamp_ms: 1,
8492            runtime_key: "runtime-a".to_string(),
8493            identity: "agent-a".to_string(),
8494            conversation_id: Some("agent-a".to_string()),
8495            session_id: Some("session-1".to_string()),
8496            kind: "user_input".to_string(),
8497            status: ConsoleFrameStatus::Delivered,
8498            payload: json!({
8499                "content": "operator steer",
8500                "handling_mode": "steer",
8501            }),
8502            source: ConsoleFrameSource {
8503                kind: ConsoleFrameSourceKind::Send,
8504                source_cursor: None,
8505            },
8506            source_event_id: None,
8507            interaction_id: Some("interaction-steer-1".to_string()),
8508            turn_id: None,
8509            run_id: None,
8510            parent_frame_id: None,
8511            caused_by_frame_id: None,
8512        };
8513        let inserted = aggregator
8514            .store()
8515            .append_if_absent(frame)
8516            .await
8517            .expect("append steer input");
8518
8519        append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
8520            .await
8521            .expect("append steer terminal");
8522
8523        let page = aggregator
8524            .query_timeline(ConsoleTimelineQuery {
8525                identity: Some("agent-a".to_string()),
8526                after: Some(inserted.frame.cursor.clone()),
8527                limit: 10,
8528                ..ConsoleTimelineQuery::default()
8529            })
8530            .await
8531            .expect("query timeline");
8532
8533        assert_eq!(page.frames.len(), 1);
8534        let terminal = &page.frames[0];
8535        assert_eq!(terminal.kind, "interaction_complete");
8536        assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
8537        assert_eq!(
8538            terminal.interaction_id.as_deref(),
8539            Some("interaction-steer-1")
8540        );
8541        assert_eq!(
8542            terminal.parent_frame_id.as_deref(),
8543            Some(inserted.frame.id.as_str())
8544        );
8545        assert_eq!(
8546            terminal.payload.get("reason").and_then(Value::as_str),
8547            Some("steer_delivered")
8548        );
8549    }
8550
8551    #[test]
8552    fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
8553        let live = transcript_fingerprint(
8554            "reasoning_complete",
8555            &json!({ "text": "I should inspect the file first." }),
8556        );
8557        let replay = transcript_fingerprint(
8558            "reasoning_complete",
8559            &json!({ "summary": "I should inspect the file first." }),
8560        );
8561        let delta = transcript_fingerprint(
8562            "reasoning_delta",
8563            &json!({ "delta": "I should inspect the file first." }),
8564        );
8565
8566        assert_eq!(live.as_deref(), Some("I should inspect the file first."));
8567        assert_eq!(live, replay);
8568        assert_eq!(live, delta);
8569    }
8570
8571    #[tokio::test]
8572    async fn history_counterpart_scan_is_not_capped_to_one_page() {
8573        let aggregator = MobKitConsoleAggregator::in_memory();
8574        for idx in 0..1_005 {
8575            aggregator
8576                .store()
8577                .append_if_absent(NewConsoleFrame {
8578                    id: None,
8579                    dedupe_key: format!("filler-{idx}"),
8580                    timestamp_ms: idx,
8581                    runtime_key: "runtime-a".to_string(),
8582                    identity: "agent-a".to_string(),
8583                    conversation_id: Some("agent-a".to_string()),
8584                    session_id: Some("session-a".to_string()),
8585                    kind: "text_delta".to_string(),
8586                    status: ConsoleFrameStatus::Completed,
8587                    payload: json!({ "delta": idx }),
8588                    source: ConsoleFrameSource {
8589                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8590                        source_cursor: None,
8591                    },
8592                    source_event_id: Some(format!("filler-{idx}")),
8593                    interaction_id: None,
8594                    turn_id: None,
8595                    run_id: None,
8596                    parent_frame_id: None,
8597                    caused_by_frame_id: None,
8598                })
8599                .await
8600                .expect("append filler");
8601        }
8602        aggregator
8603            .store()
8604            .append_if_absent(NewConsoleFrame {
8605                id: None,
8606                dedupe_key: "live-user-input".to_string(),
8607                timestamp_ms: 2_000,
8608                runtime_key: "runtime-a".to_string(),
8609                identity: "agent-a".to_string(),
8610                conversation_id: Some("agent-a".to_string()),
8611                session_id: Some("session-a".to_string()),
8612                kind: "user_input".to_string(),
8613                status: ConsoleFrameStatus::Delivered,
8614                payload: json!({ "content": "already here" }),
8615                source: ConsoleFrameSource {
8616                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8617                    source_cursor: None,
8618                },
8619                source_event_id: Some("live-user-input".to_string()),
8620                interaction_id: None,
8621                turn_id: None,
8622                run_id: None,
8623                parent_frame_id: None,
8624                caused_by_frame_id: None,
8625            })
8626            .await
8627            .expect("append live input");
8628
8629        let history = NewConsoleFrame {
8630            id: None,
8631            dedupe_key: "history-user-input".to_string(),
8632            timestamp_ms: 3_000,
8633            runtime_key: "runtime-a".to_string(),
8634            identity: "agent-a".to_string(),
8635            conversation_id: Some("agent-a".to_string()),
8636            session_id: Some("session-a".to_string()),
8637            kind: "user_input".to_string(),
8638            status: ConsoleFrameStatus::Completed,
8639            payload: json!({ "content": "already here" }),
8640            source: ConsoleFrameSource {
8641                kind: ConsoleFrameSourceKind::SessionHistory,
8642                source_cursor: Some("session-a:1006".to_string()),
8643            },
8644            source_event_id: None,
8645            interaction_id: None,
8646            turn_id: None,
8647            run_id: None,
8648            parent_frame_id: None,
8649            caused_by_frame_id: None,
8650        };
8651
8652        assert!(
8653            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8654                .await
8655                .expect("counterpart scan")
8656        );
8657    }
8658
8659    #[tokio::test]
8660    async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
8661        let aggregator = MobKitConsoleAggregator::in_memory();
8662        aggregator
8663            .store()
8664            .append_if_absent(NewConsoleFrame {
8665                id: None,
8666                dedupe_key: "live-user-input".to_string(),
8667                timestamp_ms: 2_000,
8668                runtime_key: "runtime-a".to_string(),
8669                identity: "agent-a".to_string(),
8670                conversation_id: Some("agent-a".to_string()),
8671                session_id: Some("session-a".to_string()),
8672                kind: "user_input".to_string(),
8673                status: ConsoleFrameStatus::Delivered,
8674                payload: json!({ "content": "hello from operator" }),
8675                source: ConsoleFrameSource {
8676                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8677                    source_cursor: None,
8678                },
8679                source_event_id: Some("live-user-input".to_string()),
8680                interaction_id: None,
8681                turn_id: None,
8682                run_id: None,
8683                parent_frame_id: None,
8684                caused_by_frame_id: None,
8685            })
8686            .await
8687            .expect("append live input");
8688
8689        let history = NewConsoleFrame {
8690            id: None,
8691            dedupe_key: "history-user-input".to_string(),
8692            timestamp_ms: 3_000,
8693            runtime_key: "runtime-a".to_string(),
8694            identity: "agent-a".to_string(),
8695            conversation_id: Some("agent-a".to_string()),
8696            session_id: Some("session-a".to_string()),
8697            kind: "user_input".to_string(),
8698            status: ConsoleFrameStatus::Completed,
8699            payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
8700            source: ConsoleFrameSource {
8701                kind: ConsoleFrameSourceKind::SessionHistory,
8702                source_cursor: Some("session-a:2".to_string()),
8703            },
8704            source_event_id: None,
8705            interaction_id: None,
8706            turn_id: None,
8707            run_id: None,
8708            parent_frame_id: None,
8709            caused_by_frame_id: None,
8710        };
8711
8712        assert!(
8713            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8714                .await
8715                .expect("counterpart scan")
8716        );
8717    }
8718
8719    #[tokio::test]
8720    async fn history_counterpart_scan_matches_content_block_user_prompts() {
8721        let aggregator = MobKitConsoleAggregator::in_memory();
8722        aggregator
8723            .store()
8724            .append_if_absent(NewConsoleFrame {
8725                id: None,
8726                dedupe_key: "live-user-input".to_string(),
8727                timestamp_ms: 2_000,
8728                runtime_key: "runtime-a".to_string(),
8729                identity: "agent-a".to_string(),
8730                conversation_id: Some("agent-a".to_string()),
8731                session_id: Some("session-a".to_string()),
8732                kind: "user_input".to_string(),
8733                status: ConsoleFrameStatus::Delivered,
8734                payload: json!({ "content": "hello from operator" }),
8735                source: ConsoleFrameSource {
8736                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8737                    source_cursor: None,
8738                },
8739                source_event_id: Some("live-user-input".to_string()),
8740                interaction_id: None,
8741                turn_id: None,
8742                run_id: None,
8743                parent_frame_id: None,
8744                caused_by_frame_id: None,
8745            })
8746            .await
8747            .expect("append live input");
8748
8749        let history = NewConsoleFrame {
8750            id: None,
8751            dedupe_key: "history-user-input".to_string(),
8752            timestamp_ms: 3_000,
8753            runtime_key: "runtime-a".to_string(),
8754            identity: "agent-a".to_string(),
8755            conversation_id: Some("agent-a".to_string()),
8756            session_id: Some("session-a".to_string()),
8757            kind: "user_input".to_string(),
8758            status: ConsoleFrameStatus::Completed,
8759            payload: json!({
8760                "content": [{ "type": "text", "text": "hello from operator" }]
8761            }),
8762            source: ConsoleFrameSource {
8763                kind: ConsoleFrameSourceKind::SessionHistory,
8764                source_cursor: Some("session-a:2".to_string()),
8765            },
8766            source_event_id: None,
8767            interaction_id: None,
8768            turn_id: None,
8769            run_id: None,
8770            parent_frame_id: None,
8771            caused_by_frame_id: None,
8772        };
8773
8774        assert!(
8775            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8776                .await
8777                .expect("counterpart scan")
8778        );
8779    }
8780
8781    #[tokio::test]
8782    async fn history_counterpart_scan_matches_live_tool_results() {
8783        let aggregator = MobKitConsoleAggregator::in_memory();
8784        aggregator
8785            .store()
8786            .append_if_absent(NewConsoleFrame {
8787                id: None,
8788                dedupe_key: "live-tool-result".to_string(),
8789                timestamp_ms: 2_000,
8790                runtime_key: "runtime-a".to_string(),
8791                identity: "agent-a".to_string(),
8792                conversation_id: Some("agent-a".to_string()),
8793                session_id: Some("session-a".to_string()),
8794                kind: "tool_execution_completed".to_string(),
8795                status: ConsoleFrameStatus::Delivered,
8796                payload: json!({
8797                    "id": "call-1",
8798                    "tool_call_id": "call-1",
8799                    "result": "{ \"count\": 70 }"
8800                }),
8801                source: ConsoleFrameSource {
8802                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8803                    source_cursor: None,
8804                },
8805                source_event_id: Some("live-tool-result".to_string()),
8806                interaction_id: None,
8807                turn_id: None,
8808                run_id: None,
8809                parent_frame_id: None,
8810                caused_by_frame_id: None,
8811            })
8812            .await
8813            .expect("append live tool result");
8814
8815        let history = NewConsoleFrame {
8816            id: None,
8817            dedupe_key: "history-tool-result".to_string(),
8818            timestamp_ms: 3_000,
8819            runtime_key: "runtime-a".to_string(),
8820            identity: "agent-a".to_string(),
8821            conversation_id: Some("agent-a".to_string()),
8822            session_id: Some("session-a".to_string()),
8823            kind: "tool_execution_completed".to_string(),
8824            status: ConsoleFrameStatus::Completed,
8825            payload: json!({
8826                "id": "call-1",
8827                "tool_call_id": "call-1",
8828                "result": "{ \"count\": 70 }",
8829                "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
8830            }),
8831            source: ConsoleFrameSource {
8832                kind: ConsoleFrameSourceKind::SessionHistory,
8833                source_cursor: Some("session-a:3:0".to_string()),
8834            },
8835            source_event_id: None,
8836            interaction_id: None,
8837            turn_id: None,
8838            run_id: None,
8839            parent_frame_id: None,
8840            caused_by_frame_id: None,
8841        };
8842
8843        assert!(
8844            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8845                .await
8846                .expect("counterpart scan")
8847        );
8848    }
8849
8850    #[tokio::test]
8851    async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
8852        let aggregator = MobKitConsoleAggregator::in_memory();
8853        for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
8854            aggregator
8855                .store()
8856                .append_if_absent(NewConsoleFrame {
8857                    id: None,
8858                    dedupe_key: format!("live-delta-{idx}"),
8859                    timestamp_ms: 2_000 + idx as u64,
8860                    runtime_key: "runtime-a".to_string(),
8861                    identity: "agent-a".to_string(),
8862                    conversation_id: Some("agent-a".to_string()),
8863                    session_id: Some("session-a".to_string()),
8864                    kind: "text_delta".to_string(),
8865                    status: ConsoleFrameStatus::Delivered,
8866                    payload: json!({ "delta": delta }),
8867                    source: ConsoleFrameSource {
8868                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8869                        source_cursor: None,
8870                    },
8871                    source_event_id: Some(format!("live-delta-{idx}")),
8872                    interaction_id: Some("turn-a".to_string()),
8873                    turn_id: None,
8874                    run_id: None,
8875                    parent_frame_id: None,
8876                    caused_by_frame_id: None,
8877                })
8878                .await
8879                .expect("append live delta");
8880        }
8881
8882        let history = NewConsoleFrame {
8883            id: None,
8884            dedupe_key: "history-assistant-complete".to_string(),
8885            timestamp_ms: 3_000,
8886            runtime_key: "runtime-a".to_string(),
8887            identity: "agent-a".to_string(),
8888            conversation_id: Some("agent-a".to_string()),
8889            session_id: Some("session-a".to_string()),
8890            kind: "interaction_complete".to_string(),
8891            status: ConsoleFrameStatus::Completed,
8892            payload: json!({ "result": "Ready and standing by." }),
8893            source: ConsoleFrameSource {
8894                kind: ConsoleFrameSourceKind::SessionHistory,
8895                source_cursor: Some("session-a:3".to_string()),
8896            },
8897            source_event_id: None,
8898            interaction_id: None,
8899            turn_id: None,
8900            run_id: None,
8901            parent_frame_id: None,
8902            caused_by_frame_id: None,
8903        };
8904
8905        assert!(
8906            history_frame_has_existing_counterpart(&aggregator.inner, &history)
8907                .await
8908                .expect("counterpart scan")
8909        );
8910    }
8911
8912    #[test]
8913    fn session_history_watermark_key_is_session_scoped() {
8914        assert_ne!(
8915            session_history_watermark_runtime_key("runtime-a", "session-1"),
8916            session_history_watermark_runtime_key("runtime-a", "session-2")
8917        );
8918    }
8919
8920    #[test]
8921    fn backfill_offset_advances_only_on_strict_forward_progress() {
8922        // Normal full-page advance from the start.
8923        assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
8924        // Subsequent full page from a correct cursor advances.
8925        assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
8926        // A partial page that still moves forward advances.
8927        assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
8928        // One element past the requested offset still counts as progress.
8929        assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
8930
8931        // Livelock shapes: the store re-serves a page whose (base + len) does
8932        // NOT advance past what we asked for. These must stop the loop.
8933        assert_eq!(advance_backfill_offset(500, 0, 500), None); // cursor reset to 0, full page
8934        assert_eq!(advance_backfill_offset(500, 0, 400), None); // strictly behind
8935        assert_eq!(advance_backfill_offset(1000, 500, 500), None); // lands exactly on offset
8936        assert_eq!(advance_backfill_offset(10, 10, 0), None); // empty page at same offset
8937
8938        // Saturating add: a pathological base near usize::MAX cannot panic.
8939        assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
8940    }
8941
8942    #[test]
8943    fn session_history_watermarks_are_cursor_and_ttl_aware() {
8944        let legacy = "session:with:colon:42";
8945        let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8946        let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8947
8948        assert_eq!(
8949            parse_session_history_watermark(legacy, "session:with:colon"),
8950            Some(42)
8951        );
8952        assert_eq!(
8953            parse_session_history_watermark(&checked, "session:with:colon"),
8954            Some(43)
8955        );
8956        assert!(session_history_watermark_is_fresh(
8957            &checked,
8958            "session:with:colon",
8959            1_500
8960        ));
8961        assert!(!session_history_watermark_is_fresh(
8962            &checked,
8963            "session:with:colon",
8964            1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8965        ));
8966        assert!(session_history_watermark_is_fresh(
8967            &empty_checked,
8968            "session:with:colon",
8969            1_500
8970        ));
8971        assert!(!session_history_watermark_is_fresh(
8972            &empty_checked,
8973            "session:with:colon",
8974            1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8975        ));
8976    }
8977
8978    #[test]
8979    fn session_history_messages_project_to_renderable_frames() {
8980        let user = frame_from_session_history_message(
8981            "runtime-a",
8982            "agent-a",
8983            "session-a",
8984            0,
8985            json!({
8986                "role": "user",
8987                "content": "hello",
8988                "timestamp_ms": 10
8989            }),
8990        )
8991        .expect("user history frame");
8992        // meerkat 0.7 removed the legacy flat-text `Message::Assistant`
8993        // variant: session stores serialize assistant turns only as
8994        // `block_assistant` (MessageWire), so the history projection sees the
8995        // block shape.
8996        let assistant = frame_from_session_history_message(
8997            "runtime-a",
8998            "agent-a",
8999            "session-a",
9000            1,
9001            json!({
9002                "role": "block_assistant",
9003                "blocks": [
9004                    { "block_type": "text", "data": { "text": "hi there" } }
9005                ],
9006                "stop_reason": "end_turn",
9007                "timestamp_ms": 11
9008            }),
9009        )
9010        .expect("assistant history frame");
9011
9012        assert_eq!(user.kind, "user_input");
9013        assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
9014        assert_eq!(
9015            user.payload["content"],
9016            json!([{ "type": "text", "text": "hello" }])
9017        );
9018        assert_eq!(assistant.kind, "interaction_complete");
9019        assert_eq!(assistant.payload["text"], json!("hi there"));
9020        assert!(
9021            assistant
9022                .dedupe_key
9023                .starts_with("session-history:runtime-a:session-a:1:")
9024        );
9025    }
9026
9027    #[test]
9028    fn session_history_projection_filters_scaffold_user_messages() {
9029        let spawn_notice = frame_from_session_history_message(
9030            "runtime-a",
9031            "agent-a",
9032            "session-a",
9033            0,
9034            json!({
9035                "role": "user",
9036                "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
9037                "timestamp_ms": 10
9038            }),
9039        );
9040        let peer_update = frame_from_session_history_message(
9041            "runtime-a",
9042            "agent-a",
9043            "session-a",
9044            1,
9045            json!({
9046                "role": "user",
9047                "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
9048                "timestamp_ms": 11
9049            }),
9050        );
9051        let real_user = frame_from_session_history_message(
9052            "runtime-a",
9053            "agent-a",
9054            "session-a",
9055            2,
9056            json!({
9057                "role": "user",
9058                "content": "Please review the incident notes.",
9059                "timestamp_ms": 12
9060            }),
9061        );
9062
9063        assert!(spawn_notice.is_none());
9064        assert!(peer_update.is_none());
9065        assert!(real_user.is_some());
9066    }
9067
9068    #[test]
9069    fn session_history_projection_skips_non_transcript_messages() {
9070        let skipped = frame_from_session_history_message(
9071            "runtime-a",
9072            "agent-a",
9073            "session-a",
9074            0,
9075            json!({
9076                "content": "internal system prompt"
9077            }),
9078        );
9079        assert!(skipped.is_none());
9080    }
9081
9082    #[test]
9083    fn session_history_projection_extracts_assistant_blocks() {
9084        let frame = frame_from_session_history_message(
9085            "runtime-a",
9086            "agent-a",
9087            "session-a",
9088            0,
9089            json!({
9090                "role": "block_assistant",
9091                "blocks": [
9092                    { "block_type": "text", "data": { "text": "hello " } },
9093                    { "block_type": "text", "data": { "text": "there" } }
9094                ],
9095                "stop_reason": "end_turn"
9096            }),
9097        )
9098        .expect("assistant block history frame");
9099        assert_eq!(frame.payload["text"], json!("hello there"));
9100    }
9101
9102    #[test]
9103    fn session_history_projection_extracts_nested_text_block_data() {
9104        let frame = frame_from_session_history_message(
9105            "runtime-a",
9106            "agent-a",
9107            "session-a",
9108            0,
9109            json!({
9110                "role": "block_assistant",
9111                "blocks": [
9112                    {
9113                        "block_type": "text",
9114                        "data": { "text": "Ready and standing by." }
9115                    }
9116                ],
9117                "stop_reason": "end_turn",
9118                "created_at": "1970-01-01T00:00:00.010Z"
9119            }),
9120        )
9121        .expect("assistant block history frame");
9122
9123        assert_eq!(frame.kind, "interaction_complete");
9124        assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9125    }
9126
9127    #[test]
9128    fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9129        let frame = frame_from_session_history_message(
9130            "runtime-a",
9131            "agent-a",
9132            "session-a",
9133            0,
9134            json!({
9135                "role": "block_assistant",
9136                "blocks": [
9137                    {
9138                        "block_type": "reasoning",
9139                        "data": { "text": "**Planning**\nI should not be rendered." }
9140                    },
9141                    {
9142                        "block_type": "text",
9143                        "data": { "text": "Visible answer." }
9144                    }
9145                ],
9146                "stop_reason": "end_turn",
9147                "created_at": "1970-01-01T00:00:00.010Z"
9148            }),
9149        )
9150        .expect("assistant block history frame");
9151
9152        assert_eq!(frame.kind, "interaction_complete");
9153        assert_eq!(frame.payload["result"], json!("Visible answer."));
9154        assert_eq!(frame.payload["text"], json!("Visible answer."));
9155    }
9156
9157    #[test]
9158    fn session_history_projection_leaves_reasoning_only_result_empty() {
9159        let frame = frame_from_session_history_message(
9160            "runtime-a",
9161            "agent-a",
9162            "session-a",
9163            0,
9164            json!({
9165                "role": "block_assistant",
9166                "blocks": [
9167                    {
9168                        "block_type": "reasoning",
9169                        "data": { "text": "Private planning text." }
9170                    },
9171                    {
9172                        "block_type": "tool_use",
9173                        "data": { "id": "toolu-1", "name": "peers", "args": {} }
9174                    }
9175                ],
9176                "stop_reason": "end_turn",
9177                "created_at": "1970-01-01T00:00:00.010Z"
9178            }),
9179        )
9180        .expect("assistant block history frame");
9181
9182        assert_eq!(frame.kind, "interaction_complete");
9183        assert_eq!(frame.payload["result"], json!(""));
9184        assert_eq!(frame.payload["text"], json!(""));
9185    }
9186
9187    #[test]
9188    fn session_history_projection_preserves_tool_results() {
9189        let frames = frames_from_session_history_message(
9190            "runtime-a",
9191            "agent-a",
9192            "session-a",
9193            5,
9194            json!({
9195                "role": "tool_results",
9196                "results": [
9197                    {
9198                        "tool_use_id": "call-peers",
9199                        "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9200                        "is_error": false
9201                    }
9202                ],
9203                "created_at": "1970-01-01T00:00:00.050Z"
9204            }),
9205        );
9206
9207        assert_eq!(frames.len(), 1);
9208        let frame = &frames[0];
9209        assert_eq!(frame.kind, "tool_execution_completed");
9210        assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9211        assert_eq!(
9212            frame.payload["result"],
9213            json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9214        );
9215        assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9216        assert_eq!(frame.timestamp_ms, 50);
9217    }
9218
9219    #[test]
9220    fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9221        // Regression: a generate_image tool result rebuilt from session-history
9222        // backfill must derive an `assistant_image` frame (image_id/blob_id/
9223        // media_type/width/height) just like the live ingest edge does. Without
9224        // it the image renders as raw tool JSON after a reload / fresh store.
9225        // The serialized ImageGenerationToolResult carries NO tool name, so the
9226        // backfill parses it directly.
9227        let image_result = json!({
9228            "operation_id": "00000000-0000-0000-0000-0000000000aa",
9229            "terminal": { "terminal": "generated" },
9230            "images": [
9231                {
9232                    "image_id": "00000000-0000-0000-0000-000000000051",
9233                    "blob_ref": {
9234                        "blob_id": "sha256:test-generated-image",
9235                        "media_type": "image/png"
9236                    },
9237                    "media_type": "image/png",
9238                    "width": 1024,
9239                    "height": 768
9240                }
9241            ],
9242            "provider_text": { "disposition": "not_emitted" },
9243            "revised_prompt": { "disposition": "not_requested" },
9244            "native_metadata": { "provider": "not_emitted" }
9245        });
9246        // Sanity: the fixture must be a valid ImageGenerationToolResult so the
9247        // test pins the real wire contract, not an arbitrary blob.
9248        serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9249            image_result.clone(),
9250        )
9251        .expect("fixture must be a valid ImageGenerationToolResult");
9252        let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9253
9254        let frames = frames_from_session_history_message(
9255            "runtime-a",
9256            "artist:singleton",
9257            "session-a",
9258            6,
9259            json!({
9260                "role": "tool_results",
9261                "results": [
9262                    {
9263                        "tool_use_id": "call-image",
9264                        "content": result_text,
9265                        "is_error": false
9266                    }
9267                ],
9268                "created_at": "1970-01-01T00:00:00.060Z"
9269            }),
9270        );
9271
9272        // The generic tool_execution_completed frame is still emitted...
9273        assert!(
9274            frames
9275                .iter()
9276                .any(|frame| frame.kind == "tool_execution_completed"),
9277            "tool_execution_completed frame must still be present: {frames:#?}"
9278        );
9279        // ...plus a derived assistant_image frame so the picture renders.
9280        let image_frame = frames
9281            .iter()
9282            .find(|frame| frame.kind == "assistant_image")
9283            .expect("backfill must derive an assistant_image frame");
9284        assert_eq!(
9285            image_frame.payload["blob_id"],
9286            json!("sha256:test-generated-image")
9287        );
9288        assert_eq!(
9289            image_frame.payload["image_id"],
9290            json!("00000000-0000-0000-0000-000000000051")
9291        );
9292        assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9293        assert_eq!(image_frame.payload["width"], json!(1024));
9294        assert_eq!(image_frame.payload["height"], json!(768));
9295        assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9296        assert_eq!(
9297            image_frame.source.kind,
9298            ConsoleFrameSourceKind::SessionHistory
9299        );
9300        assert_eq!(image_frame.timestamp_ms, 60);
9301    }
9302
9303    #[test]
9304    fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9305        let frames = frames_from_session_history_message(
9306            "runtime-a",
9307            "review:singleton",
9308            "session-a",
9309            7,
9310            json!({
9311                "role": "block_assistant",
9312                "blocks": [
9313                    {
9314                        "block_type": "tool_use",
9315                        "data": {
9316                            "id": "call-spawn",
9317                            "name": "mob_spawn_member",
9318                            "args": {
9319                                "member_id": "review-worker-alpha",
9320                                "profile": "review-worker",
9321                                "initial_message": "Review Initiative Alpha and save the result."
9322                            }
9323                        }
9324                    }
9325                ],
9326                "stop_reason": "tool_use",
9327                "created_at": "1970-01-01T00:00:00.070Z"
9328            }),
9329        );
9330
9331        assert_eq!(frames.len(), 2);
9332        let child = frames
9333            .iter()
9334            .find(|frame| frame.identity == "review-worker-alpha")
9335            .expect("spawned member initial message frame");
9336        assert_eq!(child.kind, "user_input");
9337        assert_eq!(child.timestamp_ms, 70);
9338        assert_eq!(
9339            child.payload["message"]["content"],
9340            "Review Initiative Alpha and save the result."
9341        );
9342        assert_eq!(child.payload["parent_identity"], "review:singleton");
9343        assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9344        assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9345    }
9346
9347    #[test]
9348    fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9349        let frames = frames_from_session_history_message(
9350            "runtime-a",
9351            "review:singleton",
9352            "session-a",
9353            8,
9354            json!({
9355                "role": "block_assistant",
9356                "blocks": [
9357                    {
9358                        "block_type": "tool_use",
9359                        "data": {
9360                            "id": "call-spawn-many",
9361                            "name": "mob_spawn_member",
9362                            "args": {
9363                                "specs": [
9364                                    {
9365                                        "agent_identity": "review-worker-alpha",
9366                                        "profile": "review-worker",
9367                                        "initial_message": "Review Initiative Alpha."
9368                                    },
9369                                    {
9370                                        "agent_identity": "review-worker-beta",
9371                                        "profile": "review-worker",
9372                                        "initial_message": "Review Initiative Beta."
9373                                    }
9374                                ]
9375                            }
9376                        }
9377                    }
9378                ],
9379                "stop_reason": "tool_use",
9380                "created_at": "1970-01-01T00:00:00.080Z"
9381            }),
9382        );
9383
9384        assert_eq!(frames.len(), 3);
9385        let child_messages = frames
9386            .iter()
9387            .filter(|frame| frame.kind == "user_input")
9388            .map(|frame| {
9389                (
9390                    frame.identity.as_str(),
9391                    frame.payload["message"]["content"]
9392                        .as_str()
9393                        .unwrap_or_default(),
9394                )
9395            })
9396            .collect::<Vec<_>>();
9397        assert_eq!(
9398            child_messages,
9399            vec![
9400                ("review-worker-alpha", "Review Initiative Alpha."),
9401                ("review-worker-beta", "Review Initiative Beta."),
9402            ]
9403        );
9404    }
9405
9406    #[tokio::test]
9407    async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
9408        let aggregator = MobKitConsoleAggregator::in_memory();
9409        let frames = frames_from_session_history_message(
9410            "runtime-a",
9411            "review:singleton",
9412            "session-a",
9413            9,
9414            json!({
9415                "role": "block_assistant",
9416                "blocks": [
9417                    {
9418                        "block_type": "tool_use",
9419                        "data": {
9420                            "id": "call-spawn-many",
9421                            "name": "mob_spawn_member",
9422                            "args": {
9423                                "specs": [
9424                                    {
9425                                        "agent_identity": "review-worker-alpha",
9426                                        "profile": "review-worker",
9427                                        "initial_message": "Review Initiative Alpha."
9428                                    },
9429                                    {
9430                                        "agent_identity": "review-worker-beta",
9431                                        "profile": "review-worker",
9432                                        "initial_message": "Review Initiative Beta."
9433                                    }
9434                                ]
9435                            }
9436                        }
9437                    }
9438                ],
9439                "stop_reason": "tool_use",
9440                "created_at": "1970-01-01T00:00:00.090Z"
9441            }),
9442        );
9443        for frame in frames {
9444            aggregator
9445                .store()
9446                .append_if_absent(frame)
9447                .await
9448                .expect("append projected session-history frame");
9449        }
9450
9451        let child_page = aggregator
9452            .query_timeline(ConsoleTimelineQuery {
9453                identity: Some("review-worker-alpha".to_string()),
9454                limit: 20,
9455                ..ConsoleTimelineQuery::default()
9456            })
9457            .await
9458            .expect("query child timeline");
9459        assert_eq!(child_page.frames.len(), 1);
9460        assert_eq!(child_page.frames[0].kind, "user_input");
9461        assert_eq!(
9462            child_page.frames[0].payload["message"]["content"],
9463            "Review Initiative Alpha."
9464        );
9465        assert_eq!(
9466            child_page.frames[0].payload["parent_identity"],
9467            "review:singleton"
9468        );
9469
9470        let parent_page = aggregator
9471            .query_timeline(ConsoleTimelineQuery {
9472                identity: Some("review:singleton".to_string()),
9473                limit: 20,
9474                ..ConsoleTimelineQuery::default()
9475            })
9476            .await
9477            .expect("query parent timeline");
9478        assert!(
9479            parent_page
9480                .frames
9481                .iter()
9482                .all(|frame| frame.identity == "review:singleton"),
9483            "child initial messages must not leak into parent timeline: {:#?}",
9484            parent_page.frames
9485        );
9486    }
9487
9488    #[test]
9489    fn session_history_projection_applies_namespace_to_spawned_child_identity() {
9490        let frames = frames_from_session_history_message_with_namespace(
9491            "runtime-a",
9492            "test/review:singleton",
9493            "test",
9494            "session-a",
9495            10,
9496            json!({
9497                "role": "block_assistant",
9498                "blocks": [
9499                    {
9500                        "block_type": "tool_use",
9501                        "data": {
9502                            "id": "call-spawn-many",
9503                            "name": "mob_spawn_member",
9504                            "args": {
9505                                "specs": [
9506                                    {
9507                                        "agent_identity": "review-worker-alpha",
9508                                        "profile": "review-worker",
9509                                        "initial_message": "Review Initiative Alpha."
9510                                    },
9511                                    {
9512                                        "agent_identity": "test/review-worker-beta",
9513                                        "profile": "review-worker",
9514                                        "initial_message": "Review Initiative Beta."
9515                                    }
9516                                ]
9517                            }
9518                        }
9519                    }
9520                ],
9521                "stop_reason": "tool_use",
9522                "created_at": "1970-01-01T00:00:00.100Z"
9523            }),
9524        );
9525
9526        let child_identities = frames
9527            .iter()
9528            .filter(|frame| frame.kind == "user_input")
9529            .map(|frame| frame.identity.as_str())
9530            .collect::<Vec<_>>();
9531        assert_eq!(
9532            child_identities,
9533            vec!["test/review-worker-alpha", "test/review-worker-beta"]
9534        );
9535    }
9536
9537    #[tokio::test]
9538    async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
9539        let aggregator = MobKitConsoleAggregator::in_memory();
9540        aggregator
9541            .inner
9542            .identity_read_model
9543            .replace(vec![
9544                identity_record_for_test("test/review:singleton"),
9545                identity_record_for_test("test/review-worker-alpha"),
9546            ])
9547            .await;
9548        let frames = frames_from_session_history_message_with_namespace(
9549            "runtime-a",
9550            "test/review:singleton",
9551            "test",
9552            "session-a",
9553            11,
9554            json!({
9555                "role": "block_assistant",
9556                "blocks": [
9557                    {
9558                        "block_type": "tool_use",
9559                        "data": {
9560                            "id": "call-spawn",
9561                            "name": "mob_spawn_member",
9562                            "args": {
9563                                "agent_identity": "review-worker-alpha",
9564                                "profile": "review-worker",
9565                                "initial_message": "Review Initiative Alpha."
9566                            }
9567                        }
9568                    }
9569                ],
9570                "stop_reason": "tool_use",
9571                "created_at": "1970-01-01T00:00:00.110Z"
9572            }),
9573        );
9574        for frame in frames {
9575            aggregator
9576                .store()
9577                .append_if_absent(frame)
9578                .await
9579                .expect("append projected namespaced session-history frame");
9580        }
9581
9582        let namespaced_child_page = aggregator
9583            .query_timeline(ConsoleTimelineQuery {
9584                identity: Some("test/review-worker-alpha".to_string()),
9585                limit: 20,
9586                ..ConsoleTimelineQuery::default()
9587            })
9588            .await
9589            .expect("query namespaced child timeline");
9590        assert_eq!(namespaced_child_page.frames.len(), 1);
9591        assert_eq!(
9592            namespaced_child_page.frames[0].identity,
9593            "test/review-worker-alpha"
9594        );
9595        assert_eq!(
9596            namespaced_child_page.frames[0].payload["message"]["content"],
9597            "Review Initiative Alpha."
9598        );
9599
9600        let raw_child_page = aggregator
9601            .query_timeline(ConsoleTimelineQuery {
9602                identity: Some("review-worker-alpha".to_string()),
9603                limit: 20,
9604                ..ConsoleTimelineQuery::default()
9605            })
9606            .await
9607            .expect("query raw child timeline");
9608        assert!(
9609            raw_child_page.frames.is_empty(),
9610            "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
9611            raw_child_page.frames
9612        );
9613    }
9614
9615    #[test]
9616    fn namespace_helpers_preserve_namespace_named_member_identity() {
9617        assert_eq!(apply_namespace("test", "test"), "test/test");
9618        assert_eq!(
9619            strip_namespace("test/test", "test").as_deref(),
9620            Some("test")
9621        );
9622        assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
9623        assert_eq!(
9624            strip_namespace("test/worker", "test").as_deref(),
9625            Some("worker")
9626        );
9627        assert_eq!(strip_namespace("test", "test"), None);
9628    }
9629
9630    #[test]
9631    fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
9632        assert!(!member_id_matches_durable_identity(
9633            "rt:review:singleton:0",
9634            "review:singleton"
9635        ));
9636        assert!(!member_id_matches_durable_identity(
9637            "review:singleton:gen1",
9638            "review:singleton"
9639        ));
9640        assert!(!member_id_matches_durable_identity(
9641            "review:singleton:1",
9642            "review:singleton"
9643        ));
9644        assert!(!member_id_matches_durable_identity(
9645            "rt:review:singleton:qa:0",
9646            "review:singleton"
9647        ));
9648        assert!(!member_id_matches_durable_identity(
9649            "review:singleton:qa",
9650            "review:singleton"
9651        ));
9652    }
9653
9654    #[test]
9655    fn session_history_projection_uses_rfc3339_created_at_timestamp() {
9656        let frame = frame_from_session_history_message(
9657            "runtime-a",
9658            "agent-a",
9659            "session-a",
9660            0,
9661            json!({
9662                "role": "user",
9663                "content": "hello",
9664                "created_at": "2026-05-12T05:00:06.564227Z"
9665            }),
9666        )
9667        .expect("user history frame");
9668
9669        assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
9670    }
9671
9672    async fn wait_for_kickoff_frames(
9673        aggregator: &MobKitConsoleAggregator,
9674        identity: &str,
9675        timeout: Duration,
9676    ) -> Vec<ConsoleFrame> {
9677        let deadline = Instant::now() + timeout;
9678        let mut observed = Vec::new();
9679        while Instant::now() < deadline {
9680            let page = aggregator
9681                .query_timeline(ConsoleTimelineQuery {
9682                    identity: Some(identity.to_string()),
9683                    limit: 50,
9684                    ..ConsoleTimelineQuery::default()
9685                })
9686                .await
9687                .expect("query timeline");
9688            observed = page.frames;
9689            if observed.iter().any(|frame| frame.kind == "user_input") {
9690                break;
9691            }
9692            tokio::time::sleep(Duration::from_millis(25)).await;
9693        }
9694        observed
9695    }
9696
9697    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9698    async fn spawn_registry_primes_access_lineage() {
9699        let runtime = build_empty_runtime("spawn-access-lineage-test").await;
9700        let sink = runtime
9701            .mob_runtime()
9702            .console_spawn_sink()
9703            .expect("unified runtime installs the console spawn sink at bootstrap");
9704
9705        runtime
9706            .spawn(SpawnMemberSpec::from_wire(
9707                "worker".to_string(),
9708                "worker-3".to_string(),
9709                None,
9710                None,
9711                None,
9712            ))
9713            .await
9714            .expect("member spawns");
9715        sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
9716            mob_id: Some("spawn-access-lineage-test".to_string()),
9717            member_id: "worker-3".to_string(),
9718            identity: "worker-3".to_string(),
9719            initial_message: None,
9720            labels: BTreeMap::new(),
9721            spawned_by: Some("ops-lead".to_string()),
9722            via_tool: "mob_spawn_member".to_string(),
9723        })
9724        .await;
9725
9726        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9727            enabled: true,
9728            admins: vec!["root@example.test".to_string()],
9729            groups: BTreeMap::new(),
9730            rules: vec![crate::access::AccessRule {
9731                id: "bob-ops-lead".to_string(),
9732                subjects: vec!["bob@example.test".to_string()],
9733                actions: vec!["agent.view".to_string()],
9734                agents: vec!["ops-lead".to_string()],
9735                ..crate::access::AccessRule::default()
9736            }],
9737        })
9738        .expect("controller");
9739
9740        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9741            .await;
9742
9743        let bob = controller.view_for_subject(Some("bob@example.test"));
9744        assert!(
9745            bob.can_view_agent("worker-3"),
9746            "spawn registry lineage must let bob see the member ops-lead spawned"
9747        );
9748        assert!(
9749            !bob.can_view_agent("unrelated-agent"),
9750            "agents outside the granted lineage stay denied"
9751        );
9752
9753        let _ = runtime.mob_handle().stop().await;
9754    }
9755
9756    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9757    async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
9758        let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
9759
9760        // Server-side spawns with caller-controlled labels and NO spawn
9761        // registry entry: lineage claims in roster labels are untrusted.
9762        let mut imposter = SpawnMemberSpec::from_wire(
9763            "worker".to_string(),
9764            "imposter".to_string(),
9765            None,
9766            None,
9767            None,
9768        );
9769        imposter.labels = Some(BTreeMap::from([(
9770            "spawned_by".to_string(),
9771            "ops-lead".to_string(),
9772        )]));
9773        runtime.spawn(imposter).await.expect("imposter spawns");
9774
9775        let mut evader = SpawnMemberSpec::from_wire(
9776            "worker".to_string(),
9777            "evader".to_string(),
9778            None,
9779            None,
9780            None,
9781        );
9782        evader.labels = Some(BTreeMap::from([(
9783            "spawned_by".to_string(),
9784            "secret-lead".to_string(),
9785        )]));
9786        runtime.spawn(evader).await.expect("evader spawns");
9787
9788        // The console identity list must not present unverified lineage.
9789        let aggregator = MobKitConsoleAggregator::in_memory();
9790        aggregator.register_runtime_handles_with_policy(
9791            "runtime-a",
9792            "",
9793            runtime.mob_runtime().clone(),
9794            None,
9795            runtime.console_events(),
9796            Arc::new(AllowAllConsoleVisibilityPolicy),
9797        );
9798        let identities = aggregator
9799            .list_identities_fresh()
9800            .await
9801            .expect("list identities");
9802        let imposter_record = identities
9803            .iter()
9804            .find(|record| record.identity == "imposter")
9805            .expect("imposter listed");
9806        assert!(
9807            !imposter_record.labels.contains_key("spawned_by"),
9808            "identity records must drop roster-claimed lineage when the spawn registry has none"
9809        );
9810
9811        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
9812            enabled: true,
9813            admins: vec!["root@example.test".to_string()],
9814            groups: BTreeMap::new(),
9815            rules: vec![
9816                crate::access::AccessRule {
9817                    id: "bob-ops-lead".to_string(),
9818                    subjects: vec!["bob@example.test".to_string()],
9819                    actions: vec!["agent.view".to_string()],
9820                    agents: vec!["ops-lead".to_string()],
9821                    ..crate::access::AccessRule::default()
9822                },
9823                crate::access::AccessRule {
9824                    id: "carol-view-all".to_string(),
9825                    subjects: vec!["carol@example.test".to_string()],
9826                    actions: vec!["agent.view".to_string()],
9827                    agents: vec!["*".to_string()],
9828                    ..crate::access::AccessRule::default()
9829                },
9830                crate::access::AccessRule {
9831                    id: "hide-secret-lead".to_string(),
9832                    effect: crate::access::AccessEffect::Deny,
9833                    actions: vec!["agent.*".to_string()],
9834                    agents: vec!["secret-lead".to_string()],
9835                    ..crate::access::AccessRule::default()
9836                },
9837            ],
9838        })
9839        .expect("controller");
9840
9841        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
9842            .await;
9843
9844        let bob = controller.view_for_subject(Some("bob@example.test"));
9845        assert!(
9846            !bob.can_view_agent("imposter"),
9847            "a roster-claimed spawned_by must not mint inherited visibility"
9848        );
9849        let carol = controller.view_for_subject(Some("carol@example.test"));
9850        assert!(
9851            carol.can_view_agent("evader"),
9852            "a roster-claimed spawned_by must not let a member hide behind a denied parent"
9853        );
9854
9855        let _ = runtime.mob_handle().stop().await;
9856    }
9857
9858    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9859    async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
9860        let runtime = build_empty_runtime("agent-spawn-console-test").await;
9861
9862        // The unified runtime must hand the agent-tool spawn path a console
9863        // sink at bootstrap — without it, agent-spawned members stay
9864        // invisible until their first runtime activity.
9865        let sink = runtime
9866            .mob_runtime()
9867            .console_spawn_sink()
9868            .expect("unified runtime installs the console spawn sink at bootstrap");
9869
9870        // The member exists in the roster exactly as an agent-tool spawn
9871        // would leave it; the server-side spawn carries no initial message
9872        // so the projected kickoff below is the only chat seed.
9873        runtime
9874            .spawn(SpawnMemberSpec::from_wire(
9875                "worker".to_string(),
9876                "worker-3".to_string(),
9877                None,
9878                None,
9879                None,
9880            ))
9881            .await
9882            .expect("member spawns");
9883
9884        let seed = crate::console_spawn::ConsoleSpawnSeed {
9885            mob_id: Some("agent-spawn-console-test".to_string()),
9886            member_id: "worker-3".to_string(),
9887            identity: "worker-3".to_string(),
9888            initial_message: Some(json!("Kick off the person hunt")),
9889            labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
9890            spawned_by: Some("ops-lead".to_string()),
9891            via_tool: "mob_spawn_member".to_string(),
9892        };
9893        sink.project_spawned_member(&seed).await;
9894        // Spawn retries must not fork or duplicate the chat.
9895        sink.project_spawned_member(&seed).await;
9896
9897        let aggregator = MobKitConsoleAggregator::in_memory();
9898        aggregator.register_runtime_handles_with_policy(
9899            "runtime-a",
9900            "test",
9901            runtime.mob_runtime().clone(),
9902            None,
9903            runtime.console_events(),
9904            Arc::new(AllowAllConsoleVisibilityPolicy),
9905        );
9906
9907        let frames =
9908            wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
9909        let kickoffs = frames
9910            .iter()
9911            .filter(|frame| frame.kind == "user_input")
9912            .collect::<Vec<_>>();
9913        assert_eq!(
9914            kickoffs.len(),
9915            1,
9916            "member timeline must contain exactly one kickoff frame; got {frames:#?}"
9917        );
9918        assert_eq!(
9919            kickoffs[0].payload["content"][0]["text"],
9920            "Kick off the person hunt"
9921        );
9922
9923        // A live runtime event for the member must aggregate into the same
9924        // identity/chat as the kickoff, not fork into a second conversation.
9925        runtime
9926            .project_console_event_from_unified(&crate::types::EventEnvelope {
9927                event_id: "evt-live-worker-3".to_string(),
9928                source: "test".to_string(),
9929                timestamp_ms: current_time_ms(),
9930                event: crate::types::UnifiedEvent::Agent {
9931                    agent_id: "worker-3:0".to_string(),
9932                    event_type: "text_delta".to_string(),
9933                    payload: Some(json!({ "delta": "on it" })),
9934                },
9935            })
9936            .await;
9937
9938        let deadline = Instant::now() + Duration::from_secs(5);
9939        let mut live_frame = None;
9940        while Instant::now() < deadline && live_frame.is_none() {
9941            let page = aggregator
9942                .query_timeline(ConsoleTimelineQuery {
9943                    identity: Some("test/worker-3".to_string()),
9944                    limit: 50,
9945                    ..ConsoleTimelineQuery::default()
9946                })
9947                .await
9948                .expect("query timeline");
9949            live_frame = page
9950                .frames
9951                .into_iter()
9952                .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
9953            if live_frame.is_none() {
9954                tokio::time::sleep(Duration::from_millis(25)).await;
9955            }
9956        }
9957        let live_frame = live_frame.expect("live runtime event lands in the member chat");
9958        assert_eq!(live_frame.identity, "test/worker-3");
9959
9960        // The identity list must carry the registered spawn labels so the
9961        // sidebar can group the member and ABAC can see its lineage.
9962        let identities = aggregator
9963            .list_identities_fresh()
9964            .await
9965            .expect("list identities");
9966        let record = identities
9967            .iter()
9968            .find(|record| record.identity == "test/worker-3")
9969            .unwrap_or_else(|| {
9970                panic!("spawned member missing from identity list: {identities:#?}")
9971            });
9972        assert_eq!(
9973            record.labels.get("group").map(String::as_str),
9974            Some("workers")
9975        );
9976        assert_eq!(
9977            record.labels.get("spawned_by").map(String::as_str),
9978            Some("ops-lead")
9979        );
9980
9981        let _ = runtime.mob_handle().stop().await;
9982    }
9983}