Skip to main content

meerkat_mobkit/console_aggregator/
mod.rs

1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::AgentIdentity;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23    MobRuntime, MobRuntimeError, assert_member_accepts_images,
24    is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30    ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31    SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32    SourceIngestionTransition,
33};
34pub use store::{
35    ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36    SqliteConsoleLogStore,
37};
38pub use types::{
39    AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40    ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41    ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42    ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43    ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59    CachedOnly,
60    IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65    inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70    pub session_history_backfill_enabled: bool,
71    pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75    fn default() -> Self {
76        Self {
77            session_history_backfill_enabled: true,
78            max_concurrent_session_backfills: 16,
79        }
80    }
81}
82
83struct AggregatorInner {
84    store: Arc<dyn ConsoleLogStore>,
85    runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86    event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87    active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88    opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89    session_backfill_permits: Arc<Semaphore>,
90    identity_read_model: ConsoleIdentityReadModel,
91    options: ConsoleAggregatorOptions,
92    /// Per-runtime shutdown signals for the live-projection tasks spawned by
93    /// [`ConsoleAggregator::register_runtime_handles_with_policy`]. Without
94    /// them, every registration leaked an immortal projection task (its
95    /// broadcast receiver can never observe `Closed` while the runtime Arc
96    /// lives) after the runtime was destroyed (issue #254 follow-up).
97    live_projection_shutdowns: std::sync::Mutex<BTreeMap<String, tokio::sync::watch::Sender<bool>>>,
98}
99
100#[derive(Clone)]
101struct ConsoleIdentityReadModel {
102    inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
103    refresh_lock: Arc<tokio::sync::Mutex<()>>,
104    primed: Arc<AtomicBool>,
105}
106
107impl Default for ConsoleIdentityReadModel {
108    fn default() -> Self {
109        Self {
110            inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
111            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
112            primed: Arc::new(AtomicBool::new(false)),
113        }
114    }
115}
116
117impl ConsoleIdentityReadModel {
118    async fn snapshot(
119        &self,
120        inner: Arc<AggregatorInner>,
121    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
122        if !self.primed.load(Ordering::Acquire) {
123            Box::pin(self.prime_now(inner)).await?;
124        }
125        Ok(self.inner.read().await.clone())
126    }
127
128    async fn current(&self) -> Vec<ConsoleIdentityRecord> {
129        self.inner.read().await.clone()
130    }
131
132    async fn refresh_now_if_idle(
133        &self,
134        inner: Arc<AggregatorInner>,
135        mode: IdentityCollectionMode,
136    ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
137        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
138            return None;
139        };
140        let result = Box::pin(collect_identity_records(&inner, mode)).await;
141        drop(guard);
142        match result {
143            Ok(identities) => {
144                self.replace(identities.clone()).await;
145                Some(Ok(identities))
146            }
147            Err(err) => Some(Err(err)),
148        }
149    }
150
151    async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
152        if self.primed.load(Ordering::Acquire) {
153            return Ok(());
154        }
155        let _guard = self.refresh_lock.clone().lock_owned().await;
156        if self.primed.load(Ordering::Acquire) {
157            return Ok(());
158        }
159        let identities = Box::pin(collect_identity_records(
160            &inner,
161            IdentityCollectionMode::CachedOnly,
162        ))
163        .await?;
164        *self.inner.write().await = identities;
165        self.primed.store(true, Ordering::Release);
166        Ok(())
167    }
168
169    fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
170        let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
171            return;
172        };
173        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
174            return;
175        };
176        let read_model = self.clone();
177        runtime_handle.spawn(async move {
178            let _guard = guard;
179            match Box::pin(collect_identity_records(
180                &inner,
181                IdentityCollectionMode::IncludeLiveMembers,
182            ))
183            .await
184            {
185                Ok(identities) => {
186                    *read_model.inner.write().await = identities;
187                    read_model.primed.store(true, Ordering::Release);
188                }
189                Err(err) => {
190                    tracing::warn!(error = %err, "console identity read-model refresh failed");
191                }
192            }
193        });
194    }
195
196    async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
197        *self.inner.write().await = identities;
198        self.primed.store(true, Ordering::Release);
199    }
200}
201
202#[derive(Clone)]
203struct RuntimeEntry {
204    runtime_key: String,
205    identity_namespace: String,
206    runtime: MobRuntime,
207    identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
208    console_events: ConsoleEventStore,
209    visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
210}
211
212#[derive(Clone)]
213struct ResolvedConsoleMember {
214    entry: RuntimeEntry,
215    handle: MobHandle,
216    member: MobMemberListEntry,
217    source_mob_id: String,
218    runtime_identity: String,
219}
220
221fn console_member_for_resolved_member(
222    resolved: &ResolvedConsoleMember,
223    record: &ConsoleIdentityRecord,
224) -> ConsoleMember {
225    ConsoleMember {
226        agent_identity: crate::member_comms_id::runtime_alias_str(
227            resolved.member.agent_identity.as_str(),
228        )
229        .into_owned(),
230        role: resolved.member.role.to_string(),
231        state: crate::mob_handle_runtime::member_status_state_string(resolved.member.status),
232        model_capabilities: Default::default(),
233        runtime_mode: Some(resolved.member.runtime_mode.to_string()),
234        session_id: record.session_id.clone(),
235        wired_to: resolved
236            .member
237            .wired_to
238            .iter()
239            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
240            .collect(),
241        labels: record.labels.clone(),
242        progress: None,
243    }
244}
245
246async fn resolved_member_visible(
247    resolved: &ResolvedConsoleMember,
248    record: &ConsoleIdentityRecord,
249) -> bool {
250    if !raw_resolved_member_visible(resolved, record) {
251        return false;
252    }
253    !Box::pin(live_record_shadowed_by_hidden_durable_binding(
254        resolved, record,
255    ))
256    .await
257}
258
259fn raw_resolved_member_visible(
260    resolved: &ResolvedConsoleMember,
261    record: &ConsoleIdentityRecord,
262) -> bool {
263    resolved.entry.visibility_policy.identity_visible(record)
264        && resolved
265            .entry
266            .visibility_policy
267            .member_visible(&console_member_for_resolved_member(resolved, record))
268}
269
270pub trait ConsoleVisibilityPolicy: Send + Sync {
271    fn include_implicit_delegate_members(&self) -> bool {
272        true
273    }
274
275    fn member_visible(&self, _member: &ConsoleMember) -> bool {
276        true
277    }
278
279    fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
280        true
281    }
282
283    fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
284        true
285    }
286
287    fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
288        None
289    }
290}
291
292#[derive(Debug, Default)]
293pub struct AllowAllConsoleVisibilityPolicy;
294
295impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
296
297#[derive(Debug, Default)]
298pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
299
300impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
301    fn include_implicit_delegate_members(&self) -> bool {
302        false
303    }
304
305    fn member_visible(&self, member: &ConsoleMember) -> bool {
306        !is_implicit_delegate_member(member.role.as_str(), &member.labels)
307    }
308
309    fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
310        !is_implicit_delegate_member(
311            record
312                .labels
313                .get("role")
314                .map(String::as_str)
315                .unwrap_or_default(),
316            &record.labels,
317        )
318    }
319}
320
321#[derive(Clone)]
322pub struct ConsoleRuntimeRegistration {
323    pub runtime_key: String,
324    pub runtime: Arc<UnifiedRuntime>,
325    pub identity_namespace: String,
326    pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
327}
328
329impl MobKitConsoleAggregator {
330    pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
331        Self::new_with_options(store, ConsoleAggregatorOptions::default())
332    }
333
334    pub fn new_with_options(
335        store: Arc<dyn ConsoleLogStore>,
336        mut options: ConsoleAggregatorOptions,
337    ) -> Self {
338        options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
339        let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
340        Self {
341            inner: Arc::new(AggregatorInner {
342                store,
343                runtimes: RwLock::new(BTreeMap::new()),
344                event_tx,
345                active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
346                opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
347                session_backfill_permits: Arc::new(Semaphore::new(
348                    options.max_concurrent_session_backfills,
349                )),
350                identity_read_model: ConsoleIdentityReadModel::default(),
351                options,
352                live_projection_shutdowns: std::sync::Mutex::new(BTreeMap::new()),
353            }),
354        }
355    }
356
357    pub fn in_memory() -> Self {
358        Self::new(Arc::new(InMemoryConsoleLogStore::new()))
359    }
360
361    pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
362        Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
363    }
364
365    pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
366        self.inner.event_tx.subscribe()
367    }
368
369    pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
370        self.inner.store.clone()
371    }
372
373    pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
374        let identity_runtime = registration.runtime.identity_runtime().cloned();
375        self.register_runtime_handles_with_policy(
376            registration.runtime_key,
377            registration.identity_namespace,
378            registration.runtime.mob_runtime().clone(),
379            identity_runtime,
380            registration.runtime.console_events(),
381            registration.visibility_policy,
382        );
383    }
384
385    pub(crate) fn register_runtime_handles_with_policy(
386        &self,
387        runtime_key: impl Into<String>,
388        identity_namespace: impl Into<String>,
389        runtime: MobRuntime,
390        identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
391        console_events: ConsoleEventStore,
392        visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
393    ) {
394        let runtime_key = runtime_key.into();
395        let identity_namespace = identity_namespace.into();
396        let entry = RuntimeEntry {
397            runtime_key: runtime_key.clone(),
398            identity_namespace,
399            runtime,
400            identity_runtime,
401            console_events: console_events.clone(),
402            visibility_policy,
403        };
404        if let Ok(mut runtimes) = self.inner.runtimes.write() {
405            runtimes.insert(runtime_key.clone(), entry);
406        }
407        // Re-registering a key replaces its live-projection task: signal the
408        // old one before spawning the new (otherwise both would project).
409        let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false);
410        if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
411            && let Some(previous) = shutdowns.insert(runtime_key.clone(), shutdown_tx)
412        {
413            let _ = previous.send(true);
414        }
415        self.inner
416            .identity_read_model
417            .refresh_soon(self.inner.clone());
418        let inner = self.inner.clone();
419        let events_for_live = console_events.clone();
420        let events_for_live_recovery = console_events.clone();
421        let runtime_key_for_live = runtime_key.clone();
422        tokio::spawn(async move {
423            let mut rx = events_for_live.subscribe();
424            loop {
425                tokio::select! {
426                    changed = shutdown_rx.changed() => {
427                        // Sender dropped or unregister signalled: stop.
428                        if changed.is_err() || *shutdown_rx.borrow() {
429                            break;
430                        }
431                    }
432                    received = rx.recv() => match received {
433                        Ok(envelope) => {
434                            let _ = project_console_event(
435                                inner.clone(),
436                                &runtime_key_for_live,
437                                envelope,
438                            )
439                            .await;
440                        }
441                        Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
442                            let _ = recover_lagged_source_events(
443                                inner.clone(),
444                                &runtime_key_for_live,
445                                &events_for_live_recovery,
446                            )
447                            .await;
448                        }
449                        Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
450                    },
451                }
452            }
453        });
454
455        let inner = self.inner.clone();
456        let events_for_replay = console_events;
457        let runtime_key_for_replay = runtime_key;
458        tokio::spawn(async move {
459            let mut ingestion_state = SourceIngestionState::Registered;
460            if let Ok((next, _effects)) =
461                ingestion_state.apply(SourceIngestionTransition::StartBackfill)
462            {
463                ingestion_state = next;
464            }
465            if let Ok(events) = events_for_replay.replay_all(None).await {
466                for envelope in events {
467                    let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
468                        .await;
469                }
470            }
471            spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
472            spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
473            if let Ok((next, _effects)) =
474                ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
475            {
476                ingestion_state = next;
477            }
478            let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
479        });
480    }
481
482    /// Unregister a runtime (issue #254 follow-up): removes the registry
483    /// entry (which also terminates the 5s session-history discovery loop on
484    /// its next tick — it checks registration), signals the live-projection
485    /// task to stop (its broadcast receiver would otherwise never observe
486    /// `Closed` while the runtime's event store lives), and refreshes the
487    /// identity read model so the dead runtime's identities leave the
488    /// console. Frames already projected into the store remain queryable —
489    /// unregister detaches the LIVE source, it does not erase history.
490    /// Idempotent; unknown keys are a no-op.
491    pub fn unregister_runtime(&self, runtime_key: &str) {
492        let removed = self
493            .inner
494            .runtimes
495            .write()
496            .ok()
497            .and_then(|mut runtimes| runtimes.remove(runtime_key))
498            .is_some();
499        if let Ok(mut shutdowns) = self.inner.live_projection_shutdowns.lock()
500            && let Some(shutdown) = shutdowns.remove(runtime_key)
501        {
502            let _ = shutdown.send(true);
503        }
504        if removed {
505            self.inner
506                .identity_read_model
507                .refresh_soon(self.inner.clone());
508        }
509    }
510
511    pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
512        if let Some(identities) = Box::pin(self.inner.identity_read_model.refresh_now_if_idle(
513            self.inner.clone(),
514            IdentityCollectionMode::IncludeLiveMembers,
515        ))
516        .await
517        {
518            let identities = identities?;
519            spawn_identity_backfills_for_records(self.inner.clone(), &identities);
520            return Ok(identities);
521        }
522        self.inner
523            .identity_read_model
524            .refresh_soon(self.inner.clone());
525        let identities =
526            Box::pin(self.inner.identity_read_model.snapshot(self.inner.clone())).await?;
527        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
528        Ok(identities)
529    }
530
531    #[cfg(test)]
532    pub(crate) async fn list_identities_fresh(
533        &self,
534    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
535        let _guard = self
536            .inner
537            .identity_read_model
538            .refresh_lock
539            .clone()
540            .lock_owned()
541            .await;
542        let identities =
543            collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
544                .await?;
545        self.inner
546            .identity_read_model
547            .replace(identities.clone())
548            .await;
549        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
550        Ok(identities)
551    }
552
553    pub async fn inspect_identity(
554        &self,
555        identity: &str,
556    ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
557        let entries = self
558            .inner
559            .runtimes
560            .read()
561            .map_err(|_| runtime_registry_lock_error())?
562            .values()
563            .cloned()
564            .collect::<Vec<_>>();
565        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
566        let mut durable_matches = Vec::new();
567        if !requested_identity_has_runtime_member_alias(identity, &entries) {
568            for entry in &entries {
569                let Some(identity_runtime) = entry.identity_runtime.clone() else {
570                    continue;
571                };
572                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
573                {
574                    let Ok(parsed_identity) =
575                        crate::identity_first::AgentIdentity::parse(&raw_identity)
576                    else {
577                        continue;
578                    };
579                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
580                        continue;
581                    };
582                    let mut record = identity_record_for_status(entry, &status);
583                    let topology_peers =
584                        identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
585                    record.topology_peers = topology_peers
586                        .get(&record.identity)
587                        .cloned()
588                        .unwrap_or_default();
589                    if Box::pin(console_identity_record_visible(entry, &record)).await {
590                        durable_matches.push((
591                            entry.clone(),
592                            identity_runtime.clone(),
593                            parsed_identity,
594                            record,
595                        ));
596                    }
597                }
598            }
599        }
600        if durable_matches.len() > 1 {
601            let candidates = durable_matches
602                .iter()
603                .map(|(_, _, _, record)| record.runtime_member_id.clone())
604                .collect::<Vec<_>>()
605                .join(", ");
606            return Err(format!(
607                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
608            )
609            .into());
610        }
611        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
612            durable_matches.into_iter().next()
613        {
614            let durable_live_records = Box::pin(self.live_records_for_durable_record(
615                &entry,
616                identity,
617                &record,
618                &live_records,
619            ))
620            .await;
621            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
622                &entry,
623                &durable_live_records,
624            ))
625            .await;
626            if let Some(ambiguous_error) =
627                ambiguous_live_alias_error(identity, &visible_durable_live_records)
628            {
629                return Err(ambiguous_error.into());
630            }
631            if let Some(rebound_record) = self_heal_durable_session_mismatch(
632                &entry,
633                identity_runtime.as_ref(),
634                &parsed_identity,
635                identity,
636                &record,
637                &durable_live_records,
638            )
639            .await
640            .map_err(ConsoleLogError::from)?
641            {
642                record = rebound_record;
643            }
644            if let Some(stale_error) =
645                stale_durable_record_error(identity, &record, &durable_live_records)
646            {
647                return Err(stale_error.into());
648            }
649            if let Some(session_id) = record.session_id.clone() {
650                spawn_session_history_backfill_target(
651                    self.inner.clone(),
652                    SessionBackfillTarget {
653                        entry,
654                        record: record.clone(),
655                        session_id,
656                    },
657                    false,
658                );
659            }
660            return Ok(Some(ConsoleIdentityInspection {
661                peers: record.topology_peers.clone(),
662                identity: record,
663            }));
664        }
665        let mut runtime_id_matches = Vec::new();
666        for entry in &entries {
667            let Some(identity_runtime) = entry.identity_runtime.clone() else {
668                continue;
669            };
670            let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
671                continue;
672            };
673            let Some(status) = identity_runtime
674                .statuses()
675                .await
676                .into_iter()
677                .find(|status| {
678                    status
679                        .agent_runtime_id
680                        .as_ref()
681                        .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
682                })
683            else {
684                continue;
685            };
686            let mut record = identity_record_for_status(entry, &status);
687            let topology_peers =
688                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
689            record.topology_peers = topology_peers
690                .get(&record.identity)
691                .cloned()
692                .unwrap_or_default();
693            if Box::pin(console_identity_record_visible(entry, &record)).await {
694                runtime_id_matches.push((entry.clone(), identity_runtime, status.identity, record));
695            }
696        }
697        if runtime_id_matches.len() > 1 {
698            let candidates = runtime_id_matches
699                .iter()
700                .map(|(_, _, _, record)| record.identity.clone())
701                .collect::<Vec<_>>()
702                .join(", ");
703            return Err(format!(
704                "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
705            )
706            .into());
707        }
708        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
709            runtime_id_matches.into_iter().next()
710        {
711            let durable_live_records = Box::pin(self.live_records_for_durable_record(
712                &entry,
713                identity,
714                &record,
715                &live_records,
716            ))
717            .await;
718            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
719                &entry,
720                &durable_live_records,
721            ))
722            .await;
723            if let Some(ambiguous_error) =
724                ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
725            {
726                return Err(ambiguous_error.into());
727            }
728            if let Some(rebound_record) = self_heal_durable_session_mismatch(
729                &entry,
730                identity_runtime.as_ref(),
731                &parsed_identity,
732                identity,
733                &record,
734                &durable_live_records,
735            )
736            .await
737            .map_err(ConsoleLogError::from)?
738            {
739                record = rebound_record;
740            }
741            if let Some(stale_error) =
742                stale_durable_record_error(identity, &record, &durable_live_records)
743            {
744                return Err(stale_error.into());
745            }
746            if let Some(session_id) = record.session_id.clone() {
747                spawn_session_history_backfill_target(
748                    self.inner.clone(),
749                    SessionBackfillTarget {
750                        entry: entry.clone(),
751                        record: record.clone(),
752                        session_id,
753                    },
754                    false,
755                );
756            }
757            return Ok(Some(ConsoleIdentityInspection {
758                peers: record.topology_peers.clone(),
759                identity: record,
760            }));
761        }
762
763        let live_matches = Box::pin(self.resolve_visible_members(identity)).await;
764        if live_matches.len() > 1 {
765            let candidates = live_matches
766                .iter()
767                .map(|resolved| resolved.runtime_identity.clone())
768                .collect::<Vec<_>>()
769                .join(", ");
770            return Err(format!(
771                "ambiguous live identity alias {identity}: candidates [{candidates}]"
772            )
773            .into());
774        }
775        if let Some(resolved) = live_matches.into_iter().next() {
776            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
777                return Ok(None);
778            };
779            if let Some(stale_error) =
780                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
781            {
782                return Err(stale_error.into());
783            }
784            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
785                && let Ok(parsed_identity) =
786                    crate::identity_first::AgentIdentity::parse(&record.identity)
787                && let Ok(status) = identity_runtime.status(&parsed_identity).await
788            {
789                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
790                let durable_live_records = Box::pin(self.live_records_for_durable_record(
791                    &resolved.entry,
792                    identity,
793                    &durable_record,
794                    &live_records,
795                ))
796                .await;
797                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
798                    &resolved.entry,
799                    &durable_live_records,
800                ))
801                .await;
802                if let Some(ambiguous_error) =
803                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
804                {
805                    return Err(ambiguous_error.into());
806                }
807                if let Some(rebound_record) = self_heal_durable_session_mismatch(
808                    &resolved.entry,
809                    identity_runtime.as_ref(),
810                    &parsed_identity,
811                    identity,
812                    &durable_record,
813                    &durable_live_records,
814                )
815                .await
816                .map_err(ConsoleLogError::from)?
817                {
818                    durable_record = rebound_record;
819                }
820                if let Some(stale_error) =
821                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
822                {
823                    return Err(stale_error.into());
824                }
825                if status
826                    .agent_runtime_id
827                    .as_ref()
828                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
829                {
830                    return Ok(None);
831                }
832            }
833            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
834                return Ok(None);
835            }
836            if let Some(session_id) = record.session_id.clone() {
837                spawn_session_history_backfill_target(
838                    self.inner.clone(),
839                    SessionBackfillTarget {
840                        entry: resolved.entry.clone(),
841                        record: record.clone(),
842                        session_id,
843                    },
844                    false,
845                );
846            }
847            let peers = resolved
848                .member
849                .wired_to
850                .iter()
851                .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
852                .collect();
853            return Ok(Some(ConsoleIdentityInspection {
854                identity: record,
855                peers,
856            }));
857        }
858
859        Ok(None)
860    }
861
862    pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
863        let entries = self
864            .inner
865            .runtimes
866            .read()
867            .map_err(|_| runtime_registry_lock_error())?
868            .clone();
869        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
870        let entries_vec = entries.values().cloned().collect::<Vec<_>>();
871        let mut durable_matches = Vec::new();
872        if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
873            for entry in entries.values() {
874                let Some(identity_runtime) = entry.identity_runtime.clone() else {
875                    continue;
876                };
877                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
878                {
879                    let Ok(parsed_identity) =
880                        crate::identity_first::AgentIdentity::parse(&raw_identity)
881                    else {
882                        continue;
883                    };
884                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
885                        continue;
886                    };
887                    let record = identity_record_for_status(entry, &status);
888                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
889                        continue;
890                    }
891                    durable_matches.push((
892                        entry.clone(),
893                        identity_runtime.clone(),
894                        parsed_identity,
895                        record,
896                    ));
897                }
898            }
899        }
900        if durable_matches.len() > 1 {
901            let candidates = durable_matches
902                .iter()
903                .map(|(_, _, identity, _)| identity.as_str().to_string())
904                .collect::<Vec<_>>()
905                .join(", ");
906            return Err(format!(
907                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
908            )
909            .into());
910        }
911        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
912            durable_matches.into_iter().next()
913        {
914            let durable_live_records = Box::pin(self.live_records_for_durable_record(
915                &entry,
916                identity,
917                &record,
918                &live_records,
919            ))
920            .await;
921            let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
922                &entry,
923                &durable_live_records,
924            ))
925            .await;
926            if let Some(ambiguous_error) =
927                ambiguous_live_alias_error(identity, &visible_durable_live_records)
928            {
929                return Err(ambiguous_error.into());
930            }
931            if let Some(rebound_record) = self_heal_durable_session_mismatch(
932                &entry,
933                identity_runtime.as_ref(),
934                &parsed_identity,
935                identity,
936                &record,
937                &durable_live_records,
938            )
939            .await
940            .map_err(ConsoleLogError::from)?
941            {
942                record = rebound_record;
943            }
944            if let Some(stale_error) =
945                stale_durable_record_error(identity, &record, &durable_live_records)
946            {
947                return Err(stale_error.into());
948            }
949            identity_runtime
950                .retire(&parsed_identity)
951                .await
952                .map_err(|err| -> ConsoleLogError {
953                    format!("retire failed for {identity}: {err}").into()
954                })?;
955            {
956                let mut durable_identities =
957                    namespace_match_candidates(identity, &entry.identity_namespace);
958                durable_identities.push(parsed_identity.as_str().to_string());
959                durable_identities.sort();
960                durable_identities.dedup();
961                if let Err(err) = Box::pin(retire_stale_console_members_for_runtime_entry(
962                    &entry,
963                    &durable_identities,
964                ))
965                .await
966                {
967                    tracing::warn!(
968                        identity,
969                        error = %err,
970                        "stale console member cleanup failed after durable identity retire"
971                    );
972                }
973            }
974            return Ok(true);
975        }
976        let matches = Box::pin(self.resolve_visible_members(identity)).await;
977        if matches.len() > 1 {
978            let candidates = matches
979                .iter()
980                .map(|resolved| resolved.runtime_identity.clone())
981                .collect::<Vec<_>>()
982                .join(", ");
983            return Err(format!(
984                "ambiguous live identity alias {identity}: candidates [{candidates}]"
985            )
986            .into());
987        }
988        let mut live_retired_any = false;
989        for resolved in matches {
990            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
991                continue;
992            };
993            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
994                continue;
995            }
996            if let Some(stale_error) =
997                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
998            {
999                return Err(stale_error.into());
1000            }
1001            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
1002                && let Ok(parsed_identity) =
1003                    crate::identity_first::AgentIdentity::parse(&record.identity)
1004                && let Ok(status) = identity_runtime.status(&parsed_identity).await
1005            {
1006                let mut durable_record = identity_record_for_status(&resolved.entry, &status);
1007                let durable_live_records = Box::pin(self.live_records_for_durable_record(
1008                    &resolved.entry,
1009                    identity,
1010                    &durable_record,
1011                    &live_records,
1012                ))
1013                .await;
1014                let visible_durable_live_records = Box::pin(visible_live_records_for_entry(
1015                    &resolved.entry,
1016                    &durable_live_records,
1017                ))
1018                .await;
1019                if let Some(ambiguous_error) =
1020                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
1021                {
1022                    return Err(ambiguous_error.into());
1023                }
1024                if let Some(rebound_record) = self_heal_durable_session_mismatch(
1025                    &resolved.entry,
1026                    identity_runtime.as_ref(),
1027                    &parsed_identity,
1028                    identity,
1029                    &durable_record,
1030                    &durable_live_records,
1031                )
1032                .await
1033                .map_err(ConsoleLogError::from)?
1034                {
1035                    durable_record = rebound_record;
1036                }
1037                if let Some(stale_error) =
1038                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
1039                {
1040                    return Err(stale_error.into());
1041                }
1042                if status
1043                    .agent_runtime_id
1044                    .as_ref()
1045                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
1046                {
1047                    continue;
1048                }
1049            }
1050            resolved
1051                .handle
1052                .retire(crate::member_comms_id::mob_member_id(
1053                    resolved.runtime_identity.as_str(),
1054                ))
1055                .await
1056                .map_err(|err| -> ConsoleLogError {
1057                    format!("retire failed for {identity}: {err}").into()
1058                })
1059                .or_else(|err| {
1060                    if lifecycle_archive_cleanup_completed(&err.to_string()) {
1061                        Ok(())
1062                    } else {
1063                        Err(err)
1064                    }
1065                })?;
1066            live_retired_any = true;
1067        }
1068        Ok(live_retired_any)
1069    }
1070
1071    pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
1072        self.inner.store.clear_frames().await
1073    }
1074
1075    pub async fn query_timeline(
1076        &self,
1077        query: ConsoleTimelineQuery,
1078    ) -> ConsoleLogResult<ConsoleTimelinePage> {
1079        let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
1080        Ok(ConsoleTimelinePage {
1081            frames: page.frames,
1082            next_cursor: page.next_cursor,
1083        })
1084    }
1085
1086    pub async fn query_timeline_windowed(
1087        &self,
1088        query: ConsoleTimelineWindowQuery,
1089    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1090        self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
1091            .await?;
1092        if query.after.is_none()
1093            && query.before.is_none()
1094            && let Some(identity) = query.identity.clone()
1095        {
1096            let mut probe_query = query.clone();
1097            probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1098            let page = self.inner.store.query_windowed_frames(probe_query).await?;
1099            if explicit_identity_query_needs_session_history_backfill(&page.frames) {
1100                spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
1101            }
1102        }
1103        Box::pin(self.query_timeline_visible(query)).await
1104    }
1105
1106    async fn reject_after_cursor_beyond_store_frontier(
1107        &self,
1108        after: Option<&ConsoleCursor>,
1109    ) -> ConsoleLogResult<()> {
1110        let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
1111            return Ok(());
1112        };
1113        let latest_seq = self
1114            .inner
1115            .store
1116            .latest_cursor()
1117            .await?
1118            .and_then(|cursor| cursor.seq())
1119            .unwrap_or(0);
1120        if after_seq > latest_seq {
1121            return Err(std::io::Error::other(
1122                "timeline replay cursor is beyond the current store frontier",
1123            )
1124            .into());
1125        }
1126        Ok(())
1127    }
1128
1129    async fn query_timeline_visible(
1130        &self,
1131        query: ConsoleTimelineWindowQuery,
1132    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
1133        let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
1134        let mut scan_query = query.clone();
1135        scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
1136        let mut visible_frames = Vec::with_capacity(requested_limit);
1137        let mut anchor_frames = Vec::new();
1138        let mut identity_visibility_cache = HashMap::new();
1139        let identity_records = self.inner.identity_read_model.current().await;
1140        let mut next_cursor = query.after.clone();
1141        let mut since_last_scanned_cursor = query.after.clone();
1142        let mut since_last_delivered_cursor = None;
1143        let mut since_stopped_at_visible_limit = false;
1144        let mut latest_cursor = None;
1145        let mut exhausted = false;
1146        let mut scanned = 0usize;
1147
1148        loop {
1149            let page = self
1150                .inner
1151                .store
1152                .query_windowed_frames(scan_query.clone())
1153                .await?;
1154            latest_cursor = latest_cursor.or(page.latest_cursor.clone());
1155            if page.frames.is_empty() {
1156                exhausted = true;
1157                break;
1158            }
1159            let raw_len = page.frames.len();
1160            scanned = scanned.saturating_add(raw_len);
1161            match query.mode {
1162                ConsoleTimelineMode::Since => {
1163                    if let Some(cursor) = page.next_cursor.clone() {
1164                        since_last_scanned_cursor = Some(cursor.clone());
1165                        scan_query.after = Some(cursor);
1166                    }
1167                }
1168                ConsoleTimelineMode::Recent => {
1169                    if let Some(first) = page.frames.first() {
1170                        scan_query.before = Some(first.cursor.clone());
1171                    }
1172                    next_cursor = page.next_cursor.clone().or(next_cursor);
1173                }
1174            }
1175
1176            for frame in page.frames {
1177                let allow_historical_identity =
1178                    query.identity.as_deref() == Some(frame.identity.as_str());
1179                if Box::pin(frame_is_visible_cached(
1180                    &self.inner,
1181                    &frame,
1182                    allow_historical_identity,
1183                    &mut identity_visibility_cache,
1184                    &identity_records,
1185                ))
1186                .await
1187                .unwrap_or(false)
1188                {
1189                    if query.mode == ConsoleTimelineMode::Recent
1190                        && query.identity.is_some()
1191                        && is_identity_timeline_anchor_frame(&frame)
1192                        && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1193                    {
1194                        anchor_frames.push(frame.clone());
1195                    }
1196                    match query.mode {
1197                        ConsoleTimelineMode::Since => {
1198                            if visible_frames.len() >= requested_limit {
1199                                since_stopped_at_visible_limit = true;
1200                                break;
1201                            }
1202                            since_last_delivered_cursor = Some(frame.cursor.clone());
1203                            visible_frames.push(frame);
1204                            if visible_frames.len() >= requested_limit {
1205                                since_stopped_at_visible_limit = true;
1206                                exhausted = false;
1207                                break;
1208                            }
1209                        }
1210                        ConsoleTimelineMode::Recent => {
1211                            visible_frames.push(frame);
1212                        }
1213                    }
1214                }
1215            }
1216            let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1217                && query.identity.is_some()
1218                && anchor_frames.is_empty()
1219                && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1220            if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1221                break;
1222            }
1223            if since_stopped_at_visible_limit {
1224                break;
1225            }
1226            if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1227                exhausted = true;
1228                break;
1229            }
1230            if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1231                break;
1232            }
1233        }
1234
1235        if query.mode == ConsoleTimelineMode::Recent {
1236            visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1237            if visible_frames.len() > requested_limit {
1238                visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1239            }
1240            if !anchor_frames.is_empty() {
1241                let anchor_cursors = anchor_frames
1242                    .iter()
1243                    .map(|frame| frame.cursor.clone())
1244                    .collect::<Vec<_>>();
1245                let mut merged = anchor_frames;
1246                for frame in visible_frames {
1247                    if !merged
1248                        .iter()
1249                        .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1250                    {
1251                        merged.push(frame);
1252                    }
1253                }
1254                merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1255                visible_frames = merged;
1256                if visible_frames.len() > requested_limit {
1257                    let mut anchors = Vec::new();
1258                    let mut tail = Vec::new();
1259                    for frame in visible_frames {
1260                        if anchor_cursors.contains(&frame.cursor) {
1261                            anchors.push(frame);
1262                        } else {
1263                            tail.push(frame);
1264                        }
1265                    }
1266                    visible_frames = if anchors.len() >= requested_limit {
1267                        anchors.split_off(anchors.len() - requested_limit)
1268                    } else {
1269                        let keep_tail = requested_limit.saturating_sub(anchors.len());
1270                        if tail.len() > keep_tail {
1271                            tail = tail.split_off(tail.len() - keep_tail);
1272                        }
1273                        anchors.extend(tail);
1274                        anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1275                        anchors
1276                    };
1277                }
1278            }
1279        }
1280
1281        Ok(ConsoleTimelineWindowPage {
1282            frames: visible_frames,
1283            next_cursor: match query.mode {
1284                ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1285                    since_last_delivered_cursor.or(since_last_scanned_cursor)
1286                }
1287                ConsoleTimelineMode::Since => since_last_scanned_cursor,
1288                ConsoleTimelineMode::Recent => next_cursor,
1289            },
1290            latest_cursor,
1291            exhausted,
1292        })
1293    }
1294
1295    pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1296        let runtime_keys = self
1297            .inner
1298            .runtimes
1299            .read()
1300            .map_err(|_| runtime_registry_lock_error())?
1301            .keys()
1302            .cloned()
1303            .collect::<Vec<_>>();
1304        let results =
1305            join_all(runtime_keys.into_iter().map(|runtime_key| {
1306                backfill_session_history(self.inner.clone(), runtime_key, true)
1307            }))
1308            .await;
1309        for result in results {
1310            result?;
1311        }
1312        Ok(())
1313    }
1314
1315    pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1316        self.inner.store.latest_cursor().await
1317    }
1318
1319    pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1320        self.timeline_event_visible_for_subscriber(event, None)
1321            .await
1322    }
1323
1324    /// Live-stream visibility gate, subscriber-aware (issue #254).
1325    ///
1326    /// The windowed query paths grant an own-identity allowance
1327    /// (`allow_historical_identity = query.identity == frame.identity`), but
1328    /// the live gate used to pass `false` unconditionally — so an
1329    /// identity-scoped SSE stream dropped every frame of a member the
1330    /// identity read model had not yet observed (`records=0 → Missing`),
1331    /// starving live tails while reconnect-with-snapshot showed full
1332    /// history. Two fixes, matching the issue's suggestions:
1333    ///
1334    /// 1. An identity-scoped subscriber gets the SAME own-identity
1335    ///    allowance the query paths grant, so its frames flow immediately.
1336    /// 2. A frame whose identity is unknown to the read model triggers a
1337    ///    debounced [`ConsoleIdentityReadModel::refresh_soon`] — the model
1338    ///    converges for UNSCOPED streams too (members spawned mid-run via
1339    ///    `ensure_member` previously stayed invisible until an unrelated
1340    ///    refresh).
1341    pub async fn timeline_event_visible_for_subscriber(
1342        &self,
1343        event: &ConsoleTimelineEvent,
1344        subscriber_identity: Option<&str>,
1345    ) -> bool {
1346        match event {
1347            ConsoleTimelineEvent::ConsoleFrame { frame }
1348            | ConsoleTimelineEvent::FrameUpdated { frame } => {
1349                let identity_records = self.inner.identity_read_model.current().await;
1350                let identity_known = identity_records.iter().any(|record| {
1351                    record.runtime_key == frame.runtime_key
1352                        && (record.identity == frame.identity
1353                            || record.runtime_member_id == frame.identity)
1354                });
1355                if !identity_known
1356                    && frame.identity != "__console__"
1357                    && frame.identity != SYSTEM_EVENT_IDENTITY
1358                {
1359                    // Self-heal: the read model has never seen this identity
1360                    // (e.g. ensure_member mid-run). Debounced internally; a
1361                    // spurious trigger for a namespaced alias mismatch just
1362                    // refreshes early.
1363                    self.inner
1364                        .identity_read_model
1365                        .refresh_soon(self.inner.clone());
1366                }
1367                let allow_historical_identity =
1368                    subscriber_identity == Some(frame.identity.as_str());
1369                Box::pin(frame_is_visible(
1370                    &self.inner,
1371                    frame,
1372                    allow_historical_identity,
1373                    &identity_records,
1374                ))
1375                .await
1376                .unwrap_or(false)
1377            }
1378            ConsoleTimelineEvent::SnapshotStarted { .. }
1379            | ConsoleTimelineEvent::SnapshotComplete { .. }
1380            | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1381        }
1382    }
1383
1384    pub async fn timeline_frame_visible_for_query(
1385        &self,
1386        frame: &ConsoleFrame,
1387        identity: Option<&str>,
1388    ) -> bool {
1389        let allow_historical_identity = identity == Some(frame.identity.as_str());
1390        let identity_records = self.inner.identity_read_model.current().await;
1391        Box::pin(frame_is_visible(
1392            &self.inner,
1393            frame,
1394            allow_historical_identity,
1395            &identity_records,
1396        ))
1397        .await
1398        .unwrap_or(false)
1399    }
1400
1401    pub async fn send(
1402        &self,
1403        request: ConsoleSendRequest,
1404    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1405        validate_send_request(&request)?;
1406        let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1407            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1408        };
1409        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1410            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1411        };
1412        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1413            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1414        }
1415        if !member_is_addressable(&resolved.member) {
1416            return Err(ConsoleSendError::NotAddressable(request.identity));
1417        }
1418        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1419            return Err(ConsoleSendError::Retired(request.identity));
1420        }
1421
1422        let content = content_input_from_value(&request.content)?;
1423        let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1424        assert_member_accepts_images(
1425            &resolved.handle,
1426            resolved.entry.runtime.session_service(),
1427            &resolved.runtime_identity,
1428            &content,
1429        )
1430        .await
1431        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1432
1433        let dedupe_key = send_dedupe_key(
1434            &resolved.entry.runtime_key,
1435            &request.identity,
1436            &request.origin,
1437            &request.idempotency_key,
1438        );
1439        let handling_mode_value = request
1440            .handling_mode
1441            .as_deref()
1442            .unwrap_or("queue")
1443            .to_string();
1444        let request_fingerprint =
1445            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1446        if let Some(existing) = self
1447            .inner
1448            .store
1449            .frame_by_dedupe_key(&dedupe_key)
1450            .await
1451            .map_err(ConsoleSendError::Log)?
1452        {
1453            let same_request = existing.source.source_cursor.as_deref()
1454                == Some(request_fingerprint.as_str())
1455                || existing.source.source_cursor.is_none()
1456                    && existing.payload.get("origin").and_then(Value::as_str)
1457                        == Some(request.origin.as_str())
1458                    && existing.payload.get("content") == Some(&request.content)
1459                    && existing
1460                        .payload
1461                        .get("handling_mode")
1462                        .and_then(Value::as_str)
1463                        == Some(handling_mode_value.as_str());
1464            if !same_request {
1465                return Err(ConsoleSendError::IdempotencyConflict(
1466                    request.idempotency_key,
1467                ));
1468            }
1469            return Ok(accepted_from_frame(&existing));
1470        }
1471
1472        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1473        resolved
1474            .entry
1475            .console_events
1476            .reserve_interaction_value(
1477                &resolved.runtime_identity,
1478                Some(resolved.runtime_identity.as_str()),
1479                &interaction_id,
1480                &request.origin,
1481                request.content.clone(),
1482            )
1483            .await
1484            .map_err(ConsoleSendError::State)?;
1485        let session_id = resolved
1486            .handle
1487            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
1488                resolved.runtime_identity.as_str(),
1489            ))
1490            .await
1491            .map(|sid| sid.to_string());
1492        let mut new_frame = NewConsoleFrame {
1493            id: None,
1494            dedupe_key,
1495            timestamp_ms: current_time_ms(),
1496            runtime_key: resolved.entry.runtime_key.clone(),
1497            identity: request.identity.clone(),
1498            conversation_id: Some(request.identity.clone()),
1499            session_id: session_id.clone(),
1500            kind: "user_input".to_string(),
1501            status: ConsoleFrameStatus::Accepted,
1502            payload: json!({
1503                "content": request.content,
1504                "origin": request.origin,
1505                "idempotency_key": request.idempotency_key,
1506                "handling_mode": handling_mode_value,
1507            }),
1508            source: ConsoleFrameSource {
1509                kind: ConsoleFrameSourceKind::Send,
1510                source_cursor: Some(request_fingerprint),
1511            },
1512            source_event_id: None,
1513            interaction_id: Some(interaction_id.clone()),
1514            turn_id: None,
1515            run_id: None,
1516            parent_frame_id: None,
1517            caused_by_frame_id: None,
1518        };
1519        if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1520            new_frame.payload = redacted;
1521            new_frame.status = ConsoleFrameStatus::Redacted;
1522        }
1523
1524        let _ = SendState::Requested
1525            .apply(SendTransition::PersistAccepted)
1526            .map_err(ConsoleSendError::State)?;
1527        let outcome = self
1528            .inner
1529            .store
1530            .append_if_absent(new_frame)
1531            .await
1532            .map_err(ConsoleSendError::Log)?;
1533        if outcome.disposition == AppendDisposition::Existing {
1534            return Ok(accepted_from_frame(&outcome.frame));
1535        }
1536        let _ = self
1537            .inner
1538            .event_tx
1539            .send(ConsoleTimelineEvent::ConsoleFrame {
1540                frame: outcome.frame.clone(),
1541            });
1542        let accepted = accepted_from_frame(&outcome.frame);
1543
1544        let (dispatching, _effects) = SendState::AcceptedPersisted
1545            .apply(SendTransition::StartDispatch)
1546            .map_err(ConsoleSendError::State)?;
1547        update_frame_status_and_emit(
1548            &self.inner,
1549            &outcome.frame.id,
1550            ConsoleFrameStatus::Dispatching,
1551        )
1552        .await
1553        .map_err(ConsoleSendError::Log)?;
1554
1555        spawn_console_send_dispatch(
1556            self.inner.clone(),
1557            resolved,
1558            content,
1559            handling_mode,
1560            dispatching,
1561            outcome.frame,
1562            interaction_id,
1563        );
1564        Ok(accepted)
1565    }
1566
1567    pub async fn reserve_identity_first_interaction(
1568        &self,
1569        request: ConsoleSendRequest,
1570        session_id: Option<&str>,
1571    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1572        validate_send_request(&request)?;
1573        let _content = content_input_from_value(&request.content)?;
1574        let runtime_key =
1575            Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1576        let handling_mode_value = request
1577            .handling_mode
1578            .as_deref()
1579            .unwrap_or("queue")
1580            .to_string();
1581        let dedupe_key = send_dedupe_key(
1582            &runtime_key,
1583            &request.identity,
1584            &request.origin,
1585            &request.idempotency_key,
1586        );
1587        let request_fingerprint =
1588            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1589        if let Some(existing) = self
1590            .inner
1591            .store
1592            .frame_by_dedupe_key(&dedupe_key)
1593            .await
1594            .map_err(ConsoleSendError::Log)?
1595        {
1596            let same_request = existing.source.source_cursor.as_deref()
1597                == Some(request_fingerprint.as_str())
1598                || existing.source.source_cursor.is_none()
1599                    && existing.payload.get("origin").and_then(Value::as_str)
1600                        == Some(request.origin.as_str())
1601                    && existing.payload.get("content") == Some(&request.content)
1602                    && existing
1603                        .payload
1604                        .get("handling_mode")
1605                        .and_then(Value::as_str)
1606                        == Some(handling_mode_value.as_str());
1607            if !same_request {
1608                return Err(ConsoleSendError::IdempotencyConflict(
1609                    request.idempotency_key,
1610                ));
1611            }
1612            return Ok(accepted_from_frame(&existing));
1613        }
1614
1615        let interaction_id = identity_first_interaction_uuid(&dedupe_key);
1616        let new_frame = NewConsoleFrame {
1617            id: None,
1618            dedupe_key,
1619            timestamp_ms: current_time_ms(),
1620            runtime_key,
1621            identity: request.identity.clone(),
1622            conversation_id: Some(request.identity.clone()),
1623            session_id: session_id.map(ToString::to_string),
1624            kind: "user_input".to_string(),
1625            status: ConsoleFrameStatus::Accepted,
1626            payload: json!({
1627                "content": request.content,
1628                "origin": request.origin,
1629                "idempotency_key": request.idempotency_key,
1630                "handling_mode": handling_mode_value,
1631            }),
1632            source: ConsoleFrameSource {
1633                kind: ConsoleFrameSourceKind::Send,
1634                source_cursor: Some(request_fingerprint),
1635            },
1636            source_event_id: None,
1637            interaction_id: Some(interaction_id),
1638            turn_id: None,
1639            run_id: None,
1640            parent_frame_id: None,
1641            caused_by_frame_id: None,
1642        };
1643        let outcome = self
1644            .inner
1645            .store
1646            .append_if_absent(new_frame)
1647            .await
1648            .map_err(ConsoleSendError::Log)?;
1649        let _ = self
1650            .inner
1651            .event_tx
1652            .send(ConsoleTimelineEvent::ConsoleFrame {
1653                frame: outcome.frame.clone(),
1654            });
1655        Ok(accepted_from_frame(&outcome.frame))
1656    }
1657
1658    async fn runtime_key_for_identity_first_send(
1659        &self,
1660        identity: &str,
1661    ) -> Result<String, ConsoleSendError> {
1662        let entries = self
1663            .inner
1664            .runtimes
1665            .read()
1666            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1667            .clone();
1668        if entries.is_empty() {
1669            return Ok("identity-first".to_string());
1670        }
1671
1672        let mut identity_runtime_keys = Vec::new();
1673        let mut hidden_durable_match = false;
1674        let mut durable_matches = Vec::new();
1675        for entry in entries.values() {
1676            let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1677                continue;
1678            };
1679            identity_runtime_keys.push(entry.runtime_key.clone());
1680            if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1681                continue;
1682            }
1683            for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1684            {
1685                let Ok(parsed_identity) =
1686                    crate::identity_first::AgentIdentity::parse(&runtime_identity)
1687                else {
1688                    continue;
1689                };
1690                if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1691                    let record = identity_record_for_status(entry, &status);
1692                    if !Box::pin(console_identity_record_visible(entry, &record)).await {
1693                        hidden_durable_match = true;
1694                        continue;
1695                    }
1696                    durable_matches.push((
1697                        entry.clone(),
1698                        identity_runtime.clone(),
1699                        parsed_identity,
1700                        record,
1701                    ));
1702                }
1703            }
1704        }
1705        if durable_matches.len() > 1 {
1706            let candidates = durable_matches
1707                .iter()
1708                .map(|(entry, _, _, record)| format!("{}@{}", record.identity, entry.runtime_key))
1709                .collect::<Vec<_>>()
1710                .join(", ");
1711            return Err(ConsoleSendError::InvalidRequest(format!(
1712                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1713            )));
1714        }
1715        if let Some((entry, identity_runtime, parsed_identity, mut record)) =
1716            durable_matches.into_iter().next()
1717        {
1718            let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1719            let durable_live_records = Box::pin(self.live_records_for_durable_record(
1720                &entry,
1721                identity,
1722                &record,
1723                &live_records,
1724            ))
1725            .await;
1726            if let Some(rebound_record) = self_heal_durable_session_mismatch(
1727                &entry,
1728                identity_runtime.as_ref(),
1729                &parsed_identity,
1730                identity,
1731                &record,
1732                &durable_live_records,
1733            )
1734            .await
1735            .map_err(ConsoleSendError::InvalidRequest)?
1736            {
1737                record = rebound_record;
1738            }
1739            if let Some(stale_error) =
1740                stale_durable_record_error(identity, &record, &durable_live_records)
1741            {
1742                return Err(ConsoleSendError::InvalidRequest(stale_error));
1743            }
1744            return Ok(entry.runtime_key);
1745        }
1746        if entries.values().any(|entry| {
1747            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1748        }) {
1749            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1750        }
1751        if hidden_durable_match {
1752            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1753        }
1754
1755        match identity_runtime_keys.as_slice() {
1756            [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1757            [runtime_key] => Ok(runtime_key.clone()),
1758            _ => Err(ConsoleSendError::InvalidRequest(format!(
1759                "identity-first send for '{identity}' did not match exactly one registered runtime"
1760            ))),
1761        }
1762    }
1763
1764    pub async fn mark_interaction_delivery_failed(
1765        &self,
1766        input_frame_id: &str,
1767    ) -> Result<(), ConsoleSendError> {
1768        update_frame_status_and_emit(
1769            &self.inner,
1770            input_frame_id,
1771            ConsoleFrameStatus::DeliveryFailed,
1772        )
1773        .await
1774        .map_err(ConsoleSendError::Log)?;
1775        Ok(())
1776    }
1777
1778    pub async fn mark_interaction_delivered(
1779        &self,
1780        input_frame_id: &str,
1781    ) -> Result<(), ConsoleSendError> {
1782        update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1783            .await
1784            .map_err(ConsoleSendError::Log)?;
1785        Ok(())
1786    }
1787
1788    pub async fn mark_steer_interaction_delivered(
1789        &self,
1790        input_frame_id: &str,
1791        interaction_id: &str,
1792    ) -> Result<(), ConsoleSendError> {
1793        let Some(updated) = update_frame_status_and_emit(
1794            &self.inner,
1795            input_frame_id,
1796            ConsoleFrameStatus::Delivered,
1797        )
1798        .await
1799        .map_err(ConsoleSendError::Log)?
1800        else {
1801            return Ok(());
1802        };
1803        append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1804            .await
1805            .map_err(ConsoleSendError::Log)?;
1806        Ok(())
1807    }
1808
1809    pub async fn binary_blob_store_for_identity(
1810        &self,
1811        identity: &str,
1812    ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1813        if identity.trim().is_empty() {
1814            return Err(ConsoleSendError::InvalidRequest(
1815                "identity must be non-empty".to_string(),
1816            ));
1817        }
1818        let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1819            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1820        };
1821        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1822            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1823        };
1824        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
1825            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1826        }
1827        if !member_is_addressable(&resolved.member) {
1828            return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1829        }
1830        if resolved.member.status == meerkat_mob::MobMemberStatus::Retiring {
1831            return Err(ConsoleSendError::Retired(identity.to_string()));
1832        }
1833        Ok(resolved.entry.runtime.binary_blob_store())
1834    }
1835
1836    pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1837        self.inner
1838            .runtimes
1839            .read()
1840            .map(|entries| {
1841                entries
1842                    .values()
1843                    .filter_map(|entry| entry.runtime.binary_blob_store())
1844                    .collect()
1845            })
1846            .unwrap_or_default()
1847    }
1848
1849    async fn resolve_send_member(
1850        &self,
1851        identity: &str,
1852    ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1853        let mut matches = Box::pin(self.resolve_visible_members(identity)).await;
1854        // Retiring generations remain visible for history and inspection, but
1855        // they must not make a durable send alias ambiguous when exactly one
1856        // active/addressable generation can receive the message. A durable
1857        // identity-runtime binding is authoritative over stale duplicate live
1858        // generations left behind by a respawn: select the exact bound runtime
1859        // member for sends while keeping inspection/control ambiguity strict.
1860        // Preserve the ambiguity error when multiple genuinely sendable
1861        // candidates have no single durable binding.
1862        let sendable_indices = sendable_resolved_member_indices(&matches);
1863        if sendable_indices.len() > 1 {
1864            if let Some(index) = Box::pin(authoritative_durable_sendable_member_index(
1865                identity,
1866                &matches,
1867                &sendable_indices,
1868            ))
1869            .await?
1870            {
1871                let resolved = matches.swap_remove(index);
1872                if let Some(record) = identity_record_for_resolved_member(&resolved).await
1873                    && let Some(stale_error) =
1874                        reconcile_stale_live_record_binding(&resolved.entry, identity, &record)
1875                            .await
1876                {
1877                    return Err(ConsoleSendError::InvalidRequest(stale_error));
1878                }
1879                if let Some(stale_error) =
1880                    Box::pin(self.durable_send_binding_error(identity)).await?
1881                {
1882                    return Err(ConsoleSendError::InvalidRequest(stale_error));
1883                }
1884                return Ok(Some(resolved));
1885            }
1886            let candidates = sendable_indices
1887                .iter()
1888                .map(|index| matches[*index].runtime_identity.clone())
1889                .collect::<Vec<_>>()
1890                .join(", ");
1891            return Err(ConsoleSendError::AmbiguousIdentity {
1892                identity: identity.to_string(),
1893                candidates,
1894            });
1895        }
1896        if let Some(index) = sendable_indices.into_iter().next() {
1897            let resolved = matches.swap_remove(index);
1898            if let Some(record) = identity_record_for_resolved_member(&resolved).await
1899                && let Some(stale_error) =
1900                    reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1901            {
1902                return Err(ConsoleSendError::InvalidRequest(stale_error));
1903            }
1904            if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1905                return Err(ConsoleSendError::InvalidRequest(stale_error));
1906            }
1907            return Ok(Some(resolved));
1908        }
1909        if matches.len() > 1 {
1910            let candidates = matches
1911                .iter()
1912                .map(|resolved| resolved.runtime_identity.clone())
1913                .collect::<Vec<_>>()
1914                .join(", ");
1915            return Err(ConsoleSendError::AmbiguousIdentity {
1916                identity: identity.to_string(),
1917                candidates,
1918            });
1919        }
1920        let Some(resolved) = matches.into_iter().next() else {
1921            if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1922                return Err(ConsoleSendError::InvalidRequest(stale_error));
1923            }
1924            return Ok(None);
1925        };
1926        if let Some(record) = identity_record_for_resolved_member(&resolved).await
1927            && let Some(stale_error) =
1928                reconcile_stale_live_record_binding(&resolved.entry, identity, &record).await
1929        {
1930            return Err(ConsoleSendError::InvalidRequest(stale_error));
1931        }
1932        if let Some(stale_error) = Box::pin(self.durable_send_binding_error(identity)).await? {
1933            return Err(ConsoleSendError::InvalidRequest(stale_error));
1934        }
1935        Ok(Some(resolved))
1936    }
1937
1938    async fn durable_send_binding_error(
1939        &self,
1940        identity: &str,
1941    ) -> Result<Option<String>, ConsoleSendError> {
1942        let entries = self
1943            .inner
1944            .runtimes
1945            .read()
1946            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1947            .clone();
1948        if entries.values().any(|entry| {
1949            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1950        }) {
1951            return Ok(None);
1952        }
1953        let live_records = Box::pin(self.live_records_for_identity(identity)).await;
1954        let mut durable_matches = Vec::new();
1955        for entry in entries.values() {
1956            let Some(identity_runtime) = entry.identity_runtime.clone() else {
1957                continue;
1958            };
1959            for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1960                let Ok(parsed_identity) =
1961                    crate::identity_first::AgentIdentity::parse(&raw_identity)
1962                else {
1963                    continue;
1964                };
1965                let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1966                    continue;
1967                };
1968                let record = identity_record_for_status(entry, &status);
1969                if !Box::pin(console_identity_record_visible(entry, &record)).await {
1970                    continue;
1971                }
1972                durable_matches.push((
1973                    entry.clone(),
1974                    identity_runtime.clone(),
1975                    parsed_identity,
1976                    record,
1977                ));
1978            }
1979        }
1980        if durable_matches.len() > 1 {
1981            let candidates = durable_matches
1982                .iter()
1983                .map(|(_, _, _, record)| record.runtime_member_id.clone())
1984                .collect::<Vec<_>>()
1985                .join(", ");
1986            return Err(ConsoleSendError::AmbiguousIdentity {
1987                identity: identity.to_string(),
1988                candidates,
1989            });
1990        }
1991        let Some((entry, identity_runtime, parsed_identity, mut durable_record)) =
1992            durable_matches.into_iter().next()
1993        else {
1994            return Ok(None);
1995        };
1996        let durable_live_records = Box::pin(self.live_records_for_durable_record(
1997            &entry,
1998            identity,
1999            &durable_record,
2000            &live_records,
2001        ))
2002        .await;
2003        if let Some(rebound_record) = self_heal_durable_session_mismatch(
2004            &entry,
2005            identity_runtime.as_ref(),
2006            &parsed_identity,
2007            identity,
2008            &durable_record,
2009            &durable_live_records,
2010        )
2011        .await
2012        .map_err(ConsoleSendError::InvalidRequest)?
2013        {
2014            durable_record = rebound_record;
2015        }
2016        Ok(stale_durable_record_error(
2017            identity,
2018            &durable_record,
2019            &durable_live_records,
2020        ))
2021    }
2022
2023    async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2024        let entries = self
2025            .inner
2026            .runtimes
2027            .read()
2028            .ok()
2029            .map(|entries| entries.clone())
2030            .unwrap_or_default();
2031        let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2032        let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
2033        for entry in entries.values() {
2034            let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2035            if raw_identities.is_empty() {
2036                continue;
2037            }
2038            let mids = raw_identities
2039                .iter()
2040                .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
2041                .collect::<Vec<_>>();
2042            for resolved in Box::pin(member_sources_for_entry(entry)).await {
2043                let session_id = resolved
2044                    .handle
2045                    .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
2046                    .await
2047                    .map(|sid| sid.to_string())
2048                    .unwrap_or_default();
2049                if mids.contains(&resolved.member.agent_identity) {
2050                    exact_matches.push((session_id, resolved));
2051                } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
2052                {
2053                    label_matches.push((session_id, resolved));
2054                }
2055            }
2056        }
2057        let mut matches = exact_matches;
2058        matches.extend(label_matches);
2059        let mut seen_members = BTreeSet::new();
2060        matches.retain(|(session_id, resolved)| {
2061            seen_members.insert((
2062                resolved.entry.runtime_key.clone(),
2063                resolved.source_mob_id.clone(),
2064                session_id.clone(),
2065                resolved.runtime_identity.clone(),
2066            ))
2067        });
2068        matches.sort_by(|left, right| right.0.cmp(&left.0));
2069        matches.into_iter().map(|(_, resolved)| resolved).collect()
2070    }
2071
2072    async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
2073        let mut visible = Vec::new();
2074        for resolved in Box::pin(self.resolve_members(identity)).await {
2075            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2076                continue;
2077            };
2078            if Box::pin(resolved_member_visible(&resolved, &record)).await {
2079                visible.push(resolved);
2080            }
2081        }
2082        visible
2083    }
2084
2085    async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
2086        let mut records = Vec::new();
2087        for resolved in Box::pin(self.resolve_members(identity)).await {
2088            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2089                continue;
2090            };
2091            if Box::pin(resolved_member_visible(&resolved, &record)).await {
2092                records.push(record);
2093            }
2094        }
2095        records
2096    }
2097
2098    async fn live_records_for_durable_record(
2099        &self,
2100        entry: &RuntimeEntry,
2101        requested_identity: &str,
2102        durable: &ConsoleIdentityRecord,
2103        requested_live_records: &[ConsoleIdentityRecord],
2104    ) -> Vec<ConsoleIdentityRecord> {
2105        let mut records = if durable.identity == requested_identity {
2106            requested_live_records.to_vec()
2107        } else {
2108            Box::pin(self.live_records_for_identity(&durable.identity)).await
2109        };
2110        for runtime_record in Box::pin(live_records_for_runtime_member(
2111            entry,
2112            &durable.runtime_member_id,
2113        ))
2114        .await
2115        {
2116            if records.iter().any(|record| {
2117                record.runtime_key == runtime_record.runtime_key
2118                    && record.runtime_member_id == runtime_record.runtime_member_id
2119                    && record.session_id == runtime_record.session_id
2120                    && record.identity == runtime_record.identity
2121            }) {
2122                continue;
2123            }
2124            records.push(runtime_record);
2125        }
2126        records
2127    }
2128}
2129
2130async fn console_identity_record_visible(
2131    entry: &RuntimeEntry,
2132    record: &ConsoleIdentityRecord,
2133) -> bool {
2134    if !entry.visibility_policy.identity_visible(record) {
2135        return false;
2136    }
2137    let mut bound_live_member_seen = false;
2138    let mut wrong_live_projection_seen = false;
2139    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2140        if resolved.runtime_identity != record.runtime_member_id {
2141            continue;
2142        }
2143        bound_live_member_seen = true;
2144        let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
2145            continue;
2146        };
2147        if live_record.identity != record.identity {
2148            wrong_live_projection_seen = true;
2149            continue;
2150        }
2151        if raw_resolved_member_visible(&resolved, &live_record) {
2152            return true;
2153        }
2154    }
2155    if wrong_live_projection_seen {
2156        return true;
2157    }
2158    !bound_live_member_seen
2159}
2160
2161async fn live_record_shadowed_by_hidden_durable_binding(
2162    resolved: &ResolvedConsoleMember,
2163    record: &ConsoleIdentityRecord,
2164) -> bool {
2165    let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
2166        return false;
2167    };
2168    let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
2169        return false;
2170    };
2171    let Ok(status) = identity_runtime.status(&identity).await else {
2172        return false;
2173    };
2174    let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
2175        return false;
2176    };
2177    if bound_runtime_id.as_str() == resolved.runtime_identity {
2178        return false;
2179    }
2180    for candidate in Box::pin(member_sources_for_entry(&resolved.entry)).await {
2181        if candidate.runtime_identity != bound_runtime_id.as_str() {
2182            continue;
2183        }
2184        let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
2185            continue;
2186        };
2187        if bound_record.identity == record.identity
2188            && !raw_resolved_member_visible(&candidate, &bound_record)
2189        {
2190            return true;
2191        }
2192    }
2193    false
2194}
2195
2196async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
2197    let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
2198        .unwrap_or_else(|| frame.identity.clone());
2199    let frame_session_id = frame.session_id.as_deref();
2200    let mut saw_hidden = false;
2201    let mut saw_visible = false;
2202    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2203        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2204            continue;
2205        };
2206        let session_matches =
2207            frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
2208        let runtime_matches = record.runtime_member_id == runtime_member_id;
2209        let identity_matches = record.identity == frame.identity;
2210        if !session_matches && !runtime_matches && !identity_matches {
2211            continue;
2212        }
2213        if session_matches && !raw_resolved_member_visible(&resolved, &record) {
2214            return true;
2215        }
2216        if Box::pin(resolved_member_visible(&resolved, &record)).await {
2217            saw_visible = true;
2218        } else {
2219            saw_hidden = true;
2220        }
2221    }
2222    saw_hidden && !saw_visible
2223}
2224
2225async fn live_records_for_runtime_member(
2226    entry: &RuntimeEntry,
2227    runtime_member_id: &str,
2228) -> Vec<ConsoleIdentityRecord> {
2229    let mut records = Vec::new();
2230    for resolved in Box::pin(member_sources_for_entry(entry)).await {
2231        if resolved.runtime_identity != runtime_member_id {
2232            continue;
2233        }
2234        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2235            continue;
2236        };
2237        records.push(record);
2238    }
2239    records
2240}
2241
2242async fn visible_live_records_for_entry(
2243    entry: &RuntimeEntry,
2244    records: &[ConsoleIdentityRecord],
2245) -> Vec<ConsoleIdentityRecord> {
2246    let mut visible = Vec::new();
2247    for record in records {
2248        if record.runtime_key != entry.runtime_key
2249            || Box::pin(console_identity_record_visible(entry, record)).await
2250        {
2251            visible.push(record.clone());
2252        }
2253    }
2254    visible
2255}
2256
2257async fn reconcile_stale_live_record_binding(
2258    entry: &RuntimeEntry,
2259    requested_identity: &str,
2260    live_record: &ConsoleIdentityRecord,
2261) -> Option<String> {
2262    let identity_runtime = entry.identity_runtime.as_ref()?;
2263    for status in identity_runtime.statuses().await {
2264        let matches_runtime = status
2265            .agent_runtime_id
2266            .as_ref()
2267            .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
2268        if !matches_runtime {
2269            continue;
2270        }
2271        let durable_record = identity_record_for_status(entry, &status);
2272        if let Some(session_mismatch) =
2273            stale_durable_session_mismatch(&durable_record, std::slice::from_ref(live_record))
2274        {
2275            let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2276                continue;
2277            };
2278            let Ok(session_id) = meerkat_core::types::SessionId::parse(live_session_id) else {
2279                return Some(format!(
2280                    "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}"
2281                ));
2282            };
2283            return match identity_runtime
2284                .rebind_session_after_live_respawn(&status.identity, session_id)
2285                .await
2286            {
2287                Ok(_) => {
2288                    tracing::warn!(
2289                        requested_identity,
2290                        identity = %durable_record.identity,
2291                        runtime_member_id = %durable_record.runtime_member_id,
2292                        old_session_id = durable_record.session_id.as_deref().unwrap_or("<none>"),
2293                        new_session_id = session_mismatch.session_id.as_deref().unwrap_or("<none>"),
2294                        "self-healed stale durable identity alias session mismatch"
2295                    );
2296                    None
2297                }
2298                Err(err) => Some(format!(
2299                    "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2300                    durable_record.identity,
2301                    session_mismatch.session_id.as_deref().unwrap_or("<none>")
2302                )),
2303            };
2304        }
2305        if let Some(stale_error) = stale_durable_record_error(
2306            requested_identity,
2307            &durable_record,
2308            std::slice::from_ref(live_record),
2309        ) {
2310            return Some(stale_error);
2311        }
2312    }
2313    None
2314}
2315
2316async fn self_heal_durable_session_mismatch(
2317    entry: &RuntimeEntry,
2318    identity_runtime: &crate::identity_first::IdentityRuntime,
2319    parsed_identity: &crate::identity_first::AgentIdentity,
2320    requested_identity: &str,
2321    durable: &ConsoleIdentityRecord,
2322    live_records: &[ConsoleIdentityRecord],
2323) -> Result<Option<ConsoleIdentityRecord>, String> {
2324    let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) else {
2325        return Ok(None);
2326    };
2327    let Some(live_session_id) = session_mismatch.session_id.as_deref() else {
2328        return Ok(None);
2329    };
2330    let session_id = meerkat_core::types::SessionId::parse(live_session_id).map_err(|err| {
2331        format!(
2332            "stale durable identity alias {requested_identity}: live console alias resolves to invalid session {live_session_id}: {err}"
2333        )
2334    })?;
2335    identity_runtime
2336        .rebind_session_after_live_respawn(parsed_identity, session_id)
2337        .await
2338        .map_err(|err| {
2339            format!(
2340                "stale durable identity alias {requested_identity}: failed to rebind {} to live session {}: {err}",
2341                durable.identity, live_session_id
2342            )
2343        })?;
2344    tracing::warn!(
2345        requested_identity,
2346        identity = %durable.identity,
2347        runtime_member_id = %durable.runtime_member_id,
2348        old_session_id = durable.session_id.as_deref().unwrap_or("<none>"),
2349        new_session_id = live_session_id,
2350        "self-healed stale durable identity alias session mismatch"
2351    );
2352    let status = identity_runtime
2353        .status(parsed_identity)
2354        .await
2355        .map_err(|err| {
2356            format!(
2357                "stale durable identity alias {requested_identity}: failed to reload rebound identity {}: {err}",
2358                durable.identity
2359            )
2360        })?;
2361    Ok(Some(identity_record_for_status(entry, &status)))
2362}
2363
2364fn stale_durable_session_mismatch<'a>(
2365    durable: &ConsoleIdentityRecord,
2366    live_records: &'a [ConsoleIdentityRecord],
2367) -> Option<&'a ConsoleIdentityRecord> {
2368    live_records.iter().find(|record| {
2369        record.identity == durable.identity
2370            && record.runtime_member_id == durable.runtime_member_id
2371            && durable.session_id.is_some()
2372            && record.session_id.is_some()
2373            && durable.session_id != record.session_id
2374    })
2375}
2376
2377fn stale_durable_record_error(
2378    requested_identity: &str,
2379    durable: &ConsoleIdentityRecord,
2380    live_records: &[ConsoleIdentityRecord],
2381) -> Option<String> {
2382    let matching_live = live_records
2383        .iter()
2384        .filter(|record| record.identity == durable.identity)
2385        .collect::<Vec<_>>();
2386    if let Some(wrong_projection) = live_records.iter().find(|record| {
2387        record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
2388    }) {
2389        return Some(format!(
2390            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
2391            durable.identity, durable.runtime_member_id, wrong_projection.identity
2392        ));
2393    }
2394    if matching_live.is_empty() {
2395        return None;
2396    }
2397    if let Some(session_mismatch) = stale_durable_session_mismatch(durable, live_records) {
2398        return Some(format!(
2399            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2400            durable.identity,
2401            durable.runtime_member_id,
2402            durable.session_id.as_deref().unwrap_or("<none>"),
2403            session_mismatch.session_id.as_deref().unwrap_or("<none>")
2404        ));
2405    }
2406    if matching_live
2407        .iter()
2408        .any(|record| record.runtime_member_id == durable.runtime_member_id)
2409    {
2410        return None;
2411    }
2412    let live_candidates = matching_live
2413        .iter()
2414        .map(|record| record.runtime_member_id.as_str())
2415        .collect::<Vec<_>>()
2416        .join(", ");
2417    Some(format!(
2418        "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2419        durable.identity, durable.runtime_member_id, live_candidates
2420    ))
2421}
2422
2423fn ambiguous_live_alias_error(
2424    requested_identity: &str,
2425    live_records: &[ConsoleIdentityRecord],
2426) -> Option<String> {
2427    if live_records.len() <= 1 {
2428        return None;
2429    }
2430    let candidates = live_records
2431        .iter()
2432        .map(|record| {
2433            format!(
2434                "{}@{}",
2435                record.runtime_member_id,
2436                record
2437                    .labels
2438                    .get("source_mob_id")
2439                    .map(String::as_str)
2440                    .unwrap_or(record.runtime_key.as_str())
2441            )
2442        })
2443        .collect::<Vec<_>>()
2444        .join(", ");
2445    Some(format!(
2446        "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2447    ))
2448}
2449
2450fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2451    // Roster ids are comms-safe encodings of public aliases (meerkat 0.7
2452    // MemberCommsName); compare in the public alias space.
2453    crate::member_comms_id::runtime_alias_str(member_id) == durable_identity
2454}
2455
2456async fn retire_stale_console_members_for_identity(
2457    handle: &MobHandle,
2458    durable_identities: &[String],
2459    keep_runtime_member_id: Option<&str>,
2460) -> Result<(), String> {
2461    let stale_members = handle
2462        .list_members_including_retiring()
2463        .await
2464        .into_iter()
2465        .filter(|member| {
2466            (durable_identities.iter().any(|durable_identity| {
2467                member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2468            }) || member.labels.get("agent_identity").is_some_and(|identity| {
2469                durable_identities
2470                    .iter()
2471                    .any(|durable_identity| identity == durable_identity)
2472            })) && keep_runtime_member_id
2473                .map(|keep| {
2474                    // `keep` is a public alias; compare decoded.
2475                    crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str())
2476                        != keep
2477                })
2478                .unwrap_or(true)
2479        })
2480        .map(|member| member.agent_identity)
2481        .collect::<Vec<_>>();
2482    for member_id in stale_members {
2483        match handle.retire(member_id).await {
2484            Ok(()) => {}
2485            Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2486            Err(err) => return Err(err.to_string()),
2487        }
2488    }
2489    Ok(())
2490}
2491
2492async fn retire_stale_console_members_for_runtime_entry(
2493    entry: &RuntimeEntry,
2494    durable_identities: &[String],
2495) -> Result<(), String> {
2496    let primary_handle = entry.runtime.handle();
2497    let primary_mob_id = primary_handle.mob_id().to_string();
2498    let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2499    if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2500        for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2501            .await
2502            .unwrap_or_default()
2503        {
2504            if mob_id.as_str() != primary_mob_id {
2505                handles.push((mob_id.to_string(), handle));
2506            }
2507        }
2508    }
2509    let mut seen_handles = BTreeSet::new();
2510    for (mob_id, handle) in handles {
2511        if !seen_handles.insert(mob_id) {
2512            continue;
2513        }
2514        retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2515    }
2516    Ok(())
2517}
2518
2519fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2520    is_recoverable_lifecycle_cleanup_error(error)
2521}
2522
2523fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2524    if frames.is_empty() {
2525        return true;
2526    }
2527    let latest_session_history_timestamp_ms = frames
2528        .iter()
2529        .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2530        .map(|frame| frame.timestamp_ms)
2531        .max();
2532    if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2533        return current_time_ms().saturating_sub(latest_timestamp_ms)
2534            >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2535    }
2536    frames.iter().any(|frame| {
2537        matches!(
2538            frame.kind.as_str(),
2539            "turn_started"
2540                | "run_started"
2541                | "reasoning_delta"
2542                | "reasoning_complete"
2543                | "tool_call_requested"
2544                | "tool_call"
2545                | "tool_execution_started"
2546                | "tool_execution_completed"
2547                | "tool_result_received"
2548        )
2549    })
2550}
2551
2552fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2553    match frame.kind.as_str() {
2554        "user_input" | "run_started" => true,
2555        "interaction_started" => frame
2556            .payload
2557            .get("content")
2558            .or_else(|| frame.payload.get("prompt"))
2559            .is_some(),
2560        _ => false,
2561    }
2562}
2563
2564fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2565    let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2566    for record in records {
2567        by_identity
2568            .entry(identity_record_dedupe_key(&record))
2569            .and_modify(|current| {
2570                if identity_record_prefer(&record, current) {
2571                    *current = record.clone();
2572                }
2573            })
2574            .or_insert(record);
2575    }
2576    by_identity.into_values().collect()
2577}
2578
2579fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2580    format!(
2581        "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2582        record.identity,
2583        record.runtime_key,
2584        record
2585            .labels
2586            .get("source_mob_id")
2587            .map(String::as_str)
2588            .unwrap_or(""),
2589        record.runtime_member_id,
2590        record.session_id.as_deref().unwrap_or("")
2591    )
2592}
2593
2594fn identity_record_prefer(
2595    candidate: &ConsoleIdentityRecord,
2596    current: &ConsoleIdentityRecord,
2597) -> bool {
2598    let candidate_is_live_label_projection =
2599        candidate
2600            .labels
2601            .get("agent_identity")
2602            .is_some_and(|identity| {
2603                identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2604                    || candidate.runtime_member_id != candidate.identity
2605            });
2606    let current_is_live_label_projection =
2607        current
2608            .labels
2609            .get("agent_identity")
2610            .is_some_and(|identity| {
2611                identity == &strip_namespace(&current.identity, "").unwrap_or_default()
2612                    || current.runtime_member_id != current.identity
2613            });
2614    if candidate_is_live_label_projection != current_is_live_label_projection {
2615        return candidate_is_live_label_projection;
2616    }
2617    let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2618    let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2619    if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2620        return candidate_has_distinct_runtime_binding;
2621    }
2622    let candidate_live = candidate.addressable && candidate.health != "retired";
2623    let current_live = current.addressable && current.health != "retired";
2624    if candidate_live != current_live {
2625        return candidate_live;
2626    }
2627    candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2628}
2629
2630async fn collect_identity_records(
2631    inner: &Arc<AggregatorInner>,
2632    mode: IdentityCollectionMode,
2633) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2634    let entries = inner
2635        .runtimes
2636        .read()
2637        .map_err(|_| runtime_registry_lock_error())?
2638        .clone();
2639    let mut identities = Vec::new();
2640    for entry in entries.values() {
2641        if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2642            let topology_peers =
2643                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2644            for status in identity_runtime.statuses().await {
2645                let mut record = identity_record_for_status(entry, &status);
2646                record.topology_peers = topology_peers
2647                    .get(&record.identity)
2648                    .cloned()
2649                    .unwrap_or_default();
2650                if Box::pin(console_identity_record_visible(entry, &record)).await {
2651                    identities.push(record);
2652                }
2653            }
2654            if mode == IdentityCollectionMode::CachedOnly {
2655                continue;
2656            }
2657        }
2658        let members = if entry.identity_runtime.is_some() {
2659            match Box::pin(tokio::time::timeout(
2660                IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2661                member_sources_for_entry(entry),
2662            ))
2663            .await
2664            {
2665                Ok(members) => members,
2666                Err(_) => {
2667                    tracing::warn!(
2668                        runtime_key = %entry.runtime_key,
2669                        "identity-first live member refresh timed out; keeping cached identity read model"
2670                    );
2671                    Vec::new()
2672                }
2673            }
2674        } else {
2675            Box::pin(member_sources_for_entry(entry)).await
2676        };
2677        for resolved in members {
2678            if let Some(record) = identity_record_for_resolved_member(&resolved).await
2679                && Box::pin(resolved_member_visible(&resolved, &record)).await
2680            {
2681                identities.push(record);
2682            }
2683        }
2684    }
2685    Ok(dedupe_identity_records(identities))
2686}
2687
2688async fn identity_runtime_topology_peers(
2689    entry: &RuntimeEntry,
2690    identity_runtime: &crate::identity_first::IdentityRuntime,
2691) -> BTreeMap<String, Vec<String>> {
2692    let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2693    for edge in identity_runtime.desired_peer_edges().await {
2694        let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2695        let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2696        peers.entry(a.clone()).or_default().insert(b.clone());
2697        peers.entry(b).or_default().insert(a);
2698    }
2699    peers
2700        .into_iter()
2701        .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2702        .collect()
2703}
2704
2705fn spawn_identity_backfills_for_records(
2706    inner: Arc<AggregatorInner>,
2707    records: &[ConsoleIdentityRecord],
2708) {
2709    if !inner.options.session_history_backfill_enabled {
2710        return;
2711    }
2712    let entries = match inner.runtimes.read() {
2713        Ok(entries) => entries.clone(),
2714        Err(_) => return,
2715    };
2716    for record in records {
2717        if record.health == "dormant" || record.health == "uninitialized" {
2718            continue;
2719        }
2720        let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2721            continue;
2722        };
2723        let Some(session_id) = record.session_id.clone() else {
2724            continue;
2725        };
2726        spawn_session_history_backfill_target(
2727            inner.clone(),
2728            SessionBackfillTarget {
2729                entry,
2730                record: record.clone(),
2731                session_id,
2732            },
2733            false,
2734        );
2735    }
2736}
2737
2738async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2739    let mut resolved = Vec::new();
2740    let primary_handle = entry.runtime.handle();
2741    let primary_mob_id = primary_handle.mob_id().to_string();
2742    for member in primary_handle.list_members_observation_snapshot().await {
2743        resolved.push(ResolvedConsoleMember {
2744            entry: entry.clone(),
2745            handle: primary_handle.clone(),
2746            // Roster ids are comms-safe encodings (meerkat 0.7); the
2747            // aggregator's runtime identity is the public alias.
2748            runtime_identity: crate::member_comms_id::runtime_alias_str(
2749                member.agent_identity.as_str(),
2750            )
2751            .into_owned(),
2752            source_mob_id: primary_mob_id.clone(),
2753            member,
2754        });
2755    }
2756
2757    let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2758        return resolved;
2759    };
2760    if !entry.visibility_policy.include_implicit_delegate_members() {
2761        return resolved;
2762    }
2763    for (mob_id, handle) in Box::pin(state.mob_handles_snapshot())
2764        .await
2765        .unwrap_or_default()
2766    {
2767        if mob_id.as_str() == primary_mob_id {
2768            continue;
2769        }
2770        for member in handle.list_members_observation_snapshot().await {
2771            resolved.push(ResolvedConsoleMember {
2772                entry: entry.clone(),
2773                handle: handle.clone(),
2774                // Roster ids are comms-safe encodings (meerkat 0.7); the
2775                // aggregator's runtime identity is the public alias.
2776                runtime_identity: crate::member_comms_id::runtime_alias_str(
2777                    member.agent_identity.as_str(),
2778                )
2779                .into_owned(),
2780                source_mob_id: mob_id.to_string(),
2781                member,
2782            });
2783        }
2784    }
2785    resolved
2786}
2787
2788async fn dispatch_message_to_resolved_member(
2789    resolved: &ResolvedConsoleMember,
2790    content: ContentInput,
2791    handling_mode: meerkat_core::types::HandlingMode,
2792) -> Result<String, String> {
2793    let mid = crate::member_comms_id::mob_member_id(resolved.runtime_identity.as_str());
2794    match send_message_on_mob_with_mode(
2795        &resolved.handle,
2796        &resolved.runtime_identity,
2797        content.clone(),
2798        handling_mode,
2799    )
2800    .await
2801    {
2802        Ok(session_id) => Ok(session_id),
2803        Err(err) if is_not_externally_addressable(&err) => {
2804            let member = resolved
2805                .handle
2806                .member(&mid)
2807                .await
2808                .map_err(|err| err.to_string())?;
2809            let _receipt = member
2810                .internal_turn(content)
2811                .await
2812                .map_err(|err| err.to_string())?;
2813            resolved
2814                .handle
2815                .resolve_bridge_session_id_observation(&mid)
2816                .await
2817                .map(|sid| sid.to_string())
2818                .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2819        }
2820        Err(err) => Err(err.to_string()),
2821    }
2822}
2823
2824fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2825    matches!(
2826        err,
2827        MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2828    )
2829}
2830
2831fn spawn_console_send_dispatch(
2832    inner: Arc<AggregatorInner>,
2833    resolved: ResolvedConsoleMember,
2834    content: ContentInput,
2835    handling_mode: meerkat_core::types::HandlingMode,
2836    dispatching: SendState,
2837    user_frame: ConsoleFrame,
2838    interaction_id: String,
2839) {
2840    tokio::spawn(async move {
2841        match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2842            Ok(_) => {
2843                let _ = dispatching.apply(SendTransition::MarkDelivered);
2844                if let Err(err) = update_frame_status_and_emit(
2845                    &inner,
2846                    &user_frame.id,
2847                    ConsoleFrameStatus::Delivered,
2848                )
2849                .await
2850                {
2851                    tracing::warn!(
2852                        frame_id = %user_frame.id,
2853                        error = %err,
2854                        "failed to update console send delivery status"
2855                    );
2856                }
2857                if handling_mode == HandlingMode::Steer
2858                    && let Err(err) =
2859                        append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2860                {
2861                    tracing::warn!(
2862                        frame_id = %user_frame.id,
2863                        interaction_id = %interaction_id,
2864                        error = %err,
2865                        "failed to append console steer terminal frame"
2866                    );
2867                }
2868            }
2869            Err(err) => {
2870                let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2871                if let Err(update_err) = update_frame_status_and_emit(
2872                    &inner,
2873                    &user_frame.id,
2874                    ConsoleFrameStatus::DeliveryFailed,
2875                )
2876                .await
2877                {
2878                    tracing::warn!(
2879                        frame_id = %user_frame.id,
2880                        error = %update_err,
2881                        "failed to update console send failure status"
2882                    );
2883                }
2884                let failure_frame = NewConsoleFrame {
2885                    id: None,
2886                    dedupe_key: format!("delivery-failed:{}", user_frame.id),
2887                    timestamp_ms: current_time_ms(),
2888                    runtime_key: user_frame.runtime_key,
2889                    identity: user_frame.identity,
2890                    conversation_id: user_frame.conversation_id,
2891                    session_id: user_frame.session_id,
2892                    kind: "message_delivery_failed".to_string(),
2893                    status: ConsoleFrameStatus::DeliveryFailed,
2894                    payload: json!({ "reason": err }),
2895                    source: ConsoleFrameSource {
2896                        kind: ConsoleFrameSourceKind::Synthetic,
2897                        source_cursor: None,
2898                    },
2899                    source_event_id: None,
2900                    interaction_id: Some(interaction_id),
2901                    turn_id: None,
2902                    run_id: None,
2903                    parent_frame_id: Some(user_frame.id.clone()),
2904                    caused_by_frame_id: Some(user_frame.id),
2905                };
2906                if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2907                    tracing::warn!(
2908                        error = %append_err,
2909                        "failed to append console send failure frame"
2910                    );
2911                }
2912            }
2913        }
2914    });
2915}
2916
2917async fn append_steer_delivery_terminal(
2918    inner: &AggregatorInner,
2919    user_frame: &ConsoleFrame,
2920    interaction_id: &str,
2921) -> ConsoleLogResult<AppendOutcome> {
2922    append_and_emit(
2923        inner,
2924        NewConsoleFrame {
2925            id: None,
2926            dedupe_key: format!("steer-delivered:{}", user_frame.id),
2927            timestamp_ms: current_time_ms(),
2928            runtime_key: user_frame.runtime_key.clone(),
2929            identity: user_frame.identity.clone(),
2930            conversation_id: user_frame.conversation_id.clone(),
2931            session_id: user_frame.session_id.clone(),
2932            kind: "interaction_complete".to_string(),
2933            status: ConsoleFrameStatus::Completed,
2934            payload: json!({
2935                "reason": "steer_delivered",
2936                "handling_mode": "steer",
2937            }),
2938            source: ConsoleFrameSource {
2939                kind: ConsoleFrameSourceKind::Synthetic,
2940                source_cursor: None,
2941            },
2942            source_event_id: None,
2943            interaction_id: Some(interaction_id.to_string()),
2944            turn_id: None,
2945            run_id: None,
2946            parent_frame_id: Some(user_frame.id.clone()),
2947            caused_by_frame_id: Some(user_frame.id.clone()),
2948        },
2949    )
2950    .await
2951}
2952
2953#[derive(Debug)]
2954pub enum ConsoleSendError {
2955    UnknownIdentity(String),
2956    AmbiguousIdentity {
2957        identity: String,
2958        candidates: String,
2959    },
2960    NotAddressable(String),
2961    Retired(String),
2962    InvalidContent(String),
2963    InvalidHandlingMode(String),
2964    InvalidRequest(String),
2965    IdempotencyConflict(String),
2966    State(&'static str),
2967    Dispatch(String),
2968    Log(ConsoleLogError),
2969}
2970
2971impl std::fmt::Display for ConsoleSendError {
2972    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2973        match self {
2974            Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2975            Self::AmbiguousIdentity {
2976                identity,
2977                candidates,
2978            } => write!(
2979                f,
2980                "ambiguous live identity alias {identity}: candidates [{candidates}]"
2981            ),
2982            Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2983            Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2984            Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2985            Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2986            Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2987            Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2988            Self::State(message) => write!(f, "console send state error: {message}"),
2989            Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2990            Self::Log(err) => write!(f, "console log error: {err}"),
2991        }
2992    }
2993}
2994
2995impl std::error::Error for ConsoleSendError {}
2996
2997async fn backfill_session_history(
2998    inner: Arc<AggregatorInner>,
2999    runtime_key: String,
3000    force_refresh: bool,
3001) -> ConsoleLogResult<()> {
3002    if !inner.options.session_history_backfill_enabled {
3003        return Ok(());
3004    }
3005    let Some(entry) = inner
3006        .runtimes
3007        .read()
3008        .ok()
3009        .and_then(|entries| entries.get(&runtime_key).cloned())
3010    else {
3011        return Ok(());
3012    };
3013    let members = Box::pin(member_sources_for_entry(&entry)).await;
3014    let mut targets = Vec::new();
3015    for resolved in members {
3016        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3017            continue;
3018        };
3019        if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3020            continue;
3021        }
3022        let Some(session_id) = record.session_id.clone() else {
3023            continue;
3024        };
3025        targets.push(SessionBackfillTarget {
3026            entry: entry.clone(),
3027            record,
3028            session_id,
3029        });
3030    }
3031    backfill_session_history_targets(inner, targets, force_refresh).await
3032}
3033
3034#[derive(Clone)]
3035struct SessionBackfillTarget {
3036    entry: RuntimeEntry,
3037    record: ConsoleIdentityRecord,
3038    session_id: String,
3039}
3040
3041async fn backfill_session_history_targets(
3042    inner: Arc<AggregatorInner>,
3043    targets: Vec<SessionBackfillTarget>,
3044    force_refresh: bool,
3045) -> ConsoleLogResult<()> {
3046    let mut tasks = tokio::task::JoinSet::new();
3047    for target in targets {
3048        tasks.spawn(backfill_one_session_history(
3049            inner.clone(),
3050            target,
3051            force_refresh,
3052        ));
3053    }
3054    let mut first_error = None;
3055    while let Some(result) = tasks.join_next().await {
3056        match result {
3057            Ok(Ok(())) => {}
3058            Ok(Err(err)) => {
3059                if first_error.is_none() {
3060                    first_error = Some(err);
3061                }
3062            }
3063            Err(err) => {
3064                if first_error.is_none() {
3065                    first_error = Some(Box::new(std::io::Error::other(format!(
3066                        "session backfill task failed: {err}"
3067                    ))) as ConsoleLogError);
3068                }
3069            }
3070        }
3071    }
3072    if let Some(err) = first_error {
3073        Err(err)
3074    } else {
3075        Ok(())
3076    }
3077}
3078
3079async fn backfill_one_session_history(
3080    inner: Arc<AggregatorInner>,
3081    target: SessionBackfillTarget,
3082    force_refresh: bool,
3083) -> ConsoleLogResult<()> {
3084    let _permit = inner
3085        .session_backfill_permits
3086        .clone()
3087        .acquire_owned()
3088        .await
3089        .map_err(|err| -> ConsoleLogError {
3090            Box::new(std::io::Error::other(format!(
3091                "session backfill limiter closed: {err}"
3092            )))
3093        })?;
3094    let SessionBackfillTarget {
3095        entry,
3096        record,
3097        session_id,
3098    } = target;
3099    let watermark_runtime_key =
3100        session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
3101    let watermark = inner
3102        .store
3103        .source_watermark(
3104            &watermark_runtime_key,
3105            ConsoleFrameSourceKind::SessionHistory,
3106        )
3107        .await?;
3108    let now_ms = current_time_ms();
3109    let mut offset = watermark
3110        .as_deref()
3111        .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
3112        .unwrap_or(0);
3113    if !force_refresh
3114        && watermark.as_deref().is_some_and(|watermark| {
3115            session_history_watermark_is_fresh(watermark, &session_id, now_ms)
3116        })
3117    {
3118        return Ok(());
3119    }
3120    loop {
3121        let page = match entry
3122            .runtime
3123            .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
3124            .await
3125        {
3126            Ok(page) => page,
3127            Err(err) => {
3128                record_session_history_backfill_failure(
3129                    &inner,
3130                    &watermark_runtime_key,
3131                    &entry.runtime_key,
3132                    &record.identity,
3133                    &session_id,
3134                    offset,
3135                    err.to_string(),
3136                )
3137                .await?;
3138                break;
3139            }
3140        };
3141        let page_value = match serde_json::to_value(page) {
3142            Ok(value) => value,
3143            Err(err) => {
3144                record_session_history_backfill_failure(
3145                    &inner,
3146                    &watermark_runtime_key,
3147                    &entry.runtime_key,
3148                    &record.identity,
3149                    &session_id,
3150                    offset,
3151                    err.to_string(),
3152                )
3153                .await?;
3154                break;
3155            }
3156        };
3157        let base_offset = page_value
3158            .get("offset")
3159            .and_then(Value::as_u64)
3160            .unwrap_or(offset as u64) as usize;
3161        let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
3162            record_session_history_backfill_failure(
3163                &inner,
3164                &watermark_runtime_key,
3165                &entry.runtime_key,
3166                &record.identity,
3167                &session_id,
3168                offset,
3169                "session history page missing messages".to_string(),
3170            )
3171            .await?;
3172            break;
3173        };
3174        if messages.is_empty() {
3175            if offset > 0 {
3176                record_session_history_watermark(
3177                    &inner,
3178                    &watermark_runtime_key,
3179                    &session_id,
3180                    offset,
3181                )
3182                .await?;
3183            }
3184            break;
3185        }
3186        for (idx, message) in messages.iter().enumerate() {
3187            let absolute_offset = base_offset + idx;
3188            let frames = frames_from_session_history_message_with_namespace(
3189                &entry.runtime_key,
3190                &record.identity,
3191                &entry.identity_namespace,
3192                &session_id,
3193                absolute_offset,
3194                message.clone(),
3195            );
3196            for mut frame in frames {
3197                if history_frame_has_existing_counterpart(&inner, &frame).await? {
3198                    continue;
3199                }
3200                if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3201                    frame.payload = redacted;
3202                    frame.status = ConsoleFrameStatus::Redacted;
3203                }
3204                append_and_emit(&inner, frame).await?;
3205            }
3206        }
3207        let Some(next_offset) = advance_backfill_offset(offset, base_offset, messages.len()) else {
3208            // Forward-progress guard: a session store that returns a page whose
3209            // (base_offset + len) does not advance past the offset we requested
3210            // — e.g. a cursor that resets to 0 while still reporting has_more —
3211            // would make this loop re-read the same page forever, re-recording
3212            // the same watermark (a livelock observed on an orphaned gateway).
3213            // Stop instead of spinning; the last good watermark stands.
3214            tracing::warn!(
3215                session_id = %session_id,
3216                requested_offset = offset,
3217                page_base_offset = base_offset,
3218                page_len = messages.len(),
3219                "session history backfill made no forward progress; stopping to avoid a livelock"
3220            );
3221            break;
3222        };
3223        offset = next_offset;
3224        record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
3225            .await?;
3226        let has_more = page_value
3227            .get("has_more")
3228            .and_then(Value::as_bool)
3229            .unwrap_or(false);
3230        if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
3231            break;
3232        }
3233    }
3234    Ok(())
3235}
3236
3237/// Compute the next session-history backfill offset, enforcing strict forward
3238/// progress. Returns `None` when the page (`base_offset + page_len`) does not
3239/// advance past `requested_offset` — a non-advancing/looping cursor — which
3240/// signals the caller to stop rather than re-read the same page forever.
3241fn advance_backfill_offset(
3242    requested_offset: usize,
3243    base_offset: usize,
3244    page_len: usize,
3245) -> Option<usize> {
3246    let next = base_offset.saturating_add(page_len);
3247    (next > requested_offset).then_some(next)
3248}
3249
3250async fn record_session_history_watermark(
3251    inner: &AggregatorInner,
3252    watermark_runtime_key: &str,
3253    session_id: &str,
3254    offset: usize,
3255) -> ConsoleLogResult<()> {
3256    inner
3257        .store
3258        .record_source_watermark(
3259            watermark_runtime_key,
3260            ConsoleFrameSourceKind::SessionHistory,
3261            &format_session_history_watermark(session_id, offset, current_time_ms()),
3262        )
3263        .await
3264}
3265
3266fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
3267    if !inner.options.session_history_backfill_enabled {
3268        return;
3269    }
3270    tokio::spawn(async move {
3271        {
3272            let mut active = inner.active_session_backfills.lock().await;
3273            if !active.insert(runtime_key.clone()) {
3274                return;
3275            }
3276        }
3277        let result = Box::pin(backfill_session_history(
3278            inner.clone(),
3279            runtime_key.clone(),
3280            false,
3281        ))
3282        .await;
3283        let mut active = inner.active_session_backfills.lock().await;
3284        active.remove(&runtime_key);
3285        drop(active);
3286        if let Err(err) = result {
3287            tracing::warn!(
3288                runtime_key = %runtime_key,
3289                error = %err,
3290                "console session-history backfill failed"
3291            );
3292        }
3293    });
3294}
3295
3296fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
3297    if !inner.options.session_history_backfill_enabled {
3298        return;
3299    }
3300    tokio::spawn(async move {
3301        let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
3302        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3303        loop {
3304            interval.tick().await;
3305            let runtime_still_registered = inner
3306                .runtimes
3307                .read()
3308                .ok()
3309                .is_some_and(|entries| entries.contains_key(&runtime_key));
3310            if !runtime_still_registered {
3311                break;
3312            }
3313            spawn_session_history_backfill(inner.clone(), runtime_key.clone());
3314        }
3315    });
3316}
3317
3318fn spawn_session_history_backfill_target(
3319    inner: Arc<AggregatorInner>,
3320    target: SessionBackfillTarget,
3321    force_refresh: bool,
3322) {
3323    if !inner.options.session_history_backfill_enabled {
3324        return;
3325    }
3326    tokio::spawn(async move {
3327        let active_key = targeted_session_history_active_key(&target, force_refresh);
3328        let result =
3329            run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
3330        if let Err(err) = result {
3331            tracing::warn!(
3332                active_key = %active_key,
3333                error = %err,
3334                "console targeted session-history backfill failed"
3335            );
3336        }
3337    });
3338}
3339
3340async fn run_targeted_session_history_backfill(
3341    inner: Arc<AggregatorInner>,
3342    target: SessionBackfillTarget,
3343    force_refresh: bool,
3344) -> ConsoleLogResult<()> {
3345    let active_key = targeted_session_history_active_key(&target, force_refresh);
3346    {
3347        let mut active = inner.active_session_backfills.lock().await;
3348        if !active.insert(active_key.clone()) {
3349            return Ok(());
3350        }
3351    }
3352    let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
3353    let mut active = inner.active_session_backfills.lock().await;
3354    active.remove(&active_key);
3355    result
3356}
3357
3358fn targeted_session_history_active_key(
3359    target: &SessionBackfillTarget,
3360    force_refresh: bool,
3361) -> String {
3362    let mode = if force_refresh { "force" } else { "refresh" };
3363    format!(
3364        "{}:session-history:{}:{mode}",
3365        target.entry.runtime_key, target.session_id
3366    )
3367}
3368
3369fn spawn_session_history_backfill_for_identity(
3370    inner: Arc<AggregatorInner>,
3371    identity: String,
3372    force_refresh: bool,
3373) {
3374    if !inner.options.session_history_backfill_enabled {
3375        return;
3376    }
3377    tokio::spawn(async move {
3378        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3379            spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
3380        }
3381    });
3382}
3383
3384fn spawn_opportunistic_session_history_backfill_for_identity(
3385    inner: Arc<AggregatorInner>,
3386    identity: String,
3387) {
3388    if !inner.options.session_history_backfill_enabled {
3389        return;
3390    }
3391    tokio::spawn(async move {
3392        for target in Box::pin(session_backfill_targets_for_identity(&inner, &identity)).await {
3393            let active_key = format!(
3394                "{}:session-history:{}",
3395                target.entry.runtime_key, target.session_id
3396            );
3397            {
3398                let mut seen = inner.opportunistic_session_backfills.lock().await;
3399                if !seen.insert(active_key) {
3400                    continue;
3401                }
3402            }
3403            spawn_session_history_backfill_target(inner.clone(), target, false);
3404        }
3405    });
3406}
3407
3408#[cfg(test)]
3409async fn session_backfill_target_for_identity(
3410    inner: &AggregatorInner,
3411    identity: &str,
3412) -> Option<SessionBackfillTarget> {
3413    session_backfill_targets_for_identity(inner, identity)
3414        .await
3415        .into_iter()
3416        .next()
3417}
3418
3419async fn session_backfill_targets_for_identity(
3420    inner: &AggregatorInner,
3421    identity: &str,
3422) -> Vec<SessionBackfillTarget> {
3423    let entries = inner
3424        .runtimes
3425        .read()
3426        .ok()
3427        .map(|entries| entries.clone())
3428        .unwrap_or_default();
3429    let mut targets = Vec::new();
3430    for entry in entries.values() {
3431        let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
3432        if raw_identities.is_empty() {
3433            continue;
3434        }
3435        let mids = raw_identities
3436            .iter()
3437            .map(|raw_identity| crate::member_comms_id::mob_member_id(raw_identity.as_str()))
3438            .collect::<Vec<_>>();
3439        for resolved in Box::pin(member_sources_for_entry(entry))
3440            .await
3441            .into_iter()
3442            .filter(|candidate| {
3443                resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
3444            })
3445        {
3446            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
3447                continue;
3448            };
3449            if !Box::pin(resolved_member_visible(&resolved, &record)).await {
3450                continue;
3451            }
3452            let Some(session_id) = record.session_id.clone() else {
3453                continue;
3454            };
3455            targets.push(SessionBackfillTarget {
3456                entry: entry.clone(),
3457                record,
3458                session_id,
3459            });
3460        }
3461    }
3462    targets
3463}
3464
3465fn resolved_member_matches_raw_identities(
3466    resolved: &ResolvedConsoleMember,
3467    raw_identities: &[String],
3468    mids: &[AgentIdentity],
3469) -> bool {
3470    mids.contains(&resolved.member.agent_identity)
3471        || resolved
3472            .member
3473            .labels
3474            .get("agent_identity")
3475            .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3476}
3477
3478async fn recover_lagged_source_events(
3479    inner: Arc<AggregatorInner>,
3480    runtime_key: &str,
3481    console_events: &ConsoleEventStore,
3482) -> ConsoleLogResult<()> {
3483    let watermark = inner
3484        .store
3485        .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3486        .await?;
3487    match console_events.replay_all(watermark.as_deref()).await {
3488        Ok(events) => {
3489            for envelope in events {
3490                project_console_event(inner.clone(), runtime_key, envelope).await?;
3491            }
3492        }
3493        Err(err) => {
3494            append_source_gap(
3495                &inner,
3496                runtime_key,
3497                format!(
3498                    "{}:{}:{}",
3499                    err.error, err.stream, err.requested_last_event_id
3500                ),
3501            )
3502            .await?;
3503        }
3504    }
3505    Ok(())
3506}
3507
3508async fn append_source_gap(
3509    inner: &AggregatorInner,
3510    runtime_key: &str,
3511    reason: String,
3512) -> ConsoleLogResult<()> {
3513    append_and_emit(
3514        inner,
3515        NewConsoleFrame {
3516            id: None,
3517            dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3518            timestamp_ms: current_time_ms(),
3519            runtime_key: runtime_key.to_string(),
3520            identity: "__console__".to_string(),
3521            conversation_id: None,
3522            session_id: None,
3523            kind: "replay_unavailable".to_string(),
3524            status: ConsoleFrameStatus::DeliveryFailed,
3525            payload: json!({
3526                "reason": reason,
3527                "source_kind": "console_event",
3528            }),
3529            source: ConsoleFrameSource {
3530                kind: ConsoleFrameSourceKind::Synthetic,
3531                source_cursor: None,
3532            },
3533            source_event_id: None,
3534            interaction_id: None,
3535            turn_id: None,
3536            run_id: None,
3537            parent_frame_id: None,
3538            caused_by_frame_id: None,
3539        },
3540    )
3541    .await?;
3542    let _ = inner
3543        .event_tx
3544        .send(ConsoleTimelineEvent::ReplayUnavailable {
3545            requested_cursor: format!("source-gap:{runtime_key}"),
3546            latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3547        });
3548    Ok(())
3549}
3550
3551/// Record a session-history backfill failure: stamp a watermark at the current
3552/// offset so the freshness gate throttles retries to once per
3553/// `SESSION_HISTORY_REFRESH_TTL_MS` (without it the 5s discovery loop re-reads
3554/// a persistently-failing session forever), and append a single gap frame.
3555async fn record_session_history_backfill_failure(
3556    inner: &AggregatorInner,
3557    watermark_runtime_key: &str,
3558    runtime_key: &str,
3559    identity: &str,
3560    session_id: &str,
3561    offset: usize,
3562    reason: String,
3563) -> ConsoleLogResult<()> {
3564    record_session_history_watermark(inner, watermark_runtime_key, session_id, offset).await?;
3565    append_backfill_gap(inner, runtime_key, identity, session_id, reason).await
3566}
3567
3568/// Stable dedupe key for a session-history backfill gap frame: a persistently
3569/// failing session collapses to a single gap frame across retries.
3570fn backfill_gap_dedupe_key(runtime_key: &str, identity: &str, session_id: &str) -> String {
3571    format!("session-backfill-gap:{runtime_key}:{identity}:{session_id}")
3572}
3573
3574async fn append_backfill_gap(
3575    inner: &AggregatorInner,
3576    runtime_key: &str,
3577    identity: &str,
3578    session_id: &str,
3579    reason: String,
3580) -> ConsoleLogResult<()> {
3581    append_and_emit(
3582        inner,
3583        NewConsoleFrame {
3584            id: None,
3585            // Stable per (runtime, identity, session): repeated failures for
3586            // the same session collapse to one gap frame instead of minting a
3587            // fresh one (timestamped) on every retry — which grew the timeline
3588            // store unbounded.
3589            dedupe_key: backfill_gap_dedupe_key(runtime_key, identity, session_id),
3590            timestamp_ms: current_time_ms(),
3591            runtime_key: runtime_key.to_string(),
3592            identity: identity.to_string(),
3593            conversation_id: Some(identity.to_string()),
3594            session_id: Some(session_id.to_string()),
3595            kind: "replay_unavailable".to_string(),
3596            status: ConsoleFrameStatus::DeliveryFailed,
3597            payload: json!({
3598                "reason": reason,
3599                "source_kind": "session_history",
3600            }),
3601            source: ConsoleFrameSource {
3602                kind: ConsoleFrameSourceKind::Synthetic,
3603                source_cursor: None,
3604            },
3605            source_event_id: None,
3606            interaction_id: None,
3607            turn_id: None,
3608            run_id: None,
3609            parent_frame_id: None,
3610            caused_by_frame_id: None,
3611        },
3612    )
3613    .await?;
3614    Ok(())
3615}
3616
3617async fn project_console_event(
3618    inner: Arc<AggregatorInner>,
3619    runtime_key: &str,
3620    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3621) -> ConsoleLogResult<()> {
3622    let Some(entry) = inner
3623        .runtimes
3624        .read()
3625        .ok()
3626        .and_then(|entries| entries.get(runtime_key).cloned())
3627    else {
3628        return Ok(());
3629    };
3630    let mut frame = frame_from_console_event(&entry, envelope);
3631    if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3632        frame.payload = redacted;
3633        frame.status = ConsoleFrameStatus::Redacted;
3634    }
3635    let source_cursor = frame
3636        .source_event_id
3637        .clone()
3638        .unwrap_or_else(|| frame.dedupe_key.clone());
3639    let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3640        Some(frame.identity.clone())
3641    } else {
3642        None
3643    };
3644    let opportunistic_refresh_identity = if refresh_identity.is_none()
3645        && console_event_should_start_session_history_backfill(&frame)
3646    {
3647        Some(frame.identity.clone())
3648    } else {
3649        None
3650    };
3651    append_and_emit(&inner, frame).await?;
3652    inner
3653        .store
3654        .record_source_watermark(
3655            &entry.runtime_key,
3656            ConsoleFrameSourceKind::ConsoleEvent,
3657            &source_cursor,
3658        )
3659        .await?;
3660    if let Some(identity) = refresh_identity {
3661        spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3662    } else if let Some(identity) = opportunistic_refresh_identity {
3663        spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3664    }
3665    Ok(())
3666}
3667
3668fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3669    matches!(
3670        frame.kind.as_str(),
3671        "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3672    ) || frame.session_id.is_some()
3673}
3674
3675fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3676    if frame.identity == SYSTEM_EVENT_IDENTITY {
3677        return false;
3678    }
3679    matches!(
3680        frame.kind.as_str(),
3681        "turn_started"
3682            | "run_started"
3683            | "reasoning_complete"
3684            | "tool_call_requested"
3685            | "tool_call"
3686            | "tool_execution_started"
3687            | "text_delta"
3688            | "system_notice"
3689    )
3690}
3691
3692async fn append_and_emit(
3693    inner: &AggregatorInner,
3694    frame: NewConsoleFrame,
3695) -> ConsoleLogResult<AppendOutcome> {
3696    let outcome = inner.store.append_if_absent(frame).await?;
3697    if outcome.disposition == AppendDisposition::Inserted {
3698        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3699            frame: outcome.frame.clone(),
3700        });
3701    }
3702    Ok(outcome)
3703}
3704
3705async fn update_frame_status_and_emit(
3706    inner: &AggregatorInner,
3707    frame_id: &str,
3708    status: ConsoleFrameStatus,
3709) -> ConsoleLogResult<Option<ConsoleFrame>> {
3710    let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3711        return Ok(None);
3712    };
3713    let update_marker = NewConsoleFrame {
3714        id: None,
3715        dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3716        timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3717        runtime_key: updated.runtime_key.clone(),
3718        identity: updated.identity.clone(),
3719        conversation_id: updated.conversation_id.clone(),
3720        session_id: updated.session_id.clone(),
3721        kind: "frame_updated".to_string(),
3722        status: updated.status,
3723        payload: json!({ "frame": updated.clone() }),
3724        source: ConsoleFrameSource {
3725            kind: ConsoleFrameSourceKind::Synthetic,
3726            source_cursor: None,
3727        },
3728        source_event_id: None,
3729        interaction_id: updated.interaction_id.clone(),
3730        turn_id: updated.turn_id.clone(),
3731        run_id: updated.run_id.clone(),
3732        parent_frame_id: Some(updated.id.clone()),
3733        caused_by_frame_id: Some(updated.id.clone()),
3734    };
3735    let outcome = inner.store.append_if_absent(update_marker).await?;
3736    if outcome.disposition == AppendDisposition::Inserted {
3737        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3738            frame: outcome.frame,
3739        });
3740    }
3741    Ok(Some(updated))
3742}
3743
3744fn frame_from_console_event(
3745    entry: &RuntimeEntry,
3746    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3747) -> NewConsoleFrame {
3748    let event_id = envelope.event_id;
3749    let interaction_id = envelope.interaction_id;
3750    let event_type = envelope.event_type;
3751    let payload = envelope.data;
3752    let turn_id = payload
3753        .get("turn_id")
3754        .and_then(Value::as_str)
3755        .map(ToString::to_string);
3756    let run_id = payload
3757        .get("run_id")
3758        .and_then(Value::as_str)
3759        .map(ToString::to_string);
3760    let status = match event_type.as_str() {
3761        "interaction_started" => ConsoleFrameStatus::Accepted,
3762        "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3763        "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3764        _ => ConsoleFrameStatus::Delivered,
3765    };
3766    // The system identity is runtime-plane, not agent-plane: namespacing it
3767    // would defeat the global-timeline visibility exemption in
3768    // `frame_is_visible_cached` on namespaced runtimes (memory.* events
3769    // attributed to `_system` would become `ns/_system` and vanish again).
3770    let identity = if envelope.identity == SYSTEM_EVENT_IDENTITY {
3771        envelope.identity
3772    } else {
3773        apply_namespace(&envelope.identity, &entry.identity_namespace)
3774    };
3775    let dedupe_key = reasoning_payload_text(&event_type, &payload)
3776        .map(|reasoning_text| {
3777            // Reasoning hashes assume console events carry full/cumulative text; if deltas become
3778            // fragmentary, key by a stable reasoning segment id or emit only the complete frame.
3779            let turn_scope = interaction_id
3780                .as_deref()
3781                .or(turn_id.as_deref())
3782                .or(run_id.as_deref())
3783                .unwrap_or(event_id.as_str());
3784            format!(
3785                "console-reasoning:{}:{}:{}",
3786                entry.runtime_key,
3787                turn_scope,
3788                hash_short(&normalize_transcript_fingerprint_text(reasoning_text))
3789            )
3790        })
3791        .unwrap_or_else(|| format!("console-event:{}:{}", entry.runtime_key, event_id));
3792    NewConsoleFrame {
3793        id: Some(event_id.clone()),
3794        dedupe_key,
3795        timestamp_ms: envelope.timestamp_ms,
3796        runtime_key: entry.runtime_key.clone(),
3797        identity: identity.clone(),
3798        conversation_id: Some(identity),
3799        session_id: payload
3800            .get("session_id")
3801            .and_then(Value::as_str)
3802            .map(ToString::to_string),
3803        kind: event_type,
3804        status,
3805        payload,
3806        source: ConsoleFrameSource {
3807            kind: ConsoleFrameSourceKind::ConsoleEvent,
3808            source_cursor: None,
3809        },
3810        source_event_id: Some(event_id),
3811        interaction_id,
3812        turn_id,
3813        run_id,
3814        parent_frame_id: None,
3815        caused_by_frame_id: None,
3816    }
3817}
3818
3819#[cfg(test)]
3820fn frame_from_session_history_message(
3821    runtime_key: &str,
3822    identity: &str,
3823    session_id: &str,
3824    offset: usize,
3825    message: Value,
3826) -> Option<NewConsoleFrame> {
3827    frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3828        .into_iter()
3829        .next()
3830}
3831
3832#[cfg(test)]
3833fn frames_from_session_history_message(
3834    runtime_key: &str,
3835    identity: &str,
3836    session_id: &str,
3837    offset: usize,
3838    message: Value,
3839) -> Vec<NewConsoleFrame> {
3840    frames_from_session_history_message_with_namespace(
3841        runtime_key,
3842        identity,
3843        "",
3844        session_id,
3845        offset,
3846        message,
3847    )
3848}
3849
3850/// Derive `assistant_image` frames from a session-history tool result's text,
3851/// mirroring the live ingest edge (`parse_generate_image_tool_result` ->
3852/// per-image `assistant_image` envelopes). The serialized
3853/// `ImageGenerationToolResult` is self-describing, so — unlike the live path —
3854/// no tool name is needed: we parse the result text directly. Returns an empty
3855/// vec for non-image tool results. The console dedupes images by blob/reference,
3856/// so an overlap with a live `assistant_image` frame collapses to one image.
3857#[allow(clippy::too_many_arguments)]
3858fn session_history_assistant_image_frames(
3859    runtime_key: &str,
3860    identity: &str,
3861    session_id: &str,
3862    offset: usize,
3863    result_idx: usize,
3864    timestamp_ms: u64,
3865    tool_use_id: &str,
3866    result_text: &str,
3867) -> Vec<NewConsoleFrame> {
3868    let Ok(image_result) = serde_json::from_str::<
3869        meerkat_core::image_generation::ImageGenerationToolResult,
3870    >(result_text) else {
3871        return Vec::new();
3872    };
3873    image_result
3874        .images
3875        .iter()
3876        .enumerate()
3877        .map(|(image_idx, image)| NewConsoleFrame {
3878            id: None,
3879            // Key on the stable image identity so a backfill frame and any live
3880            // `assistant_image` frame for the same image collapse on dedupe.
3881            dedupe_key: format!(
3882                "console-assistant-image:{runtime_key}:{}",
3883                image.blob_ref.blob_id
3884            ),
3885            timestamp_ms,
3886            runtime_key: runtime_key.to_string(),
3887            identity: identity.to_string(),
3888            conversation_id: Some(identity.to_string()),
3889            session_id: Some(session_id.to_string()),
3890            kind: "assistant_image".to_string(),
3891            status: ConsoleFrameStatus::Completed,
3892            payload: json!({
3893                "source_event_type": "session_history",
3894                "tool_call_id": tool_use_id,
3895                "image_id": image.image_id.0.to_string(),
3896                "blob_id": image.blob_ref.blob_id,
3897                "media_type": image.media_type.as_str(),
3898                "width": image.width,
3899                "height": image.height,
3900                "revised_prompt": image_result.revised_prompt.clone(),
3901                "type": "session_history",
3902            }),
3903            source: ConsoleFrameSource {
3904                kind: ConsoleFrameSourceKind::SessionHistory,
3905                source_cursor: Some(format!(
3906                    "{session_id}:{offset}:{result_idx}:image:{image_idx}"
3907                )),
3908            },
3909            source_event_id: None,
3910            interaction_id: None,
3911            turn_id: None,
3912            run_id: None,
3913            parent_frame_id: None,
3914            caused_by_frame_id: None,
3915        })
3916        .collect()
3917}
3918
3919fn frames_from_session_history_message_with_namespace(
3920    runtime_key: &str,
3921    identity: &str,
3922    identity_namespace: &str,
3923    session_id: &str,
3924    offset: usize,
3925    message: Value,
3926) -> Vec<NewConsoleFrame> {
3927    let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3928    let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3929        return Vec::new();
3930    };
3931    if let Message::ToolResults {
3932        results,
3933        created_at,
3934    } = parsed
3935    {
3936        return results
3937            .into_iter()
3938            .enumerate()
3939            .flat_map(|(idx, result)| {
3940                let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3941                let result_text = result.text_content();
3942                let tool_use_id = result.tool_use_id.clone();
3943                let timestamp_ms = created_at.timestamp_millis().max(0) as u64;
3944                let mut frames = vec![NewConsoleFrame {
3945                    id: None,
3946                    dedupe_key: format!(
3947                        "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3948                    ),
3949                    timestamp_ms,
3950                    runtime_key: runtime_key.to_string(),
3951                    identity: identity.to_string(),
3952                    conversation_id: Some(identity.to_string()),
3953                    session_id: Some(session_id.to_string()),
3954                    kind: "tool_execution_completed".to_string(),
3955                    status: ConsoleFrameStatus::Completed,
3956                    payload: json!({
3957                        "id": tool_use_id,
3958                        "tool_call_id": tool_use_id,
3959                        "result": result_text,
3960                        "content": content,
3961                        "is_error": result.is_error,
3962                        "source_event_type": "session_history",
3963                        "type": "session_history",
3964                    }),
3965                    source: ConsoleFrameSource {
3966                        kind: ConsoleFrameSourceKind::SessionHistory,
3967                        source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3968                    },
3969                    source_event_id: None,
3970                    interaction_id: None,
3971                    turn_id: None,
3972                    run_id: None,
3973                    parent_frame_id: None,
3974                    caused_by_frame_id: None,
3975                }];
3976                // Backfill parity with the live ingest edge: a generate_image
3977                // tool result serializes an `ImageGenerationToolResult` into the
3978                // result text (which carries NO tool name on meerkat 0.7.x). The
3979                // live path emits per-image `assistant_image` frames; without
3980                // this the image renders as raw tool JSON after a reload /
3981                // history backfill. Parse the result directly and mirror those
3982                // frames (the console dedupes images by blob/reference).
3983                frames.extend(session_history_assistant_image_frames(
3984                    runtime_key,
3985                    identity,
3986                    session_id,
3987                    offset,
3988                    idx,
3989                    timestamp_ms,
3990                    &tool_use_id,
3991                    &result_text,
3992                ));
3993                frames
3994            })
3995            .collect();
3996    }
3997    // meerkat 0.7.25 persists `TranscriptMessageIdentity` onto committed
3998    // transcript messages (ask 15 addendum). Stamping it here gives history
3999    // frames the SAME interaction identity the live pipeline stamped (for
4000    // identity-first sends the host-minted id round-trips through runtime
4001    // admission), so the console can join live↔history twins exactly instead
4002    // of by content heuristics.
4003    let transcript_identity = match &parsed {
4004        Message::User(user) => Some(&user.identity),
4005        Message::BlockAssistant(assistant) => Some(&assistant.identity),
4006        _ => None,
4007    };
4008    let history_interaction_id = transcript_identity
4009        .and_then(|identity| identity.interaction_id.as_ref())
4010        .map(|id| id.0.to_string());
4011    let history_run_id = transcript_identity
4012        .and_then(|identity| identity.run_id.as_ref())
4013        .map(|id| id.0.to_string());
4014    let (kind, timestamp_ms, payload) = match &parsed {
4015        Message::User(user) => {
4016            if session_history_user_message_is_scaffold(&message) {
4017                return Vec::new();
4018            }
4019            (
4020                "user_input",
4021                user.created_at.timestamp_millis().max(0) as u64,
4022                json!({
4023                    "content": user.content,
4024                    "message": message,
4025                }),
4026            )
4027        }
4028        // Meerkat 0.7 removed the legacy flat-text `Message::Assistant`
4029        // variant; all assistant history is block-shaped now.
4030        Message::BlockAssistant(assistant) => {
4031            let text = assistant.text_blocks().collect::<Vec<_>>().join("");
4032            (
4033                "interaction_complete",
4034                assistant.created_at.timestamp_millis().max(0) as u64,
4035                json!({
4036                    "result": text,
4037                    "text": text,
4038                    "message": message,
4039                    "source_event_type": "session_history",
4040                    "type": "session_history",
4041                }),
4042            )
4043        }
4044        Message::SystemNotice(notice) => (
4045            "system_notice",
4046            notice.created_at.timestamp_millis().max(0) as u64,
4047            json!({
4048                "message": message,
4049                "kind": notice.kind,
4050                "render_class": notice.kind.render_class(),
4051                "body": notice.body,
4052                "blocks": notice.blocks,
4053                "source_event_type": "session_history",
4054                "type": "session_history",
4055            }),
4056        ),
4057        Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
4058    };
4059    let mut frames = vec![NewConsoleFrame {
4060        id: None,
4061        dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
4062        timestamp_ms,
4063        runtime_key: runtime_key.to_string(),
4064        identity: identity.to_string(),
4065        conversation_id: Some(identity.to_string()),
4066        session_id: Some(session_id.to_string()),
4067        kind: kind.to_string(),
4068        status: ConsoleFrameStatus::Completed,
4069        payload,
4070        source: ConsoleFrameSource {
4071            kind: ConsoleFrameSourceKind::SessionHistory,
4072            source_cursor: Some(format!("{session_id}:{offset}")),
4073        },
4074        source_event_id: None,
4075        interaction_id: history_interaction_id,
4076        turn_id: None,
4077        run_id: history_run_id,
4078        parent_frame_id: None,
4079        caused_by_frame_id: None,
4080    }];
4081    if let Message::BlockAssistant(assistant) = &parsed {
4082        frames.extend(spawn_initial_message_frames_from_assistant(
4083            runtime_key,
4084            identity,
4085            identity_namespace,
4086            session_id,
4087            offset,
4088            assistant,
4089            &payload_hash,
4090        ));
4091    }
4092    frames
4093}
4094
4095fn spawn_initial_message_frames_from_assistant(
4096    runtime_key: &str,
4097    parent_identity: &str,
4098    identity_namespace: &str,
4099    session_id: &str,
4100    offset: usize,
4101    assistant: &meerkat_core::types::BlockAssistantMessage,
4102    payload_hash: &str,
4103) -> Vec<NewConsoleFrame> {
4104    let mut frames = Vec::new();
4105    for (tool_idx, tool) in assistant.tool_calls().enumerate() {
4106        if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
4107            continue;
4108        }
4109        let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
4110            continue;
4111        };
4112        for (spawn_idx, (target_identity, initial_message)) in
4113            spawn_initial_messages_from_tool_args(&args)
4114                .into_iter()
4115                .enumerate()
4116        {
4117            let target_identity = apply_namespace(&target_identity, identity_namespace);
4118            let message_content = match &initial_message {
4119                Value::String(text) => json!([
4120                    {
4121                        "type": "text",
4122                        "text": text,
4123                    }
4124                ]),
4125                other => other.clone(),
4126            };
4127            let message = match &initial_message {
4128                Value::String(text) => json!({
4129                    "role": "user",
4130                    "content": text,
4131                    "created_at": assistant.created_at,
4132                }),
4133                other => json!({
4134                    "role": "user",
4135                    "content": other,
4136                    "created_at": assistant.created_at,
4137                }),
4138            };
4139            frames.push(NewConsoleFrame {
4140                id: None,
4141                dedupe_key: format!(
4142                    "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
4143                ),
4144                timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
4145                runtime_key: runtime_key.to_string(),
4146                identity: target_identity.clone(),
4147                conversation_id: Some(target_identity),
4148                session_id: None,
4149                kind: "user_input".to_string(),
4150                status: ConsoleFrameStatus::Completed,
4151                payload: json!({
4152                    "content": message_content,
4153                    "message": message,
4154                    "source_event_type": "session_history_spawn_initial_message",
4155                    "type": "session_history",
4156                    "tool_call_id": tool.id,
4157                    "parent_identity": parent_identity,
4158                    "via_tool": tool.name,
4159                }),
4160                source: ConsoleFrameSource {
4161                    kind: ConsoleFrameSourceKind::SessionHistory,
4162                    source_cursor: Some(format!(
4163                        "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
4164                    )),
4165                },
4166                source_event_id: None,
4167                interaction_id: None,
4168                turn_id: None,
4169                run_id: None,
4170                parent_frame_id: None,
4171                caused_by_frame_id: None,
4172            });
4173        }
4174    }
4175    frames
4176}
4177
4178fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
4179    let mut messages = Vec::new();
4180    if let Some(message) = spawn_initial_message_from_record(args) {
4181        messages.push(message);
4182    }
4183    if let Some(specs) = args.get("specs").and_then(Value::as_array) {
4184        for spec in specs {
4185            if let Some(message) = spawn_initial_message_from_record(spec) {
4186                messages.push(message);
4187            }
4188        }
4189    }
4190    messages
4191}
4192
4193fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
4194    let target_identity = record
4195        .get("member_id")
4196        .or_else(|| record.get("agent_identity"))
4197        .and_then(Value::as_str)?
4198        .trim();
4199    if target_identity.is_empty() {
4200        return None;
4201    }
4202    let initial_message = record
4203        .get("initial_message")
4204        .or_else(|| record.get("task"))?
4205        .clone();
4206    Some((target_identity.to_string(), initial_message))
4207}
4208
4209fn session_history_user_message_is_scaffold(message: &Value) -> bool {
4210    message
4211        .get("content")
4212        .is_some_and(scaffold_message_content_is_noise)
4213}
4214
4215fn scaffold_message_content_is_noise(value: &Value) -> bool {
4216    match value {
4217        Value::String(text) => scaffold_message_text_is_noise(text),
4218        Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
4219        Value::Object(map) => ["text", "content", "message"]
4220            .iter()
4221            .filter_map(|key| map.get(*key))
4222            .any(scaffold_message_content_is_noise),
4223        _ => false,
4224    }
4225}
4226
4227fn scaffold_message_text_is_noise(text: &str) -> bool {
4228    let trimmed = text.trim_start();
4229    trimmed.starts_with("[PEER UPDATE]")
4230        || trimmed
4231            .to_ascii_lowercase()
4232            .starts_with("you have been spawned")
4233}
4234
4235fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
4236    let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
4237    rest.split(':').next()?.parse().ok()
4238}
4239
4240fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
4241    format!("{session_id}:{offset}:{checked_at_ms}")
4242}
4243
4244fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
4245    let Some(checked_at_ms) = watermark
4246        .rsplit_once(':')
4247        .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
4248    else {
4249        return false;
4250    };
4251    let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
4252    let ttl_ms = if offset > 0 {
4253        SESSION_HISTORY_GROWING_REFRESH_TTL_MS
4254    } else {
4255        SESSION_HISTORY_REFRESH_TTL_MS
4256    };
4257    now_ms.saturating_sub(checked_at_ms) < ttl_ms
4258}
4259
4260async fn history_frame_has_existing_counterpart(
4261    inner: &AggregatorInner,
4262    frame: &NewConsoleFrame,
4263) -> ConsoleLogResult<bool> {
4264    let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
4265    let Some(fingerprint) = fingerprint else {
4266        return Ok(false);
4267    };
4268    let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
4269    let mut delta_text_by_turn = BTreeMap::<String, String>::new();
4270    let mut after = None;
4271    loop {
4272        let page = inner
4273            .store
4274            .query_frames(ConsoleTimelineQuery {
4275                identity: Some(frame.identity.clone()),
4276                conversation_id: frame.conversation_id.clone(),
4277                after,
4278                limit: 1_000,
4279            })
4280            .await?;
4281        for existing in &page.frames {
4282            let same_session = existing.session_id == frame.session_id
4283                || existing.session_id.is_none()
4284                || frame.session_id.is_none();
4285            if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
4286                continue;
4287            }
4288            if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
4289                == Some(&fingerprint)
4290            {
4291                return Ok(true);
4292            }
4293            if assistant_terminal
4294                && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
4295            {
4296                let turn_key = existing
4297                    .interaction_id
4298                    .as_deref()
4299                    .or(existing.turn_id.as_deref())
4300                    .or(existing.run_id.as_deref())
4301                    .unwrap_or("session");
4302                let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
4303                aggregated.push_str(delta);
4304                if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
4305                    return Ok(true);
4306                }
4307            }
4308        }
4309        if page.frames.is_empty() || page.next_cursor.is_none() {
4310            return Ok(false);
4311        }
4312        after = page.next_cursor;
4313    }
4314}
4315
4316fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
4317    format!("{runtime_key}:session-history:{session_id}")
4318}
4319
4320fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4321    match kind {
4322        "user_input" | "interaction_started" => payload
4323            .get("content")
4324            .map(stable_value_fingerprint)
4325            .or_else(|| payload.get("message").map(stable_value_fingerprint)),
4326        "tool_execution_completed" => tool_result_fingerprint(payload),
4327        "text_delta" => {
4328            text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4329        }
4330        kind if is_reasoning_event_kind(kind) => {
4331            reasoning_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
4332        }
4333        "text_complete" | "interaction_complete" | "run_completed" => {
4334            assistant_terminal_fingerprint(kind, payload)
4335        }
4336        _ => None,
4337    }
4338}
4339
4340fn tool_result_fingerprint(payload: &Value) -> Option<String> {
4341    let id = payload
4342        .get("tool_call_id")
4343        .or_else(|| payload.get("id"))
4344        .and_then(Value::as_str)
4345        .unwrap_or("");
4346    let result = payload
4347        .get("result")
4348        .or_else(|| payload.get("content"))
4349        .map(stable_value_fingerprint)?;
4350    Some(format!("{id}:{result}"))
4351}
4352
4353fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
4354    match kind {
4355        "text_complete" | "interaction_complete" | "run_completed" => payload
4356            .get("text")
4357            .or_else(|| payload.get("result"))
4358            .or_else(|| payload.get("content"))
4359            .map(stable_value_fingerprint),
4360        _ => None,
4361    }
4362}
4363
4364fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4365    if kind != "text_delta" {
4366        return None;
4367    }
4368    payload
4369        .get("delta")
4370        .or_else(|| payload.get("text"))
4371        .or_else(|| payload.get("content"))
4372        .and_then(Value::as_str)
4373        .or_else(|| payload.as_str())
4374}
4375
4376fn reasoning_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
4377    if !is_reasoning_event_kind(kind) {
4378        return None;
4379    }
4380    if kind == "reasoning_delta" {
4381        return payload
4382            .get("delta")
4383            .or_else(|| payload.get("text"))
4384            .or_else(|| payload.get("content"))
4385            .or_else(|| payload.get("summary"))
4386            .or_else(|| payload.get("reasoning"))
4387            .and_then(Value::as_str)
4388            .or_else(|| payload.as_str());
4389    }
4390    payload
4391        .get("summary")
4392        .or_else(|| payload.get("text"))
4393        .or_else(|| payload.get("content"))
4394        .or_else(|| payload.get("delta"))
4395        .or_else(|| payload.get("reasoning"))
4396        .and_then(Value::as_str)
4397        .or_else(|| payload.as_str())
4398}
4399
4400fn is_reasoning_event_kind(kind: &str) -> bool {
4401    matches!(
4402        kind,
4403        "reasoning_delta"
4404            | "reasoning_complete"
4405            | "reasoning_summary"
4406            | "reasoning_summary_delta"
4407            | "reasoning_summary_complete"
4408    )
4409}
4410
4411fn stable_value_fingerprint(value: &Value) -> String {
4412    if let Some(text) = content_value_text(value) {
4413        return normalize_transcript_fingerprint_text(&text);
4414    }
4415    match value {
4416        Value::String(text) => normalize_transcript_fingerprint_text(text),
4417        other => serde_json::to_string(other).unwrap_or_default(),
4418    }
4419}
4420
4421fn content_value_text(value: &Value) -> Option<String> {
4422    match value {
4423        Value::String(text) => Some(text.clone()),
4424        Value::Array(items) => {
4425            let text = items
4426                .iter()
4427                .filter_map(content_value_text)
4428                .collect::<String>();
4429            (!text.is_empty()).then_some(text)
4430        }
4431        Value::Object(map) => ["text", "content", "message", "blocks"]
4432            .iter()
4433            .filter_map(|key| map.get(*key))
4434            .find_map(content_value_text),
4435        _ => None,
4436    }
4437}
4438
4439fn normalize_transcript_fingerprint_text(text: &str) -> String {
4440    let trimmed = text.trim();
4441    trimmed
4442        .strip_prefix("[EVENT via rpc] ")
4443        .unwrap_or(trimmed)
4444        .trim()
4445        .to_string()
4446}
4447
4448#[derive(Debug, Clone, Copy)]
4449enum CachedIdentityVisibility {
4450    Visible,
4451    Hidden,
4452    Missing,
4453}
4454
4455async fn frame_is_visible(
4456    inner: &AggregatorInner,
4457    frame: &ConsoleFrame,
4458    allow_historical_identity: bool,
4459    identity_records: &[ConsoleIdentityRecord],
4460) -> ConsoleLogResult<bool> {
4461    let mut identity_visibility_cache = HashMap::new();
4462    Box::pin(frame_is_visible_cached(
4463        inner,
4464        frame,
4465        allow_historical_identity,
4466        &mut identity_visibility_cache,
4467        identity_records,
4468    ))
4469    .await
4470}
4471
4472async fn frame_is_visible_cached(
4473    inner: &AggregatorInner,
4474    frame: &ConsoleFrame,
4475    allow_historical_identity: bool,
4476    identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
4477    identity_records: &[ConsoleIdentityRecord],
4478) -> ConsoleLogResult<bool> {
4479    let entry = {
4480        let entries = inner
4481            .runtimes
4482            .read()
4483            .map_err(|_| runtime_registry_lock_error())?;
4484        if entries.is_empty() {
4485            return Ok(true);
4486        }
4487        let Some(entry) = entries.get(&frame.runtime_key) else {
4488            return Ok(false);
4489        };
4490        entry.clone()
4491    };
4492    // Runtime-plane frames aren't agent frames: `__console__` (synthetic
4493    // gap markers) and `_system` (ConsoleMemoryEventSink attributes
4494    // identity-less §9.3 memory.* events here) have no roster record to
4495    // resolve, so the per-identity gate below would hide them from every
4496    // global timeline query. They skip straight to the visibility policy;
4497    // ABAC still filters them per-caller in the RPC layer
4498    // (`retain_visible_timeline_frames`).
4499    if frame.identity != "__console__" && frame.identity != SYSTEM_EVENT_IDENTITY {
4500        let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
4501        let identity_visibility =
4502            if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
4503                cached
4504            } else {
4505                let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
4506                    .unwrap_or_else(|| frame.identity.clone());
4507                let visibility = match identity_records.iter().find(|record| {
4508                    record.runtime_key == frame.runtime_key
4509                        && (record.identity == frame.identity
4510                            || record.runtime_member_id == runtime_member_id)
4511                }) {
4512                    Some(record) => {
4513                        if Box::pin(console_identity_record_visible(&entry, record)).await {
4514                            CachedIdentityVisibility::Visible
4515                        } else {
4516                            CachedIdentityVisibility::Hidden
4517                        }
4518                    }
4519                    None => CachedIdentityVisibility::Missing,
4520                };
4521                identity_visibility_cache.insert(cache_key, visibility);
4522                visibility
4523            };
4524        match identity_visibility {
4525            CachedIdentityVisibility::Visible => {
4526                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4527                    return Ok(false);
4528                }
4529            }
4530            CachedIdentityVisibility::Hidden => return Ok(false),
4531            CachedIdentityVisibility::Missing => {
4532                if Box::pin(frame_matches_hidden_member(&entry, frame)).await {
4533                    return Ok(false);
4534                }
4535                return Ok(
4536                    allow_historical_identity && entry.visibility_policy.frame_visible(frame)
4537                );
4538            }
4539        }
4540    }
4541    Ok(entry.visibility_policy.frame_visible(frame))
4542}
4543
4544async fn identity_record_for_member(
4545    entry: &RuntimeEntry,
4546    handle: &MobHandle,
4547    member: &MobMemberListEntry,
4548) -> Option<ConsoleIdentityRecord> {
4549    // Roster ids are comms-safe encodings (meerkat 0.7 MemberCommsName);
4550    // identity records carry the public alias.
4551    let runtime_member_id =
4552        crate::member_comms_id::runtime_alias_str(member.agent_identity.as_str()).into_owned();
4553    let durable_identity = member
4554        .labels
4555        .get("agent_identity")
4556        .filter(|value| !value.trim().is_empty())
4557        .map_or(runtime_member_id.as_str(), String::as_str);
4558    let identity = apply_namespace(durable_identity, &entry.identity_namespace);
4559    let addressable = member_is_addressable(member);
4560    let visibility = match member.status {
4561        meerkat_mob::MobMemberStatus::Retiring | meerkat_mob::MobMemberStatus::Completed => {
4562            ConsoleVisibility::RetiredReadable
4563        }
4564        // Broken/unknown members have no live runtime binding; mirror the
4565        // identity-first projection, which marks them unreachable.
4566        meerkat_mob::MobMemberStatus::Broken | meerkat_mob::MobMemberStatus::Unknown => {
4567            ConsoleVisibility::Unreachable
4568        }
4569        _ if addressable => ConsoleVisibility::Addressable,
4570        _ => ConsoleVisibility::Hidden,
4571    };
4572    let session_id = handle
4573        .resolve_bridge_session_id_observation(&member.agent_identity)
4574        .await
4575        .map(|sid| sid.to_string());
4576    let mut labels = member.labels.clone();
4577    labels
4578        .entry("role".to_string())
4579        .or_insert_with(|| member.role.to_string());
4580    // Spawn-time console metadata (group/display labels, spawned_by lineage)
4581    // registered by the agent-tool spawn path fills gaps the roster does not
4582    // carry; roster labels win on conflict — except lineage, which only the
4583    // spawn registry may assert (these records feed aggregate-mode ABAC
4584    // attribute priming).
4585    crate::console_spawn::sanitize_unverified_lineage_labels(&mut labels);
4586    if let Some(registered) = entry.console_events.identity_labels(durable_identity).await {
4587        crate::console_spawn::merge_registered_labels(&mut labels, &registered);
4588    }
4589    let display_name = labels
4590        .get("display_name")
4591        .cloned()
4592        .unwrap_or_else(|| runtime_member_id.clone());
4593    Some(ConsoleIdentityRecord {
4594        identity,
4595        display_name,
4596        runtime_key: entry.runtime_key.clone(),
4597        runtime_member_id,
4598        session_id,
4599        visibility,
4600        addressable,
4601        health: if addressable {
4602            "ready"
4603        } else {
4604            "hidden_by_policy"
4605        }
4606        .to_string(),
4607        topology_peers: member
4608            .wired_to
4609            .iter()
4610            .map(|peer| crate::member_comms_id::runtime_alias_str(peer.as_str()).into_owned())
4611            .collect(),
4612        labels,
4613    })
4614}
4615
4616async fn identity_record_for_resolved_member(
4617    resolved: &ResolvedConsoleMember,
4618) -> Option<ConsoleIdentityRecord> {
4619    let mut record =
4620        identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
4621    if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
4622        record
4623            .labels
4624            .entry("source_mob_id".to_string())
4625            .or_insert_with(|| resolved.source_mob_id.clone());
4626    }
4627    Some(record)
4628}
4629
4630fn identity_record_for_status(
4631    entry: &RuntimeEntry,
4632    status: &crate::identity_first::IdentityStatus,
4633) -> ConsoleIdentityRecord {
4634    let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
4635    let runtime_member_id = status
4636        .agent_runtime_id
4637        .as_ref()
4638        .map(crate::identity_first::AgentRuntimeId::as_str)
4639        .unwrap_or_else(|| status.identity.as_str())
4640        .to_string();
4641    let addressable = status.addressability
4642        == crate::identity_first::AgentAddressability::Addressable
4643        && matches!(
4644            status.state,
4645            crate::identity_first::IdentityLifecycleState::Active
4646                | crate::identity_first::IdentityLifecycleState::Dormant
4647                | crate::identity_first::IdentityLifecycleState::Uninitialized
4648        );
4649    let visibility = match status.state {
4650        crate::identity_first::IdentityLifecycleState::Retiring => {
4651            ConsoleVisibility::RetiredReadable
4652        }
4653        crate::identity_first::IdentityLifecycleState::Broken
4654        | crate::identity_first::IdentityLifecycleState::Suspended => {
4655            ConsoleVisibility::Unreachable
4656        }
4657        _ if addressable => ConsoleVisibility::Addressable,
4658        _ => ConsoleVisibility::Hidden,
4659    };
4660    let health = match status.state {
4661        crate::identity_first::IdentityLifecycleState::Active => "ready",
4662        crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
4663        crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
4664        crate::identity_first::IdentityLifecycleState::Broken => "broken",
4665        crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
4666        crate::identity_first::IdentityLifecycleState::Retiring => "retired",
4667    }
4668    .to_string();
4669    let display_name = status
4670        .display_name
4671        .as_ref()
4672        .map(crate::identity_first::DisplayName::as_str)
4673        .unwrap_or_else(|| status.identity.as_str())
4674        .to_string();
4675    ConsoleIdentityRecord {
4676        identity,
4677        display_name,
4678        runtime_key: entry.runtime_key.clone(),
4679        runtime_member_id,
4680        session_id: status.session_id.as_ref().map(ToString::to_string),
4681        visibility,
4682        addressable,
4683        health,
4684        topology_peers: Vec::new(),
4685        labels: status.labels.clone(),
4686    }
4687}
4688
4689pub(crate) fn is_implicit_delegate_member(
4690    role: &str,
4691    labels: &std::collections::BTreeMap<String, String>,
4692) -> bool {
4693    role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4694}
4695
4696fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4697    member
4698        .labels
4699        .get("addressable")
4700        .map(|value| !value.eq_ignore_ascii_case("false"))
4701        .unwrap_or(true)
4702}
4703
4704fn sendable_resolved_member_indices(matches: &[ResolvedConsoleMember]) -> Vec<usize> {
4705    matches
4706        .iter()
4707        .enumerate()
4708        .filter_map(|(index, resolved)| {
4709            (member_is_addressable(&resolved.member)
4710                && resolved.member.status == meerkat_mob::MobMemberStatus::Active)
4711                .then_some(index)
4712        })
4713        .collect()
4714}
4715
4716async fn authoritative_durable_sendable_member_index(
4717    requested_identity: &str,
4718    matches: &[ResolvedConsoleMember],
4719    sendable_indices: &[usize],
4720) -> Result<Option<usize>, ConsoleSendError> {
4721    if matches.iter().any(|resolved| {
4722        requested_identity_is_runtime_member_alias(
4723            requested_identity,
4724            &resolved.entry.identity_namespace,
4725        )
4726    }) {
4727        return Ok(None);
4728    }
4729
4730    let mut visited_runtime_keys = BTreeSet::new();
4731    let mut bindings = BTreeSet::new();
4732    for resolved in matches {
4733        let entry = &resolved.entry;
4734        if !visited_runtime_keys.insert(entry.runtime_key.clone()) {
4735            continue;
4736        }
4737        let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
4738            continue;
4739        };
4740        for raw_identity in
4741            namespace_match_candidates(requested_identity, &entry.identity_namespace)
4742        {
4743            let Ok(identity) = crate::identity_first::AgentIdentity::parse(&raw_identity) else {
4744                continue;
4745            };
4746            let Ok(status) = identity_runtime.status(&identity).await else {
4747                continue;
4748            };
4749            if status.state != crate::identity_first::IdentityLifecycleState::Active
4750                || status.addressability != crate::identity_first::AgentAddressability::Addressable
4751            {
4752                continue;
4753            }
4754            let Some(runtime_id) = status.agent_runtime_id.as_ref() else {
4755                continue;
4756            };
4757            let record = identity_record_for_status(entry, &status);
4758            if !Box::pin(console_identity_record_visible(entry, &record)).await {
4759                continue;
4760            }
4761            bindings.insert((entry.runtime_key.clone(), runtime_id.as_str().to_string()));
4762        }
4763    }
4764
4765    if bindings.len() > 1 {
4766        let candidates = bindings
4767            .iter()
4768            .map(|(runtime_key, runtime_id)| format!("{runtime_id}@{runtime_key}"))
4769            .collect::<Vec<_>>()
4770            .join(", ");
4771        return Err(ConsoleSendError::AmbiguousIdentity {
4772            identity: requested_identity.to_string(),
4773            candidates,
4774        });
4775    }
4776    let Some((runtime_key, runtime_id)) = bindings.into_iter().next() else {
4777        return Ok(None);
4778    };
4779    let bound_indices = sendable_indices
4780        .iter()
4781        .copied()
4782        .filter(|index| {
4783            matches[*index].entry.runtime_key == runtime_key
4784                && matches[*index].runtime_identity == runtime_id
4785        })
4786        .collect::<Vec<_>>();
4787    match bound_indices.as_slice() {
4788        [index] => Ok(Some(*index)),
4789        [] => Ok(None),
4790        _ => {
4791            let candidates = bound_indices
4792                .iter()
4793                .map(|index| {
4794                    format!(
4795                        "{}@{}",
4796                        matches[*index].runtime_identity, matches[*index].source_mob_id
4797                    )
4798                })
4799                .collect::<Vec<_>>()
4800                .join(", ");
4801            Err(ConsoleSendError::AmbiguousIdentity {
4802                identity: requested_identity.to_string(),
4803                candidates,
4804            })
4805        }
4806    }
4807}
4808
4809fn apply_namespace(identity: &str, namespace: &str) -> String {
4810    let namespace = namespace.trim().trim_matches('/');
4811    if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4812        identity.to_string()
4813    } else {
4814        format!("{namespace}/{identity}")
4815    }
4816}
4817
4818fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4819    let namespace = namespace.trim().trim_matches('/');
4820    if namespace.is_empty() {
4821        return Some(identity.to_string());
4822    }
4823    identity
4824        .strip_prefix(namespace)
4825        .and_then(|rest| rest.strip_prefix('/'))
4826        .map(ToString::to_string)
4827}
4828
4829fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4830    let namespace = namespace.trim().trim_matches('/');
4831    if namespace.is_empty() {
4832        return vec![identity.to_string()];
4833    }
4834    let mut candidates = Vec::new();
4835    if let Some(stripped) = strip_namespace(identity, namespace) {
4836        candidates.push(stripped);
4837        candidates.push(identity.to_string());
4838    }
4839    candidates.sort();
4840    candidates.dedup();
4841    candidates
4842}
4843
4844fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4845    namespace_match_candidates(identity, namespace)
4846        .iter()
4847        .any(|candidate| candidate.starts_with("rt:"))
4848}
4849
4850fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4851    entries.iter().any(|entry| {
4852        requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4853    })
4854}
4855
4856fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4857    if request.identity.trim().is_empty() {
4858        return Err(ConsoleSendError::InvalidRequest(
4859            "identity must be non-empty".to_string(),
4860        ));
4861    }
4862    if request.origin.trim().is_empty() {
4863        return Err(ConsoleSendError::InvalidRequest(
4864            "origin must be non-empty".to_string(),
4865        ));
4866    }
4867    if request.idempotency_key.trim().is_empty() {
4868        return Err(ConsoleSendError::InvalidRequest(
4869            "idempotency_key must be non-empty".to_string(),
4870        ));
4871    }
4872    Ok(())
4873}
4874
4875fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4876    let content: ContentInput = serde_json::from_value(value.clone())
4877        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4878    match &content {
4879        ContentInput::Text(text) if text.trim().is_empty() => Err(
4880            ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4881        ),
4882        ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4883            "content blocks must be non-empty".to_string(),
4884        )),
4885        _ => Ok(content),
4886    }
4887}
4888
4889fn parse_handling_mode(
4890    value: Option<&str>,
4891) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4892    match value.unwrap_or("queue") {
4893        "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4894        "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4895        other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4896    }
4897}
4898
4899fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4900    ConsoleInteractionAccepted {
4901        interaction_id: frame
4902            .interaction_id
4903            .clone()
4904            .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4905        identity: frame.identity.clone(),
4906        conversation_id: frame.conversation_id.clone(),
4907        session_id: frame.session_id.clone(),
4908        input_frame_id: frame.id.clone(),
4909        cursor: frame.cursor.clone(),
4910        status: frame.status,
4911    }
4912}
4913
4914fn send_dedupe_key(
4915    runtime_key: &str,
4916    identity: &str,
4917    origin: &str,
4918    idempotency_key: &str,
4919) -> String {
4920    format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4921}
4922
4923fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4924    let content_json = serde_json::to_string(content).unwrap_or_default();
4925    hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4926}
4927
4928/// Namespace for identity-first console interaction ids (UUIDv5 over the
4929/// send's dedupe key). Deterministic: an idempotent retry mints the SAME id.
4930/// UUID form is deliberate — the identity-first delivery path threads this id
4931/// into meerkat runtime admission (`WorkSpec.interaction_id`, 0.7.25 ask 15
4932/// addendum), so the turn's live frames and its persisted transcript messages
4933/// carry one joinable identity. The classic send path keeps the legacy
4934/// `console-interaction-{hash}` string: it cannot thread ids (the external
4935/// work door has no interaction parameter), and the console's dedup treats
4936/// only UUID-form ids as authoritative.
4937const CONSOLE_INTERACTION_UUID_NAMESPACE: uuid::Uuid = uuid::Uuid::from_bytes([
4938    0x6d, 0x6f, 0x62, 0x6b, 0x69, 0x74, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x6f, 0x6c, 0x65, 0x2d, 0x69,
4939]);
4940
4941fn identity_first_interaction_uuid(dedupe_key: &str) -> String {
4942    uuid::Uuid::new_v5(&CONSOLE_INTERACTION_UUID_NAMESPACE, dedupe_key.as_bytes()).to_string()
4943}
4944
4945fn hash_short(value: &str) -> String {
4946    let mut hasher = Sha256::new();
4947    hasher.update(value.as_bytes());
4948    let digest = hasher.finalize();
4949    to_hex(&digest[..8])
4950}
4951
4952fn to_hex(bytes: &[u8]) -> String {
4953    const HEX: &[u8; 16] = b"0123456789abcdef";
4954    let mut out = String::with_capacity(bytes.len() * 2);
4955    for byte in bytes {
4956        out.push(HEX[(byte >> 4) as usize] as char);
4957        out.push(HEX[(byte & 0x0f) as usize] as char);
4958    }
4959    out
4960}
4961
4962fn current_time_ms() -> u64 {
4963    match SystemTime::now().duration_since(UNIX_EPOCH) {
4964        Ok(duration) => duration.as_millis() as u64,
4965        Err(_) => 0,
4966    }
4967}
4968
4969fn runtime_registry_lock_error() -> ConsoleLogError {
4970    Box::new(std::io::Error::other(
4971        "console runtime registry lock poisoned",
4972    ))
4973}
4974
4975#[cfg(test)]
4976#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4977mod tests {
4978    /// Regression: a session that keeps failing backfill must collapse to ONE
4979    /// gap frame. The dedupe key is stable per (runtime, identity, session) —
4980    /// independent of wall-clock — so repeated failures don't mint a fresh
4981    /// timestamped frame each retry and grow the timeline store unbounded.
4982    #[test]
4983    fn backfill_gap_dedupe_key_is_stable_per_session() {
4984        use super::backfill_gap_dedupe_key;
4985        let a = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4986        let b = backfill_gap_dedupe_key("default", "worker:1", "sess-9");
4987        assert_eq!(a, b, "same target must reuse one gap frame");
4988        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:1", "sess-10"));
4989        assert_ne!(a, backfill_gap_dedupe_key("default", "worker:2", "sess-9"));
4990        // The key carries no wall-clock component (the pre-fix bug appended
4991        // current_time_ms, so every retry was a unique frame).
4992        assert_eq!(a, "session-backfill-gap:default:worker:1:sess-9");
4993    }
4994
4995    use std::sync::Arc;
4996    use std::sync::atomic::{AtomicUsize, Ordering};
4997    use std::time::Duration;
4998    use std::time::Instant;
4999
5000    use futures::StreamExt;
5001    use meerkat::{AgentFactory, Config, build_ephemeral_service};
5002    use meerkat_client::types::LlmStream;
5003    use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
5004    use meerkat_core::{
5005        AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
5006        RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
5007        SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
5008        SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
5009        StreamError,
5010    };
5011    use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
5012    use serde_json::json;
5013
5014    use super::*;
5015    use crate::mob_handle_runtime::MobBootstrapSpec;
5016
5017    #[derive(Debug)]
5018    struct HideHiddenNoiseFrames;
5019
5020    impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
5021        fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
5022            frame.kind != "hidden_noise"
5023        }
5024    }
5025
5026    #[derive(Debug)]
5027    struct HideRuntimeMemberIdentity(&'static str);
5028
5029    impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
5030        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5031            record.runtime_member_id != self.0
5032        }
5033    }
5034
5035    #[derive(Debug)]
5036    struct HideRuntimeMemberOnly(&'static str);
5037
5038    impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
5039        fn member_visible(&self, member: &ConsoleMember) -> bool {
5040            member.agent_identity != self.0
5041        }
5042    }
5043
5044    #[derive(Debug)]
5045    struct HideConsoleIdentity(&'static str);
5046
5047    impl ConsoleVisibilityPolicy for HideConsoleIdentity {
5048        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
5049            record.identity != self.0
5050        }
5051    }
5052
5053    struct CountingConsoleLogStore {
5054        inner: InMemoryConsoleLogStore,
5055        source_watermark_calls: AtomicUsize,
5056        record_watermark_calls: AtomicUsize,
5057    }
5058
5059    impl CountingConsoleLogStore {
5060        fn new() -> Self {
5061            Self {
5062                inner: InMemoryConsoleLogStore::new(),
5063                source_watermark_calls: AtomicUsize::new(0),
5064                record_watermark_calls: AtomicUsize::new(0),
5065            }
5066        }
5067
5068        fn source_watermark_calls(&self) -> usize {
5069            self.source_watermark_calls.load(Ordering::SeqCst)
5070        }
5071    }
5072
5073    struct SlowTestClient {
5074        delay: Duration,
5075    }
5076
5077    #[async_trait::async_trait]
5078    impl LlmClient for SlowTestClient {
5079        fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
5080            Ok(messages.to_vec())
5081        }
5082
5083        fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
5084            let delay = self.delay;
5085            let delayed_text = futures::stream::once(async move {
5086                tokio::time::sleep(delay).await;
5087                Ok(LlmEvent::TextDelta {
5088                    delta: "slow ok".to_string(),
5089                    meta: None,
5090                })
5091            });
5092            let done = futures::stream::once(async {
5093                Ok(LlmEvent::Done {
5094                    outcome: LlmDoneOutcome::Success {
5095                        stop_reason: StopReason::EndTurn,
5096                    },
5097                })
5098            });
5099            Box::pin(delayed_text.chain(done))
5100        }
5101
5102        fn provider(&self) -> meerkat_core::Provider {
5103            meerkat_core::Provider::Other
5104        }
5105
5106        async fn health_check(&self) -> Result<(), LlmError> {
5107            Ok(())
5108        }
5109    }
5110
5111    #[derive(Clone)]
5112    struct DelayedHistorySessionService {
5113        inner: Arc<dyn MobSessionService>,
5114        delay: Duration,
5115        read_calls: Arc<AtomicUsize>,
5116        active_reads: Arc<AtomicUsize>,
5117        max_active_reads: Arc<AtomicUsize>,
5118    }
5119
5120    impl DelayedHistorySessionService {
5121        fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
5122            Self {
5123                inner,
5124                delay,
5125                read_calls: Arc::new(AtomicUsize::new(0)),
5126                active_reads: Arc::new(AtomicUsize::new(0)),
5127                max_active_reads: Arc::new(AtomicUsize::new(0)),
5128            }
5129        }
5130
5131        fn read_calls(&self) -> usize {
5132            self.read_calls.load(Ordering::SeqCst)
5133        }
5134
5135        fn max_active_reads(&self) -> usize {
5136            self.max_active_reads.load(Ordering::SeqCst)
5137        }
5138    }
5139
5140    #[async_trait::async_trait]
5141    impl SessionService for DelayedHistorySessionService {
5142        async fn create_session(
5143            &self,
5144            req: meerkat_core::CreateSessionRequest,
5145        ) -> Result<RunResult, SessionError> {
5146            self.inner.create_session(req).await
5147        }
5148
5149        async fn start_turn(
5150            &self,
5151            id: &SessionId,
5152            req: StartTurnRequest,
5153        ) -> Result<RunResult, SessionError> {
5154            self.inner.start_turn(id, req).await
5155        }
5156
5157        async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
5158            self.inner.interrupt(id).await
5159        }
5160
5161        async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
5162            self.inner.read(id).await
5163        }
5164
5165        async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
5166            self.inner.list(query).await
5167        }
5168
5169        async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
5170            self.inner.archive(id).await
5171        }
5172
5173        async fn subscribe_session_events(
5174            &self,
5175            id: &SessionId,
5176        ) -> Result<EventStream, StreamError> {
5177            SessionService::subscribe_session_events(self.inner.as_ref(), id).await
5178        }
5179    }
5180
5181    #[async_trait::async_trait]
5182    impl SessionServiceCommsExt for DelayedHistorySessionService {
5183        async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
5184            self.inner.comms_runtime(session_id).await
5185        }
5186    }
5187
5188    #[async_trait::async_trait]
5189    impl SessionServiceControlExt for DelayedHistorySessionService {
5190        async fn append_system_context(
5191            &self,
5192            id: &SessionId,
5193            req: AppendSystemContextRequest,
5194        ) -> Result<AppendSystemContextResult, SessionControlError> {
5195            self.inner.append_system_context(id, req).await
5196        }
5197    }
5198
5199    #[async_trait::async_trait]
5200    impl SessionServiceHistoryExt for DelayedHistorySessionService {
5201        async fn read_history(
5202            &self,
5203            id: &SessionId,
5204            query: SessionHistoryQuery,
5205        ) -> Result<SessionHistoryPage, SessionError> {
5206            self.read_calls.fetch_add(1, Ordering::SeqCst);
5207            let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
5208            self.max_active_reads.fetch_max(active, Ordering::SeqCst);
5209            tokio::time::sleep(self.delay).await;
5210            let result = self.inner.read_history(id, query).await;
5211            self.active_reads.fetch_sub(1, Ordering::SeqCst);
5212            result
5213        }
5214    }
5215
5216    #[async_trait::async_trait]
5217    impl MobSessionService for DelayedHistorySessionService {
5218        fn supports_persistent_sessions(&self) -> bool {
5219            self.inner.supports_persistent_sessions()
5220        }
5221
5222        fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
5223            self.inner.runtime_adapter()
5224        }
5225
5226        async fn session_belongs_to_mob(
5227            &self,
5228            session_id: &SessionId,
5229            mob_id: &meerkat_mob::MobId,
5230        ) -> bool {
5231            self.inner.session_belongs_to_mob(session_id, mob_id).await
5232        }
5233
5234        async fn cancel_all_checkpointers(&self) {
5235            self.inner.cancel_all_checkpointers().await;
5236        }
5237
5238        async fn rearm_all_checkpointers(&self) {
5239            self.inner.rearm_all_checkpointers().await;
5240        }
5241    }
5242
5243    #[async_trait::async_trait]
5244    impl ConsoleLogStore for CountingConsoleLogStore {
5245        async fn append_if_absent(
5246            &self,
5247            frame: NewConsoleFrame,
5248        ) -> ConsoleLogResult<AppendOutcome> {
5249            self.inner.append_if_absent(frame).await
5250        }
5251
5252        async fn update_frame_status(
5253            &self,
5254            frame_id: &str,
5255            status: ConsoleFrameStatus,
5256        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5257            self.inner.update_frame_status(frame_id, status).await
5258        }
5259
5260        async fn query_frames(
5261            &self,
5262            query: ConsoleTimelineQuery,
5263        ) -> ConsoleLogResult<ConsoleTimelinePage> {
5264            self.inner.query_frames(query).await
5265        }
5266
5267        async fn query_windowed_frames(
5268            &self,
5269            query: ConsoleTimelineWindowQuery,
5270        ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
5271            self.inner.query_windowed_frames(query).await
5272        }
5273
5274        async fn frame_by_dedupe_key(
5275            &self,
5276            dedupe_key: &str,
5277        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
5278            self.inner.frame_by_dedupe_key(dedupe_key).await
5279        }
5280
5281        async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
5282            self.inner.latest_cursor().await
5283        }
5284
5285        async fn clear_frames(&self) -> ConsoleLogResult<()> {
5286            self.inner.clear_frames().await
5287        }
5288
5289        async fn record_source_watermark(
5290            &self,
5291            runtime_key: &str,
5292            source_kind: ConsoleFrameSourceKind,
5293            source_cursor: &str,
5294        ) -> ConsoleLogResult<()> {
5295            self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
5296            self.inner
5297                .record_source_watermark(runtime_key, source_kind, source_cursor)
5298                .await
5299        }
5300
5301        async fn source_watermark(
5302            &self,
5303            runtime_key: &str,
5304            source_kind: ConsoleFrameSourceKind,
5305        ) -> ConsoleLogResult<Option<String>> {
5306            self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
5307            self.inner.source_watermark(runtime_key, source_kind).await
5308        }
5309    }
5310
5311    async fn build_single_member_runtime() -> UnifiedRuntime {
5312        build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
5313    }
5314
5315    async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
5316        let definition = MobDefinition::from_toml(
5317            r#"
5318[mob]
5319id = "console-aggregator-perf-test"
5320
5321[profiles.worker]
5322model = "gpt-5.5"
5323external_addressable = true
5324
5325[profiles.worker.tools]
5326comms = true
5327"#,
5328        )
5329        .expect("definition parses");
5330        let runtime = UnifiedRuntime::builder()
5331            .definition(definition)
5332            .default_llm_client(client)
5333            .build()
5334            .await
5335            .expect("runtime builds");
5336        runtime
5337            .spawn(SpawnMemberSpec::from_wire(
5338                "worker".to_string(),
5339                "agent-a".to_string(),
5340                Some("You are agent-a.".into()),
5341                None,
5342                None,
5343            ))
5344            .await
5345            .expect("member spawns");
5346        runtime
5347    }
5348
5349    async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
5350        let definition = MobDefinition::from_toml(&format!(
5351            r#"
5352[mob]
5353id = "{mob_id}"
5354
5355[profiles.worker]
5356model = "gpt-5.5"
5357external_addressable = true
5358
5359[profiles.worker.tools]
5360comms = true
5361"#
5362        ))
5363        .expect("definition parses");
5364        UnifiedRuntime::builder()
5365            .definition(definition)
5366            .default_llm_client(Arc::new(TestClient::default()))
5367            .build()
5368            .await
5369            .expect("runtime builds")
5370    }
5371
5372    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5373    async fn send_selection_ignores_a_retiring_generation_when_one_active_generation_remains() {
5374        let runtime = build_empty_runtime("console-send-retiring-generation-test").await;
5375        let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
5376        for generation in 0..=1 {
5377            runtime
5378                .spawn(
5379                    SpawnMemberSpec::from_wire(
5380                        "worker".to_string(),
5381                        format!("rt:builder:{generation}"),
5382                        Some("You are Builder.".into()),
5383                        None,
5384                        None,
5385                    )
5386                    .with_labels(labels.clone()),
5387                )
5388                .await
5389                .expect("member spawns");
5390        }
5391
5392        let aggregator = MobKitConsoleAggregator::in_memory();
5393        aggregator.register_runtime_handles_with_policy(
5394            "runtime-a",
5395            "",
5396            runtime.mob_runtime().clone(),
5397            None,
5398            runtime.console_events(),
5399            Arc::new(AllowAllConsoleVisibilityPolicy),
5400        );
5401        let mut matches = aggregator.resolve_visible_members("builder").await;
5402        assert_eq!(matches.len(), 2);
5403        assert_eq!(sendable_resolved_member_indices(&matches).len(), 2);
5404
5405        matches[0].member.status = meerkat_mob::MobMemberStatus::Retiring;
5406        let sendable = sendable_resolved_member_indices(&matches);
5407        assert_eq!(sendable.len(), 1);
5408        assert_ne!(
5409            matches[sendable[0]].member.status,
5410            meerkat_mob::MobMemberStatus::Retiring
5411        );
5412
5413        let _ = runtime.mob_handle().stop().await;
5414    }
5415
5416    async fn build_stress_runtime(
5417        member_count: usize,
5418        history_delay: Duration,
5419    ) -> (
5420        tempfile::TempDir,
5421        Arc<UnifiedRuntime>,
5422        DelayedHistorySessionService,
5423    ) {
5424        let temp_dir = tempfile::tempdir().expect("temp dir");
5425        let session_path = temp_dir.path().join("sessions");
5426        std::fs::create_dir_all(&session_path).expect("session path");
5427        let factory = AgentFactory::new(&session_path).comms(true);
5428        let base_service = Arc::new(build_ephemeral_service(
5429            factory,
5430            Config::default(),
5431            member_count + 8,
5432        ));
5433        let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
5434        let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
5435        let definition = MobDefinition::from_toml(
5436            r#"
5437[mob]
5438id = "console-aggregator-stress-test"
5439
5440[profiles.worker]
5441model = "gpt-5.5"
5442external_addressable = true
5443
5444[profiles.worker.tools]
5445comms = true
5446"#,
5447        )
5448        .expect("definition parses");
5449        let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
5450            .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
5451                allow_ephemeral_sessions: true,
5452                notify_orchestrator_on_resume: true,
5453                default_llm_client: Some(Arc::new(TestClient::default())),
5454            });
5455        let runtime = Arc::new(
5456            UnifiedRuntime::bootstrap(
5457                spec,
5458                crate::types::MobKitConfig {
5459                    modules: Vec::new(),
5460                    discovery: crate::types::DiscoverySpec {
5461                        namespace: "stress".to_string(),
5462                        modules: Vec::new(),
5463                    },
5464                    pre_spawn: Vec::new(),
5465                },
5466                Duration::from_secs(5),
5467            )
5468            .await
5469            .expect("runtime boots"),
5470        );
5471        for idx in 0..member_count {
5472            runtime
5473                .spawn(SpawnMemberSpec::from_wire(
5474                    "worker".to_string(),
5475                    format!("agent-{idx}"),
5476                    Some(format!("You are agent-{idx}.").into()),
5477                    None,
5478                    None,
5479                ))
5480                .await
5481                .expect("member spawns");
5482        }
5483        (temp_dir, runtime, delayed_service)
5484    }
5485
5486    fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
5487        RuntimeEntry {
5488            runtime_key: runtime_key.to_string(),
5489            identity_namespace: "test".to_string(),
5490            runtime: runtime.mob_runtime().clone(),
5491            identity_runtime: runtime.identity_runtime().cloned(),
5492            console_events: runtime.console_events(),
5493            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5494        }
5495    }
5496
5497    fn console_envelope_for_test(
5498        event_id: &str,
5499        interaction_id: Option<&str>,
5500        event_type: &str,
5501        data: Value,
5502    ) -> crate::console_contracts::ConsoleIdentityEventEnvelope {
5503        crate::console_contracts::ConsoleIdentityEventEnvelope {
5504            event_id: event_id.to_string(),
5505            interaction_id: interaction_id.map(ToString::to_string),
5506            identity: "agent-a".to_string(),
5507            event_type: event_type.to_string(),
5508            timestamp_ms: 1_000,
5509            data,
5510        }
5511    }
5512
5513    fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
5514        ConsoleIdentityRecord {
5515            identity: identity.to_string(),
5516            display_name: identity.to_string(),
5517            runtime_key: "runtime-cache".to_string(),
5518            runtime_member_id: identity.to_string(),
5519            session_id: Some(format!("session-{identity}")),
5520            visibility: ConsoleVisibility::Addressable,
5521            addressable: true,
5522            health: "ready".to_string(),
5523            topology_peers: Vec::new(),
5524            labels: BTreeMap::new(),
5525        }
5526    }
5527
5528    #[test]
5529    fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
5530        let mut durable = identity_record_for_test("test/review:singleton");
5531        durable.runtime_member_id = "rt:review:singleton:1".to_string();
5532        durable.session_id = Some("session-stale".to_string());
5533        durable
5534            .labels
5535            .insert("agent_identity".to_string(), "review:singleton".to_string());
5536
5537        let mut live = identity_record_for_test("test/review:singleton");
5538        live.runtime_member_id = "rt:review:singleton:0".to_string();
5539        live.session_id = Some("session-live".to_string());
5540        live.labels
5541            .insert("agent_identity".to_string(), "review:singleton".to_string());
5542
5543        let records = dedupe_identity_records(vec![durable, live]);
5544
5545        assert_eq!(
5546            records.len(),
5547            2,
5548            "list projection must expose stale durable/live alias split-brain instead of picking one record"
5549        );
5550        assert!(
5551            records
5552                .iter()
5553                .any(|record| record.runtime_member_id == "rt:review:singleton:0")
5554        );
5555        assert!(
5556            records
5557                .iter()
5558                .any(|record| record.runtime_member_id == "rt:review:singleton:1")
5559        );
5560    }
5561
5562    #[test]
5563    fn dedupe_identity_records_preserves_duplicate_live_aliases() {
5564        let mut first = identity_record_for_test("test/review:singleton");
5565        first.runtime_member_id = "rt:review:singleton:0".to_string();
5566        first
5567            .labels
5568            .insert("agent_identity".to_string(), "review:singleton".to_string());
5569
5570        let mut second = identity_record_for_test("test/review:singleton");
5571        second.runtime_member_id = "rt:review:singleton:1".to_string();
5572        second
5573            .labels
5574            .insert("agent_identity".to_string(), "review:singleton".to_string());
5575
5576        let records = dedupe_identity_records(vec![first, second]);
5577
5578        assert_eq!(
5579            records.len(),
5580            2,
5581            "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
5582        );
5583    }
5584
5585    #[test]
5586    fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
5587        let mut first = identity_record_for_test("test/review:singleton");
5588        first.runtime_member_id = "rt:review:singleton:0".to_string();
5589        first.session_id = None;
5590        first
5591            .labels
5592            .insert("agent_identity".to_string(), "review:singleton".to_string());
5593        first
5594            .labels
5595            .insert("source_mob_id".to_string(), "mob-a".to_string());
5596
5597        let mut second = first.clone();
5598        second
5599            .labels
5600            .insert("source_mob_id".to_string(), "mob-b".to_string());
5601
5602        let records = dedupe_identity_records(vec![first, second]);
5603
5604        assert_eq!(
5605            records.len(),
5606            2,
5607            "same member/session aliases from different source mobs must remain visible"
5608        );
5609    }
5610
5611    #[test]
5612    fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
5613        let durable = identity_record_for_test("test/review:singleton");
5614        let live = durable.clone();
5615
5616        let records = dedupe_identity_records(vec![durable, live]);
5617
5618        assert_eq!(
5619            records.len(),
5620            1,
5621            "healthy durable and primary live projections should not duplicate the roster"
5622        );
5623    }
5624
5625    #[test]
5626    fn stale_durable_record_error_detects_session_split_brain() {
5627        let mut durable = identity_record_for_test("test/review:singleton");
5628        durable.runtime_member_id = "rt:review:singleton:0".to_string();
5629        durable.session_id = Some("session-durable".to_string());
5630
5631        let mut live = durable.clone();
5632        live.session_id = Some("session-live".to_string());
5633
5634        let err = stale_durable_record_error("review:singleton", &durable, &[live])
5635            .expect("session mismatch must be stale split-brain");
5636
5637        assert!(
5638            err.contains("stale durable identity alias"),
5639            "unexpected error: {err}"
5640        );
5641        assert!(
5642            err.contains("session-durable") && err.contains("session-live"),
5643            "error should name both sessions: {err}"
5644        );
5645    }
5646
5647    async fn identity_runtime_for_test(
5648        identities: &[&str],
5649    ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
5650    {
5651        let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5652            crate::identity_first::IdentityRuntimeConfig {
5653                continuity_store: Arc::new(
5654                    crate::identity_first::LocalContinuityStore::in_memory()?
5655                ),
5656                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5657                runtime_instance_id: "console-aggregator-identity-test".to_string(),
5658                has_runtime_store: true,
5659                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5660                bridge: None,
5661                default_timeout: None,
5662            },
5663        ));
5664        for identity in identities {
5665            let identity = crate::identity_first::AgentIdentity::parse(identity)?;
5666            let record = crate::identity_first::ContinuityRecord {
5667                identity: identity.clone(),
5668                agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
5669                    "rt:{}:0",
5670                    identity.as_str()
5671                ))?,
5672                session_id: SessionId::new(),
5673                generation: crate::identity_first::ContinuityGeneration::new(0),
5674                checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5675            };
5676            runtime
5677                .register(
5678                    crate::identity_first::DurableAgentSpec {
5679                        identity: identity.clone(),
5680                        profile: meerkat_mob::ProfileName::from("worker"),
5681                        addressability: crate::identity_first::AgentAddressability::Addressable,
5682                        display_name: None,
5683                        labels: BTreeMap::new(),
5684                        context: None,
5685                        additional_instructions: Vec::new(),
5686                        initial_message: None,
5687                        runtime_mode_override: None,
5688                        backend: None,
5689                        binding: None,
5690                    },
5691                    crate::identity_first::IdentityLifecycleState::Active,
5692                    Some(record),
5693                    Some(crate::identity_first::LeaseGrant {
5694                        identity,
5695                        fencing_token: crate::identity_first::FencingToken::new(1),
5696                        ttl: Duration::from_mins(5),
5697                    }),
5698                )
5699                .await;
5700        }
5701        Ok(runtime)
5702    }
5703
5704    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5705    async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
5706    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5707        let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
5708        let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
5709        let aggregator = MobKitConsoleAggregator::in_memory();
5710        aggregator.register_runtime_handles_with_policy(
5711            "runtime-a",
5712            "",
5713            runtime.mob_runtime().clone(),
5714            Some(identity_runtime),
5715            runtime.console_events(),
5716            Arc::new(AllowAllConsoleVisibilityPolicy),
5717        );
5718
5719        assert!(
5720            aggregator
5721                .inspect_identity("rt:review:singleton:0")
5722                .await?
5723                .is_none(),
5724            "runtime-member-id shaped requests must not inspect a durable rt:* identity"
5725        );
5726        assert!(
5727            !aggregator.retire_identity("rt:review:singleton:0").await?,
5728            "runtime-member-id shaped requests must not retire a durable rt:* identity"
5729        );
5730        let reserve_err = aggregator
5731            .reserve_identity_first_interaction(
5732                ConsoleSendRequest {
5733                    identity: "rt:review:singleton:0".to_string(),
5734                    content: json!("hello"),
5735                    origin: "console:test".to_string(),
5736                    idempotency_key: "rt-shadow-reserve".to_string(),
5737                    handling_mode: Some("queue".to_string()),
5738                },
5739                None,
5740            )
5741            .await
5742            .expect_err(
5743                "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
5744            );
5745        assert!(
5746            reserve_err.to_string().contains("unknown identity"),
5747            "unexpected reserve error: {reserve_err}"
5748        );
5749        let send_err = aggregator
5750            .send(ConsoleSendRequest {
5751                identity: "rt:review:singleton:0".to_string(),
5752                content: json!("hello"),
5753                origin: "console:test".to_string(),
5754                idempotency_key: "rt-shadow-send".to_string(),
5755                handling_mode: Some("queue".to_string()),
5756            })
5757            .await
5758            .expect_err(
5759                "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
5760            );
5761        assert!(
5762            send_err.to_string().contains("unknown identity"),
5763            "unexpected send error: {send_err}"
5764        );
5765        let blob_err = match aggregator
5766            .binary_blob_store_for_identity("rt:review:singleton:0")
5767            .await
5768        {
5769            Ok(_) => panic!(
5770                "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
5771            ),
5772            Err(err) => err,
5773        };
5774        assert!(
5775            blob_err.to_string().contains("unknown identity"),
5776            "unexpected blob error: {blob_err}"
5777        );
5778
5779        let _ = runtime.mob_handle().stop().await;
5780        Ok(())
5781    }
5782
5783    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5784    async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
5785    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5786        let runtime = build_empty_runtime("identity-label-test").await;
5787        let mut labels = BTreeMap::new();
5788        labels.insert(
5789            "agent_identity".to_string(),
5790            "channel:C0SMOKEOB3".to_string(),
5791        );
5792        labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
5793        runtime
5794            .spawn(
5795                SpawnMemberSpec::from_wire(
5796                    "worker".to_string(),
5797                    "rt:channel:C0SMOKEOB3:0".to_string(),
5798                    Some("You are C0SMOKEOB3.".into()),
5799                    None,
5800                    None,
5801                )
5802                .with_labels(labels),
5803            )
5804            .await
5805            .expect("member spawns");
5806
5807        let entry = RuntimeEntry {
5808            runtime_key: "runtime-a".to_string(),
5809            identity_namespace: String::new(),
5810            runtime: runtime.mob_runtime().clone(),
5811            identity_runtime: runtime.identity_runtime().cloned(),
5812            console_events: runtime.console_events(),
5813            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5814        };
5815        let aggregator = MobKitConsoleAggregator::in_memory();
5816        aggregator
5817            .inner
5818            .runtimes
5819            .write()
5820            .expect("runtime registry")
5821            .insert("runtime-a".to_string(), entry);
5822
5823        let records = aggregator.list_identities_fresh().await?;
5824        let record = records
5825            .iter()
5826            .find(|record| record.identity == "channel:C0SMOKEOB3")
5827            .expect("durable identity is exposed");
5828        assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
5829
5830        let inspection = aggregator
5831            .inspect_identity("channel:C0SMOKEOB3")
5832            .await?
5833            .expect("durable identity resolves back to runtime member");
5834        assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
5835        assert_eq!(
5836            inspection.identity.runtime_member_id,
5837            "rt:channel:C0SMOKEOB3:0"
5838        );
5839
5840        let _ = runtime.mob_handle().stop().await;
5841        Ok(())
5842    }
5843
5844    /// Issue #254 (root cause of "flows stuck forever"): the live SSE gate
5845    /// passed `allow_historical_identity = false` unconditionally, so an
5846    /// identity-scoped stream dropped every frame of a member the identity
5847    /// read model had not yet observed — while the windowed query path
5848    /// passed the same frames via the own-identity allowance. The
5849    /// subscriber-aware gate grants the same allowance, and an unknown
5850    /// identity triggers a debounced read-model refresh so UNSCOPED streams
5851    /// converge too (members spawned mid-run via ensure_member).
5852    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5853    async fn identity_scoped_live_stream_sees_members_unknown_to_the_read_model()
5854    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5855        let runtime = build_empty_runtime("live-tail-unknown-identity-test").await;
5856        let aggregator = MobKitConsoleAggregator::in_memory();
5857        aggregator.register_runtime_handles_with_policy(
5858            "runtime-a",
5859            "",
5860            runtime.mob_runtime().clone(),
5861            runtime.identity_runtime().cloned(),
5862            runtime.console_events(),
5863            Arc::new(AllowAllConsoleVisibilityPolicy),
5864        );
5865        // Prime the read model BEFORE the member exists — the exact state a
5866        // mid-run ensure_member leaves an already-connected live stream in.
5867        let _ = aggregator.list_identities_fresh().await?;
5868
5869        runtime
5870            .spawn(SpawnMemberSpec::from_wire(
5871                "worker".to_string(),
5872                "late-member".to_string(),
5873                Some("You are the late member.".into()),
5874                None,
5875                None,
5876            ))
5877            .await?;
5878        let outcome = aggregator
5879            .store()
5880            .append_if_absent(NewConsoleFrame {
5881                id: None,
5882                dedupe_key: "late-member-live-frame".to_string(),
5883                timestamp_ms: 1,
5884                runtime_key: "runtime-a".to_string(),
5885                identity: "late-member".to_string(),
5886                conversation_id: Some("late-member".to_string()),
5887                session_id: None,
5888                kind: "text_delta".to_string(),
5889                status: ConsoleFrameStatus::Completed,
5890                payload: json!({"delta": "hi"}),
5891                source: ConsoleFrameSource {
5892                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5893                    source_cursor: None,
5894                },
5895                source_event_id: None,
5896                interaction_id: None,
5897                turn_id: None,
5898                run_id: None,
5899                parent_frame_id: None,
5900                caused_by_frame_id: None,
5901            })
5902            .await?;
5903        let event = ConsoleTimelineEvent::ConsoleFrame {
5904            frame: outcome.frame,
5905        };
5906
5907        // The identity-scoped subscriber sees the frame IMMEDIATELY (the
5908        // own-identity allowance the query path always had).
5909        assert!(
5910            aggregator
5911                .timeline_event_visible_for_subscriber(&event, Some("late-member"))
5912                .await,
5913            "identity-scoped live stream must not starve on a read-model miss"
5914        );
5915        // A differently-scoped subscriber does not inherit the allowance.
5916        // Race note (CI): the spawn's own projections can converge the read
5917        // model before this line, after which the frame is legitimately
5918        // visible to everyone because the identity is KNOWN — that is not an
5919        // allowance leak. The leak we guard against is other-visibility
5920        // WHILE the unscoped gate still hides the frame (identity unknown).
5921        let visible_to_other = aggregator
5922            .timeline_event_visible_for_subscriber(&event, Some("someone-else"))
5923            .await;
5924        if visible_to_other {
5925            assert!(
5926                aggregator.timeline_event_visible(&event).await,
5927                "the allowance is strictly own-identity: someone-else saw the \
5928                 frame while the read model still treats the identity as \
5929                 unknown (unscoped gate hides it)"
5930            );
5931        }
5932
5933        // The miss triggered refresh_soon: the read model converges, after
5934        // which even the UNSCOPED gate passes the frame.
5935        let mut converged = false;
5936        for _ in 0..40 {
5937            if aggregator.timeline_event_visible(&event).await {
5938                converged = true;
5939                break;
5940            }
5941            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
5942        }
5943        assert!(
5944            converged,
5945            "the unknown-identity miss must self-heal the read model"
5946        );
5947
5948        let _ = runtime.mob_handle().stop().await;
5949        Ok(())
5950    }
5951
5952    /// Issue #254 follow-up: registrations leaked an immortal live-projection
5953    /// task (broadcast receiver never sees Closed while the runtime Arc
5954    /// lives) plus the discovery loop. unregister_runtime detaches the live
5955    /// source; already-projected frames stay queryable.
5956    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5957    async fn unregister_runtime_stops_live_projection()
5958    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5959        let runtime = build_empty_runtime("unregister-live-projection-test").await;
5960        let aggregator = MobKitConsoleAggregator::in_memory();
5961        aggregator.register_runtime_handles_with_policy(
5962            "runtime-a",
5963            "",
5964            runtime.mob_runtime().clone(),
5965            runtime.identity_runtime().cloned(),
5966            runtime.console_events(),
5967            Arc::new(AllowAllConsoleVisibilityPolicy),
5968        );
5969
5970        aggregator.unregister_runtime("runtime-a");
5971        assert!(
5972            aggregator
5973                .inner
5974                .runtimes
5975                .read()
5976                .expect("runtime registry")
5977                .is_empty(),
5978            "unregister must remove the registry entry"
5979        );
5980        assert!(
5981            aggregator
5982                .inner
5983                .live_projection_shutdowns
5984                .lock()
5985                .expect("shutdown registry")
5986                .is_empty(),
5987            "unregister must consume the live-projection shutdown handle"
5988        );
5989        // Idempotent on unknown keys.
5990        aggregator.unregister_runtime("runtime-a");
5991
5992        // Re-registration works and replaces cleanly.
5993        aggregator.register_runtime_handles_with_policy(
5994            "runtime-a",
5995            "",
5996            runtime.mob_runtime().clone(),
5997            runtime.identity_runtime().cloned(),
5998            runtime.console_events(),
5999            Arc::new(AllowAllConsoleVisibilityPolicy),
6000        );
6001        assert_eq!(
6002            aggregator
6003                .inner
6004                .live_projection_shutdowns
6005                .lock()
6006                .expect("shutdown registry")
6007                .len(),
6008            1
6009        );
6010
6011        let _ = runtime.mob_handle().stop().await;
6012        Ok(())
6013    }
6014
6015    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6016    async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
6017    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6018        let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
6019        let mut labels = BTreeMap::new();
6020        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6021        runtime
6022            .spawn(
6023                SpawnMemberSpec::from_wire(
6024                    "worker".to_string(),
6025                    "rt:review:singleton:0".to_string(),
6026                    Some("You are the Review Agent.".into()),
6027                    None,
6028                    None,
6029                )
6030                .with_labels(labels),
6031            )
6032            .await?;
6033        let hidden_session_id = runtime
6034            .mob_handle()
6035            .resolve_bridge_session_id_observation(&meerkat_mob::ids::AgentIdentity::from(
6036                "rt:review:singleton:0",
6037            ))
6038            .await
6039            .map(|sid| sid.to_string());
6040        let mut duplicate_labels = BTreeMap::new();
6041        duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6042        runtime
6043            .spawn(
6044                SpawnMemberSpec::from_wire(
6045                    "worker".to_string(),
6046                    "rt:review:singleton:1".to_string(),
6047                    Some("You are a visible duplicate Review Agent.".into()),
6048                    None,
6049                    None,
6050                )
6051                .with_labels(duplicate_labels),
6052            )
6053            .await?;
6054
6055        let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6056        let aggregator = MobKitConsoleAggregator::in_memory();
6057        aggregator.register_runtime_handles_with_policy(
6058            "runtime-a",
6059            "",
6060            runtime.mob_runtime().clone(),
6061            Some(identity_runtime),
6062            runtime.console_events(),
6063            Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
6064        );
6065
6066        assert!(
6067            aggregator
6068                .inspect_identity("review:singleton")
6069                .await?
6070                .is_none(),
6071            "member-hidden live alias must not inspect through aggregator"
6072        );
6073        let records = aggregator.list_identities_fresh().await?;
6074        assert!(
6075            records
6076                .iter()
6077                .all(|record| record.identity != "review:singleton"),
6078            "member-hidden durable/live alias must not appear in identity list: {records:#?}"
6079        );
6080        let appended = aggregator
6081            .store()
6082            .append_if_absent(NewConsoleFrame {
6083                id: None,
6084                dedupe_key: "member-hidden-frame".to_string(),
6085                timestamp_ms: 1,
6086                runtime_key: "runtime-a".to_string(),
6087                identity: "review:singleton".to_string(),
6088                conversation_id: Some("review:singleton".to_string()),
6089                session_id: hidden_session_id,
6090                kind: "text_delta".to_string(),
6091                status: ConsoleFrameStatus::Delivered,
6092                payload: json!({ "delta": "hidden" }),
6093                source: ConsoleFrameSource {
6094                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6095                    source_cursor: None,
6096                },
6097                source_event_id: Some("member-hidden-frame".to_string()),
6098                interaction_id: None,
6099                turn_id: None,
6100                run_id: None,
6101                parent_frame_id: None,
6102                caused_by_frame_id: None,
6103            })
6104            .await?;
6105        assert!(
6106            !aggregator
6107                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
6108                    frame: appended.frame.clone()
6109                })
6110                .await,
6111            "member-hidden live alias must not pass timeline event visibility"
6112        );
6113        let identity_only = aggregator
6114            .store()
6115            .append_if_absent(NewConsoleFrame {
6116                id: None,
6117                dedupe_key: "member-hidden-identity-only-frame".to_string(),
6118                timestamp_ms: 2,
6119                runtime_key: "runtime-a".to_string(),
6120                identity: "review:singleton".to_string(),
6121                conversation_id: Some("review:singleton".to_string()),
6122                session_id: None,
6123                kind: "text_delta".to_string(),
6124                status: ConsoleFrameStatus::Delivered,
6125                payload: json!({ "delta": "identity-only-hidden" }),
6126                source: ConsoleFrameSource {
6127                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6128                    source_cursor: None,
6129                },
6130                source_event_id: Some("member-hidden-identity-only-frame".to_string()),
6131                interaction_id: None,
6132                turn_id: None,
6133                run_id: None,
6134                parent_frame_id: None,
6135                caused_by_frame_id: None,
6136            })
6137            .await?;
6138        assert!(
6139            !aggregator
6140                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
6141                    frame: identity_only.frame.clone()
6142                })
6143                .await,
6144            "member-hidden durable alias must not leak identity-only timeline frames"
6145        );
6146        let page = aggregator
6147            .query_timeline(ConsoleTimelineQuery {
6148                identity: Some("review:singleton".to_string()),
6149                limit: 10,
6150                ..ConsoleTimelineQuery::default()
6151            })
6152            .await?;
6153        assert!(
6154            page.frames.is_empty(),
6155            "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
6156        );
6157        assert!(
6158            !aggregator.retire_identity("review:singleton").await?,
6159            "member-hidden live alias must not retire through aggregator"
6160        );
6161        let reserve_err = aggregator
6162            .reserve_identity_first_interaction(
6163                ConsoleSendRequest {
6164                    identity: "review:singleton".to_string(),
6165                    content: json!("hello"),
6166                    origin: "console:test".to_string(),
6167                    idempotency_key: "member-hidden-reserve".to_string(),
6168                    handling_mode: Some("queue".to_string()),
6169                },
6170                None,
6171            )
6172            .await
6173            .expect_err("member-hidden durable alias must not reserve identity-first interaction");
6174        assert!(
6175            reserve_err.to_string().contains("unknown identity"),
6176            "unexpected reserve error: {reserve_err}"
6177        );
6178        let send_err = aggregator
6179            .send(ConsoleSendRequest {
6180                identity: "review:singleton".to_string(),
6181                content: json!("hello"),
6182                origin: "console:test".to_string(),
6183                idempotency_key: "member-hidden-send".to_string(),
6184                handling_mode: Some("queue".to_string()),
6185            })
6186            .await
6187            .expect_err("member-hidden live alias must not receive sends");
6188        assert!(
6189            send_err.to_string().contains("unknown identity"),
6190            "unexpected send error: {send_err}"
6191        );
6192        let blob_result = aggregator
6193            .binary_blob_store_for_identity("review:singleton")
6194            .await;
6195        let Err(blob_err) = blob_result else {
6196            panic!("member-hidden live alias must not expose blob store");
6197        };
6198        assert!(
6199            blob_err.to_string().contains("unknown identity"),
6200            "unexpected blob error: {blob_err}"
6201        );
6202
6203        let _ = runtime.mob_handle().stop().await;
6204        Ok(())
6205    }
6206
6207    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6208    async fn explicit_session_backfill_target_resolves_durable_identity_label()
6209    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6210        let runtime = build_empty_runtime("identity-label-backfill-test").await;
6211        let mut labels = BTreeMap::new();
6212        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6213        runtime
6214            .spawn(
6215                SpawnMemberSpec::from_wire(
6216                    "worker".to_string(),
6217                    "rt:review:singleton:0".to_string(),
6218                    Some("You are the Review Agent.".into()),
6219                    None,
6220                    None,
6221                )
6222                .with_labels(labels),
6223            )
6224            .await
6225            .expect("member spawns");
6226
6227        let entry = RuntimeEntry {
6228            runtime_key: "runtime-a".to_string(),
6229            identity_namespace: String::new(),
6230            runtime: runtime.mob_runtime().clone(),
6231            identity_runtime: runtime.identity_runtime().cloned(),
6232            console_events: runtime.console_events(),
6233            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6234        };
6235        let aggregator = MobKitConsoleAggregator::in_memory();
6236        aggregator
6237            .inner
6238            .runtimes
6239            .write()
6240            .expect("runtime registry")
6241            .insert("runtime-a".to_string(), entry);
6242
6243        let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
6244            .await
6245            .expect("durable label should resolve targeted session backfill");
6246        assert_eq!(target.record.identity, "review:singleton");
6247        assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
6248
6249        let _ = runtime.mob_handle().stop().await;
6250        Ok(())
6251    }
6252
6253    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6254    async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
6255    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6256        let runtime = build_empty_runtime("identity-stale-durable-test").await;
6257        let mut labels = BTreeMap::new();
6258        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6259        runtime
6260            .spawn(
6261                SpawnMemberSpec::from_wire(
6262                    "worker".to_string(),
6263                    "rt:review:singleton:0".to_string(),
6264                    Some("You are the live Review Agent.".into()),
6265                    None,
6266                    None,
6267                )
6268                .with_labels(labels),
6269            )
6270            .await
6271            .expect("member spawns");
6272
6273        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6274            crate::identity_first::IdentityRuntimeConfig {
6275                continuity_store: Arc::new(
6276                    crate::identity_first::LocalContinuityStore::in_memory()?
6277                ),
6278                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6279                runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
6280                has_runtime_store: true,
6281                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6282                bridge: None,
6283                default_timeout: None,
6284            },
6285        ));
6286        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6287        let record = crate::identity_first::ContinuityRecord {
6288            identity: identity.clone(),
6289            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6290                "rt:review:singleton:1",
6291            )?,
6292            session_id: SessionId::new(),
6293            generation: crate::identity_first::ContinuityGeneration::new(1),
6294            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6295        };
6296        identity_runtime
6297            .register(
6298                crate::identity_first::DurableAgentSpec {
6299                    identity: identity.clone(),
6300                    profile: meerkat_mob::ProfileName::from("worker"),
6301                    addressability: crate::identity_first::AgentAddressability::Addressable,
6302                    display_name: None,
6303                    labels: BTreeMap::new(),
6304                    context: None,
6305                    additional_instructions: Vec::new(),
6306                    initial_message: None,
6307                    runtime_mode_override: None,
6308                    backend: None,
6309                    binding: None,
6310                },
6311                crate::identity_first::IdentityLifecycleState::Active,
6312                Some(record),
6313                Some(crate::identity_first::LeaseGrant {
6314                    identity,
6315                    fencing_token: crate::identity_first::FencingToken::new(7),
6316                    ttl: Duration::from_mins(5),
6317                }),
6318            )
6319            .await;
6320
6321        let aggregator = MobKitConsoleAggregator::in_memory();
6322        aggregator
6323            .inner
6324            .runtimes
6325            .write()
6326            .expect("runtime registry")
6327            .insert(
6328                "runtime-a".to_string(),
6329                RuntimeEntry {
6330                    runtime_key: "runtime-a".to_string(),
6331                    identity_namespace: String::new(),
6332                    runtime: runtime.mob_runtime().clone(),
6333                    identity_runtime: Some(identity_runtime),
6334                    console_events: runtime.console_events(),
6335                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6336                },
6337            );
6338
6339        let err = aggregator
6340            .inspect_identity("review:singleton")
6341            .await
6342            .expect_err("stale durable binding must not inspect as healthy");
6343        assert!(
6344            err.to_string().contains("stale durable identity alias"),
6345            "unexpected error: {err}"
6346        );
6347        let retire_err = aggregator
6348            .retire_identity("review:singleton")
6349            .await
6350            .expect_err("stale durable binding must not retire as healthy");
6351        assert!(
6352            retire_err
6353                .to_string()
6354                .contains("stale durable identity alias"),
6355            "unexpected retire error: {retire_err}"
6356        );
6357        let send_err = aggregator
6358            .send(ConsoleSendRequest {
6359                identity: "review:singleton".to_string(),
6360                content: json!("hello"),
6361                origin: "console:test".to_string(),
6362                idempotency_key: "stale-durable-send".to_string(),
6363                handling_mode: Some("queue".to_string()),
6364            })
6365            .await
6366            .expect_err("stale durable binding must not send to the wrong live member");
6367        assert!(
6368            send_err
6369                .to_string()
6370                .contains("stale durable identity alias"),
6371            "unexpected send error: {send_err}"
6372        );
6373        let reserve_err = aggregator
6374            .reserve_identity_first_interaction(
6375                ConsoleSendRequest {
6376                    identity: "review:singleton".to_string(),
6377                    content: json!("hello"),
6378                    origin: "console:test".to_string(),
6379                    idempotency_key: "stale-durable-reserve".to_string(),
6380                    handling_mode: Some("queue".to_string()),
6381                },
6382                None,
6383            )
6384            .await
6385            .expect_err("identity-first reserve must not bypass stale durable binding");
6386        assert!(
6387            reserve_err
6388                .to_string()
6389                .contains("stale durable identity alias"),
6390            "unexpected reserve error: {reserve_err}"
6391        );
6392        let blob_err = match aggregator
6393            .binary_blob_store_for_identity("review:singleton")
6394            .await
6395        {
6396            Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
6397            Err(err) => err,
6398        };
6399        assert!(
6400            blob_err
6401                .to_string()
6402                .contains("stale durable identity alias"),
6403            "unexpected blob error: {blob_err}"
6404        );
6405        let records = aggregator.list_identities_fresh().await?;
6406        assert!(
6407            records
6408                .iter()
6409                .any(|record| record.identity == "review:singleton"
6410                    && record.runtime_member_id == "rt:review:singleton:0"),
6411            "live alias should still be listed; records: {records:#?}"
6412        );
6413        assert!(
6414            records
6415                .iter()
6416                .any(|record| record.identity == "review:singleton"
6417                    && record.runtime_member_id == "rt:review:singleton:1"),
6418            "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
6419        );
6420
6421        let _ = runtime.mob_handle().stop().await;
6422        Ok(())
6423    }
6424
6425    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6426    async fn identity_first_alias_self_heals_session_only_split_brain()
6427    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6428        let runtime = build_empty_runtime("identity-session-rebind-test").await;
6429        let mut labels = BTreeMap::new();
6430        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6431        runtime
6432            .spawn(
6433                SpawnMemberSpec::from_wire(
6434                    "worker".to_string(),
6435                    "rt:review:singleton:0".to_string(),
6436                    Some("You are the live Review Agent.".into()),
6437                    None,
6438                    None,
6439                )
6440                .with_labels(labels),
6441            )
6442            .await
6443            .expect("member spawns");
6444        let live_session_id = runtime
6445            .mob_handle()
6446            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
6447                "rt:review:singleton:0",
6448            ))
6449            .await
6450            .expect("live member has bridge session");
6451
6452        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6453            crate::identity_first::IdentityRuntimeConfig {
6454                continuity_store: Arc::new(
6455                    crate::identity_first::LocalContinuityStore::in_memory()?
6456                ),
6457                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6458                runtime_instance_id: "console-aggregator-session-rebind-test".to_string(),
6459                has_runtime_store: true,
6460                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6461                bridge: None,
6462                default_timeout: None,
6463            },
6464        ));
6465        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6466        let stale_session_id = SessionId::new();
6467        assert_ne!(stale_session_id, live_session_id);
6468        let record = crate::identity_first::ContinuityRecord {
6469            identity: identity.clone(),
6470            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6471                "rt:review:singleton:0",
6472            )?,
6473            session_id: stale_session_id,
6474            generation: crate::identity_first::ContinuityGeneration::new(1),
6475            checkpoint_version: crate::identity_first::CheckpointVersion::new(3),
6476        };
6477        identity_runtime
6478            .register(
6479                crate::identity_first::DurableAgentSpec {
6480                    identity: identity.clone(),
6481                    profile: meerkat_mob::ProfileName::from("worker"),
6482                    addressability: crate::identity_first::AgentAddressability::Addressable,
6483                    display_name: None,
6484                    labels: BTreeMap::new(),
6485                    context: None,
6486                    additional_instructions: Vec::new(),
6487                    initial_message: None,
6488                    runtime_mode_override: None,
6489                    backend: None,
6490                    binding: None,
6491                },
6492                crate::identity_first::IdentityLifecycleState::Active,
6493                Some(record),
6494                Some(crate::identity_first::LeaseGrant {
6495                    identity: identity.clone(),
6496                    fencing_token: crate::identity_first::FencingToken::new(7),
6497                    ttl: Duration::from_mins(5),
6498                }),
6499            )
6500            .await;
6501
6502        let aggregator = MobKitConsoleAggregator::in_memory();
6503        aggregator
6504            .inner
6505            .runtimes
6506            .write()
6507            .expect("runtime registry")
6508            .insert(
6509                "runtime-a".to_string(),
6510                RuntimeEntry {
6511                    runtime_key: "runtime-a".to_string(),
6512                    identity_namespace: String::new(),
6513                    runtime: runtime.mob_runtime().clone(),
6514                    identity_runtime: Some(identity_runtime.clone()),
6515                    console_events: runtime.console_events(),
6516                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6517                },
6518            );
6519
6520        aggregator
6521            .reserve_identity_first_interaction(
6522                ConsoleSendRequest {
6523                    identity: "review:singleton".to_string(),
6524                    content: json!("hello"),
6525                    origin: "console:test".to_string(),
6526                    idempotency_key: "session-rebind-reserve".to_string(),
6527                    handling_mode: Some("queue".to_string()),
6528                },
6529                None,
6530            )
6531            .await
6532            .expect("session-only split-brain should self-heal before reserve");
6533
6534        let rebound_status = identity_runtime.status(&identity).await?;
6535        assert_eq!(rebound_status.session_id, Some(live_session_id.clone()));
6536
6537        let inspection = aggregator
6538            .inspect_identity("review:singleton")
6539            .await?
6540            .expect("rebound identity should inspect");
6541        let live_session_string = live_session_id.to_string();
6542        assert_eq!(
6543            inspection.identity.session_id.as_deref(),
6544            Some(live_session_string.as_str())
6545        );
6546
6547        let _ = runtime.mob_handle().stop().await;
6548        Ok(())
6549    }
6550
6551    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6552    async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
6553    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6554        let runtime = build_empty_runtime("identity-wrong-projection-test").await;
6555        let mut labels = BTreeMap::new();
6556        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6557        runtime
6558            .spawn(
6559                SpawnMemberSpec::from_wire(
6560                    "worker".to_string(),
6561                    "rt:review:singleton:0".to_string(),
6562                    Some("You are the mislabeled Review Agent.".into()),
6563                    None,
6564                    None,
6565                )
6566                .with_labels(labels),
6567            )
6568            .await
6569            .expect("member spawns");
6570
6571        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6572            crate::identity_first::IdentityRuntimeConfig {
6573                continuity_store: Arc::new(
6574                    crate::identity_first::LocalContinuityStore::in_memory()?
6575                ),
6576                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6577                runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
6578                has_runtime_store: true,
6579                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6580                bridge: None,
6581                default_timeout: None,
6582            },
6583        ));
6584        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6585        let record = crate::identity_first::ContinuityRecord {
6586            identity: identity.clone(),
6587            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6588                "rt:review:singleton:0",
6589            )?,
6590            session_id: SessionId::new(),
6591            generation: crate::identity_first::ContinuityGeneration::new(1),
6592            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6593        };
6594        identity_runtime
6595            .register(
6596                crate::identity_first::DurableAgentSpec {
6597                    identity: identity.clone(),
6598                    profile: meerkat_mob::ProfileName::from("worker"),
6599                    addressability: crate::identity_first::AgentAddressability::Addressable,
6600                    display_name: None,
6601                    labels: BTreeMap::new(),
6602                    context: None,
6603                    additional_instructions: Vec::new(),
6604                    initial_message: None,
6605                    runtime_mode_override: None,
6606                    backend: None,
6607                    binding: None,
6608                },
6609                crate::identity_first::IdentityLifecycleState::Active,
6610                Some(record),
6611                Some(crate::identity_first::LeaseGrant {
6612                    identity,
6613                    fencing_token: crate::identity_first::FencingToken::new(7),
6614                    ttl: Duration::from_mins(5),
6615                }),
6616            )
6617            .await;
6618
6619        let aggregator = MobKitConsoleAggregator::in_memory();
6620        aggregator
6621            .inner
6622            .runtimes
6623            .write()
6624            .expect("runtime registry")
6625            .insert(
6626                "runtime-a".to_string(),
6627                RuntimeEntry {
6628                    runtime_key: "runtime-a".to_string(),
6629                    identity_namespace: String::new(),
6630                    runtime: runtime.mob_runtime().clone(),
6631                    identity_runtime: Some(identity_runtime),
6632                    console_events: runtime.console_events(),
6633                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6634                },
6635            );
6636
6637        let err = aggregator
6638            .inspect_identity("review:singleton")
6639            .await
6640            .expect_err("wrong projected live identity must not inspect as healthy");
6641        assert!(
6642            err.to_string()
6643                .contains("projects identity other:singleton"),
6644            "unexpected error: {err}"
6645        );
6646        let retire_err = aggregator
6647            .retire_identity("review:singleton")
6648            .await
6649            .expect_err("wrong projected live identity must not retire as healthy");
6650        assert!(
6651            retire_err
6652                .to_string()
6653                .contains("projects identity other:singleton"),
6654            "unexpected retire error: {retire_err}"
6655        );
6656        let send_err = aggregator
6657            .send(ConsoleSendRequest {
6658                identity: "review:singleton".to_string(),
6659                content: json!("hello"),
6660                origin: "console:test".to_string(),
6661                idempotency_key: "wrong-projection-send".to_string(),
6662                handling_mode: Some("queue".to_string()),
6663            })
6664            .await
6665            .expect_err("wrong projected live identity must not send as unknown");
6666        assert!(
6667            send_err
6668                .to_string()
6669                .contains("projects identity other:singleton"),
6670            "unexpected send error: {send_err}"
6671        );
6672        let reserve_err = aggregator
6673            .reserve_identity_first_interaction(
6674                ConsoleSendRequest {
6675                    identity: "review:singleton".to_string(),
6676                    content: json!("hello"),
6677                    origin: "console:test".to_string(),
6678                    idempotency_key: "wrong-projection-reserve".to_string(),
6679                    handling_mode: Some("queue".to_string()),
6680                },
6681                None,
6682            )
6683            .await
6684            .expect_err("identity-first reserve must not bypass wrong live projection");
6685        assert!(
6686            reserve_err
6687                .to_string()
6688                .contains("projects identity other:singleton"),
6689            "unexpected reserve error: {reserve_err}"
6690        );
6691        let blob_err = match aggregator
6692            .binary_blob_store_for_identity("review:singleton")
6693            .await
6694        {
6695            Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
6696            Err(err) => err,
6697        };
6698        assert!(
6699            blob_err
6700                .to_string()
6701                .contains("projects identity other:singleton"),
6702            "unexpected blob error: {blob_err}"
6703        );
6704        let projected_err = aggregator
6705            .inspect_identity("other:singleton")
6706            .await
6707            .expect_err("wrong projected alias must not inspect through live-only fallback");
6708        assert!(
6709            projected_err
6710                .to_string()
6711                .contains("identity runtime binding for review:singleton"),
6712            "unexpected projected-alias error: {projected_err}"
6713        );
6714        let projected_retire_err = aggregator
6715            .retire_identity("other:singleton")
6716            .await
6717            .expect_err("wrong projected alias must not retire through live-only fallback");
6718        assert!(
6719            projected_retire_err
6720                .to_string()
6721                .contains("identity runtime binding for review:singleton"),
6722            "unexpected projected-alias retire error: {projected_retire_err}"
6723        );
6724
6725        let _ = runtime.mob_handle().stop().await;
6726        Ok(())
6727    }
6728
6729    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6730    async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
6731    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6732        let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
6733        let mut labels = BTreeMap::new();
6734        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6735        runtime
6736            .spawn(
6737                SpawnMemberSpec::from_wire(
6738                    "worker".to_string(),
6739                    "rt:review:singleton:0".to_string(),
6740                    Some("You are a hidden wrong projection.".into()),
6741                    None,
6742                    None,
6743                )
6744                .with_labels(labels),
6745            )
6746            .await
6747            .expect("member spawns");
6748
6749        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6750            crate::identity_first::IdentityRuntimeConfig {
6751                continuity_store: Arc::new(
6752                    crate::identity_first::LocalContinuityStore::in_memory()?
6753                ),
6754                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6755                runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
6756                has_runtime_store: true,
6757                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6758                bridge: None,
6759                default_timeout: None,
6760            },
6761        ));
6762        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6763        identity_runtime
6764            .register(
6765                crate::identity_first::DurableAgentSpec {
6766                    identity: identity.clone(),
6767                    profile: meerkat_mob::ProfileName::from("worker"),
6768                    addressability: crate::identity_first::AgentAddressability::Addressable,
6769                    display_name: None,
6770                    labels: BTreeMap::new(),
6771                    context: None,
6772                    additional_instructions: Vec::new(),
6773                    initial_message: None,
6774                    runtime_mode_override: None,
6775                    backend: None,
6776                    binding: None,
6777                },
6778                crate::identity_first::IdentityLifecycleState::Active,
6779                Some(crate::identity_first::ContinuityRecord {
6780                    identity: identity.clone(),
6781                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6782                        "rt:review:singleton:0",
6783                    )?,
6784                    session_id: SessionId::new(),
6785                    generation: crate::identity_first::ContinuityGeneration::new(1),
6786                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6787                }),
6788                Some(crate::identity_first::LeaseGrant {
6789                    identity,
6790                    fencing_token: crate::identity_first::FencingToken::new(7),
6791                    ttl: Duration::from_mins(5),
6792                }),
6793            )
6794            .await;
6795
6796        let aggregator = MobKitConsoleAggregator::in_memory();
6797        aggregator
6798            .inner
6799            .runtimes
6800            .write()
6801            .expect("runtime registry")
6802            .insert(
6803                "runtime-a".to_string(),
6804                RuntimeEntry {
6805                    runtime_key: "runtime-a".to_string(),
6806                    identity_namespace: String::new(),
6807                    runtime: runtime.mob_runtime().clone(),
6808                    identity_runtime: Some(identity_runtime),
6809                    console_events: runtime.console_events(),
6810                    visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
6811                },
6812            );
6813
6814        let err = aggregator
6815            .inspect_identity("review:singleton")
6816            .await
6817            .expect_err("hidden wrong projection must still fail durable binding validation");
6818        assert!(
6819            err.to_string()
6820                .contains("projects identity other:singleton"),
6821            "unexpected hidden wrong-projection error: {err}"
6822        );
6823        let reserve_err = aggregator
6824            .reserve_identity_first_interaction(
6825                ConsoleSendRequest {
6826                    identity: "review:singleton".to_string(),
6827                    content: json!("hello"),
6828                    origin: "console:test".to_string(),
6829                    idempotency_key: "hidden-wrong-projection-reserve".to_string(),
6830                    handling_mode: Some("queue".to_string()),
6831                },
6832                None,
6833            )
6834            .await
6835            .expect_err("reserve must not hide wrong projection from durable validation");
6836        assert!(
6837            reserve_err
6838                .to_string()
6839                .contains("projects identity other:singleton"),
6840            "unexpected reserve error: {reserve_err}"
6841        );
6842
6843        let _ = runtime.mob_handle().stop().await;
6844        Ok(())
6845    }
6846
6847    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6848    async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
6849    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6850        let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
6851        let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
6852        let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6853            crate::identity_first::IdentityRuntimeConfig {
6854                continuity_store: Arc::new(
6855                    crate::identity_first::LocalContinuityStore::in_memory()?
6856                ),
6857                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6858                runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
6859                has_runtime_store: true,
6860                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6861                bridge: None,
6862                default_timeout: None,
6863            },
6864        ));
6865        let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6866            crate::identity_first::IdentityRuntimeConfig {
6867                continuity_store: Arc::new(
6868                    crate::identity_first::LocalContinuityStore::in_memory()?
6869                ),
6870                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
6871                runtime_instance_id: "runtime-id-visible-match-test".to_string(),
6872                has_runtime_store: true,
6873                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
6874                bridge: None,
6875                default_timeout: None,
6876            },
6877        ));
6878        for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
6879            let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
6880            identity_runtime
6881                .register(
6882                    crate::identity_first::DurableAgentSpec {
6883                        identity: identity.clone(),
6884                        profile: meerkat_mob::ProfileName::from("worker"),
6885                        addressability: crate::identity_first::AgentAddressability::Addressable,
6886                        display_name: None,
6887                        labels: BTreeMap::new(),
6888                        context: None,
6889                        additional_instructions: Vec::new(),
6890                        initial_message: None,
6891                        runtime_mode_override: None,
6892                        backend: None,
6893                        binding: None,
6894                    },
6895                    crate::identity_first::IdentityLifecycleState::Active,
6896                    Some(crate::identity_first::ContinuityRecord {
6897                        identity: identity.clone(),
6898                        agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
6899                            "rt:review:singleton:0",
6900                        )?,
6901                        session_id: SessionId::new(),
6902                        generation: crate::identity_first::ContinuityGeneration::new(1),
6903                        checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
6904                    }),
6905                    Some(crate::identity_first::LeaseGrant {
6906                        identity,
6907                        fencing_token: crate::identity_first::FencingToken::new(7),
6908                        ttl: Duration::from_mins(5),
6909                    }),
6910                )
6911                .await;
6912        }
6913
6914        let aggregator = MobKitConsoleAggregator::in_memory();
6915        {
6916            let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
6917            runtimes.insert(
6918                "a-hidden".to_string(),
6919                RuntimeEntry {
6920                    runtime_key: "a-hidden".to_string(),
6921                    identity_namespace: String::new(),
6922                    runtime: hidden_runtime.mob_runtime().clone(),
6923                    identity_runtime: Some(hidden_identity_runtime),
6924                    console_events: hidden_runtime.console_events(),
6925                    visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6926                },
6927            );
6928            runtimes.insert(
6929                "b-visible".to_string(),
6930                RuntimeEntry {
6931                    runtime_key: "b-visible".to_string(),
6932                    identity_namespace: String::new(),
6933                    runtime: visible_runtime.mob_runtime().clone(),
6934                    identity_runtime: Some(visible_identity_runtime),
6935                    console_events: visible_runtime.console_events(),
6936                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6937                },
6938            );
6939        }
6940
6941        let inspection = aggregator
6942            .inspect_identity("rt:review:singleton:0")
6943            .await?
6944            .expect("visible runtime-id match should be returned");
6945        assert_eq!(inspection.identity.runtime_key, "b-visible");
6946
6947        let _ = hidden_runtime.mob_handle().stop().await;
6948        let _ = visible_runtime.mob_handle().stop().await;
6949        Ok(())
6950    }
6951
6952    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6953    async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
6954    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6955        let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
6956        let mut primary_labels = BTreeMap::new();
6957        primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6958        runtime
6959            .spawn(
6960                SpawnMemberSpec::from_wire(
6961                    "worker".to_string(),
6962                    "rt:review:singleton:0".to_string(),
6963                    Some("You are the primary Review Agent.".into()),
6964                    None,
6965                    None,
6966                )
6967                .with_labels(primary_labels),
6968            )
6969            .await
6970            .expect("primary member spawns");
6971
6972        let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
6973        let mut delegate_labels = BTreeMap::new();
6974        delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
6975        delegate_runtime
6976            .spawn(
6977                SpawnMemberSpec::from_wire(
6978                    "worker".to_string(),
6979                    "rt:review:singleton:0".to_string(),
6980                    Some("You are the wrong projected Review Agent.".into()),
6981                    None,
6982                    None,
6983                )
6984                .with_labels(delegate_labels),
6985            )
6986            .await
6987            .expect("delegate member spawns");
6988        runtime
6989            .mob_runtime()
6990            .agent_mob_mcp_state()
6991            .expect("runtime should expose mob MCP state")
6992            .mob_insert_handle(
6993                meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
6994                delegate_runtime.mob_handle().clone(),
6995            )
6996            .await;
6997
6998        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
6999            crate::identity_first::IdentityRuntimeConfig {
7000                continuity_store: Arc::new(
7001                    crate::identity_first::LocalContinuityStore::in_memory()?
7002                ),
7003                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7004                runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
7005                has_runtime_store: true,
7006                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7007                bridge: None,
7008                default_timeout: None,
7009            },
7010        ));
7011        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7012        let record = crate::identity_first::ContinuityRecord {
7013            identity: identity.clone(),
7014            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7015                "rt:review:singleton:0",
7016            )?,
7017            session_id: SessionId::new(),
7018            generation: crate::identity_first::ContinuityGeneration::new(1),
7019            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7020        };
7021        identity_runtime
7022            .register(
7023                crate::identity_first::DurableAgentSpec {
7024                    identity: identity.clone(),
7025                    profile: meerkat_mob::ProfileName::from("worker"),
7026                    addressability: crate::identity_first::AgentAddressability::Addressable,
7027                    display_name: None,
7028                    labels: BTreeMap::new(),
7029                    context: None,
7030                    additional_instructions: Vec::new(),
7031                    initial_message: None,
7032                    runtime_mode_override: None,
7033                    backend: None,
7034                    binding: None,
7035                },
7036                crate::identity_first::IdentityLifecycleState::Active,
7037                Some(record),
7038                Some(crate::identity_first::LeaseGrant {
7039                    identity,
7040                    fencing_token: crate::identity_first::FencingToken::new(7),
7041                    ttl: Duration::from_mins(5),
7042                }),
7043            )
7044            .await;
7045
7046        let aggregator = MobKitConsoleAggregator::in_memory();
7047        aggregator
7048            .inner
7049            .runtimes
7050            .write()
7051            .expect("runtime registry")
7052            .insert(
7053                "runtime-a".to_string(),
7054                RuntimeEntry {
7055                    runtime_key: "runtime-a".to_string(),
7056                    identity_namespace: String::new(),
7057                    runtime: runtime.mob_runtime().clone(),
7058                    identity_runtime: Some(identity_runtime),
7059                    console_events: runtime.console_events(),
7060                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7061                },
7062            );
7063
7064        let err = aggregator
7065            .inspect_identity("review:singleton")
7066            .await
7067            .expect_err("cross-source wrong projection must not be hidden by primary source");
7068        assert!(
7069            err.to_string()
7070                .contains("projects identity other:singleton")
7071                || err.to_string().contains("ambiguous live identity alias"),
7072            "unexpected error: {err}"
7073        );
7074
7075        let _ = delegate_runtime.mob_handle().stop().await;
7076        let _ = runtime.mob_handle().stop().await;
7077        Ok(())
7078    }
7079
7080    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7081    async fn durable_send_selects_bound_generation_when_stale_generation_is_still_active()
7082    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7083        let runtime = build_empty_runtime("identity-respawn-send-authority-test").await;
7084        let labels = BTreeMap::from([("agent_identity".to_string(), "builder".to_string())]);
7085        for generation in 0..=1 {
7086            runtime
7087                .spawn(
7088                    SpawnMemberSpec::from_wire(
7089                        "worker".to_string(),
7090                        format!("rt:builder:{generation}"),
7091                        Some(format!("You are Builder generation {generation}.").into()),
7092                        None,
7093                        None,
7094                    )
7095                    .with_labels(labels.clone()),
7096                )
7097                .await
7098                .expect("both live generations spawn");
7099        }
7100        let bound_session_id = runtime
7101            .mob_handle()
7102            .resolve_bridge_session_id_observation(&crate::member_comms_id::mob_member_id(
7103                "rt:builder:1",
7104            ))
7105            .await
7106            .expect("bound generation has a bridge session");
7107
7108        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7109            crate::identity_first::IdentityRuntimeConfig {
7110                continuity_store: Arc::new(
7111                    crate::identity_first::LocalContinuityStore::in_memory()?
7112                ),
7113                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7114                runtime_instance_id: "console-aggregator-respawn-send-authority-test".to_string(),
7115                has_runtime_store: true,
7116                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7117                bridge: None,
7118                default_timeout: None,
7119            },
7120        ));
7121        let identity = crate::identity_first::AgentIdentity::parse("builder")?;
7122        identity_runtime
7123            .register(
7124                crate::identity_first::DurableAgentSpec {
7125                    identity: identity.clone(),
7126                    profile: meerkat_mob::ProfileName::from("worker"),
7127                    addressability: crate::identity_first::AgentAddressability::Addressable,
7128                    display_name: None,
7129                    labels: BTreeMap::new(),
7130                    context: None,
7131                    additional_instructions: Vec::new(),
7132                    initial_message: None,
7133                    runtime_mode_override: None,
7134                    backend: None,
7135                    binding: None,
7136                },
7137                crate::identity_first::IdentityLifecycleState::Active,
7138                Some(crate::identity_first::ContinuityRecord {
7139                    identity: identity.clone(),
7140                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse("rt:builder:1")?,
7141                    session_id: bound_session_id.clone(),
7142                    generation: crate::identity_first::ContinuityGeneration::new(1),
7143                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7144                }),
7145                Some(crate::identity_first::LeaseGrant {
7146                    identity,
7147                    fencing_token: crate::identity_first::FencingToken::new(9),
7148                    ttl: Duration::from_mins(5),
7149                }),
7150            )
7151            .await;
7152
7153        let aggregator = MobKitConsoleAggregator::in_memory();
7154        aggregator
7155            .inner
7156            .runtimes
7157            .write()
7158            .expect("runtime registry")
7159            .insert(
7160                "default".to_string(),
7161                RuntimeEntry {
7162                    runtime_key: "default".to_string(),
7163                    identity_namespace: String::new(),
7164                    runtime: runtime.mob_runtime().clone(),
7165                    identity_runtime: Some(identity_runtime),
7166                    console_events: runtime.console_events(),
7167                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7168                },
7169            );
7170
7171        let matches = aggregator.resolve_visible_members("builder").await;
7172        assert_eq!(
7173            matches.len(),
7174            2,
7175            "both Active generations remain observable"
7176        );
7177        assert_eq!(
7178            sendable_resolved_member_indices(&matches).len(),
7179            2,
7180            "the stale generation reproduces the pre-fix send ambiguity"
7181        );
7182        let resolved = aggregator
7183            .resolve_send_member("builder")
7184            .await?
7185            .expect("durable builder alias resolves");
7186        assert_eq!(
7187            resolved.runtime_identity, "rt:builder:1",
7188            "continuity binding must shadow the stale Active generation for sends"
7189        );
7190
7191        let inspect_error = aggregator
7192            .inspect_identity("builder")
7193            .await
7194            .expect_err("inspection must keep duplicate live aliases explicit");
7195        assert!(
7196            inspect_error
7197                .to_string()
7198                .contains("ambiguous live identity alias")
7199        );
7200        let retire_error = aggregator
7201            .retire_identity("builder")
7202            .await
7203            .expect_err("control must not silently retire through a durable alias");
7204        assert!(
7205            retire_error
7206                .to_string()
7207                .contains("ambiguous live identity alias")
7208        );
7209
7210        let accepted = aggregator
7211            .send(ConsoleSendRequest {
7212                identity: "builder".to_string(),
7213                content: json!("continue after respawn"),
7214                origin: "console:test".to_string(),
7215                idempotency_key: "respawn-bound-generation-send".to_string(),
7216                handling_mode: Some("queue".to_string()),
7217            })
7218            .await?;
7219        assert_eq!(accepted.session_id, Some(bound_session_id.to_string()));
7220
7221        let _ = runtime.mob_handle().stop().await;
7222        Ok(())
7223    }
7224
7225    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7226    async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
7227    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7228        let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
7229        let mut labels = BTreeMap::new();
7230        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7231        runtime
7232            .spawn(
7233                SpawnMemberSpec::from_wire(
7234                    "worker".to_string(),
7235                    "rt:review:singleton:0".to_string(),
7236                    Some("You are the first Review Agent.".into()),
7237                    None,
7238                    None,
7239                )
7240                .with_labels(labels.clone()),
7241            )
7242            .await
7243            .expect("first member spawns");
7244        runtime
7245            .spawn(
7246                SpawnMemberSpec::from_wire(
7247                    "worker".to_string(),
7248                    "rt:review:singleton:1".to_string(),
7249                    Some("You are the second Review Agent.".into()),
7250                    None,
7251                    None,
7252                )
7253                .with_labels(labels),
7254            )
7255            .await
7256            .expect("second member spawns");
7257
7258        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
7259            crate::identity_first::IdentityRuntimeConfig {
7260                continuity_store: Arc::new(
7261                    crate::identity_first::LocalContinuityStore::in_memory()?
7262                ),
7263                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
7264                runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
7265                has_runtime_store: true,
7266                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
7267                bridge: None,
7268                default_timeout: None,
7269            },
7270        ));
7271        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
7272        identity_runtime
7273            .register(
7274                crate::identity_first::DurableAgentSpec {
7275                    identity: identity.clone(),
7276                    profile: meerkat_mob::ProfileName::from("worker"),
7277                    addressability: crate::identity_first::AgentAddressability::Addressable,
7278                    display_name: None,
7279                    labels: BTreeMap::new(),
7280                    context: None,
7281                    additional_instructions: Vec::new(),
7282                    initial_message: None,
7283                    runtime_mode_override: None,
7284                    backend: None,
7285                    binding: None,
7286                },
7287                crate::identity_first::IdentityLifecycleState::Active,
7288                Some(crate::identity_first::ContinuityRecord {
7289                    identity: identity.clone(),
7290                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
7291                        "rt:review:singleton:0",
7292                    )?,
7293                    session_id: SessionId::new(),
7294                    generation: crate::identity_first::ContinuityGeneration::new(1),
7295                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
7296                }),
7297                Some(crate::identity_first::LeaseGrant {
7298                    identity,
7299                    fencing_token: crate::identity_first::FencingToken::new(8),
7300                    ttl: Duration::from_mins(5),
7301                }),
7302            )
7303            .await;
7304
7305        let aggregator = MobKitConsoleAggregator::in_memory();
7306        aggregator
7307            .inner
7308            .runtimes
7309            .write()
7310            .expect("runtime registry")
7311            .insert(
7312                "runtime-a".to_string(),
7313                RuntimeEntry {
7314                    runtime_key: "runtime-a".to_string(),
7315                    identity_namespace: String::new(),
7316                    runtime: runtime.mob_runtime().clone(),
7317                    identity_runtime: Some(identity_runtime),
7318                    console_events: runtime.console_events(),
7319                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7320                },
7321            );
7322
7323        let err = aggregator
7324            .inspect_identity("review:singleton")
7325            .await
7326            .expect_err("duplicate live alias must not inspect via durable fast path");
7327        assert!(
7328            err.to_string().contains("ambiguous live identity alias"),
7329            "unexpected error: {err}"
7330        );
7331        let retire_err = aggregator
7332            .retire_identity("review:singleton")
7333            .await
7334            .expect_err("duplicate live alias must not retire via durable fast path");
7335        assert!(
7336            retire_err
7337                .to_string()
7338                .contains("ambiguous live identity alias"),
7339            "unexpected retire error: {retire_err}"
7340        );
7341        let runtime_id_err = aggregator
7342            .inspect_identity("rt:review:singleton:0")
7343            .await
7344            .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
7345        assert!(
7346            runtime_id_err
7347                .to_string()
7348                .contains("ambiguous live identity alias"),
7349            "unexpected runtime-id inspect error: {runtime_id_err}"
7350        );
7351        let runtime_id_retire_err = aggregator
7352            .retire_identity("rt:review:singleton:0")
7353            .await
7354            .expect_err("runtime-id retire must still reject duplicate durable live aliases");
7355        assert!(
7356            runtime_id_retire_err
7357                .to_string()
7358                .contains("ambiguous live identity alias")
7359                || runtime_id_retire_err
7360                    .to_string()
7361                    .contains("stale durable identity alias"),
7362            "unexpected runtime-id retire error: {runtime_id_retire_err}"
7363        );
7364
7365        aggregator
7366            .reserve_identity_first_interaction(
7367                ConsoleSendRequest {
7368                    identity: "review:singleton".to_string(),
7369                    content: json!("hello"),
7370                    origin: "console:test".to_string(),
7371                    idempotency_key: "duplicate-live-alias-durable-reserve".to_string(),
7372                    handling_mode: Some("queue".to_string()),
7373                },
7374                None,
7375            )
7376            .await
7377            .expect("durable identity binding remains authoritative for interaction reserve");
7378
7379        let _ = runtime.mob_handle().stop().await;
7380        Ok(())
7381    }
7382
7383    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7384    async fn inspect_identity_ignores_hidden_live_alias_candidates()
7385    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7386        let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
7387        let mut labels = BTreeMap::new();
7388        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
7389        runtime
7390            .spawn(
7391                SpawnMemberSpec::from_wire(
7392                    "worker".to_string(),
7393                    "rt:review:singleton:0".to_string(),
7394                    Some("You are the visible Review Agent.".into()),
7395                    None,
7396                    None,
7397                )
7398                .with_labels(labels.clone()),
7399            )
7400            .await
7401            .expect("visible member spawns");
7402        runtime
7403            .spawn(
7404                SpawnMemberSpec::from_wire(
7405                    "worker".to_string(),
7406                    "rt:review:singleton:1".to_string(),
7407                    Some("You are hidden maintenance noise.".into()),
7408                    None,
7409                    None,
7410                )
7411                .with_labels(labels),
7412            )
7413            .await
7414            .expect("hidden member spawns");
7415
7416        let aggregator = MobKitConsoleAggregator::in_memory();
7417        aggregator.register_runtime_handles_with_policy(
7418            "runtime-a",
7419            "",
7420            runtime.mob_runtime().clone(),
7421            None,
7422            runtime.console_events(),
7423            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
7424        );
7425
7426        let inspection = aggregator
7427            .inspect_identity("review:singleton")
7428            .await?
7429            .expect("visible live alias should inspect without hidden ambiguity");
7430        assert_eq!(inspection.identity.identity, "review:singleton");
7431        assert_eq!(
7432            inspection.identity.runtime_member_id,
7433            "rt:review:singleton:0"
7434        );
7435
7436        let retired = aggregator.retire_identity("review:singleton").await?;
7437        assert!(
7438            retired,
7439            "visible live alias should retire without hidden ambiguity"
7440        );
7441
7442        let _ = runtime.mob_handle().stop().await;
7443        Ok(())
7444    }
7445
7446    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7447    async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
7448    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7449        let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
7450        let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
7451        let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
7452        let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
7453
7454        let aggregator = MobKitConsoleAggregator::in_memory();
7455        aggregator.register_runtime_handles_with_policy(
7456            "a-hidden",
7457            "",
7458            hidden_runtime.mob_runtime().clone(),
7459            Some(hidden_identity_runtime),
7460            hidden_runtime.console_events(),
7461            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
7462        );
7463        aggregator.register_runtime_handles_with_policy(
7464            "b-visible",
7465            "",
7466            visible_runtime.mob_runtime().clone(),
7467            Some(visible_identity_runtime),
7468            visible_runtime.console_events(),
7469            Arc::new(AllowAllConsoleVisibilityPolicy),
7470        );
7471
7472        let accepted = aggregator
7473            .reserve_identity_first_interaction(
7474                ConsoleSendRequest {
7475                    identity: "review:singleton".to_string(),
7476                    content: json!("hello"),
7477                    origin: "console:test".to_string(),
7478                    idempotency_key: "visible-after-hidden-reserve".to_string(),
7479                    handling_mode: Some("queue".to_string()),
7480                },
7481                None,
7482            )
7483            .await?;
7484        let page = aggregator
7485            .query_timeline(ConsoleTimelineQuery {
7486                identity: Some("review:singleton".to_string()),
7487                ..ConsoleTimelineQuery::default()
7488            })
7489            .await?;
7490        let frame = page
7491            .frames
7492            .iter()
7493            .find(|frame| frame.id == accepted.input_frame_id)
7494            .ok_or("accepted frame missing from timeline")?;
7495        assert_eq!(frame.runtime_key, "b-visible");
7496
7497        let _ = hidden_runtime.mob_handle().stop().await;
7498        let _ = visible_runtime.mob_handle().stop().await;
7499        Ok(())
7500    }
7501
7502    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7503    async fn identity_first_list_identities_projects_cached_topology_peers()
7504    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7505        let runtime = build_empty_runtime("identity-topology-cache-test").await;
7506        let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
7507        identity_runtime
7508            .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
7509                crate::identity_first::AgentIdentity::parse("agent:alpha")?,
7510                crate::identity_first::AgentIdentity::parse("agent:beta")?,
7511            )?])
7512            .await;
7513
7514        let aggregator = MobKitConsoleAggregator::in_memory();
7515        aggregator.register_runtime_handles_with_policy(
7516            "identity-first",
7517            "",
7518            runtime.mob_runtime().clone(),
7519            Some(identity_runtime),
7520            runtime.console_events(),
7521            Arc::new(AllowAllConsoleVisibilityPolicy),
7522        );
7523
7524        let identities = aggregator.list_identities().await?;
7525        let alpha = identities
7526            .iter()
7527            .find(|record| record.identity == "agent:alpha")
7528            .ok_or("agent:alpha identity missing")?;
7529        assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
7530        let inspection = aggregator
7531            .inspect_identity("agent:alpha")
7532            .await?
7533            .ok_or("agent:alpha inspection missing")?;
7534        assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
7535
7536        let _ = runtime.mob_handle().stop().await;
7537        Ok(())
7538    }
7539
7540    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7541    async fn identity_first_list_identities_includes_member_only_spawned_workers()
7542    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
7543        let runtime = build_empty_runtime("identity-member-only-test").await;
7544        let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
7545        let aggregator = MobKitConsoleAggregator::in_memory();
7546        aggregator.register_runtime_handles_with_policy(
7547            "identity-first",
7548            "",
7549            runtime.mob_runtime().clone(),
7550            Some(identity_runtime),
7551            runtime.console_events(),
7552            Arc::new(AllowAllConsoleVisibilityPolicy),
7553        );
7554
7555        runtime
7556            .spawn(SpawnMemberSpec::from_wire(
7557                "worker".to_string(),
7558                "agent:beta".to_string(),
7559                Some("You are beta.".into()),
7560                None,
7561                None,
7562            ))
7563            .await?;
7564
7565        let identities = aggregator.list_identities().await?;
7566        assert!(
7567            identities
7568                .iter()
7569                .any(|record| record.identity == "agent:beta"),
7570            "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
7571        );
7572
7573        let _ = runtime.mob_handle().stop().await;
7574        Ok(())
7575    }
7576
7577    #[tokio::test]
7578    async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
7579        let aggregator = MobKitConsoleAggregator::in_memory();
7580        let record = identity_record_for_test("agent-cached");
7581        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
7582        aggregator
7583            .inner
7584            .identity_read_model
7585            .primed
7586            .store(true, Ordering::Release);
7587        let _guard = aggregator
7588            .inner
7589            .identity_read_model
7590            .refresh_lock
7591            .clone()
7592            .lock_owned()
7593            .await;
7594
7595        let identities =
7596            tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
7597                .await
7598                .expect("hot identity list should not wait for refresh lock")
7599                .expect("identity list succeeds");
7600
7601        assert_eq!(identities, vec![record]);
7602    }
7603
7604    #[tokio::test]
7605    async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
7606        let aggregator = MobKitConsoleAggregator::in_memory();
7607        let guard = aggregator
7608            .inner
7609            .identity_read_model
7610            .refresh_lock
7611            .clone()
7612            .lock_owned()
7613            .await;
7614        let waiter = tokio::spawn({
7615            let aggregator = aggregator.clone();
7616            async move { aggregator.list_identities().await }
7617        });
7618
7619        tokio::time::sleep(Duration::from_millis(20)).await;
7620        assert!(
7621            !waiter.is_finished(),
7622            "cold identity list should wait for the in-flight refresh to finish"
7623        );
7624
7625        let record = identity_record_for_test("agent-primed");
7626        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
7627        aggregator
7628            .inner
7629            .identity_read_model
7630            .primed
7631            .store(true, Ordering::Release);
7632        drop(guard);
7633
7634        let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
7635            .await
7636            .expect("cold identity list waiter should resume")
7637            .expect("waiter joins")
7638            .expect("identity list succeeds");
7639        assert_eq!(identities, vec![record]);
7640    }
7641
7642    #[tokio::test]
7643    async fn query_timeline_reads_from_aggregate_store() {
7644        let aggregator = MobKitConsoleAggregator::in_memory();
7645        let frame = NewConsoleFrame {
7646            id: None,
7647            dedupe_key: "event-1".to_string(),
7648            timestamp_ms: 1,
7649            runtime_key: "runtime-a".to_string(),
7650            identity: "agent-a".to_string(),
7651            conversation_id: Some("agent-a".to_string()),
7652            session_id: None,
7653            kind: "text_delta".to_string(),
7654            status: ConsoleFrameStatus::Delivered,
7655            payload: json!({ "delta": "hello" }),
7656            source: ConsoleFrameSource {
7657                kind: ConsoleFrameSourceKind::ConsoleEvent,
7658                source_cursor: None,
7659            },
7660            source_event_id: Some("event-1".to_string()),
7661            interaction_id: None,
7662            turn_id: None,
7663            run_id: None,
7664            parent_frame_id: None,
7665            caused_by_frame_id: None,
7666        };
7667        aggregator
7668            .store()
7669            .append_if_absent(frame)
7670            .await
7671            .expect("append frame");
7672
7673        let page = aggregator
7674            .query_timeline(ConsoleTimelineQuery {
7675                identity: Some("agent-a".to_string()),
7676                limit: 10,
7677                ..ConsoleTimelineQuery::default()
7678            })
7679            .await
7680            .expect("query timeline");
7681        assert_eq!(page.frames.len(), 1);
7682        assert_eq!(page.frames[0].kind, "text_delta");
7683    }
7684
7685    /// §9.3: memory.* events with no affected identity are attributed to
7686    /// `_system` by the ConsoleMemoryEventSink. `_system` has no roster
7687    /// record, so without an explicit exemption the per-identity visibility
7688    /// gate hides them from every GLOBAL timeline query (the pre-fix bug:
7689    /// the rail and the Memory panel live strip never saw dream/quarantine/
7690    /// promotion events). The exemption must be exact — an arbitrary
7691    /// unknown identity stays hidden — and must survive identity
7692    /// namespacing (`frame_from_console_event` must not rewrite `_system`
7693    /// to `ns/_system`). ABAC is unaffected: under an enforced config the
7694    /// RPC layer still filters `_system` frames per caller via
7695    /// `retain_visible_timeline_frames` (covered in tests/access_control.rs).
7696    #[tokio::test]
7697    async fn system_identity_frames_are_visible_on_global_timeline_queries() {
7698        let aggregator = MobKitConsoleAggregator::in_memory();
7699        let runtime = build_single_member_runtime().await;
7700        // Register a runtime so the per-identity visibility gate engages
7701        // (an empty registry short-circuits every frame to visible). The
7702        // test entry carries the "test" identity namespace on purpose.
7703        aggregator
7704            .inner
7705            .runtimes
7706            .write()
7707            .expect("runtime registry")
7708            .insert(
7709                "runtime-a".to_string(),
7710                runtime_entry_for_test("runtime-a", &runtime),
7711            );
7712        let entry = runtime_entry_for_test("runtime-a", &runtime);
7713
7714        let envelope = |event_id: &str, identity: &str, event_type: &str| {
7715            crate::console_contracts::ConsoleIdentityEventEnvelope {
7716                event_id: event_id.to_string(),
7717                interaction_id: None,
7718                identity: identity.to_string(),
7719                event_type: event_type.to_string(),
7720                timestamp_ms: 1_000,
7721                data: json!({ "realm": "default" }),
7722            }
7723        };
7724
7725        let system_frame = frame_from_console_event(
7726            &entry,
7727            envelope("evt-mem-1", SYSTEM_EVENT_IDENTITY, "memory.dream.completed"),
7728        );
7729        assert_eq!(
7730            system_frame.identity, SYSTEM_EVENT_IDENTITY,
7731            "projection must not namespace the system identity"
7732        );
7733        let ghost_frame = frame_from_console_event(
7734            &entry,
7735            envelope("evt-ghost-1", "ghost-agent", "memory.dream.completed"),
7736        );
7737        assert_eq!(
7738            ghost_frame.identity, "test/ghost-agent",
7739            "ordinary identities keep the namespace"
7740        );
7741        for frame in [system_frame, ghost_frame] {
7742            aggregator
7743                .store()
7744                .append_if_absent(frame)
7745                .await
7746                .expect("append frame");
7747        }
7748
7749        // GLOBAL query: no identity requested, so allow_historical_identity
7750        // never applies — exactly the lane the pre-fix bug hid frames from.
7751        let page = aggregator
7752            .query_timeline(ConsoleTimelineQuery {
7753                identity: None,
7754                limit: 10,
7755                ..ConsoleTimelineQuery::default()
7756            })
7757            .await
7758            .expect("query timeline");
7759        let identities: Vec<&str> = page
7760            .frames
7761            .iter()
7762            .map(|frame| frame.identity.as_str())
7763            .collect();
7764        assert!(
7765            identities.contains(&SYSTEM_EVENT_IDENTITY),
7766            "system-attributed memory event must be globally visible: {identities:?}"
7767        );
7768        assert!(
7769            !identities.contains(&"test/ghost-agent"),
7770            "unknown identities must stay hidden — the exemption is exact: {identities:?}"
7771        );
7772
7773        let _ = runtime.mob_handle().stop().await;
7774    }
7775
7776    #[tokio::test]
7777    async fn reasoning_console_events_with_identical_text_share_a_frame_key() {
7778        let aggregator = MobKitConsoleAggregator::in_memory();
7779        let runtime = build_single_member_runtime().await;
7780        let entry = runtime_entry_for_test("runtime-a", &runtime);
7781        for event_type in ["reasoning_delta", "reasoning_complete"] {
7782            let frame = frame_from_console_event(
7783                &entry,
7784                console_envelope_for_test(
7785                    event_type,
7786                    Some("interaction-a"),
7787                    event_type,
7788                    json!({
7789                        "delta": "I should inspect the file first.",
7790                        "text": "I should inspect the file first.",
7791                        "turn_id": "turn-a",
7792                    }),
7793                ),
7794            );
7795            aggregator
7796                .store()
7797                .append_if_absent(frame)
7798                .await
7799                .expect("append reasoning frame");
7800        }
7801
7802        let page = aggregator
7803            .query_timeline(ConsoleTimelineQuery {
7804                identity: Some("test/agent-a".to_string()),
7805                limit: 10,
7806                ..ConsoleTimelineQuery::default()
7807            })
7808            .await
7809            .expect("query timeline");
7810
7811        assert_eq!(
7812            page.frames.len(),
7813            1,
7814            "identical reasoning re-emissions in one turn should collapse"
7815        );
7816        assert_eq!(page.frames[0].kind, "reasoning_delta");
7817        assert_eq!(
7818            page.frames[0].dedupe_key,
7819            format!(
7820                "console-reasoning:runtime-a:interaction-a:{}",
7821                hash_short("I should inspect the file first.")
7822            )
7823        );
7824    }
7825
7826    #[tokio::test]
7827    async fn reasoning_console_events_with_different_text_keep_distinct_frames() {
7828        let aggregator = MobKitConsoleAggregator::in_memory();
7829        let runtime = build_single_member_runtime().await;
7830        let entry = runtime_entry_for_test("runtime-a", &runtime);
7831        for (event_id, text) in [
7832            ("reasoning-1", "I should inspect the file first."),
7833            ("reasoning-2", "Now I should run the focused test."),
7834        ] {
7835            let frame = frame_from_console_event(
7836                &entry,
7837                console_envelope_for_test(
7838                    event_id,
7839                    Some("interaction-a"),
7840                    "reasoning_complete",
7841                    json!({
7842                        "text": text,
7843                        "turn_id": "turn-a",
7844                    }),
7845                ),
7846            );
7847            aggregator
7848                .store()
7849                .append_if_absent(frame)
7850                .await
7851                .expect("append reasoning frame");
7852        }
7853
7854        let page = aggregator
7855            .query_timeline(ConsoleTimelineQuery {
7856                identity: Some("test/agent-a".to_string()),
7857                limit: 10,
7858                ..ConsoleTimelineQuery::default()
7859            })
7860            .await
7861            .expect("query timeline");
7862
7863        assert_eq!(
7864            page.frames.len(),
7865            2,
7866            "distinct same-turn reasoning segments should not collapse"
7867        );
7868        assert_ne!(page.frames[0].dedupe_key, page.frames[1].dedupe_key);
7869    }
7870
7871    #[tokio::test]
7872    async fn identity_recent_anchor_respects_query_limit() {
7873        let aggregator = MobKitConsoleAggregator::in_memory();
7874        aggregator
7875            .store()
7876            .append_if_absent(NewConsoleFrame {
7877                id: None,
7878                dedupe_key: "anchored-user-input".to_string(),
7879                timestamp_ms: 1,
7880                runtime_key: "runtime-a".to_string(),
7881                identity: "agent-a".to_string(),
7882                conversation_id: Some("agent-a".to_string()),
7883                session_id: None,
7884                kind: "user_input".to_string(),
7885                status: ConsoleFrameStatus::Delivered,
7886                payload: json!({ "content": "anchor me" }),
7887                source: ConsoleFrameSource {
7888                    kind: ConsoleFrameSourceKind::Synthetic,
7889                    source_cursor: None,
7890                },
7891                source_event_id: Some("anchored-user-input".to_string()),
7892                interaction_id: Some("turn-a".to_string()),
7893                turn_id: None,
7894                run_id: None,
7895                parent_frame_id: None,
7896                caused_by_frame_id: None,
7897            })
7898            .await
7899            .expect("append anchor");
7900        for idx in 2..=40 {
7901            aggregator
7902                .store()
7903                .append_if_absent(NewConsoleFrame {
7904                    id: None,
7905                    dedupe_key: format!("noisy-tail-{idx}"),
7906                    timestamp_ms: idx,
7907                    runtime_key: "runtime-a".to_string(),
7908                    identity: "agent-a".to_string(),
7909                    conversation_id: Some("agent-a".to_string()),
7910                    session_id: None,
7911                    kind: "reasoning_delta".to_string(),
7912                    status: ConsoleFrameStatus::Delivered,
7913                    payload: json!({ "delta": idx }),
7914                    source: ConsoleFrameSource {
7915                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7916                        source_cursor: None,
7917                    },
7918                    source_event_id: Some(format!("noisy-tail-{idx}")),
7919                    interaction_id: Some("turn-a".to_string()),
7920                    turn_id: None,
7921                    run_id: None,
7922                    parent_frame_id: None,
7923                    caused_by_frame_id: None,
7924                })
7925                .await
7926                .expect("append noisy tail");
7927        }
7928
7929        let page = aggregator
7930            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7931                identity: Some("agent-a".to_string()),
7932                mode: ConsoleTimelineMode::Recent,
7933                limit: 5,
7934                ..ConsoleTimelineWindowQuery::default()
7935            })
7936            .await
7937            .expect("query recent identity timeline");
7938
7939        assert_eq!(
7940            page.frames.len(),
7941            5,
7942            "identity anchor merge must not exceed the requested limit"
7943        );
7944        assert!(
7945            page.frames
7946                .iter()
7947                .any(|frame| frame.dedupe_key == "anchored-user-input"),
7948            "the bounded result should still retain the useful turn anchor: {:#?}",
7949            page.frames
7950        );
7951        assert_eq!(
7952            page.frames.last().and_then(|frame| frame.cursor.seq()),
7953            Some(40)
7954        );
7955    }
7956
7957    #[tokio::test]
7958    async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
7959        let aggregator = MobKitConsoleAggregator::in_memory();
7960        let runtime = build_single_member_runtime().await;
7961        let mut entry = runtime_entry_for_test("runtime-a", &runtime);
7962        entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
7963        aggregator
7964            .inner
7965            .runtimes
7966            .write()
7967            .expect("runtime registry")
7968            .insert("runtime-a".to_string(), entry);
7969
7970        for idx in 0..1_500 {
7971            aggregator
7972                .store()
7973                .append_if_absent(NewConsoleFrame {
7974                    id: None,
7975                    dedupe_key: format!("hidden-gap-{idx}"),
7976                    timestamp_ms: idx,
7977                    runtime_key: "runtime-a".to_string(),
7978                    identity: "agent-a".to_string(),
7979                    conversation_id: Some("agent-a".to_string()),
7980                    session_id: None,
7981                    kind: "hidden_noise".to_string(),
7982                    status: ConsoleFrameStatus::Delivered,
7983                    payload: json!({ "delta": idx }),
7984                    source: ConsoleFrameSource {
7985                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7986                        source_cursor: None,
7987                    },
7988                    source_event_id: Some(format!("hidden-gap-{idx}")),
7989                    interaction_id: None,
7990                    turn_id: None,
7991                    run_id: None,
7992                    parent_frame_id: None,
7993                    caused_by_frame_id: None,
7994                })
7995                .await
7996                .expect("append hidden frame");
7997        }
7998        aggregator
7999            .store()
8000            .append_if_absent(NewConsoleFrame {
8001                id: None,
8002                dedupe_key: "visible-after-gap".to_string(),
8003                timestamp_ms: 1_501,
8004                runtime_key: "runtime-a".to_string(),
8005                identity: "agent-a".to_string(),
8006                conversation_id: Some("agent-a".to_string()),
8007                session_id: None,
8008                kind: "interaction_complete".to_string(),
8009                status: ConsoleFrameStatus::Delivered,
8010                payload: json!({ "text": "visible after hidden gap" }),
8011                source: ConsoleFrameSource {
8012                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8013                    source_cursor: None,
8014                },
8015                source_event_id: Some("visible-after-gap".to_string()),
8016                interaction_id: Some("turn-visible".to_string()),
8017                turn_id: None,
8018                run_id: None,
8019                parent_frame_id: None,
8020                caused_by_frame_id: None,
8021            })
8022            .await
8023            .expect("append visible frame");
8024
8025        let page = aggregator
8026            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8027                identity: Some("agent-a".to_string()),
8028                mode: ConsoleTimelineMode::Since,
8029                limit: 1,
8030                ..ConsoleTimelineWindowQuery::default()
8031            })
8032            .await
8033            .expect("query timeline");
8034        assert_eq!(page.frames.len(), 1);
8035        assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
8036        assert_eq!(
8037            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8038            Some(1_501)
8039        );
8040        let _ = runtime.mob_handle().stop().await;
8041    }
8042
8043    #[tokio::test]
8044    async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
8045        let aggregator = MobKitConsoleAggregator::in_memory();
8046        for idx in 1..=300 {
8047            aggregator
8048                .store()
8049                .append_if_absent(NewConsoleFrame {
8050                    id: None,
8051                    dedupe_key: format!("visible-{idx}"),
8052                    timestamp_ms: idx,
8053                    runtime_key: "runtime-a".to_string(),
8054                    identity: "agent-a".to_string(),
8055                    conversation_id: Some("agent-a".to_string()),
8056                    session_id: None,
8057                    kind: "interaction_complete".to_string(),
8058                    status: ConsoleFrameStatus::Delivered,
8059                    payload: json!({ "text": format!("visible {idx}") }),
8060                    source: ConsoleFrameSource {
8061                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8062                        source_cursor: None,
8063                    },
8064                    source_event_id: Some(format!("visible-{idx}")),
8065                    interaction_id: Some(format!("turn-{idx}")),
8066                    turn_id: None,
8067                    run_id: None,
8068                    parent_frame_id: None,
8069                    caused_by_frame_id: None,
8070                })
8071                .await
8072                .expect("append visible frame");
8073        }
8074
8075        let first = aggregator
8076            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8077                identity: Some("agent-a".to_string()),
8078                mode: ConsoleTimelineMode::Since,
8079                limit: 10,
8080                ..ConsoleTimelineWindowQuery::default()
8081            })
8082            .await
8083            .expect("query first page");
8084        assert_eq!(first.frames.len(), 10);
8085        assert_eq!(
8086            first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8087            Some(10)
8088        );
8089
8090        let second = aggregator
8091            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8092                identity: Some("agent-a".to_string()),
8093                mode: ConsoleTimelineMode::Since,
8094                after: first.next_cursor,
8095                limit: 10,
8096                ..ConsoleTimelineWindowQuery::default()
8097            })
8098            .await
8099            .expect("query second page");
8100        assert_eq!(second.frames.len(), 10);
8101        assert_eq!(second.frames[0].dedupe_key, "visible-11");
8102        assert_eq!(
8103            second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8104            Some(20)
8105        );
8106    }
8107
8108    #[tokio::test]
8109    async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
8110        let store = Arc::new(CountingConsoleLogStore::new());
8111        let aggregator = MobKitConsoleAggregator::new(store.clone());
8112        let runtime = build_single_member_runtime().await;
8113        aggregator
8114            .inner
8115            .runtimes
8116            .write()
8117            .expect("runtime registry")
8118            .insert(
8119                "runtime-a".to_string(),
8120                runtime_entry_for_test("runtime-a", &runtime),
8121            );
8122        let inserted = store
8123            .append_if_absent(NewConsoleFrame {
8124                id: None,
8125                dedupe_key: "event-1".to_string(),
8126                timestamp_ms: 1,
8127                runtime_key: "runtime-a".to_string(),
8128                identity: "agent-a".to_string(),
8129                conversation_id: Some("agent-a".to_string()),
8130                session_id: None,
8131                kind: "text_delta".to_string(),
8132                status: ConsoleFrameStatus::Delivered,
8133                payload: json!({ "delta": "hello" }),
8134                source: ConsoleFrameSource {
8135                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8136                    source_cursor: None,
8137                },
8138                source_event_id: Some("event-1".to_string()),
8139                interaction_id: None,
8140                turn_id: None,
8141                run_id: None,
8142                parent_frame_id: None,
8143                caused_by_frame_id: None,
8144            })
8145            .await
8146            .expect("append frame");
8147
8148        let page = aggregator
8149            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8150                identity: Some("agent-a".to_string()),
8151                mode: ConsoleTimelineMode::Since,
8152                after: Some(inserted.frame.cursor),
8153                limit: 10,
8154                ..ConsoleTimelineWindowQuery::default()
8155            })
8156            .await
8157            .expect("query continuation");
8158
8159        assert!(page.frames.is_empty());
8160        assert_eq!(
8161            store.source_watermark_calls(),
8162            0,
8163            "empty since continuation must not synchronously force session-history backfill"
8164        );
8165        let _ = runtime.mob_handle().stop().await;
8166    }
8167
8168    #[test]
8169    fn legacy_timeline_struct_literals_remain_source_compatible() {
8170        let query = ConsoleTimelineQuery {
8171            identity: Some("agent-a".to_string()),
8172            conversation_id: None,
8173            after: Some(ConsoleCursor::from("console:1")),
8174            limit: 10,
8175        };
8176        let page = ConsoleTimelinePage {
8177            frames: Vec::new(),
8178            next_cursor: query.after.clone(),
8179        };
8180
8181        assert_eq!(query.limit, 10);
8182        assert_eq!(
8183            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
8184            Some(1)
8185        );
8186    }
8187
8188    #[tokio::test]
8189    async fn query_timeline_is_store_local_for_registered_runtimes() {
8190        let store = Arc::new(CountingConsoleLogStore::new());
8191        let aggregator = MobKitConsoleAggregator::new(store.clone());
8192        let runtime = build_single_member_runtime().await;
8193        aggregator
8194            .inner
8195            .runtimes
8196            .write()
8197            .expect("runtime registry")
8198            .insert(
8199                "runtime-a".to_string(),
8200                runtime_entry_for_test("runtime-a", &runtime),
8201            );
8202        store
8203            .append_if_absent(NewConsoleFrame {
8204                id: None,
8205                dedupe_key: "event-1".to_string(),
8206                timestamp_ms: 1,
8207                runtime_key: "runtime-a".to_string(),
8208                identity: "agent-a".to_string(),
8209                conversation_id: Some("agent-a".to_string()),
8210                session_id: None,
8211                kind: "text_delta".to_string(),
8212                status: ConsoleFrameStatus::Delivered,
8213                payload: json!({ "delta": "hello" }),
8214                source: ConsoleFrameSource {
8215                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8216                    source_cursor: None,
8217                },
8218                source_event_id: Some("event-1".to_string()),
8219                interaction_id: None,
8220                turn_id: None,
8221                run_id: None,
8222                parent_frame_id: None,
8223                caused_by_frame_id: None,
8224            })
8225            .await
8226            .expect("append frame");
8227
8228        let page = tokio::time::timeout(
8229            Duration::from_millis(250),
8230            aggregator.query_timeline(ConsoleTimelineQuery {
8231                identity: Some("agent-a".to_string()),
8232                limit: 10,
8233                ..ConsoleTimelineQuery::default()
8234            }),
8235        )
8236        .await
8237        .expect("timeline query should not wait for session history")
8238        .expect("timeline query succeeds");
8239
8240        assert_eq!(page.frames.len(), 1);
8241        assert_eq!(
8242            store.source_watermark_calls(),
8243            0,
8244            "query_timeline must not synchronously touch session-history watermarks"
8245        );
8246        let _ = runtime.mob_handle().stop().await;
8247    }
8248
8249    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8250    async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
8251        let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
8252            delay: Duration::from_secs(2),
8253        }))
8254        .await;
8255        let aggregator = MobKitConsoleAggregator::in_memory();
8256        aggregator
8257            .inner
8258            .runtimes
8259            .write()
8260            .expect("runtime registry")
8261            .insert(
8262                "runtime-a".to_string(),
8263                runtime_entry_for_test("runtime-a", &runtime),
8264            );
8265
8266        let start = Instant::now();
8267        let accepted = tokio::time::timeout(
8268            Duration::from_millis(300),
8269            aggregator.send(ConsoleSendRequest {
8270                identity: "test/agent-a".to_string(),
8271                content: json!("hello slow agent"),
8272                origin: "console:test".to_string(),
8273                idempotency_key: "nonblocking-send".to_string(),
8274                handling_mode: Some("queue".to_string()),
8275            }),
8276        )
8277        .await
8278        .expect("console send should return once the input is accepted")
8279        .expect("send succeeds");
8280
8281        assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
8282        assert!(
8283            start.elapsed() < Duration::from_secs(1),
8284            "console send should not wait for the delayed LLM turn"
8285        );
8286
8287        wait_for_session_history_text(
8288            &aggregator,
8289            "test/agent-a",
8290            "slow ok",
8291            Duration::from_secs(5),
8292        )
8293        .await
8294        .expect("background dispatch should still complete and project history");
8295        let _ = runtime.mob_handle().stop().await;
8296    }
8297
8298    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8299    async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
8300    {
8301        let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
8302        let aggregator = MobKitConsoleAggregator::in_memory();
8303        aggregator.register_runtime(ConsoleRuntimeRegistration {
8304            runtime_key: "runtime-late".to_string(),
8305            runtime: runtime.clone(),
8306            identity_namespace: "late".to_string(),
8307            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
8308        });
8309
8310        runtime
8311            .spawn(SpawnMemberSpec::from_wire(
8312                "worker".to_string(),
8313                "agent-late".to_string(),
8314                Some("You are agent-late.".into()),
8315                None,
8316                None,
8317            ))
8318            .await
8319            .expect("late member spawns");
8320        let session_id = send_message_on_mob_with_mode(
8321            &runtime.mob_handle(),
8322            "agent-late",
8323            ContentInput::Text("hello after registration".to_string()),
8324            meerkat_core::types::HandlingMode::Queue,
8325        )
8326        .await
8327        .expect("direct member send succeeds");
8328        wait_for_identity_record(
8329            &aggregator,
8330            "late/agent-late",
8331            Some(session_id.as_str()),
8332            Duration::from_secs(5),
8333        )
8334        .await?;
8335
8336        wait_for_session_history_text(
8337            &aggregator,
8338            "late/agent-late",
8339            "You are agent-late.",
8340            Duration::from_secs(5),
8341        )
8342        .await?;
8343        let _ = runtime.mob_handle().stop().await;
8344        Ok(())
8345    }
8346
8347    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8348    async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
8349    -> Result<(), String> {
8350        let runtime = build_single_member_runtime().await;
8351        let entry = runtime_entry_for_test("runtime-a", &runtime);
8352        let resolved = member_sources_for_entry(&entry)
8353            .await
8354            .into_iter()
8355            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
8356            .expect("agent-a member exists");
8357        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
8358            .await
8359            .expect("identity record exists");
8360        let session_id = record.session_id.expect("agent-a has a session");
8361        wait_for_runtime_session_history_text(
8362            &runtime,
8363            &session_id,
8364            "You are agent-a.",
8365            Duration::from_secs(5),
8366        )
8367        .await?;
8368
8369        let aggregator = MobKitConsoleAggregator::in_memory();
8370        aggregator
8371            .inner
8372            .runtimes
8373            .write()
8374            .expect("runtime registry")
8375            .insert("runtime-a".to_string(), entry);
8376        let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
8377        aggregator
8378            .store()
8379            .record_source_watermark(
8380                &watermark_key,
8381                ConsoleFrameSourceKind::SessionHistory,
8382                &format_session_history_watermark(&session_id, 0, current_time_ms()),
8383            )
8384            .await
8385            .expect("record fresh empty watermark");
8386
8387        // Hang/stall backstop only — generous so CPU starvation under full
8388        // parallel CI load can't false-trip it (matches the sibling timeline
8389        // query test). The real assertions are on the returned frames below.
8390        let page = tokio::time::timeout(
8391            Duration::from_mins(1),
8392            aggregator.query_timeline(ConsoleTimelineQuery {
8393                identity: Some("test/agent-a".to_string()),
8394                limit: 20,
8395                ..ConsoleTimelineQuery::default()
8396            }),
8397        )
8398        .await
8399        .expect("explicit identity query should not stall")
8400        .expect("query succeeds");
8401
8402        if !page.frames.is_empty() {
8403            assert!(
8404                page.frames.iter().any(|frame| {
8405                    frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8406                        && session_history_content_text(frame).as_deref()
8407                            == Some("You are agent-a.")
8408                }),
8409                "query returned frames before the polling assertion; expected them to be the forced session-history refresh, frames: {:#?}",
8410                page.frames
8411            );
8412        }
8413        wait_for_session_history_text(
8414            &aggregator,
8415            "test/agent-a",
8416            "You are agent-a.",
8417            Duration::from_secs(5),
8418        )
8419        .await?;
8420        let _ = runtime.mob_handle().stop().await;
8421        Ok(())
8422    }
8423
8424    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8425    async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
8426    {
8427        let store = Arc::new(CountingConsoleLogStore::new());
8428        let aggregator = MobKitConsoleAggregator::new(store.clone());
8429        let runtime = build_single_member_runtime().await;
8430        let entry = runtime_entry_for_test("runtime-a", &runtime);
8431        let resolved = member_sources_for_entry(&entry)
8432            .await
8433            .into_iter()
8434            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
8435            .expect("agent-a member exists");
8436        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
8437            .await
8438            .expect("identity record exists");
8439        let session_id = record.session_id.expect("agent-a has a session");
8440        wait_for_runtime_session_history_text(
8441            &runtime,
8442            &session_id,
8443            "You are agent-a.",
8444            Duration::from_secs(5),
8445        )
8446        .await?;
8447        aggregator
8448            .inner
8449            .runtimes
8450            .write()
8451            .expect("runtime registry")
8452            .insert("runtime-a".to_string(), entry);
8453
8454        store
8455            .append_if_absent(NewConsoleFrame {
8456                id: None,
8457                dedupe_key: "stale-session-history-agent-a".to_string(),
8458                timestamp_ms: 10,
8459                runtime_key: "runtime-a".to_string(),
8460                identity: "test/agent-a".to_string(),
8461                conversation_id: Some("test/agent-a".to_string()),
8462                session_id: Some(session_id.clone()),
8463                kind: "user_input".to_string(),
8464                status: ConsoleFrameStatus::Delivered,
8465                payload: json!({
8466                    "text": "stale projected history",
8467                    "type": "user_input",
8468                }),
8469                source: ConsoleFrameSource {
8470                    kind: ConsoleFrameSourceKind::SessionHistory,
8471                    source_cursor: Some("stale-session-history-agent-a".to_string()),
8472                },
8473                source_event_id: Some("stale-session-history-agent-a".to_string()),
8474                interaction_id: None,
8475                turn_id: None,
8476                run_id: None,
8477                parent_frame_id: None,
8478                caused_by_frame_id: None,
8479            })
8480            .await
8481            .expect("append stale history frame");
8482
8483        let fresh_prompt = "fresh prompt after stale history";
8484        let sent_session_id = send_message_on_mob_with_mode(
8485            &runtime.mob_handle(),
8486            "agent-a",
8487            ContentInput::Text(fresh_prompt.to_string()),
8488            meerkat_core::types::HandlingMode::Queue,
8489        )
8490        .await
8491        .expect("direct member send succeeds");
8492        assert_eq!(sent_session_id, session_id);
8493        wait_for_runtime_session_history_text(
8494            &runtime,
8495            &session_id,
8496            fresh_prompt,
8497            Duration::from_secs(5),
8498        )
8499        .await?;
8500
8501        let page = aggregator
8502            .query_timeline(ConsoleTimelineQuery {
8503                identity: Some("test/agent-a".to_string()),
8504                limit: 50,
8505                ..ConsoleTimelineQuery::default()
8506            })
8507            .await
8508            .expect("query child timeline");
8509
8510        assert!(
8511            page.frames.iter().any(|frame| {
8512                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8513                    && session_history_content_text(frame).as_deref()
8514                        == Some("stale projected history")
8515            }),
8516            "explicit identity query should return existing history before async refresh; frames: {:#?}",
8517            page.frames
8518        );
8519        wait_for_session_history_text(
8520            &aggregator,
8521            "test/agent-a",
8522            fresh_prompt,
8523            Duration::from_secs(5),
8524        )
8525        .await?;
8526        assert!(
8527            store.source_watermark_calls() > 0,
8528            "stale existing session history should force a targeted source refresh"
8529        );
8530        let _ = runtime.mob_handle().stop().await;
8531        Ok(())
8532    }
8533
8534    async fn wait_for_session_history_text(
8535        aggregator: &MobKitConsoleAggregator,
8536        identity: &str,
8537        expected: &str,
8538        timeout: Duration,
8539    ) -> Result<(), String> {
8540        let deadline = Instant::now() + timeout;
8541        let mut observed = Vec::new();
8542        while Instant::now() < deadline {
8543            let _ = aggregator.list_identities().await;
8544            let page = aggregator
8545                .query_timeline(ConsoleTimelineQuery {
8546                    identity: Some(identity.to_string()),
8547                    limit: 20,
8548                    ..ConsoleTimelineQuery::default()
8549                })
8550                .await
8551                .expect("query timeline");
8552            observed = page.frames;
8553            if observed.iter().any(|frame| {
8554                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8555                    && session_history_content_text(frame).as_deref() == Some(expected)
8556            }) {
8557                return Ok(());
8558            }
8559            tokio::time::sleep(Duration::from_millis(25)).await;
8560        }
8561
8562        Err(format!(
8563            "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
8564        ))
8565    }
8566
8567    async fn wait_for_session_backfill_target(
8568        aggregator: &MobKitConsoleAggregator,
8569        identity: &str,
8570        timeout: Duration,
8571    ) -> Result<String, String> {
8572        let deadline = Instant::now() + timeout;
8573        while Instant::now() < deadline {
8574            if let Some(target) =
8575                session_backfill_target_for_identity(&aggregator.inner, identity).await
8576            {
8577                return Ok(target.session_id);
8578            }
8579            tokio::time::sleep(Duration::from_millis(25)).await;
8580        }
8581        Err(format!(
8582            "session backfill target for {identity:?} was not resolvable"
8583        ))
8584    }
8585
8586    async fn wait_for_runtime_session_history_text(
8587        runtime: &UnifiedRuntime,
8588        session_id: &str,
8589        expected: &str,
8590        timeout: Duration,
8591    ) -> Result<(), String> {
8592        let deadline = Instant::now() + timeout;
8593        let mut observed = Vec::new();
8594        while Instant::now() < deadline {
8595            let page = runtime
8596                .mob_runtime()
8597                .read_session_history(session_id, 0, Some(20))
8598                .await
8599                .map_err(|err| err.to_string())?;
8600            observed = page.messages;
8601            if observed.iter().enumerate().any(|(idx, message)| {
8602                let Some(message) = serde_json::to_value(message).ok() else {
8603                    return false;
8604                };
8605                frames_from_session_history_message(
8606                    "runtime-a",
8607                    "test/agent-a",
8608                    session_id,
8609                    idx,
8610                    message,
8611                )
8612                .iter()
8613                .any(|frame| {
8614                    matches!(
8615                        frame.kind.as_str(),
8616                        "user_input" | "system_notice" | "interaction_complete"
8617                    ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
8618                })
8619            }) {
8620                return Ok(());
8621            }
8622            tokio::time::sleep(Duration::from_millis(25)).await;
8623        }
8624
8625        Err(format!(
8626            "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
8627        ))
8628    }
8629
8630    async fn wait_for_identity_record(
8631        aggregator: &MobKitConsoleAggregator,
8632        identity: &str,
8633        session_id: Option<&str>,
8634        timeout: Duration,
8635    ) -> Result<(), String> {
8636        let deadline = Instant::now() + timeout;
8637        let mut observed = Vec::new();
8638        while Instant::now() < deadline {
8639            observed = aggregator
8640                .list_identities()
8641                .await
8642                .map_err(|err| err.to_string())?;
8643            if observed.iter().any(|record| {
8644                record.identity == identity && record.session_id.as_deref() == session_id
8645            }) {
8646                return Ok(());
8647            }
8648            tokio::time::sleep(Duration::from_millis(25)).await;
8649        }
8650
8651        Err(format!(
8652            "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
8653        ))
8654    }
8655
8656    fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
8657        session_history_payload_text(&frame.payload)
8658    }
8659
8660    fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
8661        session_history_payload_text(&frame.payload)
8662    }
8663
8664    fn session_history_payload_text(payload: &Value) -> Option<String> {
8665        if let Some(text) = payload.get("text").and_then(Value::as_str) {
8666            return Some(text.to_string());
8667        }
8668        if let Some(text) = payload.get("result").and_then(Value::as_str) {
8669            return Some(text.to_string());
8670        }
8671        if let Some(text) = payload.get("body").and_then(Value::as_str) {
8672            return Some(text.to_string());
8673        }
8674        match payload.get("content")? {
8675            Value::String(text) => Some(text.clone()),
8676            Value::Array(blocks) => Some(
8677                blocks
8678                    .iter()
8679                    .filter_map(|block| block.get("text").and_then(Value::as_str))
8680                    .collect::<Vec<_>>()
8681                    .join(""),
8682            ),
8683            _ => None,
8684        }
8685    }
8686
8687    #[tokio::test]
8688    async fn query_timeline_handles_large_store_without_backfill_calls() {
8689        let store = Arc::new(CountingConsoleLogStore::new());
8690        let aggregator = MobKitConsoleAggregator::new(store.clone());
8691        for idx in 0..5_000 {
8692            store
8693                .append_if_absent(NewConsoleFrame {
8694                    id: None,
8695                    dedupe_key: format!("event-{idx}"),
8696                    timestamp_ms: idx,
8697                    runtime_key: "runtime-a".to_string(),
8698                    identity: "agent-a".to_string(),
8699                    conversation_id: Some("agent-a".to_string()),
8700                    session_id: Some("session-a".to_string()),
8701                    kind: "text_delta".to_string(),
8702                    status: ConsoleFrameStatus::Delivered,
8703                    payload: json!({ "delta": idx }),
8704                    source: ConsoleFrameSource {
8705                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8706                        source_cursor: None,
8707                    },
8708                    source_event_id: Some(format!("event-{idx}")),
8709                    interaction_id: None,
8710                    turn_id: None,
8711                    run_id: None,
8712                    parent_frame_id: None,
8713                    caused_by_frame_id: None,
8714                })
8715                .await
8716                .expect("append frame");
8717        }
8718
8719        let page = aggregator
8720            .query_timeline(ConsoleTimelineQuery {
8721                identity: Some("agent-a".to_string()),
8722                limit: 1_000,
8723                ..ConsoleTimelineQuery::default()
8724            })
8725            .await
8726            .expect("large query");
8727
8728        assert_eq!(page.frames.len(), 1_000);
8729        assert_eq!(store.source_watermark_calls(), 0);
8730    }
8731
8732    #[tokio::test]
8733    async fn query_timeline_rejects_future_cursor_after_store_reset() {
8734        let aggregator = MobKitConsoleAggregator::in_memory();
8735        let err = aggregator
8736            .query_timeline_windowed(ConsoleTimelineWindowQuery {
8737                after: Some(ConsoleCursor::from("console:99")),
8738                limit: 10,
8739                ..ConsoleTimelineWindowQuery::default()
8740            })
8741            .await
8742            .expect_err("future cursor on empty/reset store must be replay-unavailable");
8743
8744        assert!(
8745            err.to_string()
8746                .contains("beyond the current store frontier"),
8747            "unexpected error: {err}"
8748        );
8749    }
8750
8751    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8752    async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
8753        let store = Arc::new(CountingConsoleLogStore::new());
8754        let aggregator = MobKitConsoleAggregator::new(store.clone());
8755        let runtime = build_single_member_runtime().await;
8756        aggregator
8757            .inner
8758            .runtimes
8759            .write()
8760            .expect("runtime registry")
8761            .insert(
8762                "runtime-a".to_string(),
8763                runtime_entry_for_test("runtime-a", &runtime),
8764            );
8765        let session_id =
8766            wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
8767                .await
8768                .expect("spawned member is resolvable for targeted backfill");
8769        wait_for_runtime_session_history_text(
8770            &runtime,
8771            &session_id,
8772            "You are agent-a.",
8773            Duration::from_secs(5),
8774        )
8775        .await
8776        .expect("spawned member kickoff is readable before live-frame race");
8777        store
8778            .append_if_absent(NewConsoleFrame {
8779                id: None,
8780                dedupe_key: "live-tool-before-history".to_string(),
8781                timestamp_ms: 10,
8782                runtime_key: "runtime-a".to_string(),
8783                identity: "test/agent-a".to_string(),
8784                conversation_id: Some("test/agent-a".to_string()),
8785                session_id: None,
8786                kind: "tool_execution_started".to_string(),
8787                status: ConsoleFrameStatus::Delivered,
8788                payload: json!({
8789                    "id": "call-live",
8790                    "name": "king_search",
8791                    "source_event_type": "tool_execution_started",
8792                    "type": "tool_execution_started",
8793                }),
8794                source: ConsoleFrameSource {
8795                    kind: ConsoleFrameSourceKind::ConsoleEvent,
8796                    source_cursor: Some("live-tool-before-history".to_string()),
8797                },
8798                source_event_id: Some("live-tool-before-history".to_string()),
8799                interaction_id: None,
8800                turn_id: None,
8801                run_id: None,
8802                parent_frame_id: None,
8803                caused_by_frame_id: None,
8804            })
8805            .await
8806            .expect("append live tool frame");
8807
8808        let page = aggregator
8809            .query_timeline(ConsoleTimelineQuery {
8810                identity: Some("test/agent-a".to_string()),
8811                limit: 20,
8812                ..ConsoleTimelineQuery::default()
8813            })
8814            .await
8815            .expect("query child timeline");
8816
8817        assert!(
8818            page.frames
8819                .iter()
8820                .any(|frame| frame.kind == "tool_execution_started"),
8821            "initial explicit child timeline should return existing store frames without waiting for session history"
8822        );
8823
8824        let mut observed_backfill = false;
8825        for _ in 0..80 {
8826            let page = aggregator
8827                .query_timeline(ConsoleTimelineQuery {
8828                    identity: Some("test/agent-a".to_string()),
8829                    limit: 20,
8830                    ..ConsoleTimelineQuery::default()
8831                })
8832                .await
8833                .expect("query child timeline after scheduled backfill");
8834            if page.frames.iter().any(|frame| {
8835                frame.kind == "user_input"
8836                    && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
8837                    && frame.payload.to_string().contains("You are agent-a.")
8838            }) {
8839                observed_backfill = true;
8840                break;
8841            }
8842            tokio::time::sleep(Duration::from_millis(25)).await;
8843        }
8844        assert!(
8845            observed_backfill,
8846            "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
8847        );
8848        let _ = runtime.mob_handle().stop().await;
8849    }
8850
8851    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8852    async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
8853        let store = Arc::new(CountingConsoleLogStore::new());
8854        let aggregator = MobKitConsoleAggregator::new(store.clone());
8855        let (_temp, runtime, _delayed_service) =
8856            build_stress_runtime(64, Duration::from_millis(0)).await;
8857        aggregator
8858            .inner
8859            .runtimes
8860            .write()
8861            .expect("runtime registry")
8862            .insert(
8863                "runtime-a".to_string(),
8864                runtime_entry_for_test("runtime-a", runtime.as_ref()),
8865            );
8866        for idx in 0..1_000 {
8867            store
8868                .append_if_absent(NewConsoleFrame {
8869                    id: None,
8870                    dedupe_key: format!("event-{idx}"),
8871                    timestamp_ms: idx,
8872                    runtime_key: "runtime-a".to_string(),
8873                    identity: "test/agent-0".to_string(),
8874                    conversation_id: Some("test/agent-0".to_string()),
8875                    session_id: Some("session-a".to_string()),
8876                    kind: "text_delta".to_string(),
8877                    status: ConsoleFrameStatus::Delivered,
8878                    payload: json!({ "delta": idx }),
8879                    source: ConsoleFrameSource {
8880                        kind: ConsoleFrameSourceKind::ConsoleEvent,
8881                        source_cursor: None,
8882                    },
8883                    source_event_id: Some(format!("event-{idx}")),
8884                    interaction_id: None,
8885                    turn_id: None,
8886                    run_id: None,
8887                    parent_frame_id: None,
8888                    caused_by_frame_id: None,
8889                })
8890                .await
8891                .expect("append frame");
8892        }
8893
8894        // The per-frame roster-rediscovery regression is asserted exactly by
8895        // `source_watermark_calls() == 0` below (and would also blow this query
8896        // up to O(frames) async roster resolutions). The timeout is only a
8897        // hang/O(n^2)-blowup backstop — generous so it never false-trips on CPU
8898        // starvation under full parallel CI load (the old 2s ceiling flaked the
8899        // suite); a real regression takes minutes, not seconds, on 1000 frames.
8900        let page = tokio::time::timeout(
8901            Duration::from_mins(1),
8902            aggregator.query_timeline(ConsoleTimelineQuery {
8903                identity: Some("test/agent-0".to_string()),
8904                limit: 1_000,
8905                ..ConsoleTimelineQuery::default()
8906            }),
8907        )
8908        .await
8909        .expect("identity timeline query should not hang or rediscover the roster per frame")
8910        .expect("timeline query succeeds");
8911
8912        assert_eq!(page.frames.len(), 1_000);
8913        assert_eq!(store.source_watermark_calls(), 0);
8914        let _ = runtime.mob_handle().stop().await;
8915    }
8916
8917    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8918    async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
8919        const MEMBER_COUNT: usize = 32;
8920        let (_temp, runtime, delayed_service) =
8921            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
8922        let aggregator = MobKitConsoleAggregator::in_memory();
8923        aggregator
8924            .inner
8925            .runtimes
8926            .write()
8927            .expect("runtime registry")
8928            .insert(
8929                "runtime-stress".to_string(),
8930                runtime_entry_for_test("runtime-stress", &runtime),
8931            );
8932
8933        let started = Instant::now();
8934        aggregator
8935            .refresh_session_history()
8936            .await
8937            .expect("stress refresh");
8938        let elapsed = started.elapsed();
8939
8940        assert!(
8941            delayed_service.read_calls() >= MEMBER_COUNT,
8942            "expected at least one history read per member, saw {}",
8943            delayed_service.read_calls()
8944        );
8945        assert!(
8946            delayed_service.max_active_reads() > 1,
8947            "session history backfill should fan out instead of reading members serially"
8948        );
8949        assert!(
8950            delayed_service.max_active_reads()
8951                <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
8952            "session history backfill should respect the default concurrency limit"
8953        );
8954        assert!(
8955            elapsed < Duration::from_millis(600),
8956            "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
8957            MEMBER_COUNT * 40
8958        );
8959        let _ = runtime.mob_handle().stop().await;
8960    }
8961
8962    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8963    async fn session_history_backfill_respects_configured_concurrency_limit() {
8964        const MEMBER_COUNT: usize = 16;
8965        let (_temp, runtime, delayed_service) =
8966            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
8967        let aggregator =
8968            MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
8969                max_concurrent_session_backfills: 4,
8970                ..ConsoleAggregatorOptions::default()
8971            });
8972        aggregator
8973            .inner
8974            .runtimes
8975            .write()
8976            .expect("runtime registry")
8977            .insert(
8978                "runtime-stress".to_string(),
8979                runtime_entry_for_test("runtime-stress", &runtime),
8980            );
8981
8982        aggregator
8983            .refresh_session_history()
8984            .await
8985            .expect("stress refresh");
8986
8987        assert!(
8988            delayed_service.max_active_reads() <= 4,
8989            "configured concurrency limit should bound session history reads, saw {}",
8990            delayed_service.max_active_reads()
8991        );
8992        let _ = runtime.mob_handle().stop().await;
8993    }
8994
8995    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
8996    async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
8997        const MEMBER_COUNT: usize = 24;
8998        const LIVE_EVENT_COUNT: usize = 2_048;
8999        let (_temp, runtime, _delayed_service) =
9000            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
9001        let aggregator = MobKitConsoleAggregator::in_memory();
9002        aggregator.register_runtime(ConsoleRuntimeRegistration {
9003            runtime_key: "runtime-burst".to_string(),
9004            runtime: runtime.clone(),
9005            identity_namespace: "stress".to_string(),
9006            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
9007        });
9008        let console_events = runtime.console_events();
9009        for idx in 0..LIVE_EVENT_COUNT {
9010            console_events
9011                .append(
9012                    "agent-0",
9013                    Some("burst-turn".to_string()),
9014                    "text_delta",
9015                    json!({ "delta": format!("frame-{idx}") }),
9016                )
9017                .await;
9018        }
9019
9020        // Generous deadline: the loop breaks the instant all live frames land,
9021        // so a large ceiling is free in the normal case and only prevents a
9022        // false drop-detection when the live pump is CPU-starved under full
9023        // parallel CI load (the old 5s deadline flaked the suite).
9024        let deadline = Instant::now() + Duration::from_secs(30);
9025        let mut observed = 0;
9026        while Instant::now() < deadline {
9027            observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
9028            if observed >= LIVE_EVENT_COUNT {
9029                break;
9030            }
9031            tokio::time::sleep(Duration::from_millis(25)).await;
9032        }
9033
9034        assert_eq!(
9035            observed, LIVE_EVENT_COUNT,
9036            "live pump should not drop frames while slow background backfill is running"
9037        );
9038        let _ = runtime.mob_handle().stop().await;
9039    }
9040
9041    async fn count_console_event_frames(
9042        aggregator: &MobKitConsoleAggregator,
9043        identity: &str,
9044    ) -> usize {
9045        let mut after = None;
9046        let mut count = 0;
9047        loop {
9048            let page = aggregator
9049                .query_timeline(ConsoleTimelineQuery {
9050                    identity: Some(identity.to_string()),
9051                    after,
9052                    limit: 1_000,
9053                    ..ConsoleTimelineQuery::default()
9054                })
9055                .await
9056                .expect("query burst timeline");
9057            if page.frames.is_empty() {
9058                break;
9059            }
9060            count += page
9061                .frames
9062                .iter()
9063                .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
9064                .count();
9065            after = page.next_cursor;
9066            if after.is_none() {
9067                break;
9068            }
9069        }
9070        count
9071    }
9072
9073    #[tokio::test]
9074    async fn status_updates_get_replayable_aggregate_cursors() {
9075        let aggregator = MobKitConsoleAggregator::in_memory();
9076        let frame = NewConsoleFrame {
9077            id: None,
9078            dedupe_key: "send-1".to_string(),
9079            timestamp_ms: 1,
9080            runtime_key: "runtime-a".to_string(),
9081            identity: "agent-a".to_string(),
9082            conversation_id: Some("agent-a".to_string()),
9083            session_id: Some("session-1".to_string()),
9084            kind: "user_input".to_string(),
9085            status: ConsoleFrameStatus::Accepted,
9086            payload: json!({ "content": "hello" }),
9087            source: ConsoleFrameSource {
9088                kind: ConsoleFrameSourceKind::Send,
9089                source_cursor: None,
9090            },
9091            source_event_id: None,
9092            interaction_id: Some("interaction-1".to_string()),
9093            turn_id: None,
9094            run_id: None,
9095            parent_frame_id: None,
9096            caused_by_frame_id: None,
9097        };
9098        let inserted = aggregator
9099            .store()
9100            .append_if_absent(frame)
9101            .await
9102            .expect("append frame");
9103
9104        update_frame_status_and_emit(
9105            &aggregator.inner,
9106            &inserted.frame.id,
9107            ConsoleFrameStatus::Delivered,
9108        )
9109        .await
9110        .expect("update status");
9111
9112        let page = aggregator
9113            .query_timeline(ConsoleTimelineQuery {
9114                identity: Some("agent-a".to_string()),
9115                after: Some(inserted.frame.cursor.clone()),
9116                limit: 10,
9117                ..ConsoleTimelineQuery::default()
9118            })
9119            .await
9120            .expect("query timeline");
9121        assert_eq!(page.frames.len(), 1);
9122        assert_eq!(page.frames[0].kind, "frame_updated");
9123        assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
9124        assert_eq!(
9125            page.frames[0]
9126                .payload
9127                .get("frame")
9128                .and_then(|frame| frame.get("status"))
9129                .and_then(Value::as_str),
9130            Some("delivered")
9131        );
9132    }
9133
9134    #[tokio::test]
9135    async fn steer_delivery_appends_terminal_control_frame() {
9136        let aggregator = MobKitConsoleAggregator::in_memory();
9137        let frame = NewConsoleFrame {
9138            id: None,
9139            dedupe_key: "send-steer-1".to_string(),
9140            timestamp_ms: 1,
9141            runtime_key: "runtime-a".to_string(),
9142            identity: "agent-a".to_string(),
9143            conversation_id: Some("agent-a".to_string()),
9144            session_id: Some("session-1".to_string()),
9145            kind: "user_input".to_string(),
9146            status: ConsoleFrameStatus::Delivered,
9147            payload: json!({
9148                "content": "operator steer",
9149                "handling_mode": "steer",
9150            }),
9151            source: ConsoleFrameSource {
9152                kind: ConsoleFrameSourceKind::Send,
9153                source_cursor: None,
9154            },
9155            source_event_id: None,
9156            interaction_id: Some("interaction-steer-1".to_string()),
9157            turn_id: None,
9158            run_id: None,
9159            parent_frame_id: None,
9160            caused_by_frame_id: None,
9161        };
9162        let inserted = aggregator
9163            .store()
9164            .append_if_absent(frame)
9165            .await
9166            .expect("append steer input");
9167
9168        append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
9169            .await
9170            .expect("append steer terminal");
9171
9172        let page = aggregator
9173            .query_timeline(ConsoleTimelineQuery {
9174                identity: Some("agent-a".to_string()),
9175                after: Some(inserted.frame.cursor.clone()),
9176                limit: 10,
9177                ..ConsoleTimelineQuery::default()
9178            })
9179            .await
9180            .expect("query timeline");
9181
9182        assert_eq!(page.frames.len(), 1);
9183        let terminal = &page.frames[0];
9184        assert_eq!(terminal.kind, "interaction_complete");
9185        assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
9186        assert_eq!(
9187            terminal.interaction_id.as_deref(),
9188            Some("interaction-steer-1")
9189        );
9190        assert_eq!(
9191            terminal.parent_frame_id.as_deref(),
9192            Some(inserted.frame.id.as_str())
9193        );
9194        assert_eq!(
9195            terminal.payload.get("reason").and_then(Value::as_str),
9196            Some("steer_delivered")
9197        );
9198    }
9199
9200    #[test]
9201    fn transcript_fingerprint_matches_equivalent_reasoning_payloads() {
9202        let live = transcript_fingerprint(
9203            "reasoning_complete",
9204            &json!({ "text": "I should inspect the file first." }),
9205        );
9206        let replay = transcript_fingerprint(
9207            "reasoning_complete",
9208            &json!({ "summary": "I should inspect the file first." }),
9209        );
9210        let delta = transcript_fingerprint(
9211            "reasoning_delta",
9212            &json!({ "delta": "I should inspect the file first." }),
9213        );
9214
9215        assert_eq!(live.as_deref(), Some("I should inspect the file first."));
9216        assert_eq!(live, replay);
9217        assert_eq!(live, delta);
9218    }
9219
9220    #[tokio::test]
9221    async fn history_counterpart_scan_is_not_capped_to_one_page() {
9222        let aggregator = MobKitConsoleAggregator::in_memory();
9223        for idx in 0..1_005 {
9224            aggregator
9225                .store()
9226                .append_if_absent(NewConsoleFrame {
9227                    id: None,
9228                    dedupe_key: format!("filler-{idx}"),
9229                    timestamp_ms: idx,
9230                    runtime_key: "runtime-a".to_string(),
9231                    identity: "agent-a".to_string(),
9232                    conversation_id: Some("agent-a".to_string()),
9233                    session_id: Some("session-a".to_string()),
9234                    kind: "text_delta".to_string(),
9235                    status: ConsoleFrameStatus::Completed,
9236                    payload: json!({ "delta": idx }),
9237                    source: ConsoleFrameSource {
9238                        kind: ConsoleFrameSourceKind::ConsoleEvent,
9239                        source_cursor: None,
9240                    },
9241                    source_event_id: Some(format!("filler-{idx}")),
9242                    interaction_id: None,
9243                    turn_id: None,
9244                    run_id: None,
9245                    parent_frame_id: None,
9246                    caused_by_frame_id: None,
9247                })
9248                .await
9249                .expect("append filler");
9250        }
9251        aggregator
9252            .store()
9253            .append_if_absent(NewConsoleFrame {
9254                id: None,
9255                dedupe_key: "live-user-input".to_string(),
9256                timestamp_ms: 2_000,
9257                runtime_key: "runtime-a".to_string(),
9258                identity: "agent-a".to_string(),
9259                conversation_id: Some("agent-a".to_string()),
9260                session_id: Some("session-a".to_string()),
9261                kind: "user_input".to_string(),
9262                status: ConsoleFrameStatus::Delivered,
9263                payload: json!({ "content": "already here" }),
9264                source: ConsoleFrameSource {
9265                    kind: ConsoleFrameSourceKind::ConsoleEvent,
9266                    source_cursor: None,
9267                },
9268                source_event_id: Some("live-user-input".to_string()),
9269                interaction_id: None,
9270                turn_id: None,
9271                run_id: None,
9272                parent_frame_id: None,
9273                caused_by_frame_id: None,
9274            })
9275            .await
9276            .expect("append live input");
9277
9278        let history = NewConsoleFrame {
9279            id: None,
9280            dedupe_key: "history-user-input".to_string(),
9281            timestamp_ms: 3_000,
9282            runtime_key: "runtime-a".to_string(),
9283            identity: "agent-a".to_string(),
9284            conversation_id: Some("agent-a".to_string()),
9285            session_id: Some("session-a".to_string()),
9286            kind: "user_input".to_string(),
9287            status: ConsoleFrameStatus::Completed,
9288            payload: json!({ "content": "already here" }),
9289            source: ConsoleFrameSource {
9290                kind: ConsoleFrameSourceKind::SessionHistory,
9291                source_cursor: Some("session-a:1006".to_string()),
9292            },
9293            source_event_id: None,
9294            interaction_id: None,
9295            turn_id: None,
9296            run_id: None,
9297            parent_frame_id: None,
9298            caused_by_frame_id: None,
9299        };
9300
9301        assert!(
9302            history_frame_has_existing_counterpart(&aggregator.inner, &history)
9303                .await
9304                .expect("counterpart scan")
9305        );
9306    }
9307
9308    #[tokio::test]
9309    async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
9310        let aggregator = MobKitConsoleAggregator::in_memory();
9311        aggregator
9312            .store()
9313            .append_if_absent(NewConsoleFrame {
9314                id: None,
9315                dedupe_key: "live-user-input".to_string(),
9316                timestamp_ms: 2_000,
9317                runtime_key: "runtime-a".to_string(),
9318                identity: "agent-a".to_string(),
9319                conversation_id: Some("agent-a".to_string()),
9320                session_id: Some("session-a".to_string()),
9321                kind: "user_input".to_string(),
9322                status: ConsoleFrameStatus::Delivered,
9323                payload: json!({ "content": "hello from operator" }),
9324                source: ConsoleFrameSource {
9325                    kind: ConsoleFrameSourceKind::ConsoleEvent,
9326                    source_cursor: None,
9327                },
9328                source_event_id: Some("live-user-input".to_string()),
9329                interaction_id: None,
9330                turn_id: None,
9331                run_id: None,
9332                parent_frame_id: None,
9333                caused_by_frame_id: None,
9334            })
9335            .await
9336            .expect("append live input");
9337
9338        let history = NewConsoleFrame {
9339            id: None,
9340            dedupe_key: "history-user-input".to_string(),
9341            timestamp_ms: 3_000,
9342            runtime_key: "runtime-a".to_string(),
9343            identity: "agent-a".to_string(),
9344            conversation_id: Some("agent-a".to_string()),
9345            session_id: Some("session-a".to_string()),
9346            kind: "user_input".to_string(),
9347            status: ConsoleFrameStatus::Completed,
9348            payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
9349            source: ConsoleFrameSource {
9350                kind: ConsoleFrameSourceKind::SessionHistory,
9351                source_cursor: Some("session-a:2".to_string()),
9352            },
9353            source_event_id: None,
9354            interaction_id: None,
9355            turn_id: None,
9356            run_id: None,
9357            parent_frame_id: None,
9358            caused_by_frame_id: None,
9359        };
9360
9361        assert!(
9362            history_frame_has_existing_counterpart(&aggregator.inner, &history)
9363                .await
9364                .expect("counterpart scan")
9365        );
9366    }
9367
9368    #[tokio::test]
9369    async fn history_counterpart_scan_matches_content_block_user_prompts() {
9370        let aggregator = MobKitConsoleAggregator::in_memory();
9371        aggregator
9372            .store()
9373            .append_if_absent(NewConsoleFrame {
9374                id: None,
9375                dedupe_key: "live-user-input".to_string(),
9376                timestamp_ms: 2_000,
9377                runtime_key: "runtime-a".to_string(),
9378                identity: "agent-a".to_string(),
9379                conversation_id: Some("agent-a".to_string()),
9380                session_id: Some("session-a".to_string()),
9381                kind: "user_input".to_string(),
9382                status: ConsoleFrameStatus::Delivered,
9383                payload: json!({ "content": "hello from operator" }),
9384                source: ConsoleFrameSource {
9385                    kind: ConsoleFrameSourceKind::ConsoleEvent,
9386                    source_cursor: None,
9387                },
9388                source_event_id: Some("live-user-input".to_string()),
9389                interaction_id: None,
9390                turn_id: None,
9391                run_id: None,
9392                parent_frame_id: None,
9393                caused_by_frame_id: None,
9394            })
9395            .await
9396            .expect("append live input");
9397
9398        let history = NewConsoleFrame {
9399            id: None,
9400            dedupe_key: "history-user-input".to_string(),
9401            timestamp_ms: 3_000,
9402            runtime_key: "runtime-a".to_string(),
9403            identity: "agent-a".to_string(),
9404            conversation_id: Some("agent-a".to_string()),
9405            session_id: Some("session-a".to_string()),
9406            kind: "user_input".to_string(),
9407            status: ConsoleFrameStatus::Completed,
9408            payload: json!({
9409                "content": [{ "type": "text", "text": "hello from operator" }]
9410            }),
9411            source: ConsoleFrameSource {
9412                kind: ConsoleFrameSourceKind::SessionHistory,
9413                source_cursor: Some("session-a:2".to_string()),
9414            },
9415            source_event_id: None,
9416            interaction_id: None,
9417            turn_id: None,
9418            run_id: None,
9419            parent_frame_id: None,
9420            caused_by_frame_id: None,
9421        };
9422
9423        assert!(
9424            history_frame_has_existing_counterpart(&aggregator.inner, &history)
9425                .await
9426                .expect("counterpart scan")
9427        );
9428    }
9429
9430    #[tokio::test]
9431    async fn history_counterpart_scan_matches_live_tool_results() {
9432        let aggregator = MobKitConsoleAggregator::in_memory();
9433        aggregator
9434            .store()
9435            .append_if_absent(NewConsoleFrame {
9436                id: None,
9437                dedupe_key: "live-tool-result".to_string(),
9438                timestamp_ms: 2_000,
9439                runtime_key: "runtime-a".to_string(),
9440                identity: "agent-a".to_string(),
9441                conversation_id: Some("agent-a".to_string()),
9442                session_id: Some("session-a".to_string()),
9443                kind: "tool_execution_completed".to_string(),
9444                status: ConsoleFrameStatus::Delivered,
9445                payload: json!({
9446                    "id": "call-1",
9447                    "tool_call_id": "call-1",
9448                    "result": "{ \"count\": 70 }"
9449                }),
9450                source: ConsoleFrameSource {
9451                    kind: ConsoleFrameSourceKind::ConsoleEvent,
9452                    source_cursor: None,
9453                },
9454                source_event_id: Some("live-tool-result".to_string()),
9455                interaction_id: None,
9456                turn_id: None,
9457                run_id: None,
9458                parent_frame_id: None,
9459                caused_by_frame_id: None,
9460            })
9461            .await
9462            .expect("append live tool result");
9463
9464        let history = NewConsoleFrame {
9465            id: None,
9466            dedupe_key: "history-tool-result".to_string(),
9467            timestamp_ms: 3_000,
9468            runtime_key: "runtime-a".to_string(),
9469            identity: "agent-a".to_string(),
9470            conversation_id: Some("agent-a".to_string()),
9471            session_id: Some("session-a".to_string()),
9472            kind: "tool_execution_completed".to_string(),
9473            status: ConsoleFrameStatus::Completed,
9474            payload: json!({
9475                "id": "call-1",
9476                "tool_call_id": "call-1",
9477                "result": "{ \"count\": 70 }",
9478                "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
9479            }),
9480            source: ConsoleFrameSource {
9481                kind: ConsoleFrameSourceKind::SessionHistory,
9482                source_cursor: Some("session-a:3:0".to_string()),
9483            },
9484            source_event_id: None,
9485            interaction_id: None,
9486            turn_id: None,
9487            run_id: None,
9488            parent_frame_id: None,
9489            caused_by_frame_id: None,
9490        };
9491
9492        assert!(
9493            history_frame_has_existing_counterpart(&aggregator.inner, &history)
9494                .await
9495                .expect("counterpart scan")
9496        );
9497    }
9498
9499    #[tokio::test]
9500    async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
9501        let aggregator = MobKitConsoleAggregator::in_memory();
9502        for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
9503            aggregator
9504                .store()
9505                .append_if_absent(NewConsoleFrame {
9506                    id: None,
9507                    dedupe_key: format!("live-delta-{idx}"),
9508                    timestamp_ms: 2_000 + idx as u64,
9509                    runtime_key: "runtime-a".to_string(),
9510                    identity: "agent-a".to_string(),
9511                    conversation_id: Some("agent-a".to_string()),
9512                    session_id: Some("session-a".to_string()),
9513                    kind: "text_delta".to_string(),
9514                    status: ConsoleFrameStatus::Delivered,
9515                    payload: json!({ "delta": delta }),
9516                    source: ConsoleFrameSource {
9517                        kind: ConsoleFrameSourceKind::ConsoleEvent,
9518                        source_cursor: None,
9519                    },
9520                    source_event_id: Some(format!("live-delta-{idx}")),
9521                    interaction_id: Some("turn-a".to_string()),
9522                    turn_id: None,
9523                    run_id: None,
9524                    parent_frame_id: None,
9525                    caused_by_frame_id: None,
9526                })
9527                .await
9528                .expect("append live delta");
9529        }
9530
9531        let history = NewConsoleFrame {
9532            id: None,
9533            dedupe_key: "history-assistant-complete".to_string(),
9534            timestamp_ms: 3_000,
9535            runtime_key: "runtime-a".to_string(),
9536            identity: "agent-a".to_string(),
9537            conversation_id: Some("agent-a".to_string()),
9538            session_id: Some("session-a".to_string()),
9539            kind: "interaction_complete".to_string(),
9540            status: ConsoleFrameStatus::Completed,
9541            payload: json!({ "result": "Ready and standing by." }),
9542            source: ConsoleFrameSource {
9543                kind: ConsoleFrameSourceKind::SessionHistory,
9544                source_cursor: Some("session-a:3".to_string()),
9545            },
9546            source_event_id: None,
9547            interaction_id: None,
9548            turn_id: None,
9549            run_id: None,
9550            parent_frame_id: None,
9551            caused_by_frame_id: None,
9552        };
9553
9554        assert!(
9555            history_frame_has_existing_counterpart(&aggregator.inner, &history)
9556                .await
9557                .expect("counterpart scan")
9558        );
9559    }
9560
9561    #[test]
9562    fn session_history_watermark_key_is_session_scoped() {
9563        assert_ne!(
9564            session_history_watermark_runtime_key("runtime-a", "session-1"),
9565            session_history_watermark_runtime_key("runtime-a", "session-2")
9566        );
9567    }
9568
9569    #[test]
9570    fn backfill_offset_advances_only_on_strict_forward_progress() {
9571        // Normal full-page advance from the start.
9572        assert_eq!(advance_backfill_offset(0, 0, 500), Some(500));
9573        // Subsequent full page from a correct cursor advances.
9574        assert_eq!(advance_backfill_offset(500, 500, 500), Some(1000));
9575        // A partial page that still moves forward advances.
9576        assert_eq!(advance_backfill_offset(500, 500, 3), Some(503));
9577        // One element past the requested offset still counts as progress.
9578        assert_eq!(advance_backfill_offset(499, 0, 500), Some(500));
9579
9580        // Livelock shapes: the store re-serves a page whose (base + len) does
9581        // NOT advance past what we asked for. These must stop the loop.
9582        assert_eq!(advance_backfill_offset(500, 0, 500), None); // cursor reset to 0, full page
9583        assert_eq!(advance_backfill_offset(500, 0, 400), None); // strictly behind
9584        assert_eq!(advance_backfill_offset(1000, 500, 500), None); // lands exactly on offset
9585        assert_eq!(advance_backfill_offset(10, 10, 0), None); // empty page at same offset
9586
9587        // Saturating add: a pathological base near usize::MAX cannot panic.
9588        assert_eq!(advance_backfill_offset(0, usize::MAX, 5), Some(usize::MAX));
9589    }
9590
9591    #[test]
9592    fn session_history_watermarks_are_cursor_and_ttl_aware() {
9593        let legacy = "session:with:colon:42";
9594        let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
9595        let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
9596
9597        assert_eq!(
9598            parse_session_history_watermark(legacy, "session:with:colon"),
9599            Some(42)
9600        );
9601        assert_eq!(
9602            parse_session_history_watermark(&checked, "session:with:colon"),
9603            Some(43)
9604        );
9605        assert!(session_history_watermark_is_fresh(
9606            &checked,
9607            "session:with:colon",
9608            1_500
9609        ));
9610        assert!(!session_history_watermark_is_fresh(
9611            &checked,
9612            "session:with:colon",
9613            1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
9614        ));
9615        assert!(session_history_watermark_is_fresh(
9616            &empty_checked,
9617            "session:with:colon",
9618            1_500
9619        ));
9620        assert!(!session_history_watermark_is_fresh(
9621            &empty_checked,
9622            "session:with:colon",
9623            1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
9624        ));
9625    }
9626
9627    #[test]
9628    fn session_history_messages_project_to_renderable_frames() {
9629        let user = frame_from_session_history_message(
9630            "runtime-a",
9631            "agent-a",
9632            "session-a",
9633            0,
9634            json!({
9635                "role": "user",
9636                "content": "hello",
9637                "timestamp_ms": 10
9638            }),
9639        )
9640        .expect("user history frame");
9641        // meerkat 0.7 removed the legacy flat-text `Message::Assistant`
9642        // variant: session stores serialize assistant turns only as
9643        // `block_assistant` (MessageWire), so the history projection sees the
9644        // block shape.
9645        let assistant = frame_from_session_history_message(
9646            "runtime-a",
9647            "agent-a",
9648            "session-a",
9649            1,
9650            json!({
9651                "role": "block_assistant",
9652                "blocks": [
9653                    { "block_type": "text", "data": { "text": "hi there" } }
9654                ],
9655                "stop_reason": "end_turn",
9656                "timestamp_ms": 11
9657            }),
9658        )
9659        .expect("assistant history frame");
9660
9661        assert_eq!(user.kind, "user_input");
9662        assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
9663        assert_eq!(
9664            user.payload["content"],
9665            json!([{ "type": "text", "text": "hello" }])
9666        );
9667        assert_eq!(assistant.kind, "interaction_complete");
9668        assert_eq!(assistant.payload["text"], json!("hi there"));
9669        assert!(
9670            assistant
9671                .dedupe_key
9672                .starts_with("session-history:runtime-a:session-a:1:")
9673        );
9674    }
9675
9676    #[test]
9677    fn session_history_projection_filters_scaffold_user_messages() {
9678        let spawn_notice = frame_from_session_history_message(
9679            "runtime-a",
9680            "agent-a",
9681            "session-a",
9682            0,
9683            json!({
9684                "role": "user",
9685                "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
9686                "timestamp_ms": 10
9687            }),
9688        );
9689        let peer_update = frame_from_session_history_message(
9690            "runtime-a",
9691            "agent-a",
9692            "session-a",
9693            1,
9694            json!({
9695                "role": "user",
9696                "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
9697                "timestamp_ms": 11
9698            }),
9699        );
9700        let real_user = frame_from_session_history_message(
9701            "runtime-a",
9702            "agent-a",
9703            "session-a",
9704            2,
9705            json!({
9706                "role": "user",
9707                "content": "Please review the incident notes.",
9708                "timestamp_ms": 12
9709            }),
9710        );
9711
9712        assert!(spawn_notice.is_none());
9713        assert!(peer_update.is_none());
9714        assert!(real_user.is_some());
9715    }
9716
9717    #[test]
9718    fn session_history_projection_skips_non_transcript_messages() {
9719        let skipped = frame_from_session_history_message(
9720            "runtime-a",
9721            "agent-a",
9722            "session-a",
9723            0,
9724            json!({
9725                "content": "internal system prompt"
9726            }),
9727        );
9728        assert!(skipped.is_none());
9729    }
9730
9731    #[test]
9732    fn session_history_projection_extracts_assistant_blocks() {
9733        let frame = frame_from_session_history_message(
9734            "runtime-a",
9735            "agent-a",
9736            "session-a",
9737            0,
9738            json!({
9739                "role": "block_assistant",
9740                "blocks": [
9741                    { "block_type": "text", "data": { "text": "hello " } },
9742                    { "block_type": "text", "data": { "text": "there" } }
9743                ],
9744                "stop_reason": "end_turn"
9745            }),
9746        )
9747        .expect("assistant block history frame");
9748        assert_eq!(frame.payload["text"], json!("hello there"));
9749    }
9750
9751    #[test]
9752    fn session_history_projection_extracts_nested_text_block_data() {
9753        let frame = frame_from_session_history_message(
9754            "runtime-a",
9755            "agent-a",
9756            "session-a",
9757            0,
9758            json!({
9759                "role": "block_assistant",
9760                "blocks": [
9761                    {
9762                        "block_type": "text",
9763                        "data": { "text": "Ready and standing by." }
9764                    }
9765                ],
9766                "stop_reason": "end_turn",
9767                "created_at": "1970-01-01T00:00:00.010Z"
9768            }),
9769        )
9770        .expect("assistant block history frame");
9771
9772        assert_eq!(frame.kind, "interaction_complete");
9773        assert_eq!(frame.payload["result"], json!("Ready and standing by."));
9774    }
9775
9776    #[test]
9777    fn session_history_projection_drops_reasoning_blocks_from_result_text() {
9778        let frame = frame_from_session_history_message(
9779            "runtime-a",
9780            "agent-a",
9781            "session-a",
9782            0,
9783            json!({
9784                "role": "block_assistant",
9785                "blocks": [
9786                    {
9787                        "block_type": "reasoning",
9788                        "data": { "text": "**Planning**\nI should not be rendered." }
9789                    },
9790                    {
9791                        "block_type": "text",
9792                        "data": { "text": "Visible answer." }
9793                    }
9794                ],
9795                "stop_reason": "end_turn",
9796                "created_at": "1970-01-01T00:00:00.010Z"
9797            }),
9798        )
9799        .expect("assistant block history frame");
9800
9801        assert_eq!(frame.kind, "interaction_complete");
9802        assert_eq!(frame.payload["result"], json!("Visible answer."));
9803        assert_eq!(frame.payload["text"], json!("Visible answer."));
9804    }
9805
9806    #[test]
9807    fn session_history_projection_leaves_reasoning_only_result_empty() {
9808        let frame = frame_from_session_history_message(
9809            "runtime-a",
9810            "agent-a",
9811            "session-a",
9812            0,
9813            json!({
9814                "role": "block_assistant",
9815                "blocks": [
9816                    {
9817                        "block_type": "reasoning",
9818                        "data": { "text": "Private planning text." }
9819                    },
9820                    {
9821                        "block_type": "tool_use",
9822                        "data": { "id": "toolu-1", "name": "peers", "args": {} }
9823                    }
9824                ],
9825                "stop_reason": "end_turn",
9826                "created_at": "1970-01-01T00:00:00.010Z"
9827            }),
9828        )
9829        .expect("assistant block history frame");
9830
9831        assert_eq!(frame.kind, "interaction_complete");
9832        assert_eq!(frame.payload["result"], json!(""));
9833        assert_eq!(frame.payload["text"], json!(""));
9834    }
9835
9836    #[test]
9837    fn session_history_projection_preserves_tool_results() {
9838        let frames = frames_from_session_history_message(
9839            "runtime-a",
9840            "agent-a",
9841            "session-a",
9842            5,
9843            json!({
9844                "role": "tool_results",
9845                "results": [
9846                    {
9847                        "tool_use_id": "call-peers",
9848                        "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
9849                        "is_error": false
9850                    }
9851                ],
9852                "created_at": "1970-01-01T00:00:00.050Z"
9853            }),
9854        );
9855
9856        assert_eq!(frames.len(), 1);
9857        let frame = &frames[0];
9858        assert_eq!(frame.kind, "tool_execution_completed");
9859        assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
9860        assert_eq!(
9861            frame.payload["result"],
9862            json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
9863        );
9864        assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
9865        assert_eq!(frame.timestamp_ms, 50);
9866    }
9867
9868    #[test]
9869    fn session_history_projection_derives_assistant_image_from_image_tool_result() {
9870        // Regression: a generate_image tool result rebuilt from session-history
9871        // backfill must derive an `assistant_image` frame (image_id/blob_id/
9872        // media_type/width/height) just like the live ingest edge does. Without
9873        // it the image renders as raw tool JSON after a reload / fresh store.
9874        // The serialized ImageGenerationToolResult carries NO tool name, so the
9875        // backfill parses it directly.
9876        let image_result = json!({
9877            "operation_id": "00000000-0000-0000-0000-0000000000aa",
9878            "terminal": { "terminal": "generated" },
9879            "images": [
9880                {
9881                    "image_id": "00000000-0000-0000-0000-000000000051",
9882                    "blob_ref": {
9883                        "blob_id": "sha256:test-generated-image",
9884                        "media_type": "image/png"
9885                    },
9886                    "media_type": "image/png",
9887                    "width": 1024,
9888                    "height": 768
9889                }
9890            ],
9891            "provider_text": { "disposition": "not_emitted" },
9892            "revised_prompt": { "disposition": "not_requested" },
9893            "native_metadata": { "provider": "not_emitted" }
9894        });
9895        // Sanity: the fixture must be a valid ImageGenerationToolResult so the
9896        // test pins the real wire contract, not an arbitrary blob.
9897        serde_json::from_value::<meerkat_core::image_generation::ImageGenerationToolResult>(
9898            image_result.clone(),
9899        )
9900        .expect("fixture must be a valid ImageGenerationToolResult");
9901        let result_text = serde_json::to_string(&image_result).expect("serialize image result");
9902
9903        let frames = frames_from_session_history_message(
9904            "runtime-a",
9905            "artist:singleton",
9906            "session-a",
9907            6,
9908            json!({
9909                "role": "tool_results",
9910                "results": [
9911                    {
9912                        "tool_use_id": "call-image",
9913                        "content": result_text,
9914                        "is_error": false
9915                    }
9916                ],
9917                "created_at": "1970-01-01T00:00:00.060Z"
9918            }),
9919        );
9920
9921        // The generic tool_execution_completed frame is still emitted...
9922        assert!(
9923            frames
9924                .iter()
9925                .any(|frame| frame.kind == "tool_execution_completed"),
9926            "tool_execution_completed frame must still be present: {frames:#?}"
9927        );
9928        // ...plus a derived assistant_image frame so the picture renders.
9929        let image_frame = frames
9930            .iter()
9931            .find(|frame| frame.kind == "assistant_image")
9932            .expect("backfill must derive an assistant_image frame");
9933        assert_eq!(
9934            image_frame.payload["blob_id"],
9935            json!("sha256:test-generated-image")
9936        );
9937        assert_eq!(
9938            image_frame.payload["image_id"],
9939            json!("00000000-0000-0000-0000-000000000051")
9940        );
9941        assert_eq!(image_frame.payload["media_type"], json!("image/png"));
9942        assert_eq!(image_frame.payload["width"], json!(1024));
9943        assert_eq!(image_frame.payload["height"], json!(768));
9944        assert_eq!(image_frame.payload["tool_call_id"], json!("call-image"));
9945        assert_eq!(
9946            image_frame.source.kind,
9947            ConsoleFrameSourceKind::SessionHistory
9948        );
9949        assert_eq!(image_frame.timestamp_ms, 60);
9950    }
9951
9952    #[test]
9953    fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
9954        let frames = frames_from_session_history_message(
9955            "runtime-a",
9956            "review:singleton",
9957            "session-a",
9958            7,
9959            json!({
9960                "role": "block_assistant",
9961                "blocks": [
9962                    {
9963                        "block_type": "tool_use",
9964                        "data": {
9965                            "id": "call-spawn",
9966                            "name": "mob_spawn_member",
9967                            "args": {
9968                                "member_id": "review-worker-alpha",
9969                                "profile": "review-worker",
9970                                "initial_message": "Review Initiative Alpha and save the result."
9971                            }
9972                        }
9973                    }
9974                ],
9975                "stop_reason": "tool_use",
9976                "created_at": "1970-01-01T00:00:00.070Z"
9977            }),
9978        );
9979
9980        assert_eq!(frames.len(), 2);
9981        let child = frames
9982            .iter()
9983            .find(|frame| frame.identity == "review-worker-alpha")
9984            .expect("spawned member initial message frame");
9985        assert_eq!(child.kind, "user_input");
9986        assert_eq!(child.timestamp_ms, 70);
9987        assert_eq!(
9988            child.payload["message"]["content"],
9989            "Review Initiative Alpha and save the result."
9990        );
9991        assert_eq!(child.payload["parent_identity"], "review:singleton");
9992        assert_eq!(child.payload["via_tool"], "mob_spawn_member");
9993        assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
9994    }
9995
9996    #[test]
9997    fn session_history_projection_surfaces_specs_spawn_initial_messages() {
9998        let frames = frames_from_session_history_message(
9999            "runtime-a",
10000            "review:singleton",
10001            "session-a",
10002            8,
10003            json!({
10004                "role": "block_assistant",
10005                "blocks": [
10006                    {
10007                        "block_type": "tool_use",
10008                        "data": {
10009                            "id": "call-spawn-many",
10010                            "name": "mob_spawn_member",
10011                            "args": {
10012                                "specs": [
10013                                    {
10014                                        "agent_identity": "review-worker-alpha",
10015                                        "profile": "review-worker",
10016                                        "initial_message": "Review Initiative Alpha."
10017                                    },
10018                                    {
10019                                        "agent_identity": "review-worker-beta",
10020                                        "profile": "review-worker",
10021                                        "initial_message": "Review Initiative Beta."
10022                                    }
10023                                ]
10024                            }
10025                        }
10026                    }
10027                ],
10028                "stop_reason": "tool_use",
10029                "created_at": "1970-01-01T00:00:00.080Z"
10030            }),
10031        );
10032
10033        assert_eq!(frames.len(), 3);
10034        let child_messages = frames
10035            .iter()
10036            .filter(|frame| frame.kind == "user_input")
10037            .map(|frame| {
10038                (
10039                    frame.identity.as_str(),
10040                    frame.payload["message"]["content"]
10041                        .as_str()
10042                        .unwrap_or_default(),
10043                )
10044            })
10045            .collect::<Vec<_>>();
10046        assert_eq!(
10047            child_messages,
10048            vec![
10049                ("review-worker-alpha", "Review Initiative Alpha."),
10050                ("review-worker-beta", "Review Initiative Beta."),
10051            ]
10052        );
10053    }
10054
10055    #[tokio::test]
10056    async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
10057        let aggregator = MobKitConsoleAggregator::in_memory();
10058        let frames = frames_from_session_history_message(
10059            "runtime-a",
10060            "review:singleton",
10061            "session-a",
10062            9,
10063            json!({
10064                "role": "block_assistant",
10065                "blocks": [
10066                    {
10067                        "block_type": "tool_use",
10068                        "data": {
10069                            "id": "call-spawn-many",
10070                            "name": "mob_spawn_member",
10071                            "args": {
10072                                "specs": [
10073                                    {
10074                                        "agent_identity": "review-worker-alpha",
10075                                        "profile": "review-worker",
10076                                        "initial_message": "Review Initiative Alpha."
10077                                    },
10078                                    {
10079                                        "agent_identity": "review-worker-beta",
10080                                        "profile": "review-worker",
10081                                        "initial_message": "Review Initiative Beta."
10082                                    }
10083                                ]
10084                            }
10085                        }
10086                    }
10087                ],
10088                "stop_reason": "tool_use",
10089                "created_at": "1970-01-01T00:00:00.090Z"
10090            }),
10091        );
10092        for frame in frames {
10093            aggregator
10094                .store()
10095                .append_if_absent(frame)
10096                .await
10097                .expect("append projected session-history frame");
10098        }
10099
10100        let child_page = aggregator
10101            .query_timeline(ConsoleTimelineQuery {
10102                identity: Some("review-worker-alpha".to_string()),
10103                limit: 20,
10104                ..ConsoleTimelineQuery::default()
10105            })
10106            .await
10107            .expect("query child timeline");
10108        assert_eq!(child_page.frames.len(), 1);
10109        assert_eq!(child_page.frames[0].kind, "user_input");
10110        assert_eq!(
10111            child_page.frames[0].payload["message"]["content"],
10112            "Review Initiative Alpha."
10113        );
10114        assert_eq!(
10115            child_page.frames[0].payload["parent_identity"],
10116            "review:singleton"
10117        );
10118
10119        let parent_page = aggregator
10120            .query_timeline(ConsoleTimelineQuery {
10121                identity: Some("review:singleton".to_string()),
10122                limit: 20,
10123                ..ConsoleTimelineQuery::default()
10124            })
10125            .await
10126            .expect("query parent timeline");
10127        assert!(
10128            parent_page
10129                .frames
10130                .iter()
10131                .all(|frame| frame.identity == "review:singleton"),
10132            "child initial messages must not leak into parent timeline: {:#?}",
10133            parent_page.frames
10134        );
10135    }
10136
10137    #[test]
10138    fn session_history_frames_carry_transcript_interaction_identity() {
10139        // meerkat 0.7.25 ask 15 addendum: persisted TranscriptMessageIdentity
10140        // must reach the projected frames so the console can join
10141        // live/history twins by id instead of content heuristics.
10142        let interaction = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
10143        let run = "886313e1-3b8a-5372-9b90-0c9aee199e5d";
10144        let frames = frames_from_session_history_message_with_namespace(
10145            "runtime-a",
10146            "helper",
10147            "",
10148            "session-a",
10149            3,
10150            json!({
10151                "role": "user",
10152                "content": "please ack",
10153                "identity": { "interaction_id": interaction, "run_id": run },
10154                "created_at": "1970-01-01T00:00:00.100Z"
10155            }),
10156        );
10157        assert_eq!(frames.len(), 1, "{frames:#?}");
10158        assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
10159        assert_eq!(frames[0].run_id.as_deref(), Some(run));
10160
10161        let frames = frames_from_session_history_message_with_namespace(
10162            "runtime-a",
10163            "helper",
10164            "",
10165            "session-a",
10166            4,
10167            json!({
10168                "role": "block_assistant",
10169                "blocks": [ { "block_type": "text", "data": { "text": "ACK" } } ],
10170                "identity": { "interaction_id": interaction },
10171                "stop_reason": "end_turn",
10172                "created_at": "1970-01-01T00:00:00.200Z"
10173            }),
10174        );
10175        assert!(!frames.is_empty(), "assistant history frame expected");
10176        assert_eq!(frames[0].interaction_id.as_deref(), Some(interaction));
10177        assert_eq!(frames[0].run_id, None);
10178
10179        // Messages without a persisted identity stay unstamped (pre-0.7.25
10180        // transcripts deserialize with the empty default).
10181        let frames = frames_from_session_history_message_with_namespace(
10182            "runtime-a",
10183            "helper",
10184            "",
10185            "session-a",
10186            5,
10187            json!({
10188                "role": "user",
10189                "content": "legacy row",
10190                "created_at": "1970-01-01T00:00:00.300Z"
10191            }),
10192        );
10193        assert_eq!(frames.len(), 1);
10194        assert_eq!(frames[0].interaction_id, None);
10195    }
10196
10197    #[test]
10198    fn identity_first_interaction_ids_are_deterministic_uuids() {
10199        let a = identity_first_interaction_uuid("send:key:1");
10200        let b = identity_first_interaction_uuid("send:key:1");
10201        let c = identity_first_interaction_uuid("send:key:2");
10202        assert_eq!(a, b, "idempotent retries must mint the same id");
10203        assert_ne!(a, c);
10204        assert!(
10205            a.parse::<uuid::Uuid>().is_ok(),
10206            "identity-first interaction ids must be UUID-form (the console's \
10207             authoritative twin-identity scheme): {a}"
10208        );
10209    }
10210
10211    #[test]
10212    fn session_history_projection_applies_namespace_to_spawned_child_identity() {
10213        let frames = frames_from_session_history_message_with_namespace(
10214            "runtime-a",
10215            "test/review:singleton",
10216            "test",
10217            "session-a",
10218            10,
10219            json!({
10220                "role": "block_assistant",
10221                "blocks": [
10222                    {
10223                        "block_type": "tool_use",
10224                        "data": {
10225                            "id": "call-spawn-many",
10226                            "name": "mob_spawn_member",
10227                            "args": {
10228                                "specs": [
10229                                    {
10230                                        "agent_identity": "review-worker-alpha",
10231                                        "profile": "review-worker",
10232                                        "initial_message": "Review Initiative Alpha."
10233                                    },
10234                                    {
10235                                        "agent_identity": "test/review-worker-beta",
10236                                        "profile": "review-worker",
10237                                        "initial_message": "Review Initiative Beta."
10238                                    }
10239                                ]
10240                            }
10241                        }
10242                    }
10243                ],
10244                "stop_reason": "tool_use",
10245                "created_at": "1970-01-01T00:00:00.100Z"
10246            }),
10247        );
10248
10249        let child_identities = frames
10250            .iter()
10251            .filter(|frame| frame.kind == "user_input")
10252            .map(|frame| frame.identity.as_str())
10253            .collect::<Vec<_>>();
10254        assert_eq!(
10255            child_identities,
10256            vec!["test/review-worker-alpha", "test/review-worker-beta"]
10257        );
10258    }
10259
10260    #[tokio::test]
10261    async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
10262        let aggregator = MobKitConsoleAggregator::in_memory();
10263        aggregator
10264            .inner
10265            .identity_read_model
10266            .replace(vec![
10267                identity_record_for_test("test/review:singleton"),
10268                identity_record_for_test("test/review-worker-alpha"),
10269            ])
10270            .await;
10271        let frames = frames_from_session_history_message_with_namespace(
10272            "runtime-a",
10273            "test/review:singleton",
10274            "test",
10275            "session-a",
10276            11,
10277            json!({
10278                "role": "block_assistant",
10279                "blocks": [
10280                    {
10281                        "block_type": "tool_use",
10282                        "data": {
10283                            "id": "call-spawn",
10284                            "name": "mob_spawn_member",
10285                            "args": {
10286                                "agent_identity": "review-worker-alpha",
10287                                "profile": "review-worker",
10288                                "initial_message": "Review Initiative Alpha."
10289                            }
10290                        }
10291                    }
10292                ],
10293                "stop_reason": "tool_use",
10294                "created_at": "1970-01-01T00:00:00.110Z"
10295            }),
10296        );
10297        for frame in frames {
10298            aggregator
10299                .store()
10300                .append_if_absent(frame)
10301                .await
10302                .expect("append projected namespaced session-history frame");
10303        }
10304
10305        let namespaced_child_page = aggregator
10306            .query_timeline(ConsoleTimelineQuery {
10307                identity: Some("test/review-worker-alpha".to_string()),
10308                limit: 20,
10309                ..ConsoleTimelineQuery::default()
10310            })
10311            .await
10312            .expect("query namespaced child timeline");
10313        assert_eq!(namespaced_child_page.frames.len(), 1);
10314        assert_eq!(
10315            namespaced_child_page.frames[0].identity,
10316            "test/review-worker-alpha"
10317        );
10318        assert_eq!(
10319            namespaced_child_page.frames[0].payload["message"]["content"],
10320            "Review Initiative Alpha."
10321        );
10322
10323        let raw_child_page = aggregator
10324            .query_timeline(ConsoleTimelineQuery {
10325                identity: Some("review-worker-alpha".to_string()),
10326                limit: 20,
10327                ..ConsoleTimelineQuery::default()
10328            })
10329            .await
10330            .expect("query raw child timeline");
10331        assert!(
10332            raw_child_page.frames.is_empty(),
10333            "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
10334            raw_child_page.frames
10335        );
10336    }
10337
10338    #[test]
10339    fn namespace_helpers_preserve_namespace_named_member_identity() {
10340        assert_eq!(apply_namespace("test", "test"), "test/test");
10341        assert_eq!(
10342            strip_namespace("test/test", "test").as_deref(),
10343            Some("test")
10344        );
10345        assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
10346        assert_eq!(
10347            strip_namespace("test/worker", "test").as_deref(),
10348            Some("worker")
10349        );
10350        assert_eq!(strip_namespace("test", "test"), None);
10351    }
10352
10353    #[test]
10354    fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
10355        assert!(!member_id_matches_durable_identity(
10356            "rt:review:singleton:0",
10357            "review:singleton"
10358        ));
10359        assert!(!member_id_matches_durable_identity(
10360            "review:singleton:gen1",
10361            "review:singleton"
10362        ));
10363        assert!(!member_id_matches_durable_identity(
10364            "review:singleton:1",
10365            "review:singleton"
10366        ));
10367        assert!(!member_id_matches_durable_identity(
10368            "rt:review:singleton:qa:0",
10369            "review:singleton"
10370        ));
10371        assert!(!member_id_matches_durable_identity(
10372            "review:singleton:qa",
10373            "review:singleton"
10374        ));
10375    }
10376
10377    #[test]
10378    fn session_history_projection_uses_rfc3339_created_at_timestamp() {
10379        let frame = frame_from_session_history_message(
10380            "runtime-a",
10381            "agent-a",
10382            "session-a",
10383            0,
10384            json!({
10385                "role": "user",
10386                "content": "hello",
10387                "created_at": "2026-05-12T05:00:06.564227Z"
10388            }),
10389        )
10390        .expect("user history frame");
10391
10392        assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
10393    }
10394
10395    async fn wait_for_kickoff_frames(
10396        aggregator: &MobKitConsoleAggregator,
10397        identity: &str,
10398        timeout: Duration,
10399    ) -> Vec<ConsoleFrame> {
10400        let deadline = Instant::now() + timeout;
10401        let mut observed = Vec::new();
10402        while Instant::now() < deadline {
10403            let page = aggregator
10404                .query_timeline(ConsoleTimelineQuery {
10405                    identity: Some(identity.to_string()),
10406                    limit: 50,
10407                    ..ConsoleTimelineQuery::default()
10408                })
10409                .await
10410                .expect("query timeline");
10411            observed = page.frames;
10412            if observed.iter().any(|frame| frame.kind == "user_input") {
10413                break;
10414            }
10415            tokio::time::sleep(Duration::from_millis(25)).await;
10416        }
10417        observed
10418    }
10419
10420    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10421    async fn spawn_registry_primes_access_lineage() {
10422        let runtime = build_empty_runtime("spawn-access-lineage-test").await;
10423        let sink = runtime
10424            .mob_runtime()
10425            .console_spawn_sink()
10426            .expect("unified runtime installs the console spawn sink at bootstrap");
10427
10428        runtime
10429            .spawn(SpawnMemberSpec::from_wire(
10430                "worker".to_string(),
10431                "worker-3".to_string(),
10432                None,
10433                None,
10434                None,
10435            ))
10436            .await
10437            .expect("member spawns");
10438        sink.project_spawned_member(&crate::console_spawn::ConsoleSpawnSeed {
10439            mob_id: Some("spawn-access-lineage-test".to_string()),
10440            member_id: "worker-3".to_string(),
10441            identity: "worker-3".to_string(),
10442            initial_message: None,
10443            labels: BTreeMap::new(),
10444            spawned_by: Some("ops-lead".to_string()),
10445            via_tool: "mob_spawn_member".to_string(),
10446        })
10447        .await;
10448
10449        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
10450            enabled: true,
10451            admins: vec!["root@example.test".to_string()],
10452            groups: BTreeMap::new(),
10453            rules: vec![crate::access::AccessRule {
10454                id: "bob-ops-lead".to_string(),
10455                subjects: vec!["bob@example.test".to_string()],
10456                actions: vec!["agent.view".to_string()],
10457                agents: vec!["ops-lead".to_string()],
10458                ..crate::access::AccessRule::default()
10459            }],
10460        })
10461        .expect("controller");
10462
10463        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
10464            .await;
10465
10466        let bob = controller.view_for_subject(Some("bob@example.test"));
10467        assert!(
10468            bob.can_view_agent("worker-3"),
10469            "spawn registry lineage must let bob see the member ops-lead spawned"
10470        );
10471        assert!(
10472            !bob.can_view_agent("unrelated-agent"),
10473            "agents outside the granted lineage stay denied"
10474        );
10475
10476        let _ = runtime.mob_handle().stop().await;
10477    }
10478
10479    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10480    async fn roster_label_spoof_cannot_mint_or_evade_lineage() {
10481        let runtime = build_empty_runtime("spawn-lineage-spoof-test").await;
10482
10483        // Server-side spawns with caller-controlled labels and NO spawn
10484        // registry entry: lineage claims in roster labels are untrusted.
10485        let mut imposter = SpawnMemberSpec::from_wire(
10486            "worker".to_string(),
10487            "imposter".to_string(),
10488            None,
10489            None,
10490            None,
10491        );
10492        imposter.labels = Some(BTreeMap::from([(
10493            "spawned_by".to_string(),
10494            "ops-lead".to_string(),
10495        )]));
10496        runtime.spawn(imposter).await.expect("imposter spawns");
10497
10498        let mut evader = SpawnMemberSpec::from_wire(
10499            "worker".to_string(),
10500            "evader".to_string(),
10501            None,
10502            None,
10503            None,
10504        );
10505        evader.labels = Some(BTreeMap::from([(
10506            "spawned_by".to_string(),
10507            "secret-lead".to_string(),
10508        )]));
10509        runtime.spawn(evader).await.expect("evader spawns");
10510
10511        // The console identity list must not present unverified lineage.
10512        let aggregator = MobKitConsoleAggregator::in_memory();
10513        aggregator.register_runtime_handles_with_policy(
10514            "runtime-a",
10515            "",
10516            runtime.mob_runtime().clone(),
10517            None,
10518            runtime.console_events(),
10519            Arc::new(AllowAllConsoleVisibilityPolicy),
10520        );
10521        let identities = aggregator
10522            .list_identities_fresh()
10523            .await
10524            .expect("list identities");
10525        let imposter_record = identities
10526            .iter()
10527            .find(|record| record.identity == "imposter")
10528            .expect("imposter listed");
10529        assert!(
10530            !imposter_record.labels.contains_key("spawned_by"),
10531            "identity records must drop roster-claimed lineage when the spawn registry has none"
10532        );
10533
10534        let controller = crate::access::AccessController::new(crate::access::AccessControlConfig {
10535            enabled: true,
10536            admins: vec!["root@example.test".to_string()],
10537            groups: BTreeMap::new(),
10538            rules: vec![
10539                crate::access::AccessRule {
10540                    id: "bob-ops-lead".to_string(),
10541                    subjects: vec!["bob@example.test".to_string()],
10542                    actions: vec!["agent.view".to_string()],
10543                    agents: vec!["ops-lead".to_string()],
10544                    ..crate::access::AccessRule::default()
10545                },
10546                crate::access::AccessRule {
10547                    id: "carol-view-all".to_string(),
10548                    subjects: vec!["carol@example.test".to_string()],
10549                    actions: vec!["agent.view".to_string()],
10550                    agents: vec!["*".to_string()],
10551                    ..crate::access::AccessRule::default()
10552                },
10553                crate::access::AccessRule {
10554                    id: "hide-secret-lead".to_string(),
10555                    effect: crate::access::AccessEffect::Deny,
10556                    actions: vec!["agent.*".to_string()],
10557                    agents: vec!["secret-lead".to_string()],
10558                    ..crate::access::AccessRule::default()
10559                },
10560            ],
10561        })
10562        .expect("controller");
10563
10564        crate::http_console::prime_access_cache_from_runtime(runtime.mob_runtime(), &controller)
10565            .await;
10566
10567        let bob = controller.view_for_subject(Some("bob@example.test"));
10568        assert!(
10569            !bob.can_view_agent("imposter"),
10570            "a roster-claimed spawned_by must not mint inherited visibility"
10571        );
10572        let carol = controller.view_for_subject(Some("carol@example.test"));
10573        assert!(
10574            carol.can_view_agent("evader"),
10575            "a roster-claimed spawned_by must not let a member hide behind a denied parent"
10576        );
10577
10578        let _ = runtime.mob_handle().stop().await;
10579    }
10580
10581    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10582    async fn agent_spawned_member_kickoff_and_live_events_share_one_chat() {
10583        let runtime = build_empty_runtime("agent-spawn-console-test").await;
10584
10585        // The unified runtime must hand the agent-tool spawn path a console
10586        // sink at bootstrap — without it, agent-spawned members stay
10587        // invisible until their first runtime activity.
10588        let sink = runtime
10589            .mob_runtime()
10590            .console_spawn_sink()
10591            .expect("unified runtime installs the console spawn sink at bootstrap");
10592
10593        // The member exists in the roster exactly as an agent-tool spawn
10594        // would leave it; the server-side spawn carries no initial message
10595        // so the projected kickoff below is the only chat seed.
10596        runtime
10597            .spawn(SpawnMemberSpec::from_wire(
10598                "worker".to_string(),
10599                "worker-3".to_string(),
10600                None,
10601                None,
10602                None,
10603            ))
10604            .await
10605            .expect("member spawns");
10606
10607        let seed = crate::console_spawn::ConsoleSpawnSeed {
10608            mob_id: Some("agent-spawn-console-test".to_string()),
10609            member_id: "worker-3".to_string(),
10610            identity: "worker-3".to_string(),
10611            initial_message: Some(json!("Kick off the person hunt")),
10612            labels: BTreeMap::from([("group".to_string(), "workers".to_string())]),
10613            spawned_by: Some("ops-lead".to_string()),
10614            via_tool: "mob_spawn_member".to_string(),
10615        };
10616        sink.project_spawned_member(&seed).await;
10617        // Spawn retries must not fork or duplicate the chat.
10618        sink.project_spawned_member(&seed).await;
10619
10620        let aggregator = MobKitConsoleAggregator::in_memory();
10621        aggregator.register_runtime_handles_with_policy(
10622            "runtime-a",
10623            "test",
10624            runtime.mob_runtime().clone(),
10625            None,
10626            runtime.console_events(),
10627            Arc::new(AllowAllConsoleVisibilityPolicy),
10628        );
10629
10630        let frames =
10631            wait_for_kickoff_frames(&aggregator, "test/worker-3", Duration::from_secs(5)).await;
10632        let kickoffs = frames
10633            .iter()
10634            .filter(|frame| frame.kind == "user_input")
10635            .collect::<Vec<_>>();
10636        assert_eq!(
10637            kickoffs.len(),
10638            1,
10639            "member timeline must contain exactly one kickoff frame; got {frames:#?}"
10640        );
10641        assert_eq!(
10642            kickoffs[0].payload["content"][0]["text"],
10643            "Kick off the person hunt"
10644        );
10645
10646        // A live runtime event for the member must aggregate into the same
10647        // identity/chat as the kickoff, not fork into a second conversation.
10648        runtime
10649            .project_console_event_from_unified(&crate::types::EventEnvelope {
10650                event_id: "evt-live-worker-3".to_string(),
10651                source: "test".to_string(),
10652                timestamp_ms: current_time_ms(),
10653                event: crate::types::UnifiedEvent::Agent {
10654                    agent_id: "worker-3:0".to_string(),
10655                    event_type: "text_delta".to_string(),
10656                    payload: Some(json!({ "delta": "on it" })),
10657                },
10658            })
10659            .await;
10660
10661        let deadline = Instant::now() + Duration::from_secs(5);
10662        let mut live_frame = None;
10663        while Instant::now() < deadline && live_frame.is_none() {
10664            let page = aggregator
10665                .query_timeline(ConsoleTimelineQuery {
10666                    identity: Some("test/worker-3".to_string()),
10667                    limit: 50,
10668                    ..ConsoleTimelineQuery::default()
10669                })
10670                .await
10671                .expect("query timeline");
10672            live_frame = page
10673                .frames
10674                .into_iter()
10675                .find(|frame| frame.source_event_id.as_deref() == Some("evt-live-worker-3"));
10676            if live_frame.is_none() {
10677                tokio::time::sleep(Duration::from_millis(25)).await;
10678            }
10679        }
10680        let live_frame = live_frame.expect("live runtime event lands in the member chat");
10681        assert_eq!(live_frame.identity, "test/worker-3");
10682
10683        // The identity list must carry the registered spawn labels so the
10684        // sidebar can group the member and ABAC can see its lineage.
10685        let identities = aggregator
10686            .list_identities_fresh()
10687            .await
10688            .expect("list identities");
10689        let record = identities
10690            .iter()
10691            .find(|record| record.identity == "test/worker-3")
10692            .unwrap_or_else(|| {
10693                panic!("spawned member missing from identity list: {identities:#?}")
10694            });
10695        assert_eq!(
10696            record.labels.get("group").map(String::as_str),
10697            Some("workers")
10698        );
10699        assert_eq!(
10700            record.labels.get("spawned_by").map(String::as_str),
10701            Some("ops-lead")
10702        );
10703
10704        let _ = runtime.mob_handle().stop().await;
10705    }
10706}