Skip to main content

meerkat_mobkit/console_aggregator/
mod.rs

1mod state;
2mod store;
3mod types;
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::sync::atomic::{AtomicBool, Ordering};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use futures::future::join_all;
11use meerkat_core::types::HandlingMode;
12use meerkat_core::{ContentInput, Message};
13use meerkat_mob::ids::MeerkatId;
14use meerkat_mob::runtime::MobMemberListEntry;
15use meerkat_mob::{MobError, MobHandle};
16use serde_json::{Value, json};
17use sha2::{Digest, Sha256};
18use tokio::sync::{Semaphore, broadcast};
19
20use crate::blob_store::BinaryBlobStore;
21use crate::console_contracts::SYSTEM_EVENT_IDENTITY;
22use crate::mob_handle_runtime::{
23    MobRuntime, MobRuntimeError, assert_member_accepts_images,
24    is_recoverable_lifecycle_cleanup_error, send_message_on_mob_with_mode,
25};
26use crate::runtime::ConsoleMember;
27use crate::unified_runtime::{ConsoleEventStore, UnifiedRuntime};
28
29pub use state::{
30    ReplaySubscriptionEffect, ReplaySubscriptionState, ReplaySubscriptionTransition, SendEffect,
31    SendState, SendTransition, SourceIngestionEffect, SourceIngestionState,
32    SourceIngestionTransition,
33};
34pub use store::{
35    ConsoleLogError, ConsoleLogResult, ConsoleLogStore, InMemoryConsoleLogStore,
36    SqliteConsoleLogStore,
37};
38pub use types::{
39    AppendDisposition, AppendOutcome, ConsoleCursor, ConsoleFrame, ConsoleFrameSource,
40    ConsoleFrameSourceKind, ConsoleFrameStatus, ConsoleIdentityInspection, ConsoleIdentityRecord,
41    ConsoleInteractionAccepted, ConsoleReplayUnavailable, ConsoleSendRequest, ConsoleTimelineEvent,
42    ConsoleTimelineMode, ConsoleTimelinePage, ConsoleTimelineQuery, ConsoleTimelineWindowPage,
43    ConsoleTimelineWindowQuery, ConsoleVisibility, NewConsoleFrame,
44};
45
46const TIMELINE_CHANNEL_CAP: usize = 1024;
47const SESSION_HISTORY_PAGE_LIMIT: usize = 500;
48const SESSION_HISTORY_REFRESH_TTL_MS: u64 = 30_000;
49const SESSION_HISTORY_GROWING_REFRESH_TTL_MS: u64 = 2_000;
50const SESSION_HISTORY_DISCOVERY_INTERVAL: Duration = Duration::from_secs(5);
51const IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT: Duration = Duration::from_millis(250);
52const TIMELINE_RAW_SCAN_PAGE_LIMIT: usize = 1_000;
53const TIMELINE_MAX_RAW_SCAN_FRAMES: usize = 100_000;
54const TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT: usize = 5_000;
55const IDENTITY_RECENT_ANCHOR_LIMIT: usize = 8;
56
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58enum IdentityCollectionMode {
59    CachedOnly,
60    IncludeLiveMembers,
61}
62
63#[derive(Clone)]
64pub struct MobKitConsoleAggregator {
65    inner: Arc<AggregatorInner>,
66}
67
68#[derive(Debug, Clone, Copy)]
69pub struct ConsoleAggregatorOptions {
70    pub session_history_backfill_enabled: bool,
71    pub max_concurrent_session_backfills: usize,
72}
73
74impl Default for ConsoleAggregatorOptions {
75    fn default() -> Self {
76        Self {
77            session_history_backfill_enabled: true,
78            max_concurrent_session_backfills: 16,
79        }
80    }
81}
82
83struct AggregatorInner {
84    store: Arc<dyn ConsoleLogStore>,
85    runtimes: RwLock<BTreeMap<String, RuntimeEntry>>,
86    event_tx: broadcast::Sender<ConsoleTimelineEvent>,
87    active_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
88    opportunistic_session_backfills: tokio::sync::Mutex<BTreeSet<String>>,
89    session_backfill_permits: Arc<Semaphore>,
90    identity_read_model: ConsoleIdentityReadModel,
91    options: ConsoleAggregatorOptions,
92}
93
94#[derive(Clone)]
95struct ConsoleIdentityReadModel {
96    inner: Arc<tokio::sync::RwLock<Vec<ConsoleIdentityRecord>>>,
97    refresh_lock: Arc<tokio::sync::Mutex<()>>,
98    primed: Arc<AtomicBool>,
99}
100
101impl Default for ConsoleIdentityReadModel {
102    fn default() -> Self {
103        Self {
104            inner: Arc::new(tokio::sync::RwLock::new(Vec::new())),
105            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
106            primed: Arc::new(AtomicBool::new(false)),
107        }
108    }
109}
110
111impl ConsoleIdentityReadModel {
112    async fn snapshot(
113        &self,
114        inner: Arc<AggregatorInner>,
115    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
116        if !self.primed.load(Ordering::Acquire) {
117            self.prime_now(inner).await?;
118        }
119        Ok(self.inner.read().await.clone())
120    }
121
122    async fn current(&self) -> Vec<ConsoleIdentityRecord> {
123        self.inner.read().await.clone()
124    }
125
126    async fn refresh_now_if_idle(
127        &self,
128        inner: Arc<AggregatorInner>,
129        mode: IdentityCollectionMode,
130    ) -> Option<ConsoleLogResult<Vec<ConsoleIdentityRecord>>> {
131        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
132            return None;
133        };
134        let result = collect_identity_records(&inner, mode).await;
135        drop(guard);
136        match result {
137            Ok(identities) => {
138                self.replace(identities.clone()).await;
139                Some(Ok(identities))
140            }
141            Err(err) => Some(Err(err)),
142        }
143    }
144
145    async fn prime_now(&self, inner: Arc<AggregatorInner>) -> ConsoleLogResult<()> {
146        if self.primed.load(Ordering::Acquire) {
147            return Ok(());
148        }
149        let _guard = self.refresh_lock.clone().lock_owned().await;
150        if self.primed.load(Ordering::Acquire) {
151            return Ok(());
152        }
153        let identities =
154            collect_identity_records(&inner, IdentityCollectionMode::CachedOnly).await?;
155        *self.inner.write().await = identities;
156        self.primed.store(true, Ordering::Release);
157        Ok(())
158    }
159
160    fn refresh_soon(&self, inner: Arc<AggregatorInner>) {
161        let Ok(runtime_handle) = tokio::runtime::Handle::try_current() else {
162            return;
163        };
164        let Ok(guard) = self.refresh_lock.clone().try_lock_owned() else {
165            return;
166        };
167        let read_model = self.clone();
168        runtime_handle.spawn(async move {
169            let _guard = guard;
170            match collect_identity_records(&inner, IdentityCollectionMode::IncludeLiveMembers).await
171            {
172                Ok(identities) => {
173                    *read_model.inner.write().await = identities;
174                    read_model.primed.store(true, Ordering::Release);
175                }
176                Err(err) => {
177                    tracing::warn!(error = %err, "console identity read-model refresh failed");
178                }
179            }
180        });
181    }
182
183    async fn replace(&self, identities: Vec<ConsoleIdentityRecord>) {
184        *self.inner.write().await = identities;
185        self.primed.store(true, Ordering::Release);
186    }
187}
188
189#[derive(Clone)]
190struct RuntimeEntry {
191    runtime_key: String,
192    identity_namespace: String,
193    runtime: MobRuntime,
194    identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
195    console_events: ConsoleEventStore,
196    visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
197}
198
199#[derive(Clone)]
200struct ResolvedConsoleMember {
201    entry: RuntimeEntry,
202    handle: MobHandle,
203    member: MobMemberListEntry,
204    source_mob_id: String,
205    runtime_identity: String,
206}
207
208fn console_member_for_resolved_member(
209    resolved: &ResolvedConsoleMember,
210    record: &ConsoleIdentityRecord,
211) -> ConsoleMember {
212    ConsoleMember {
213        agent_identity: resolved.member.agent_identity.to_string(),
214        role: resolved.member.role.to_string(),
215        state: format!("{:?}", resolved.member.state),
216        model_capabilities: Default::default(),
217        runtime_mode: Some(resolved.member.runtime_mode.to_string()),
218        session_id: record.session_id.clone(),
219        wired_to: resolved
220            .member
221            .wired_to
222            .iter()
223            .map(ToString::to_string)
224            .collect(),
225        labels: record.labels.clone(),
226    }
227}
228
229async fn resolved_member_visible(
230    resolved: &ResolvedConsoleMember,
231    record: &ConsoleIdentityRecord,
232) -> bool {
233    if !raw_resolved_member_visible(resolved, record) {
234        return false;
235    }
236    !live_record_shadowed_by_hidden_durable_binding(resolved, record).await
237}
238
239fn raw_resolved_member_visible(
240    resolved: &ResolvedConsoleMember,
241    record: &ConsoleIdentityRecord,
242) -> bool {
243    resolved.entry.visibility_policy.identity_visible(record)
244        && resolved
245            .entry
246            .visibility_policy
247            .member_visible(&console_member_for_resolved_member(resolved, record))
248}
249
250pub trait ConsoleVisibilityPolicy: Send + Sync {
251    fn include_implicit_delegate_members(&self) -> bool {
252        true
253    }
254
255    fn member_visible(&self, _member: &ConsoleMember) -> bool {
256        true
257    }
258
259    fn identity_visible(&self, _record: &ConsoleIdentityRecord) -> bool {
260        true
261    }
262
263    fn frame_visible(&self, _frame: &ConsoleFrame) -> bool {
264        true
265    }
266
267    fn redact_payload(&self, _frame: &NewConsoleFrame) -> Option<Value> {
268        None
269    }
270}
271
272#[derive(Debug, Default)]
273pub struct AllowAllConsoleVisibilityPolicy;
274
275impl ConsoleVisibilityPolicy for AllowAllConsoleVisibilityPolicy {}
276
277#[derive(Debug, Default)]
278pub struct HideImplicitDelegateMembersConsoleVisibilityPolicy;
279
280impl ConsoleVisibilityPolicy for HideImplicitDelegateMembersConsoleVisibilityPolicy {
281    fn include_implicit_delegate_members(&self) -> bool {
282        false
283    }
284
285    fn member_visible(&self, member: &ConsoleMember) -> bool {
286        !is_implicit_delegate_member(member.role.as_str(), &member.labels)
287    }
288
289    fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
290        !is_implicit_delegate_member(
291            record
292                .labels
293                .get("role")
294                .map(String::as_str)
295                .unwrap_or_default(),
296            &record.labels,
297        )
298    }
299}
300
301#[derive(Clone)]
302pub struct ConsoleRuntimeRegistration {
303    pub runtime_key: String,
304    pub runtime: Arc<UnifiedRuntime>,
305    pub identity_namespace: String,
306    pub visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
307}
308
309impl MobKitConsoleAggregator {
310    pub fn new(store: Arc<dyn ConsoleLogStore>) -> Self {
311        Self::new_with_options(store, ConsoleAggregatorOptions::default())
312    }
313
314    pub fn new_with_options(
315        store: Arc<dyn ConsoleLogStore>,
316        mut options: ConsoleAggregatorOptions,
317    ) -> Self {
318        options.max_concurrent_session_backfills = options.max_concurrent_session_backfills.max(1);
319        let (event_tx, _) = broadcast::channel(TIMELINE_CHANNEL_CAP);
320        Self {
321            inner: Arc::new(AggregatorInner {
322                store,
323                runtimes: RwLock::new(BTreeMap::new()),
324                event_tx,
325                active_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
326                opportunistic_session_backfills: tokio::sync::Mutex::new(BTreeSet::new()),
327                session_backfill_permits: Arc::new(Semaphore::new(
328                    options.max_concurrent_session_backfills,
329                )),
330                identity_read_model: ConsoleIdentityReadModel::default(),
331                options,
332            }),
333        }
334    }
335
336    pub fn in_memory() -> Self {
337        Self::new(Arc::new(InMemoryConsoleLogStore::new()))
338    }
339
340    pub fn in_memory_with_options(options: ConsoleAggregatorOptions) -> Self {
341        Self::new_with_options(Arc::new(InMemoryConsoleLogStore::new()), options)
342    }
343
344    pub fn subscribe(&self) -> broadcast::Receiver<ConsoleTimelineEvent> {
345        self.inner.event_tx.subscribe()
346    }
347
348    pub fn store(&self) -> Arc<dyn ConsoleLogStore> {
349        self.inner.store.clone()
350    }
351
352    pub fn register_runtime(&self, registration: ConsoleRuntimeRegistration) {
353        let identity_runtime = registration.runtime.identity_runtime().cloned();
354        self.register_runtime_handles_with_policy(
355            registration.runtime_key,
356            registration.identity_namespace,
357            registration.runtime.mob_runtime().clone(),
358            identity_runtime,
359            registration.runtime.console_events(),
360            registration.visibility_policy,
361        );
362    }
363
364    pub(crate) fn register_runtime_handles_with_policy(
365        &self,
366        runtime_key: impl Into<String>,
367        identity_namespace: impl Into<String>,
368        runtime: MobRuntime,
369        identity_runtime: Option<Arc<crate::identity_first::IdentityRuntime>>,
370        console_events: ConsoleEventStore,
371        visibility_policy: Arc<dyn ConsoleVisibilityPolicy>,
372    ) {
373        let runtime_key = runtime_key.into();
374        let identity_namespace = identity_namespace.into();
375        let entry = RuntimeEntry {
376            runtime_key: runtime_key.clone(),
377            identity_namespace,
378            runtime,
379            identity_runtime,
380            console_events: console_events.clone(),
381            visibility_policy,
382        };
383        if let Ok(mut runtimes) = self.inner.runtimes.write() {
384            runtimes.insert(runtime_key.clone(), entry);
385        }
386        self.inner
387            .identity_read_model
388            .refresh_soon(self.inner.clone());
389        let inner = self.inner.clone();
390        let events_for_live = console_events.clone();
391        let events_for_live_recovery = console_events.clone();
392        let runtime_key_for_live = runtime_key.clone();
393        tokio::spawn(async move {
394            let mut rx = events_for_live.subscribe();
395            loop {
396                match rx.recv().await {
397                    Ok(envelope) => {
398                        let _ =
399                            project_console_event(inner.clone(), &runtime_key_for_live, envelope)
400                                .await;
401                    }
402                    Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
403                        let _ = recover_lagged_source_events(
404                            inner.clone(),
405                            &runtime_key_for_live,
406                            &events_for_live_recovery,
407                        )
408                        .await;
409                    }
410                    Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
411                }
412            }
413        });
414
415        let inner = self.inner.clone();
416        let events_for_replay = console_events;
417        let runtime_key_for_replay = runtime_key;
418        tokio::spawn(async move {
419            let mut ingestion_state = SourceIngestionState::Registered;
420            if let Ok((next, _effects)) =
421                ingestion_state.apply(SourceIngestionTransition::StartBackfill)
422            {
423                ingestion_state = next;
424            }
425            if let Ok(events) = events_for_replay.replay_all(None).await {
426                for envelope in events {
427                    let _ = project_console_event(inner.clone(), &runtime_key_for_replay, envelope)
428                        .await;
429                }
430            }
431            spawn_session_history_backfill(inner.clone(), runtime_key_for_replay.clone());
432            spawn_session_history_discovery_loop(inner.clone(), runtime_key_for_replay.clone());
433            if let Ok((next, _effects)) =
434                ingestion_state.apply(SourceIngestionTransition::BackfillComplete)
435            {
436                ingestion_state = next;
437            }
438            let _ = ingestion_state.apply(SourceIngestionTransition::StartLive);
439        });
440    }
441
442    pub async fn list_identities(&self) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
443        if let Some(identities) = self
444            .inner
445            .identity_read_model
446            .refresh_now_if_idle(
447                self.inner.clone(),
448                IdentityCollectionMode::IncludeLiveMembers,
449            )
450            .await
451        {
452            let identities = identities?;
453            spawn_identity_backfills_for_records(self.inner.clone(), &identities);
454            return Ok(identities);
455        }
456        self.inner
457            .identity_read_model
458            .refresh_soon(self.inner.clone());
459        let identities = self
460            .inner
461            .identity_read_model
462            .snapshot(self.inner.clone())
463            .await?;
464        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
465        Ok(identities)
466    }
467
468    #[cfg(test)]
469    pub(crate) async fn list_identities_fresh(
470        &self,
471    ) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
472        let _guard = self
473            .inner
474            .identity_read_model
475            .refresh_lock
476            .clone()
477            .lock_owned()
478            .await;
479        let identities =
480            collect_identity_records(&self.inner, IdentityCollectionMode::IncludeLiveMembers)
481                .await?;
482        self.inner
483            .identity_read_model
484            .replace(identities.clone())
485            .await;
486        spawn_identity_backfills_for_records(self.inner.clone(), &identities);
487        Ok(identities)
488    }
489
490    pub async fn inspect_identity(
491        &self,
492        identity: &str,
493    ) -> ConsoleLogResult<Option<ConsoleIdentityInspection>> {
494        let entries = self
495            .inner
496            .runtimes
497            .read()
498            .map_err(|_| runtime_registry_lock_error())?
499            .values()
500            .cloned()
501            .collect::<Vec<_>>();
502        let live_records = self.live_records_for_identity(identity).await;
503        let mut durable_matches = Vec::new();
504        if !requested_identity_has_runtime_member_alias(identity, &entries) {
505            for entry in &entries {
506                let Some(identity_runtime) = entry.identity_runtime.clone() else {
507                    continue;
508                };
509                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
510                {
511                    let Ok(parsed_identity) =
512                        crate::identity_first::AgentIdentity::parse(&raw_identity)
513                    else {
514                        continue;
515                    };
516                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
517                        continue;
518                    };
519                    let mut record = identity_record_for_status(entry, &status);
520                    let topology_peers =
521                        identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
522                    record.topology_peers = topology_peers
523                        .get(&record.identity)
524                        .cloned()
525                        .unwrap_or_default();
526                    if console_identity_record_visible(entry, &record).await {
527                        durable_matches.push((entry.clone(), record));
528                    }
529                }
530            }
531        }
532        if durable_matches.len() > 1 {
533            let candidates = durable_matches
534                .iter()
535                .map(|(_, record)| record.runtime_member_id.clone())
536                .collect::<Vec<_>>()
537                .join(", ");
538            return Err(format!(
539                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
540            )
541            .into());
542        }
543        if let Some((entry, record)) = durable_matches.into_iter().next() {
544            let durable_live_records = self
545                .live_records_for_durable_record(&entry, identity, &record, &live_records)
546                .await;
547            let visible_durable_live_records =
548                visible_live_records_for_entry(&entry, &durable_live_records).await;
549            if let Some(ambiguous_error) =
550                ambiguous_live_alias_error(identity, &visible_durable_live_records)
551            {
552                return Err(ambiguous_error.into());
553            }
554            if let Some(stale_error) =
555                stale_durable_record_error(identity, &record, &durable_live_records)
556            {
557                return Err(stale_error.into());
558            }
559            if let Some(session_id) = record.session_id.clone() {
560                spawn_session_history_backfill_target(
561                    self.inner.clone(),
562                    SessionBackfillTarget {
563                        entry,
564                        record: record.clone(),
565                        session_id,
566                    },
567                    false,
568                );
569            }
570            return Ok(Some(ConsoleIdentityInspection {
571                peers: record.topology_peers.clone(),
572                identity: record,
573            }));
574        }
575        let mut runtime_id_matches = Vec::new();
576        for entry in &entries {
577            let Some(identity_runtime) = entry.identity_runtime.clone() else {
578                continue;
579            };
580            let Some(raw_identity) = strip_namespace(identity, &entry.identity_namespace) else {
581                continue;
582            };
583            let Some(status) = identity_runtime
584                .statuses()
585                .await
586                .into_iter()
587                .find(|status| {
588                    status
589                        .agent_runtime_id
590                        .as_ref()
591                        .is_some_and(|runtime_id| runtime_id.as_str() == raw_identity)
592                })
593            else {
594                continue;
595            };
596            let mut record = identity_record_for_status(entry, &status);
597            let topology_peers =
598                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
599            record.topology_peers = topology_peers
600                .get(&record.identity)
601                .cloned()
602                .unwrap_or_default();
603            if console_identity_record_visible(entry, &record).await {
604                runtime_id_matches.push((entry.clone(), record));
605            }
606        }
607        if runtime_id_matches.len() > 1 {
608            let candidates = runtime_id_matches
609                .iter()
610                .map(|(_, record)| record.identity.clone())
611                .collect::<Vec<_>>()
612                .join(", ");
613            return Err(format!(
614                "ambiguous runtime identity alias {identity}: candidates [{candidates}]"
615            )
616            .into());
617        }
618        if let Some((entry, record)) = runtime_id_matches.into_iter().next() {
619            let durable_live_records = self
620                .live_records_for_durable_record(&entry, identity, &record, &live_records)
621                .await;
622            let visible_durable_live_records =
623                visible_live_records_for_entry(&entry, &durable_live_records).await;
624            if let Some(ambiguous_error) =
625                ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
626            {
627                return Err(ambiguous_error.into());
628            }
629            if let Some(stale_error) =
630                stale_durable_record_error(identity, &record, &durable_live_records)
631            {
632                return Err(stale_error.into());
633            }
634            if let Some(session_id) = record.session_id.clone() {
635                spawn_session_history_backfill_target(
636                    self.inner.clone(),
637                    SessionBackfillTarget {
638                        entry: entry.clone(),
639                        record: record.clone(),
640                        session_id,
641                    },
642                    false,
643                );
644            }
645            return Ok(Some(ConsoleIdentityInspection {
646                peers: record.topology_peers.clone(),
647                identity: record,
648            }));
649        }
650
651        let live_matches = self.resolve_visible_members(identity).await;
652        if live_matches.len() > 1 {
653            let candidates = live_matches
654                .iter()
655                .map(|resolved| resolved.runtime_identity.clone())
656                .collect::<Vec<_>>()
657                .join(", ");
658            return Err(format!(
659                "ambiguous live identity alias {identity}: candidates [{candidates}]"
660            )
661            .into());
662        }
663        if let Some(resolved) = live_matches.into_iter().next() {
664            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
665                return Ok(None);
666            };
667            if let Some(stale_error) =
668                stale_live_record_binding_error(&resolved.entry, identity, &record).await
669            {
670                return Err(stale_error.into());
671            }
672            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
673                && let Ok(parsed_identity) =
674                    crate::identity_first::AgentIdentity::parse(&record.identity)
675                && let Ok(status) = identity_runtime.status(&parsed_identity).await
676            {
677                let durable_record = identity_record_for_status(&resolved.entry, &status);
678                let durable_live_records = self
679                    .live_records_for_durable_record(
680                        &resolved.entry,
681                        identity,
682                        &durable_record,
683                        &live_records,
684                    )
685                    .await;
686                let visible_durable_live_records =
687                    visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
688                if let Some(ambiguous_error) =
689                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
690                {
691                    return Err(ambiguous_error.into());
692                }
693                if let Some(stale_error) =
694                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
695                {
696                    return Err(stale_error.into());
697                }
698                if status
699                    .agent_runtime_id
700                    .as_ref()
701                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
702                {
703                    return Ok(None);
704                }
705            }
706            if !resolved_member_visible(&resolved, &record).await {
707                return Ok(None);
708            }
709            if let Some(session_id) = record.session_id.clone() {
710                spawn_session_history_backfill_target(
711                    self.inner.clone(),
712                    SessionBackfillTarget {
713                        entry: resolved.entry.clone(),
714                        record: record.clone(),
715                        session_id,
716                    },
717                    false,
718                );
719            }
720            let peers = resolved
721                .member
722                .wired_to
723                .iter()
724                .map(ToString::to_string)
725                .collect();
726            return Ok(Some(ConsoleIdentityInspection {
727                identity: record,
728                peers,
729            }));
730        }
731
732        Ok(None)
733    }
734
735    pub async fn retire_identity(&self, identity: &str) -> ConsoleLogResult<bool> {
736        let entries = self
737            .inner
738            .runtimes
739            .read()
740            .map_err(|_| runtime_registry_lock_error())?
741            .clone();
742        let live_records = self.live_records_for_identity(identity).await;
743        let entries_vec = entries.values().cloned().collect::<Vec<_>>();
744        let mut durable_matches = Vec::new();
745        if !requested_identity_has_runtime_member_alias(identity, &entries_vec) {
746            for entry in entries.values() {
747                let Some(identity_runtime) = entry.identity_runtime.clone() else {
748                    continue;
749                };
750                for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace)
751                {
752                    let Ok(parsed_identity) =
753                        crate::identity_first::AgentIdentity::parse(&raw_identity)
754                    else {
755                        continue;
756                    };
757                    let Ok(status) = identity_runtime.status(&parsed_identity).await else {
758                        continue;
759                    };
760                    let record = identity_record_for_status(entry, &status);
761                    if !console_identity_record_visible(entry, &record).await {
762                        continue;
763                    }
764                    durable_matches.push((
765                        entry.clone(),
766                        identity_runtime.clone(),
767                        parsed_identity,
768                        record,
769                    ));
770                }
771            }
772        }
773        if durable_matches.len() > 1 {
774            let candidates = durable_matches
775                .iter()
776                .map(|(_, _, identity, _)| identity.as_str().to_string())
777                .collect::<Vec<_>>()
778                .join(", ");
779            return Err(format!(
780                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
781            )
782            .into());
783        }
784        if let Some((entry, identity_runtime, parsed_identity, record)) =
785            durable_matches.into_iter().next()
786        {
787            let durable_live_records = self
788                .live_records_for_durable_record(&entry, identity, &record, &live_records)
789                .await;
790            let visible_durable_live_records =
791                visible_live_records_for_entry(&entry, &durable_live_records).await;
792            if let Some(ambiguous_error) =
793                ambiguous_live_alias_error(identity, &visible_durable_live_records)
794            {
795                return Err(ambiguous_error.into());
796            }
797            if let Some(stale_error) =
798                stale_durable_record_error(identity, &record, &durable_live_records)
799            {
800                return Err(stale_error.into());
801            }
802            identity_runtime
803                .retire(&parsed_identity)
804                .await
805                .map_err(|err| -> ConsoleLogError {
806                    format!("retire failed for {identity}: {err}").into()
807                })?;
808            {
809                let mut durable_identities =
810                    namespace_match_candidates(identity, &entry.identity_namespace);
811                durable_identities.push(parsed_identity.as_str().to_string());
812                durable_identities.sort();
813                durable_identities.dedup();
814                if let Err(err) =
815                    retire_stale_console_members_for_runtime_entry(&entry, &durable_identities)
816                        .await
817                {
818                    tracing::warn!(
819                        identity,
820                        error = %err,
821                        "stale console member cleanup failed after durable identity retire"
822                    );
823                }
824            }
825            return Ok(true);
826        }
827        let matches = self.resolve_visible_members(identity).await;
828        if matches.len() > 1 {
829            let candidates = matches
830                .iter()
831                .map(|resolved| resolved.runtime_identity.clone())
832                .collect::<Vec<_>>()
833                .join(", ");
834            return Err(format!(
835                "ambiguous live identity alias {identity}: candidates [{candidates}]"
836            )
837            .into());
838        }
839        let mut live_retired_any = false;
840        for resolved in matches {
841            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
842                continue;
843            };
844            if !resolved_member_visible(&resolved, &record).await {
845                continue;
846            }
847            if let Some(stale_error) =
848                stale_live_record_binding_error(&resolved.entry, identity, &record).await
849            {
850                return Err(stale_error.into());
851            }
852            if let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref()
853                && let Ok(parsed_identity) =
854                    crate::identity_first::AgentIdentity::parse(&record.identity)
855                && let Ok(status) = identity_runtime.status(&parsed_identity).await
856            {
857                let durable_record = identity_record_for_status(&resolved.entry, &status);
858                let durable_live_records = self
859                    .live_records_for_durable_record(
860                        &resolved.entry,
861                        identity,
862                        &durable_record,
863                        &live_records,
864                    )
865                    .await;
866                let visible_durable_live_records =
867                    visible_live_records_for_entry(&resolved.entry, &durable_live_records).await;
868                if let Some(ambiguous_error) =
869                    ambiguous_live_alias_error(&record.identity, &visible_durable_live_records)
870                {
871                    return Err(ambiguous_error.into());
872                }
873                if let Some(stale_error) =
874                    stale_durable_record_error(identity, &durable_record, &durable_live_records)
875                {
876                    return Err(stale_error.into());
877                }
878                if status
879                    .agent_runtime_id
880                    .as_ref()
881                    .is_some_and(|runtime_id| runtime_id.as_str() != resolved.runtime_identity)
882                {
883                    continue;
884                }
885            }
886            resolved
887                .handle
888                .retire(MeerkatId::from(resolved.runtime_identity.as_str()))
889                .await
890                .map_err(|err| -> ConsoleLogError {
891                    format!("retire failed for {identity}: {err}").into()
892                })
893                .or_else(|err| {
894                    if lifecycle_archive_cleanup_completed(&err.to_string()) {
895                        Ok(())
896                    } else {
897                        Err(err)
898                    }
899                })?;
900            live_retired_any = true;
901        }
902        Ok(live_retired_any)
903    }
904
905    pub async fn clear_timeline_frames(&self) -> ConsoleLogResult<()> {
906        self.inner.store.clear_frames().await
907    }
908
909    pub async fn query_timeline(
910        &self,
911        query: ConsoleTimelineQuery,
912    ) -> ConsoleLogResult<ConsoleTimelinePage> {
913        let page = Box::pin(self.query_timeline_windowed(query.into())).await?;
914        Ok(ConsoleTimelinePage {
915            frames: page.frames,
916            next_cursor: page.next_cursor,
917        })
918    }
919
920    pub async fn query_timeline_windowed(
921        &self,
922        query: ConsoleTimelineWindowQuery,
923    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
924        self.reject_after_cursor_beyond_store_frontier(query.after.as_ref())
925            .await?;
926        if query.after.is_none()
927            && query.before.is_none()
928            && let Some(identity) = query.identity.clone()
929        {
930            let mut probe_query = query.clone();
931            probe_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
932            let page = self.inner.store.query_windowed_frames(probe_query).await?;
933            if explicit_identity_query_needs_session_history_backfill(&page.frames) {
934                spawn_session_history_backfill_for_identity(self.inner.clone(), identity, true);
935            }
936        }
937        Box::pin(self.query_timeline_visible(query)).await
938    }
939
940    async fn reject_after_cursor_beyond_store_frontier(
941        &self,
942        after: Option<&ConsoleCursor>,
943    ) -> ConsoleLogResult<()> {
944        let Some(after_seq) = after.and_then(ConsoleCursor::seq) else {
945            return Ok(());
946        };
947        let latest_seq = self
948            .inner
949            .store
950            .latest_cursor()
951            .await?
952            .and_then(|cursor| cursor.seq())
953            .unwrap_or(0);
954        if after_seq > latest_seq {
955            return Err(std::io::Error::other(
956                "timeline replay cursor is beyond the current store frontier",
957            )
958            .into());
959        }
960        Ok(())
961    }
962
963    async fn query_timeline_visible(
964        &self,
965        query: ConsoleTimelineWindowQuery,
966    ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
967        let requested_limit = query.limit.clamp(1, TIMELINE_RAW_SCAN_PAGE_LIMIT);
968        let mut scan_query = query.clone();
969        scan_query.limit = TIMELINE_RAW_SCAN_PAGE_LIMIT;
970        let mut visible_frames = Vec::with_capacity(requested_limit);
971        let mut anchor_frames = Vec::new();
972        let mut identity_visibility_cache = HashMap::new();
973        let identity_records = self.inner.identity_read_model.current().await;
974        let mut next_cursor = query.after.clone();
975        let mut since_last_scanned_cursor = query.after.clone();
976        let mut since_last_delivered_cursor = None;
977        let mut since_stopped_at_visible_limit = false;
978        let mut latest_cursor = None;
979        let mut exhausted = false;
980        let mut scanned = 0usize;
981
982        loop {
983            let page = self
984                .inner
985                .store
986                .query_windowed_frames(scan_query.clone())
987                .await?;
988            latest_cursor = latest_cursor.or(page.latest_cursor.clone());
989            if page.frames.is_empty() {
990                exhausted = true;
991                break;
992            }
993            let raw_len = page.frames.len();
994            scanned = scanned.saturating_add(raw_len);
995            match query.mode {
996                ConsoleTimelineMode::Since => {
997                    if let Some(cursor) = page.next_cursor.clone() {
998                        since_last_scanned_cursor = Some(cursor.clone());
999                        scan_query.after = Some(cursor);
1000                    }
1001                }
1002                ConsoleTimelineMode::Recent => {
1003                    if let Some(first) = page.frames.first() {
1004                        scan_query.before = Some(first.cursor.clone());
1005                    }
1006                    next_cursor = page.next_cursor.clone().or(next_cursor);
1007                }
1008            }
1009
1010            for frame in page.frames {
1011                let allow_historical_identity =
1012                    query.identity.as_deref() == Some(frame.identity.as_str());
1013                if frame_is_visible_cached(
1014                    &self.inner,
1015                    &frame,
1016                    allow_historical_identity,
1017                    &mut identity_visibility_cache,
1018                    &identity_records,
1019                )
1020                .await
1021                .unwrap_or(false)
1022                {
1023                    if query.mode == ConsoleTimelineMode::Recent
1024                        && query.identity.is_some()
1025                        && is_identity_timeline_anchor_frame(&frame)
1026                        && anchor_frames.len() < IDENTITY_RECENT_ANCHOR_LIMIT
1027                    {
1028                        anchor_frames.push(frame.clone());
1029                    }
1030                    match query.mode {
1031                        ConsoleTimelineMode::Since => {
1032                            if visible_frames.len() >= requested_limit {
1033                                since_stopped_at_visible_limit = true;
1034                                break;
1035                            }
1036                            since_last_delivered_cursor = Some(frame.cursor.clone());
1037                            visible_frames.push(frame);
1038                            if visible_frames.len() >= requested_limit {
1039                                since_stopped_at_visible_limit = true;
1040                                exhausted = false;
1041                                break;
1042                            }
1043                        }
1044                        ConsoleTimelineMode::Recent => {
1045                            visible_frames.push(frame);
1046                        }
1047                    }
1048                }
1049            }
1050            let needs_identity_anchor = query.mode == ConsoleTimelineMode::Recent
1051                && query.identity.is_some()
1052                && anchor_frames.is_empty()
1053                && scanned < TIMELINE_RECENT_ANCHOR_RAW_SCAN_LIMIT;
1054            if visible_frames.len() >= requested_limit && !needs_identity_anchor {
1055                break;
1056            }
1057            if since_stopped_at_visible_limit {
1058                break;
1059            }
1060            if page.exhausted || raw_len < TIMELINE_RAW_SCAN_PAGE_LIMIT {
1061                exhausted = true;
1062                break;
1063            }
1064            if scanned >= TIMELINE_MAX_RAW_SCAN_FRAMES {
1065                break;
1066            }
1067        }
1068
1069        if query.mode == ConsoleTimelineMode::Recent {
1070            visible_frames.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1071            if visible_frames.len() > requested_limit {
1072                visible_frames = visible_frames.split_off(visible_frames.len() - requested_limit);
1073            }
1074            if !anchor_frames.is_empty() {
1075                let anchor_cursors = anchor_frames
1076                    .iter()
1077                    .map(|frame| frame.cursor.clone())
1078                    .collect::<Vec<_>>();
1079                let mut merged = anchor_frames;
1080                for frame in visible_frames {
1081                    if !merged
1082                        .iter()
1083                        .any(|existing| existing.cursor == frame.cursor || existing.id == frame.id)
1084                    {
1085                        merged.push(frame);
1086                    }
1087                }
1088                merged.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1089                visible_frames = merged;
1090                if visible_frames.len() > requested_limit {
1091                    let mut anchors = Vec::new();
1092                    let mut tail = Vec::new();
1093                    for frame in visible_frames {
1094                        if anchor_cursors.contains(&frame.cursor) {
1095                            anchors.push(frame);
1096                        } else {
1097                            tail.push(frame);
1098                        }
1099                    }
1100                    visible_frames = if anchors.len() >= requested_limit {
1101                        anchors.split_off(anchors.len() - requested_limit)
1102                    } else {
1103                        let keep_tail = requested_limit.saturating_sub(anchors.len());
1104                        if tail.len() > keep_tail {
1105                            tail = tail.split_off(tail.len() - keep_tail);
1106                        }
1107                        anchors.extend(tail);
1108                        anchors.sort_by_key(|frame| frame.cursor.seq().unwrap_or(u64::MAX));
1109                        anchors
1110                    };
1111                }
1112            }
1113        }
1114
1115        Ok(ConsoleTimelineWindowPage {
1116            frames: visible_frames,
1117            next_cursor: match query.mode {
1118                ConsoleTimelineMode::Since if since_stopped_at_visible_limit => {
1119                    since_last_delivered_cursor.or(since_last_scanned_cursor)
1120                }
1121                ConsoleTimelineMode::Since => since_last_scanned_cursor,
1122                ConsoleTimelineMode::Recent => next_cursor,
1123            },
1124            latest_cursor,
1125            exhausted,
1126        })
1127    }
1128
1129    pub async fn refresh_session_history(&self) -> ConsoleLogResult<()> {
1130        let runtime_keys = self
1131            .inner
1132            .runtimes
1133            .read()
1134            .map_err(|_| runtime_registry_lock_error())?
1135            .keys()
1136            .cloned()
1137            .collect::<Vec<_>>();
1138        let results =
1139            join_all(runtime_keys.into_iter().map(|runtime_key| {
1140                backfill_session_history(self.inner.clone(), runtime_key, true)
1141            }))
1142            .await;
1143        for result in results {
1144            result?;
1145        }
1146        Ok(())
1147    }
1148
1149    pub async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
1150        self.inner.store.latest_cursor().await
1151    }
1152
1153    pub async fn timeline_event_visible(&self, event: &ConsoleTimelineEvent) -> bool {
1154        match event {
1155            ConsoleTimelineEvent::ConsoleFrame { frame }
1156            | ConsoleTimelineEvent::FrameUpdated { frame } => {
1157                let identity_records = self.inner.identity_read_model.current().await;
1158                frame_is_visible(&self.inner, frame, false, &identity_records)
1159                    .await
1160                    .unwrap_or(false)
1161            }
1162            ConsoleTimelineEvent::SnapshotStarted { .. }
1163            | ConsoleTimelineEvent::SnapshotComplete { .. }
1164            | ConsoleTimelineEvent::ReplayUnavailable { .. } => true,
1165        }
1166    }
1167
1168    pub async fn timeline_frame_visible_for_query(
1169        &self,
1170        frame: &ConsoleFrame,
1171        identity: Option<&str>,
1172    ) -> bool {
1173        let allow_historical_identity = identity == Some(frame.identity.as_str());
1174        let identity_records = self.inner.identity_read_model.current().await;
1175        frame_is_visible(
1176            &self.inner,
1177            frame,
1178            allow_historical_identity,
1179            &identity_records,
1180        )
1181        .await
1182        .unwrap_or(false)
1183    }
1184
1185    pub async fn send(
1186        &self,
1187        request: ConsoleSendRequest,
1188    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1189        validate_send_request(&request)?;
1190        let Some(resolved) = Box::pin(self.resolve_send_member(&request.identity)).await? else {
1191            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1192        };
1193        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1194            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1195        };
1196        if !resolved_member_visible(&resolved, &record).await {
1197            return Err(ConsoleSendError::UnknownIdentity(request.identity));
1198        }
1199        if !member_is_addressable(&resolved.member) {
1200            return Err(ConsoleSendError::NotAddressable(request.identity));
1201        }
1202        if resolved.member.state == meerkat_mob::MemberState::Retiring {
1203            return Err(ConsoleSendError::Retired(request.identity));
1204        }
1205
1206        let content = content_input_from_value(&request.content)?;
1207        let handling_mode = parse_handling_mode(request.handling_mode.as_deref())?;
1208        assert_member_accepts_images(
1209            &resolved.handle,
1210            resolved.entry.runtime.session_service(),
1211            &resolved.runtime_identity,
1212            &content,
1213        )
1214        .await
1215        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
1216
1217        let dedupe_key = send_dedupe_key(
1218            &resolved.entry.runtime_key,
1219            &request.identity,
1220            &request.origin,
1221            &request.idempotency_key,
1222        );
1223        let handling_mode_value = request
1224            .handling_mode
1225            .as_deref()
1226            .unwrap_or("queue")
1227            .to_string();
1228        let request_fingerprint =
1229            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1230        if let Some(existing) = self
1231            .inner
1232            .store
1233            .frame_by_dedupe_key(&dedupe_key)
1234            .await
1235            .map_err(ConsoleSendError::Log)?
1236        {
1237            let same_request = existing.source.source_cursor.as_deref()
1238                == Some(request_fingerprint.as_str())
1239                || existing.source.source_cursor.is_none()
1240                    && existing.payload.get("origin").and_then(Value::as_str)
1241                        == Some(request.origin.as_str())
1242                    && existing.payload.get("content") == Some(&request.content)
1243                    && existing
1244                        .payload
1245                        .get("handling_mode")
1246                        .and_then(Value::as_str)
1247                        == Some(handling_mode_value.as_str());
1248            if !same_request {
1249                return Err(ConsoleSendError::IdempotencyConflict(
1250                    request.idempotency_key,
1251                ));
1252            }
1253            return Ok(accepted_from_frame(&existing));
1254        }
1255
1256        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1257        resolved
1258            .entry
1259            .console_events
1260            .reserve_interaction_value(
1261                &resolved.runtime_identity,
1262                Some(resolved.runtime_identity.as_str()),
1263                &interaction_id,
1264                &request.origin,
1265                request.content.clone(),
1266            )
1267            .await
1268            .map_err(ConsoleSendError::State)?;
1269        let session_id = resolved
1270            .handle
1271            .resolve_bridge_session_id_observation(&MeerkatId::from(
1272                resolved.runtime_identity.as_str(),
1273            ))
1274            .await
1275            .map(|sid| sid.to_string());
1276        let mut new_frame = NewConsoleFrame {
1277            id: None,
1278            dedupe_key,
1279            timestamp_ms: current_time_ms(),
1280            runtime_key: resolved.entry.runtime_key.clone(),
1281            identity: request.identity.clone(),
1282            conversation_id: Some(request.identity.clone()),
1283            session_id: session_id.clone(),
1284            kind: "user_input".to_string(),
1285            status: ConsoleFrameStatus::Accepted,
1286            payload: json!({
1287                "content": request.content,
1288                "origin": request.origin,
1289                "idempotency_key": request.idempotency_key,
1290                "handling_mode": handling_mode_value,
1291            }),
1292            source: ConsoleFrameSource {
1293                kind: ConsoleFrameSourceKind::Send,
1294                source_cursor: Some(request_fingerprint),
1295            },
1296            source_event_id: None,
1297            interaction_id: Some(interaction_id.clone()),
1298            turn_id: None,
1299            run_id: None,
1300            parent_frame_id: None,
1301            caused_by_frame_id: None,
1302        };
1303        if let Some(redacted) = resolved.entry.visibility_policy.redact_payload(&new_frame) {
1304            new_frame.payload = redacted;
1305            new_frame.status = ConsoleFrameStatus::Redacted;
1306        }
1307
1308        let _ = SendState::Requested
1309            .apply(SendTransition::PersistAccepted)
1310            .map_err(ConsoleSendError::State)?;
1311        let outcome = self
1312            .inner
1313            .store
1314            .append_if_absent(new_frame)
1315            .await
1316            .map_err(ConsoleSendError::Log)?;
1317        if outcome.disposition == AppendDisposition::Existing {
1318            return Ok(accepted_from_frame(&outcome.frame));
1319        }
1320        let _ = self
1321            .inner
1322            .event_tx
1323            .send(ConsoleTimelineEvent::ConsoleFrame {
1324                frame: outcome.frame.clone(),
1325            });
1326        let accepted = accepted_from_frame(&outcome.frame);
1327
1328        let (dispatching, _effects) = SendState::AcceptedPersisted
1329            .apply(SendTransition::StartDispatch)
1330            .map_err(ConsoleSendError::State)?;
1331        update_frame_status_and_emit(
1332            &self.inner,
1333            &outcome.frame.id,
1334            ConsoleFrameStatus::Dispatching,
1335        )
1336        .await
1337        .map_err(ConsoleSendError::Log)?;
1338
1339        spawn_console_send_dispatch(
1340            self.inner.clone(),
1341            resolved,
1342            content,
1343            handling_mode,
1344            dispatching,
1345            outcome.frame,
1346            interaction_id,
1347        );
1348        Ok(accepted)
1349    }
1350
1351    pub async fn reserve_identity_first_interaction(
1352        &self,
1353        request: ConsoleSendRequest,
1354        session_id: Option<&str>,
1355    ) -> Result<ConsoleInteractionAccepted, ConsoleSendError> {
1356        validate_send_request(&request)?;
1357        let _content = content_input_from_value(&request.content)?;
1358        let runtime_key =
1359            Box::pin(self.runtime_key_for_identity_first_send(&request.identity)).await?;
1360        let handling_mode_value = request
1361            .handling_mode
1362            .as_deref()
1363            .unwrap_or("queue")
1364            .to_string();
1365        let dedupe_key = send_dedupe_key(
1366            &runtime_key,
1367            &request.identity,
1368            &request.origin,
1369            &request.idempotency_key,
1370        );
1371        let request_fingerprint =
1372            send_request_fingerprint(&request.origin, &request.content, &handling_mode_value);
1373        if let Some(existing) = self
1374            .inner
1375            .store
1376            .frame_by_dedupe_key(&dedupe_key)
1377            .await
1378            .map_err(ConsoleSendError::Log)?
1379        {
1380            let same_request = existing.source.source_cursor.as_deref()
1381                == Some(request_fingerprint.as_str())
1382                || existing.source.source_cursor.is_none()
1383                    && existing.payload.get("origin").and_then(Value::as_str)
1384                        == Some(request.origin.as_str())
1385                    && existing.payload.get("content") == Some(&request.content)
1386                    && existing
1387                        .payload
1388                        .get("handling_mode")
1389                        .and_then(Value::as_str)
1390                        == Some(handling_mode_value.as_str());
1391            if !same_request {
1392                return Err(ConsoleSendError::IdempotencyConflict(
1393                    request.idempotency_key,
1394                ));
1395            }
1396            return Ok(accepted_from_frame(&existing));
1397        }
1398
1399        let interaction_id = format!("console-interaction-{}", hash_short(&dedupe_key));
1400        let new_frame = NewConsoleFrame {
1401            id: None,
1402            dedupe_key,
1403            timestamp_ms: current_time_ms(),
1404            runtime_key,
1405            identity: request.identity.clone(),
1406            conversation_id: Some(request.identity.clone()),
1407            session_id: session_id.map(ToString::to_string),
1408            kind: "user_input".to_string(),
1409            status: ConsoleFrameStatus::Accepted,
1410            payload: json!({
1411                "content": request.content,
1412                "origin": request.origin,
1413                "idempotency_key": request.idempotency_key,
1414                "handling_mode": handling_mode_value,
1415            }),
1416            source: ConsoleFrameSource {
1417                kind: ConsoleFrameSourceKind::Send,
1418                source_cursor: Some(request_fingerprint),
1419            },
1420            source_event_id: None,
1421            interaction_id: Some(interaction_id),
1422            turn_id: None,
1423            run_id: None,
1424            parent_frame_id: None,
1425            caused_by_frame_id: None,
1426        };
1427        let outcome = self
1428            .inner
1429            .store
1430            .append_if_absent(new_frame)
1431            .await
1432            .map_err(ConsoleSendError::Log)?;
1433        let _ = self
1434            .inner
1435            .event_tx
1436            .send(ConsoleTimelineEvent::ConsoleFrame {
1437                frame: outcome.frame.clone(),
1438            });
1439        Ok(accepted_from_frame(&outcome.frame))
1440    }
1441
1442    async fn runtime_key_for_identity_first_send(
1443        &self,
1444        identity: &str,
1445    ) -> Result<String, ConsoleSendError> {
1446        let entries = self
1447            .inner
1448            .runtimes
1449            .read()
1450            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1451            .clone();
1452        if entries.is_empty() {
1453            return Ok("identity-first".to_string());
1454        }
1455
1456        let mut identity_runtime_keys = Vec::new();
1457        let mut hidden_durable_match = false;
1458        let mut durable_matches = Vec::new();
1459        for entry in entries.values() {
1460            let Some(identity_runtime) = entry.identity_runtime.as_ref() else {
1461                continue;
1462            };
1463            identity_runtime_keys.push(entry.runtime_key.clone());
1464            if requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace) {
1465                continue;
1466            }
1467            for runtime_identity in namespace_match_candidates(identity, &entry.identity_namespace)
1468            {
1469                let Ok(parsed_identity) =
1470                    crate::identity_first::AgentIdentity::parse(&runtime_identity)
1471                else {
1472                    continue;
1473                };
1474                if let Ok(status) = identity_runtime.status(&parsed_identity).await {
1475                    let record = identity_record_for_status(entry, &status);
1476                    if !console_identity_record_visible(entry, &record).await {
1477                        hidden_durable_match = true;
1478                        continue;
1479                    }
1480                    durable_matches.push((entry.clone(), record));
1481                }
1482            }
1483        }
1484        if durable_matches.len() > 1 {
1485            let candidates = durable_matches
1486                .iter()
1487                .map(|(entry, record)| format!("{}@{}", record.identity, entry.runtime_key))
1488                .collect::<Vec<_>>()
1489                .join(", ");
1490            return Err(ConsoleSendError::InvalidRequest(format!(
1491                "ambiguous durable identity alias {identity}: candidates [{candidates}]"
1492            )));
1493        }
1494        if let Some((entry, record)) = durable_matches.into_iter().next() {
1495            let live_records = self.live_records_for_identity(identity).await;
1496            let durable_live_records = self
1497                .live_records_for_durable_record(&entry, identity, &record, &live_records)
1498                .await;
1499            let visible_durable_live_records =
1500                visible_live_records_for_entry(&entry, &durable_live_records).await;
1501            if let Some(ambiguous_error) =
1502                ambiguous_live_alias_error(identity, &visible_durable_live_records)
1503            {
1504                return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1505            }
1506            if let Some(stale_error) =
1507                stale_durable_record_error(identity, &record, &durable_live_records)
1508            {
1509                return Err(ConsoleSendError::InvalidRequest(stale_error));
1510            }
1511            return Ok(entry.runtime_key);
1512        }
1513        if entries.values().any(|entry| {
1514            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1515        }) {
1516            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1517        }
1518        if hidden_durable_match {
1519            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1520        }
1521
1522        match identity_runtime_keys.as_slice() {
1523            [] => Err(ConsoleSendError::UnknownIdentity(identity.to_string())),
1524            [runtime_key] => Ok(runtime_key.clone()),
1525            _ => Err(ConsoleSendError::InvalidRequest(format!(
1526                "identity-first send for '{identity}' did not match exactly one registered runtime"
1527            ))),
1528        }
1529    }
1530
1531    pub async fn mark_interaction_delivery_failed(
1532        &self,
1533        input_frame_id: &str,
1534    ) -> Result<(), ConsoleSendError> {
1535        update_frame_status_and_emit(
1536            &self.inner,
1537            input_frame_id,
1538            ConsoleFrameStatus::DeliveryFailed,
1539        )
1540        .await
1541        .map_err(ConsoleSendError::Log)?;
1542        Ok(())
1543    }
1544
1545    pub async fn mark_interaction_delivered(
1546        &self,
1547        input_frame_id: &str,
1548    ) -> Result<(), ConsoleSendError> {
1549        update_frame_status_and_emit(&self.inner, input_frame_id, ConsoleFrameStatus::Delivered)
1550            .await
1551            .map_err(ConsoleSendError::Log)?;
1552        Ok(())
1553    }
1554
1555    pub async fn mark_steer_interaction_delivered(
1556        &self,
1557        input_frame_id: &str,
1558        interaction_id: &str,
1559    ) -> Result<(), ConsoleSendError> {
1560        let Some(updated) = update_frame_status_and_emit(
1561            &self.inner,
1562            input_frame_id,
1563            ConsoleFrameStatus::Delivered,
1564        )
1565        .await
1566        .map_err(ConsoleSendError::Log)?
1567        else {
1568            return Ok(());
1569        };
1570        append_steer_delivery_terminal(&self.inner, &updated, interaction_id)
1571            .await
1572            .map_err(ConsoleSendError::Log)?;
1573        Ok(())
1574    }
1575
1576    pub async fn binary_blob_store_for_identity(
1577        &self,
1578        identity: &str,
1579    ) -> Result<Option<Arc<dyn BinaryBlobStore>>, ConsoleSendError> {
1580        if identity.trim().is_empty() {
1581            return Err(ConsoleSendError::InvalidRequest(
1582                "identity must be non-empty".to_string(),
1583            ));
1584        }
1585        let Some(resolved) = Box::pin(self.resolve_send_member(identity)).await? else {
1586            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1587        };
1588        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1589            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1590        };
1591        if !resolved_member_visible(&resolved, &record).await {
1592            return Err(ConsoleSendError::UnknownIdentity(identity.to_string()));
1593        }
1594        if !member_is_addressable(&resolved.member) {
1595            return Err(ConsoleSendError::NotAddressable(identity.to_string()));
1596        }
1597        if resolved.member.state == meerkat_mob::MemberState::Retiring {
1598            return Err(ConsoleSendError::Retired(identity.to_string()));
1599        }
1600        Ok(resolved.entry.runtime.binary_blob_store())
1601    }
1602
1603    pub fn binary_blob_stores(&self) -> Vec<Arc<dyn BinaryBlobStore>> {
1604        self.inner
1605            .runtimes
1606            .read()
1607            .map(|entries| {
1608                entries
1609                    .values()
1610                    .filter_map(|entry| entry.runtime.binary_blob_store())
1611                    .collect()
1612            })
1613            .unwrap_or_default()
1614    }
1615
1616    async fn resolve_send_member(
1617        &self,
1618        identity: &str,
1619    ) -> Result<Option<ResolvedConsoleMember>, ConsoleSendError> {
1620        let matches = self.resolve_visible_members(identity).await;
1621        if matches.len() > 1 {
1622            let candidates = matches
1623                .iter()
1624                .map(|resolved| resolved.runtime_identity.clone())
1625                .collect::<Vec<_>>()
1626                .join(", ");
1627            return Err(ConsoleSendError::AmbiguousIdentity {
1628                identity: identity.to_string(),
1629                candidates,
1630            });
1631        }
1632        let Some(resolved) = matches.into_iter().next() else {
1633            if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1634                return Err(ConsoleSendError::InvalidRequest(stale_error));
1635            }
1636            return Ok(None);
1637        };
1638        if let Some(record) = identity_record_for_resolved_member(&resolved).await
1639            && let Some(stale_error) =
1640                stale_live_record_binding_error(&resolved.entry, identity, &record).await
1641        {
1642            return Err(ConsoleSendError::InvalidRequest(stale_error));
1643        }
1644        if let Some(stale_error) = self.durable_send_binding_error(identity).await? {
1645            return Err(ConsoleSendError::InvalidRequest(stale_error));
1646        }
1647        Ok(Some(resolved))
1648    }
1649
1650    async fn durable_send_binding_error(
1651        &self,
1652        identity: &str,
1653    ) -> Result<Option<String>, ConsoleSendError> {
1654        let entries = self
1655            .inner
1656            .runtimes
1657            .read()
1658            .map_err(|_| ConsoleSendError::Log(runtime_registry_lock_error()))?
1659            .clone();
1660        if entries.values().any(|entry| {
1661            requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
1662        }) {
1663            return Ok(None);
1664        }
1665        let live_records = self.live_records_for_identity(identity).await;
1666        let mut durable_matches = Vec::new();
1667        for entry in entries.values() {
1668            let Some(identity_runtime) = entry.identity_runtime.clone() else {
1669                continue;
1670            };
1671            for raw_identity in namespace_match_candidates(identity, &entry.identity_namespace) {
1672                let Ok(parsed_identity) =
1673                    crate::identity_first::AgentIdentity::parse(&raw_identity)
1674                else {
1675                    continue;
1676                };
1677                let Ok(status) = identity_runtime.status(&parsed_identity).await else {
1678                    continue;
1679                };
1680                let record = identity_record_for_status(entry, &status);
1681                if !console_identity_record_visible(entry, &record).await {
1682                    continue;
1683                }
1684                durable_matches.push((entry.clone(), record));
1685            }
1686        }
1687        if durable_matches.len() > 1 {
1688            let candidates = durable_matches
1689                .iter()
1690                .map(|(_, record)| record.runtime_member_id.clone())
1691                .collect::<Vec<_>>()
1692                .join(", ");
1693            return Err(ConsoleSendError::AmbiguousIdentity {
1694                identity: identity.to_string(),
1695                candidates,
1696            });
1697        }
1698        let Some((entry, durable_record)) = durable_matches.into_iter().next() else {
1699            return Ok(None);
1700        };
1701        let durable_live_records = self
1702            .live_records_for_durable_record(&entry, identity, &durable_record, &live_records)
1703            .await;
1704        let visible_durable_live_records =
1705            visible_live_records_for_entry(&entry, &durable_live_records).await;
1706        if let Some(ambiguous_error) =
1707            ambiguous_live_alias_error(&durable_record.identity, &visible_durable_live_records)
1708        {
1709            return Err(ConsoleSendError::InvalidRequest(ambiguous_error));
1710        }
1711        Ok(stale_durable_record_error(
1712            identity,
1713            &durable_record,
1714            &durable_live_records,
1715        ))
1716    }
1717
1718    async fn resolve_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1719        let entries = self
1720            .inner
1721            .runtimes
1722            .read()
1723            .ok()
1724            .map(|entries| entries.clone())
1725            .unwrap_or_default();
1726        let mut exact_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1727        let mut label_matches: Vec<(String, ResolvedConsoleMember)> = Vec::new();
1728        for entry in entries.values() {
1729            let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
1730            if raw_identities.is_empty() {
1731                continue;
1732            }
1733            let mids = raw_identities
1734                .iter()
1735                .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
1736                .collect::<Vec<_>>();
1737            for resolved in member_sources_for_entry(entry).await {
1738                let session_id = resolved
1739                    .handle
1740                    .resolve_bridge_session_id_observation(&resolved.member.agent_identity)
1741                    .await
1742                    .map(|sid| sid.to_string())
1743                    .unwrap_or_default();
1744                if mids.contains(&resolved.member.agent_identity) {
1745                    exact_matches.push((session_id, resolved));
1746                } else if resolved_member_matches_raw_identities(&resolved, &raw_identities, &mids)
1747                {
1748                    label_matches.push((session_id, resolved));
1749                }
1750            }
1751        }
1752        let mut matches = exact_matches;
1753        matches.extend(label_matches);
1754        let mut seen_members = BTreeSet::new();
1755        matches.retain(|(session_id, resolved)| {
1756            seen_members.insert((
1757                resolved.entry.runtime_key.clone(),
1758                resolved.source_mob_id.clone(),
1759                session_id.clone(),
1760                resolved.runtime_identity.clone(),
1761            ))
1762        });
1763        matches.sort_by(|left, right| right.0.cmp(&left.0));
1764        matches.into_iter().map(|(_, resolved)| resolved).collect()
1765    }
1766
1767    async fn resolve_visible_members(&self, identity: &str) -> Vec<ResolvedConsoleMember> {
1768        let mut visible = Vec::new();
1769        for resolved in self.resolve_members(identity).await {
1770            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1771                continue;
1772            };
1773            if resolved_member_visible(&resolved, &record).await {
1774                visible.push(resolved);
1775            }
1776        }
1777        visible
1778    }
1779
1780    async fn live_records_for_identity(&self, identity: &str) -> Vec<ConsoleIdentityRecord> {
1781        let mut records = Vec::new();
1782        for resolved in self.resolve_members(identity).await {
1783            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1784                continue;
1785            };
1786            if resolved_member_visible(&resolved, &record).await {
1787                records.push(record);
1788            }
1789        }
1790        records
1791    }
1792
1793    async fn live_records_for_durable_record(
1794        &self,
1795        entry: &RuntimeEntry,
1796        requested_identity: &str,
1797        durable: &ConsoleIdentityRecord,
1798        requested_live_records: &[ConsoleIdentityRecord],
1799    ) -> Vec<ConsoleIdentityRecord> {
1800        let mut records = if durable.identity == requested_identity {
1801            requested_live_records.to_vec()
1802        } else {
1803            self.live_records_for_identity(&durable.identity).await
1804        };
1805        for runtime_record in
1806            live_records_for_runtime_member(entry, &durable.runtime_member_id).await
1807        {
1808            if records.iter().any(|record| {
1809                record.runtime_key == runtime_record.runtime_key
1810                    && record.runtime_member_id == runtime_record.runtime_member_id
1811                    && record.session_id == runtime_record.session_id
1812                    && record.identity == runtime_record.identity
1813            }) {
1814                continue;
1815            }
1816            records.push(runtime_record);
1817        }
1818        records
1819    }
1820}
1821
1822async fn console_identity_record_visible(
1823    entry: &RuntimeEntry,
1824    record: &ConsoleIdentityRecord,
1825) -> bool {
1826    if !entry.visibility_policy.identity_visible(record) {
1827        return false;
1828    }
1829    let mut bound_live_member_seen = false;
1830    let mut wrong_live_projection_seen = false;
1831    for resolved in member_sources_for_entry(entry).await {
1832        if resolved.runtime_identity != record.runtime_member_id {
1833            continue;
1834        }
1835        bound_live_member_seen = true;
1836        let Some(live_record) = identity_record_for_resolved_member(&resolved).await else {
1837            continue;
1838        };
1839        if live_record.identity != record.identity {
1840            wrong_live_projection_seen = true;
1841            continue;
1842        }
1843        if raw_resolved_member_visible(&resolved, &live_record) {
1844            return true;
1845        }
1846    }
1847    if wrong_live_projection_seen {
1848        return true;
1849    }
1850    !bound_live_member_seen
1851}
1852
1853async fn live_record_shadowed_by_hidden_durable_binding(
1854    resolved: &ResolvedConsoleMember,
1855    record: &ConsoleIdentityRecord,
1856) -> bool {
1857    let Some(identity_runtime) = resolved.entry.identity_runtime.as_ref() else {
1858        return false;
1859    };
1860    let Ok(identity) = crate::identity_first::AgentIdentity::parse(&record.identity) else {
1861        return false;
1862    };
1863    let Ok(status) = identity_runtime.status(&identity).await else {
1864        return false;
1865    };
1866    let Some(bound_runtime_id) = status.agent_runtime_id.as_ref() else {
1867        return false;
1868    };
1869    if bound_runtime_id.as_str() == resolved.runtime_identity {
1870        return false;
1871    }
1872    for candidate in member_sources_for_entry(&resolved.entry).await {
1873        if candidate.runtime_identity != bound_runtime_id.as_str() {
1874            continue;
1875        }
1876        let Some(bound_record) = identity_record_for_resolved_member(&candidate).await else {
1877            continue;
1878        };
1879        if bound_record.identity == record.identity
1880            && !raw_resolved_member_visible(&candidate, &bound_record)
1881        {
1882            return true;
1883        }
1884    }
1885    false
1886}
1887
1888async fn frame_matches_hidden_member(entry: &RuntimeEntry, frame: &ConsoleFrame) -> bool {
1889    let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
1890        .unwrap_or_else(|| frame.identity.clone());
1891    let frame_session_id = frame.session_id.as_deref();
1892    let mut saw_hidden = false;
1893    let mut saw_visible = false;
1894    for resolved in member_sources_for_entry(entry).await {
1895        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1896            continue;
1897        };
1898        let session_matches =
1899            frame_session_id.is_some() && record.session_id.as_deref() == frame_session_id;
1900        let runtime_matches = record.runtime_member_id == runtime_member_id;
1901        let identity_matches = record.identity == frame.identity;
1902        if !session_matches && !runtime_matches && !identity_matches {
1903            continue;
1904        }
1905        if session_matches && !raw_resolved_member_visible(&resolved, &record) {
1906            return true;
1907        }
1908        if resolved_member_visible(&resolved, &record).await {
1909            saw_visible = true;
1910        } else {
1911            saw_hidden = true;
1912        }
1913    }
1914    saw_hidden && !saw_visible
1915}
1916
1917async fn live_records_for_runtime_member(
1918    entry: &RuntimeEntry,
1919    runtime_member_id: &str,
1920) -> Vec<ConsoleIdentityRecord> {
1921    let mut records = Vec::new();
1922    for resolved in member_sources_for_entry(entry).await {
1923        if resolved.runtime_identity != runtime_member_id {
1924            continue;
1925        }
1926        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
1927            continue;
1928        };
1929        records.push(record);
1930    }
1931    records
1932}
1933
1934async fn visible_live_records_for_entry(
1935    entry: &RuntimeEntry,
1936    records: &[ConsoleIdentityRecord],
1937) -> Vec<ConsoleIdentityRecord> {
1938    let mut visible = Vec::new();
1939    for record in records {
1940        if record.runtime_key != entry.runtime_key
1941            || console_identity_record_visible(entry, record).await
1942        {
1943            visible.push(record.clone());
1944        }
1945    }
1946    visible
1947}
1948
1949async fn stale_live_record_binding_error(
1950    entry: &RuntimeEntry,
1951    requested_identity: &str,
1952    live_record: &ConsoleIdentityRecord,
1953) -> Option<String> {
1954    let identity_runtime = entry.identity_runtime.as_ref()?;
1955    for status in identity_runtime.statuses().await {
1956        let matches_runtime = status
1957            .agent_runtime_id
1958            .as_ref()
1959            .is_some_and(|runtime_id| runtime_id.as_str() == live_record.runtime_member_id);
1960        if !matches_runtime {
1961            continue;
1962        }
1963        let durable_record = identity_record_for_status(entry, &status);
1964        if let Some(stale_error) = stale_durable_record_error(
1965            requested_identity,
1966            &durable_record,
1967            std::slice::from_ref(live_record),
1968        ) {
1969            return Some(stale_error);
1970        }
1971    }
1972    None
1973}
1974
1975fn stale_durable_record_error(
1976    requested_identity: &str,
1977    durable: &ConsoleIdentityRecord,
1978    live_records: &[ConsoleIdentityRecord],
1979) -> Option<String> {
1980    let matching_live = live_records
1981        .iter()
1982        .filter(|record| record.identity == durable.identity)
1983        .collect::<Vec<_>>();
1984    if let Some(wrong_projection) = live_records.iter().find(|record| {
1985        record.runtime_member_id == durable.runtime_member_id && record.identity != durable.identity
1986    }) {
1987        return Some(format!(
1988            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias projects identity {}",
1989            durable.identity, durable.runtime_member_id, wrong_projection.identity
1990        ));
1991    }
1992    if matching_live.is_empty() {
1993        return None;
1994    }
1995    if let Some(session_mismatch) = matching_live.iter().find(|record| {
1996        record.runtime_member_id == durable.runtime_member_id
1997            && durable.session_id.is_some()
1998            && record.session_id.is_some()
1999            && durable.session_id != record.session_id
2000    }) {
2001        return Some(format!(
2002            "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {} session {}, but live console alias resolves to session {}",
2003            durable.identity,
2004            durable.runtime_member_id,
2005            durable.session_id.as_deref().unwrap_or("<none>"),
2006            session_mismatch.session_id.as_deref().unwrap_or("<none>")
2007        ));
2008    }
2009    if matching_live
2010        .iter()
2011        .any(|record| record.runtime_member_id == durable.runtime_member_id)
2012    {
2013        return None;
2014    }
2015    let live_candidates = matching_live
2016        .iter()
2017        .map(|record| record.runtime_member_id.as_str())
2018        .collect::<Vec<_>>()
2019        .join(", ");
2020    Some(format!(
2021        "stale durable identity alias {requested_identity}: identity runtime binding for {} points at {}, but live console alias resolves to [{}]",
2022        durable.identity, durable.runtime_member_id, live_candidates
2023    ))
2024}
2025
2026fn ambiguous_live_alias_error(
2027    requested_identity: &str,
2028    live_records: &[ConsoleIdentityRecord],
2029) -> Option<String> {
2030    if live_records.len() <= 1 {
2031        return None;
2032    }
2033    let candidates = live_records
2034        .iter()
2035        .map(|record| {
2036            format!(
2037                "{}@{}",
2038                record.runtime_member_id,
2039                record
2040                    .labels
2041                    .get("source_mob_id")
2042                    .map(String::as_str)
2043                    .unwrap_or(record.runtime_key.as_str())
2044            )
2045        })
2046        .collect::<Vec<_>>()
2047        .join(", ");
2048    Some(format!(
2049        "ambiguous live identity alias {requested_identity}: candidates [{candidates}]"
2050    ))
2051}
2052
2053fn member_id_matches_durable_identity(member_id: &str, durable_identity: &str) -> bool {
2054    member_id == durable_identity
2055}
2056
2057async fn retire_stale_console_members_for_identity(
2058    handle: &MobHandle,
2059    durable_identities: &[String],
2060    keep_runtime_member_id: Option<&str>,
2061) -> Result<(), String> {
2062    let stale_members = handle
2063        .list_members_including_retiring()
2064        .await
2065        .into_iter()
2066        .filter(|member| {
2067            (durable_identities.iter().any(|durable_identity| {
2068                member_id_matches_durable_identity(member.agent_identity.as_str(), durable_identity)
2069            }) || member.labels.get("agent_identity").is_some_and(|identity| {
2070                durable_identities
2071                    .iter()
2072                    .any(|durable_identity| identity == durable_identity)
2073            })) && keep_runtime_member_id
2074                .map(|keep| member.agent_identity.as_str() != keep)
2075                .unwrap_or(true)
2076        })
2077        .map(|member| member.agent_identity)
2078        .collect::<Vec<_>>();
2079    for member_id in stale_members {
2080        match handle.retire(member_id).await {
2081            Ok(()) => {}
2082            Err(err) if lifecycle_archive_cleanup_completed(&err.to_string()) => {}
2083            Err(err) => return Err(err.to_string()),
2084        }
2085    }
2086    Ok(())
2087}
2088
2089async fn retire_stale_console_members_for_runtime_entry(
2090    entry: &RuntimeEntry,
2091    durable_identities: &[String],
2092) -> Result<(), String> {
2093    let primary_handle = entry.runtime.handle();
2094    let primary_mob_id = primary_handle.mob_id().to_string();
2095    let mut handles = vec![(primary_mob_id.clone(), primary_handle)];
2096    if let Some(state) = entry.runtime.agent_mob_mcp_state() {
2097        for (mob_id, handle) in state.mob_handles_snapshot().await {
2098            if mob_id.as_str() != primary_mob_id {
2099                handles.push((mob_id.to_string(), handle));
2100            }
2101        }
2102    }
2103    let mut seen_handles = BTreeSet::new();
2104    for (mob_id, handle) in handles {
2105        if !seen_handles.insert(mob_id) {
2106            continue;
2107        }
2108        retire_stale_console_members_for_identity(&handle, durable_identities, None).await?;
2109    }
2110    Ok(())
2111}
2112
2113fn lifecycle_archive_cleanup_completed(error: &str) -> bool {
2114    is_recoverable_lifecycle_cleanup_error(error)
2115}
2116
2117fn explicit_identity_query_needs_session_history_backfill(frames: &[ConsoleFrame]) -> bool {
2118    if frames.is_empty() {
2119        return true;
2120    }
2121    let latest_session_history_timestamp_ms = frames
2122        .iter()
2123        .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::SessionHistory)
2124        .map(|frame| frame.timestamp_ms)
2125        .max();
2126    if let Some(latest_timestamp_ms) = latest_session_history_timestamp_ms {
2127        return current_time_ms().saturating_sub(latest_timestamp_ms)
2128            >= SESSION_HISTORY_GROWING_REFRESH_TTL_MS;
2129    }
2130    frames.iter().any(|frame| {
2131        matches!(
2132            frame.kind.as_str(),
2133            "turn_started"
2134                | "run_started"
2135                | "reasoning_delta"
2136                | "reasoning_complete"
2137                | "tool_call_requested"
2138                | "tool_call"
2139                | "tool_execution_started"
2140                | "tool_execution_completed"
2141                | "tool_result_received"
2142        )
2143    })
2144}
2145
2146fn is_identity_timeline_anchor_frame(frame: &ConsoleFrame) -> bool {
2147    match frame.kind.as_str() {
2148        "user_input" | "run_started" => true,
2149        "interaction_started" => frame
2150            .payload
2151            .get("content")
2152            .or_else(|| frame.payload.get("prompt"))
2153            .is_some(),
2154        _ => false,
2155    }
2156}
2157
2158fn dedupe_identity_records(records: Vec<ConsoleIdentityRecord>) -> Vec<ConsoleIdentityRecord> {
2159    let mut by_identity: BTreeMap<String, ConsoleIdentityRecord> = BTreeMap::new();
2160    for record in records {
2161        by_identity
2162            .entry(identity_record_dedupe_key(&record))
2163            .and_modify(|current| {
2164                if identity_record_prefer(&record, current) {
2165                    *current = record.clone();
2166                }
2167            })
2168            .or_insert(record);
2169    }
2170    by_identity.into_values().collect()
2171}
2172
2173fn identity_record_dedupe_key(record: &ConsoleIdentityRecord) -> String {
2174    format!(
2175        "{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}",
2176        record.identity,
2177        record.runtime_key,
2178        record
2179            .labels
2180            .get("source_mob_id")
2181            .map(String::as_str)
2182            .unwrap_or(""),
2183        record.runtime_member_id,
2184        record.session_id.as_deref().unwrap_or("")
2185    )
2186}
2187
2188fn identity_record_prefer(
2189    candidate: &ConsoleIdentityRecord,
2190    current: &ConsoleIdentityRecord,
2191) -> bool {
2192    let candidate_is_live_label_projection =
2193        candidate
2194            .labels
2195            .get("agent_identity")
2196            .is_some_and(|identity| {
2197                identity == &strip_namespace(&candidate.identity, "").unwrap_or_default()
2198                    || candidate.runtime_member_id != candidate.identity
2199            });
2200    let current_is_live_label_projection =
2201        current
2202            .labels
2203            .get("agent_identity")
2204            .is_some_and(|identity| {
2205                identity == &strip_namespace(&current.identity, "").unwrap_or_default()
2206                    || current.runtime_member_id != current.identity
2207            });
2208    if candidate_is_live_label_projection != current_is_live_label_projection {
2209        return candidate_is_live_label_projection;
2210    }
2211    let candidate_has_distinct_runtime_binding = candidate.runtime_member_id != candidate.identity;
2212    let current_has_distinct_runtime_binding = current.runtime_member_id != current.identity;
2213    if candidate_has_distinct_runtime_binding != current_has_distinct_runtime_binding {
2214        return candidate_has_distinct_runtime_binding;
2215    }
2216    let candidate_live = candidate.addressable && candidate.health != "retired";
2217    let current_live = current.addressable && current.health != "retired";
2218    if candidate_live != current_live {
2219        return candidate_live;
2220    }
2221    candidate.session_id.as_deref().unwrap_or("") > current.session_id.as_deref().unwrap_or("")
2222}
2223
2224async fn collect_identity_records(
2225    inner: &Arc<AggregatorInner>,
2226    mode: IdentityCollectionMode,
2227) -> ConsoleLogResult<Vec<ConsoleIdentityRecord>> {
2228    let entries = inner
2229        .runtimes
2230        .read()
2231        .map_err(|_| runtime_registry_lock_error())?
2232        .clone();
2233    let mut identities = Vec::new();
2234    for entry in entries.values() {
2235        if let Some(identity_runtime) = entry.identity_runtime.as_ref() {
2236            let topology_peers =
2237                identity_runtime_topology_peers(entry, identity_runtime.as_ref()).await;
2238            for status in identity_runtime.statuses().await {
2239                let mut record = identity_record_for_status(entry, &status);
2240                record.topology_peers = topology_peers
2241                    .get(&record.identity)
2242                    .cloned()
2243                    .unwrap_or_default();
2244                if console_identity_record_visible(entry, &record).await {
2245                    identities.push(record);
2246                }
2247            }
2248            if mode == IdentityCollectionMode::CachedOnly {
2249                continue;
2250            }
2251        }
2252        let members = if entry.identity_runtime.is_some() {
2253            match tokio::time::timeout(
2254                IDENTITY_FIRST_LIVE_MEMBER_REFRESH_WAIT,
2255                member_sources_for_entry(entry),
2256            )
2257            .await
2258            {
2259                Ok(members) => members,
2260                Err(_) => {
2261                    tracing::warn!(
2262                        runtime_key = %entry.runtime_key,
2263                        "identity-first live member refresh timed out; keeping cached identity read model"
2264                    );
2265                    Vec::new()
2266                }
2267            }
2268        } else {
2269            member_sources_for_entry(entry).await
2270        };
2271        for resolved in members {
2272            if let Some(record) = identity_record_for_resolved_member(&resolved).await
2273                && resolved_member_visible(&resolved, &record).await
2274            {
2275                identities.push(record);
2276            }
2277        }
2278    }
2279    Ok(dedupe_identity_records(identities))
2280}
2281
2282async fn identity_runtime_topology_peers(
2283    entry: &RuntimeEntry,
2284    identity_runtime: &crate::identity_first::IdentityRuntime,
2285) -> BTreeMap<String, Vec<String>> {
2286    let mut peers: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
2287    for edge in identity_runtime.desired_peer_edges().await {
2288        let a = apply_namespace(edge.a().as_str(), &entry.identity_namespace);
2289        let b = apply_namespace(edge.b().as_str(), &entry.identity_namespace);
2290        peers.entry(a.clone()).or_default().insert(b.clone());
2291        peers.entry(b).or_default().insert(a);
2292    }
2293    peers
2294        .into_iter()
2295        .map(|(identity, peers)| (identity, peers.into_iter().collect()))
2296        .collect()
2297}
2298
2299fn spawn_identity_backfills_for_records(
2300    inner: Arc<AggregatorInner>,
2301    records: &[ConsoleIdentityRecord],
2302) {
2303    if !inner.options.session_history_backfill_enabled {
2304        return;
2305    }
2306    let entries = match inner.runtimes.read() {
2307        Ok(entries) => entries.clone(),
2308        Err(_) => return,
2309    };
2310    for record in records {
2311        if record.health == "dormant" || record.health == "uninitialized" {
2312            continue;
2313        }
2314        let Some(entry) = entries.get(&record.runtime_key).cloned() else {
2315            continue;
2316        };
2317        let Some(session_id) = record.session_id.clone() else {
2318            continue;
2319        };
2320        spawn_session_history_backfill_target(
2321            inner.clone(),
2322            SessionBackfillTarget {
2323                entry,
2324                record: record.clone(),
2325                session_id,
2326            },
2327            false,
2328        );
2329    }
2330}
2331
2332async fn member_sources_for_entry(entry: &RuntimeEntry) -> Vec<ResolvedConsoleMember> {
2333    let mut resolved = Vec::new();
2334    let primary_handle = entry.runtime.handle();
2335    let primary_mob_id = primary_handle.mob_id().to_string();
2336    for member in primary_handle.list_members_observation_snapshot().await {
2337        resolved.push(ResolvedConsoleMember {
2338            entry: entry.clone(),
2339            handle: primary_handle.clone(),
2340            runtime_identity: member.agent_identity.to_string(),
2341            source_mob_id: primary_mob_id.clone(),
2342            member,
2343        });
2344    }
2345
2346    let Some(state) = entry.runtime.agent_mob_mcp_state() else {
2347        return resolved;
2348    };
2349    if !entry.visibility_policy.include_implicit_delegate_members() {
2350        return resolved;
2351    }
2352    for (mob_id, handle) in state.mob_handles_snapshot().await {
2353        if mob_id.as_str() == primary_mob_id {
2354            continue;
2355        }
2356        for member in handle.list_members_observation_snapshot().await {
2357            resolved.push(ResolvedConsoleMember {
2358                entry: entry.clone(),
2359                handle: handle.clone(),
2360                runtime_identity: member.agent_identity.to_string(),
2361                source_mob_id: mob_id.to_string(),
2362                member,
2363            });
2364        }
2365    }
2366    resolved
2367}
2368
2369async fn dispatch_message_to_resolved_member(
2370    resolved: &ResolvedConsoleMember,
2371    content: ContentInput,
2372    handling_mode: meerkat_core::types::HandlingMode,
2373) -> Result<String, String> {
2374    let mid = MeerkatId::from(resolved.runtime_identity.as_str());
2375    match send_message_on_mob_with_mode(
2376        &resolved.handle,
2377        &resolved.runtime_identity,
2378        content.clone(),
2379        handling_mode,
2380    )
2381    .await
2382    {
2383        Ok(session_id) => Ok(session_id),
2384        Err(err) if is_not_externally_addressable(&err) => {
2385            let member = resolved
2386                .handle
2387                .member(&mid)
2388                .await
2389                .map_err(|err| err.to_string())?;
2390            let _receipt = member
2391                .internal_turn(content)
2392                .await
2393                .map_err(|err| err.to_string())?;
2394            resolved
2395                .handle
2396                .resolve_bridge_session_id_observation(&mid)
2397                .await
2398                .map(|sid| sid.to_string())
2399                .ok_or_else(|| "member has no bridge session after internal turn".to_string())
2400        }
2401        Err(err) => Err(err.to_string()),
2402    }
2403}
2404
2405fn is_not_externally_addressable(err: &MobRuntimeError) -> bool {
2406    matches!(
2407        err,
2408        MobRuntimeError::Mob(MobError::NotExternallyAddressable(_))
2409    )
2410}
2411
2412fn spawn_console_send_dispatch(
2413    inner: Arc<AggregatorInner>,
2414    resolved: ResolvedConsoleMember,
2415    content: ContentInput,
2416    handling_mode: meerkat_core::types::HandlingMode,
2417    dispatching: SendState,
2418    user_frame: ConsoleFrame,
2419    interaction_id: String,
2420) {
2421    tokio::spawn(async move {
2422        match dispatch_message_to_resolved_member(&resolved, content, handling_mode).await {
2423            Ok(_) => {
2424                let _ = dispatching.apply(SendTransition::MarkDelivered);
2425                if let Err(err) = update_frame_status_and_emit(
2426                    &inner,
2427                    &user_frame.id,
2428                    ConsoleFrameStatus::Delivered,
2429                )
2430                .await
2431                {
2432                    tracing::warn!(
2433                        frame_id = %user_frame.id,
2434                        error = %err,
2435                        "failed to update console send delivery status"
2436                    );
2437                }
2438                if handling_mode == HandlingMode::Steer
2439                    && let Err(err) =
2440                        append_steer_delivery_terminal(&inner, &user_frame, &interaction_id).await
2441                {
2442                    tracing::warn!(
2443                        frame_id = %user_frame.id,
2444                        interaction_id = %interaction_id,
2445                        error = %err,
2446                        "failed to append console steer terminal frame"
2447                    );
2448                }
2449            }
2450            Err(err) => {
2451                let _ = dispatching.apply(SendTransition::MarkDeliveryFailed);
2452                if let Err(update_err) = update_frame_status_and_emit(
2453                    &inner,
2454                    &user_frame.id,
2455                    ConsoleFrameStatus::DeliveryFailed,
2456                )
2457                .await
2458                {
2459                    tracing::warn!(
2460                        frame_id = %user_frame.id,
2461                        error = %update_err,
2462                        "failed to update console send failure status"
2463                    );
2464                }
2465                let failure_frame = NewConsoleFrame {
2466                    id: None,
2467                    dedupe_key: format!("delivery-failed:{}", user_frame.id),
2468                    timestamp_ms: current_time_ms(),
2469                    runtime_key: user_frame.runtime_key,
2470                    identity: user_frame.identity,
2471                    conversation_id: user_frame.conversation_id,
2472                    session_id: user_frame.session_id,
2473                    kind: "message_delivery_failed".to_string(),
2474                    status: ConsoleFrameStatus::DeliveryFailed,
2475                    payload: json!({ "reason": err }),
2476                    source: ConsoleFrameSource {
2477                        kind: ConsoleFrameSourceKind::Synthetic,
2478                        source_cursor: None,
2479                    },
2480                    source_event_id: None,
2481                    interaction_id: Some(interaction_id),
2482                    turn_id: None,
2483                    run_id: None,
2484                    parent_frame_id: Some(user_frame.id.clone()),
2485                    caused_by_frame_id: Some(user_frame.id),
2486                };
2487                if let Err(append_err) = append_and_emit(&inner, failure_frame).await {
2488                    tracing::warn!(
2489                        error = %append_err,
2490                        "failed to append console send failure frame"
2491                    );
2492                }
2493            }
2494        }
2495    });
2496}
2497
2498async fn append_steer_delivery_terminal(
2499    inner: &AggregatorInner,
2500    user_frame: &ConsoleFrame,
2501    interaction_id: &str,
2502) -> ConsoleLogResult<AppendOutcome> {
2503    append_and_emit(
2504        inner,
2505        NewConsoleFrame {
2506            id: None,
2507            dedupe_key: format!("steer-delivered:{}", user_frame.id),
2508            timestamp_ms: current_time_ms(),
2509            runtime_key: user_frame.runtime_key.clone(),
2510            identity: user_frame.identity.clone(),
2511            conversation_id: user_frame.conversation_id.clone(),
2512            session_id: user_frame.session_id.clone(),
2513            kind: "interaction_complete".to_string(),
2514            status: ConsoleFrameStatus::Completed,
2515            payload: json!({
2516                "reason": "steer_delivered",
2517                "handling_mode": "steer",
2518            }),
2519            source: ConsoleFrameSource {
2520                kind: ConsoleFrameSourceKind::Synthetic,
2521                source_cursor: None,
2522            },
2523            source_event_id: None,
2524            interaction_id: Some(interaction_id.to_string()),
2525            turn_id: None,
2526            run_id: None,
2527            parent_frame_id: Some(user_frame.id.clone()),
2528            caused_by_frame_id: Some(user_frame.id.clone()),
2529        },
2530    )
2531    .await
2532}
2533
2534#[derive(Debug)]
2535pub enum ConsoleSendError {
2536    UnknownIdentity(String),
2537    AmbiguousIdentity {
2538        identity: String,
2539        candidates: String,
2540    },
2541    NotAddressable(String),
2542    Retired(String),
2543    InvalidContent(String),
2544    InvalidHandlingMode(String),
2545    InvalidRequest(String),
2546    IdempotencyConflict(String),
2547    State(&'static str),
2548    Dispatch(String),
2549    Log(ConsoleLogError),
2550}
2551
2552impl std::fmt::Display for ConsoleSendError {
2553    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2554        match self {
2555            Self::UnknownIdentity(identity) => write!(f, "unknown identity: {identity}"),
2556            Self::AmbiguousIdentity {
2557                identity,
2558                candidates,
2559            } => write!(
2560                f,
2561                "ambiguous live identity alias {identity}: candidates [{candidates}]"
2562            ),
2563            Self::NotAddressable(identity) => write!(f, "not addressable: {identity}"),
2564            Self::Retired(identity) => write!(f, "identity retired: {identity}"),
2565            Self::InvalidContent(message) => write!(f, "invalid content: {message}"),
2566            Self::InvalidHandlingMode(mode) => write!(f, "invalid handling mode: {mode}"),
2567            Self::InvalidRequest(message) => write!(f, "invalid request: {message}"),
2568            Self::IdempotencyConflict(key) => write!(f, "idempotency key conflict: {key}"),
2569            Self::State(message) => write!(f, "console send state error: {message}"),
2570            Self::Dispatch(message) => write!(f, "dispatch failed: {message}"),
2571            Self::Log(err) => write!(f, "console log error: {err}"),
2572        }
2573    }
2574}
2575
2576impl std::error::Error for ConsoleSendError {}
2577
2578async fn backfill_session_history(
2579    inner: Arc<AggregatorInner>,
2580    runtime_key: String,
2581    force_refresh: bool,
2582) -> ConsoleLogResult<()> {
2583    if !inner.options.session_history_backfill_enabled {
2584        return Ok(());
2585    }
2586    let Some(entry) = inner
2587        .runtimes
2588        .read()
2589        .ok()
2590        .and_then(|entries| entries.get(&runtime_key).cloned())
2591    else {
2592        return Ok(());
2593    };
2594    let members = member_sources_for_entry(&entry).await;
2595    let mut targets = Vec::new();
2596    for resolved in members {
2597        let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2598            continue;
2599        };
2600        if !resolved_member_visible(&resolved, &record).await {
2601            continue;
2602        }
2603        let Some(session_id) = record.session_id.clone() else {
2604            continue;
2605        };
2606        targets.push(SessionBackfillTarget {
2607            entry: entry.clone(),
2608            record,
2609            session_id,
2610        });
2611    }
2612    backfill_session_history_targets(inner, targets, force_refresh).await
2613}
2614
2615#[derive(Clone)]
2616struct SessionBackfillTarget {
2617    entry: RuntimeEntry,
2618    record: ConsoleIdentityRecord,
2619    session_id: String,
2620}
2621
2622async fn backfill_session_history_targets(
2623    inner: Arc<AggregatorInner>,
2624    targets: Vec<SessionBackfillTarget>,
2625    force_refresh: bool,
2626) -> ConsoleLogResult<()> {
2627    let mut tasks = tokio::task::JoinSet::new();
2628    for target in targets {
2629        tasks.spawn(backfill_one_session_history(
2630            inner.clone(),
2631            target,
2632            force_refresh,
2633        ));
2634    }
2635    let mut first_error = None;
2636    while let Some(result) = tasks.join_next().await {
2637        match result {
2638            Ok(Ok(())) => {}
2639            Ok(Err(err)) => {
2640                if first_error.is_none() {
2641                    first_error = Some(err);
2642                }
2643            }
2644            Err(err) => {
2645                if first_error.is_none() {
2646                    first_error = Some(Box::new(std::io::Error::other(format!(
2647                        "session backfill task failed: {err}"
2648                    ))) as ConsoleLogError);
2649                }
2650            }
2651        }
2652    }
2653    if let Some(err) = first_error {
2654        Err(err)
2655    } else {
2656        Ok(())
2657    }
2658}
2659
2660async fn backfill_one_session_history(
2661    inner: Arc<AggregatorInner>,
2662    target: SessionBackfillTarget,
2663    force_refresh: bool,
2664) -> ConsoleLogResult<()> {
2665    let _permit = inner
2666        .session_backfill_permits
2667        .clone()
2668        .acquire_owned()
2669        .await
2670        .map_err(|err| -> ConsoleLogError {
2671            Box::new(std::io::Error::other(format!(
2672                "session backfill limiter closed: {err}"
2673            )))
2674        })?;
2675    let SessionBackfillTarget {
2676        entry,
2677        record,
2678        session_id,
2679    } = target;
2680    let watermark_runtime_key =
2681        session_history_watermark_runtime_key(&entry.runtime_key, &session_id);
2682    let watermark = inner
2683        .store
2684        .source_watermark(
2685            &watermark_runtime_key,
2686            ConsoleFrameSourceKind::SessionHistory,
2687        )
2688        .await?;
2689    let now_ms = current_time_ms();
2690    let mut offset = watermark
2691        .as_deref()
2692        .and_then(|watermark| parse_session_history_watermark(watermark, &session_id))
2693        .unwrap_or(0);
2694    if !force_refresh
2695        && watermark.as_deref().is_some_and(|watermark| {
2696            session_history_watermark_is_fresh(watermark, &session_id, now_ms)
2697        })
2698    {
2699        return Ok(());
2700    }
2701    loop {
2702        let page = match entry
2703            .runtime
2704            .read_session_history(&session_id, offset, Some(SESSION_HISTORY_PAGE_LIMIT))
2705            .await
2706        {
2707            Ok(page) => page,
2708            Err(err) => {
2709                append_backfill_gap(
2710                    &inner,
2711                    &entry.runtime_key,
2712                    &record.identity,
2713                    err.to_string(),
2714                )
2715                .await?;
2716                break;
2717            }
2718        };
2719        let page_value = match serde_json::to_value(page) {
2720            Ok(value) => value,
2721            Err(err) => {
2722                append_backfill_gap(
2723                    &inner,
2724                    &entry.runtime_key,
2725                    &record.identity,
2726                    err.to_string(),
2727                )
2728                .await?;
2729                break;
2730            }
2731        };
2732        let base_offset = page_value
2733            .get("offset")
2734            .and_then(Value::as_u64)
2735            .unwrap_or(offset as u64) as usize;
2736        let Some(messages) = page_value.get("messages").and_then(Value::as_array) else {
2737            append_backfill_gap(
2738                &inner,
2739                &entry.runtime_key,
2740                &record.identity,
2741                "session history page missing messages".to_string(),
2742            )
2743            .await?;
2744            break;
2745        };
2746        if messages.is_empty() {
2747            if offset > 0 {
2748                record_session_history_watermark(
2749                    &inner,
2750                    &watermark_runtime_key,
2751                    &session_id,
2752                    offset,
2753                )
2754                .await?;
2755            }
2756            break;
2757        }
2758        for (idx, message) in messages.iter().enumerate() {
2759            let absolute_offset = base_offset + idx;
2760            let frames = frames_from_session_history_message_with_namespace(
2761                &entry.runtime_key,
2762                &record.identity,
2763                &entry.identity_namespace,
2764                &session_id,
2765                absolute_offset,
2766                message.clone(),
2767            );
2768            for mut frame in frames {
2769                if history_frame_has_existing_counterpart(&inner, &frame).await? {
2770                    continue;
2771                }
2772                if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
2773                    frame.payload = redacted;
2774                    frame.status = ConsoleFrameStatus::Redacted;
2775                }
2776                append_and_emit(&inner, frame).await?;
2777            }
2778        }
2779        offset = base_offset + messages.len();
2780        record_session_history_watermark(&inner, &watermark_runtime_key, &session_id, offset)
2781            .await?;
2782        let has_more = page_value
2783            .get("has_more")
2784            .and_then(Value::as_bool)
2785            .unwrap_or(false);
2786        if !has_more || messages.len() < SESSION_HISTORY_PAGE_LIMIT {
2787            break;
2788        }
2789    }
2790    Ok(())
2791}
2792
2793async fn record_session_history_watermark(
2794    inner: &AggregatorInner,
2795    watermark_runtime_key: &str,
2796    session_id: &str,
2797    offset: usize,
2798) -> ConsoleLogResult<()> {
2799    inner
2800        .store
2801        .record_source_watermark(
2802            watermark_runtime_key,
2803            ConsoleFrameSourceKind::SessionHistory,
2804            &format_session_history_watermark(session_id, offset, current_time_ms()),
2805        )
2806        .await
2807}
2808
2809fn spawn_session_history_backfill(inner: Arc<AggregatorInner>, runtime_key: String) {
2810    if !inner.options.session_history_backfill_enabled {
2811        return;
2812    }
2813    tokio::spawn(async move {
2814        {
2815            let mut active = inner.active_session_backfills.lock().await;
2816            if !active.insert(runtime_key.clone()) {
2817                return;
2818            }
2819        }
2820        let result = backfill_session_history(inner.clone(), runtime_key.clone(), false).await;
2821        let mut active = inner.active_session_backfills.lock().await;
2822        active.remove(&runtime_key);
2823        drop(active);
2824        if let Err(err) = result {
2825            tracing::warn!(
2826                runtime_key = %runtime_key,
2827                error = %err,
2828                "console session-history backfill failed"
2829            );
2830        }
2831    });
2832}
2833
2834fn spawn_session_history_discovery_loop(inner: Arc<AggregatorInner>, runtime_key: String) {
2835    if !inner.options.session_history_backfill_enabled {
2836        return;
2837    }
2838    tokio::spawn(async move {
2839        let mut interval = tokio::time::interval(SESSION_HISTORY_DISCOVERY_INTERVAL);
2840        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2841        loop {
2842            interval.tick().await;
2843            let runtime_still_registered = inner
2844                .runtimes
2845                .read()
2846                .ok()
2847                .is_some_and(|entries| entries.contains_key(&runtime_key));
2848            if !runtime_still_registered {
2849                break;
2850            }
2851            spawn_session_history_backfill(inner.clone(), runtime_key.clone());
2852        }
2853    });
2854}
2855
2856fn spawn_session_history_backfill_target(
2857    inner: Arc<AggregatorInner>,
2858    target: SessionBackfillTarget,
2859    force_refresh: bool,
2860) {
2861    if !inner.options.session_history_backfill_enabled {
2862        return;
2863    }
2864    tokio::spawn(async move {
2865        let active_key = targeted_session_history_active_key(&target, force_refresh);
2866        let result =
2867            run_targeted_session_history_backfill(inner.clone(), target, force_refresh).await;
2868        if let Err(err) = result {
2869            tracing::warn!(
2870                active_key = %active_key,
2871                error = %err,
2872                "console targeted session-history backfill failed"
2873            );
2874        }
2875    });
2876}
2877
2878async fn run_targeted_session_history_backfill(
2879    inner: Arc<AggregatorInner>,
2880    target: SessionBackfillTarget,
2881    force_refresh: bool,
2882) -> ConsoleLogResult<()> {
2883    let active_key = targeted_session_history_active_key(&target, force_refresh);
2884    {
2885        let mut active = inner.active_session_backfills.lock().await;
2886        if !active.insert(active_key.clone()) {
2887            return Ok(());
2888        }
2889    }
2890    let result = backfill_one_session_history(inner.clone(), target, force_refresh).await;
2891    let mut active = inner.active_session_backfills.lock().await;
2892    active.remove(&active_key);
2893    result
2894}
2895
2896fn targeted_session_history_active_key(
2897    target: &SessionBackfillTarget,
2898    force_refresh: bool,
2899) -> String {
2900    let mode = if force_refresh { "force" } else { "refresh" };
2901    format!(
2902        "{}:session-history:{}:{mode}",
2903        target.entry.runtime_key, target.session_id
2904    )
2905}
2906
2907fn spawn_session_history_backfill_for_identity(
2908    inner: Arc<AggregatorInner>,
2909    identity: String,
2910    force_refresh: bool,
2911) {
2912    if !inner.options.session_history_backfill_enabled {
2913        return;
2914    }
2915    tokio::spawn(async move {
2916        for target in session_backfill_targets_for_identity(&inner, &identity).await {
2917            spawn_session_history_backfill_target(inner.clone(), target, force_refresh);
2918        }
2919    });
2920}
2921
2922fn spawn_opportunistic_session_history_backfill_for_identity(
2923    inner: Arc<AggregatorInner>,
2924    identity: String,
2925) {
2926    if !inner.options.session_history_backfill_enabled {
2927        return;
2928    }
2929    tokio::spawn(async move {
2930        for target in session_backfill_targets_for_identity(&inner, &identity).await {
2931            let active_key = format!(
2932                "{}:session-history:{}",
2933                target.entry.runtime_key, target.session_id
2934            );
2935            {
2936                let mut seen = inner.opportunistic_session_backfills.lock().await;
2937                if !seen.insert(active_key) {
2938                    continue;
2939                }
2940            }
2941            spawn_session_history_backfill_target(inner.clone(), target, false);
2942        }
2943    });
2944}
2945
2946#[cfg(test)]
2947async fn session_backfill_target_for_identity(
2948    inner: &AggregatorInner,
2949    identity: &str,
2950) -> Option<SessionBackfillTarget> {
2951    session_backfill_targets_for_identity(inner, identity)
2952        .await
2953        .into_iter()
2954        .next()
2955}
2956
2957async fn session_backfill_targets_for_identity(
2958    inner: &AggregatorInner,
2959    identity: &str,
2960) -> Vec<SessionBackfillTarget> {
2961    let entries = inner
2962        .runtimes
2963        .read()
2964        .ok()
2965        .map(|entries| entries.clone())
2966        .unwrap_or_default();
2967    let mut targets = Vec::new();
2968    for entry in entries.values() {
2969        let raw_identities = namespace_match_candidates(identity, &entry.identity_namespace);
2970        if raw_identities.is_empty() {
2971            continue;
2972        }
2973        let mids = raw_identities
2974            .iter()
2975            .map(|raw_identity| MeerkatId::from(raw_identity.as_str()))
2976            .collect::<Vec<_>>();
2977        for resolved in member_sources_for_entry(entry)
2978            .await
2979            .into_iter()
2980            .filter(|candidate| {
2981                resolved_member_matches_raw_identities(candidate, &raw_identities, &mids)
2982            })
2983        {
2984            let Some(record) = identity_record_for_resolved_member(&resolved).await else {
2985                continue;
2986            };
2987            if !resolved_member_visible(&resolved, &record).await {
2988                continue;
2989            }
2990            let Some(session_id) = record.session_id.clone() else {
2991                continue;
2992            };
2993            targets.push(SessionBackfillTarget {
2994                entry: entry.clone(),
2995                record,
2996                session_id,
2997            });
2998        }
2999    }
3000    targets
3001}
3002
3003fn resolved_member_matches_raw_identities(
3004    resolved: &ResolvedConsoleMember,
3005    raw_identities: &[String],
3006    mids: &[MeerkatId],
3007) -> bool {
3008    mids.contains(&resolved.member.agent_identity)
3009        || resolved
3010            .member
3011            .labels
3012            .get("agent_identity")
3013            .is_some_and(|agent_identity| raw_identities.iter().any(|raw| agent_identity == raw))
3014}
3015
3016async fn recover_lagged_source_events(
3017    inner: Arc<AggregatorInner>,
3018    runtime_key: &str,
3019    console_events: &ConsoleEventStore,
3020) -> ConsoleLogResult<()> {
3021    let watermark = inner
3022        .store
3023        .source_watermark(runtime_key, ConsoleFrameSourceKind::ConsoleEvent)
3024        .await?;
3025    match console_events.replay_all(watermark.as_deref()).await {
3026        Ok(events) => {
3027            for envelope in events {
3028                project_console_event(inner.clone(), runtime_key, envelope).await?;
3029            }
3030        }
3031        Err(err) => {
3032            append_source_gap(
3033                &inner,
3034                runtime_key,
3035                format!(
3036                    "{}:{}:{}",
3037                    err.error, err.stream, err.requested_last_event_id
3038                ),
3039            )
3040            .await?;
3041        }
3042    }
3043    Ok(())
3044}
3045
3046async fn append_source_gap(
3047    inner: &AggregatorInner,
3048    runtime_key: &str,
3049    reason: String,
3050) -> ConsoleLogResult<()> {
3051    append_and_emit(
3052        inner,
3053        NewConsoleFrame {
3054            id: None,
3055            dedupe_key: format!("source-gap:{runtime_key}:{}", current_time_ms()),
3056            timestamp_ms: current_time_ms(),
3057            runtime_key: runtime_key.to_string(),
3058            identity: "__console__".to_string(),
3059            conversation_id: None,
3060            session_id: None,
3061            kind: "replay_unavailable".to_string(),
3062            status: ConsoleFrameStatus::DeliveryFailed,
3063            payload: json!({
3064                "reason": reason,
3065                "source_kind": "console_event",
3066            }),
3067            source: ConsoleFrameSource {
3068                kind: ConsoleFrameSourceKind::Synthetic,
3069                source_cursor: None,
3070            },
3071            source_event_id: None,
3072            interaction_id: None,
3073            turn_id: None,
3074            run_id: None,
3075            parent_frame_id: None,
3076            caused_by_frame_id: None,
3077        },
3078    )
3079    .await?;
3080    let _ = inner
3081        .event_tx
3082        .send(ConsoleTimelineEvent::ReplayUnavailable {
3083            requested_cursor: format!("source-gap:{runtime_key}"),
3084            latest_cursor: inner.store.latest_cursor().await.ok().flatten(),
3085        });
3086    Ok(())
3087}
3088
3089async fn append_backfill_gap(
3090    inner: &AggregatorInner,
3091    runtime_key: &str,
3092    identity: &str,
3093    reason: String,
3094) -> ConsoleLogResult<()> {
3095    append_and_emit(
3096        inner,
3097        NewConsoleFrame {
3098            id: None,
3099            dedupe_key: format!(
3100                "session-backfill-gap:{runtime_key}:{identity}:{}",
3101                current_time_ms()
3102            ),
3103            timestamp_ms: current_time_ms(),
3104            runtime_key: runtime_key.to_string(),
3105            identity: identity.to_string(),
3106            conversation_id: Some(identity.to_string()),
3107            session_id: None,
3108            kind: "replay_unavailable".to_string(),
3109            status: ConsoleFrameStatus::DeliveryFailed,
3110            payload: json!({
3111                "reason": reason,
3112                "source_kind": "session_history",
3113            }),
3114            source: ConsoleFrameSource {
3115                kind: ConsoleFrameSourceKind::Synthetic,
3116                source_cursor: None,
3117            },
3118            source_event_id: None,
3119            interaction_id: None,
3120            turn_id: None,
3121            run_id: None,
3122            parent_frame_id: None,
3123            caused_by_frame_id: None,
3124        },
3125    )
3126    .await?;
3127    Ok(())
3128}
3129
3130async fn project_console_event(
3131    inner: Arc<AggregatorInner>,
3132    runtime_key: &str,
3133    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3134) -> ConsoleLogResult<()> {
3135    let Some(entry) = inner
3136        .runtimes
3137        .read()
3138        .ok()
3139        .and_then(|entries| entries.get(runtime_key).cloned())
3140    else {
3141        return Ok(());
3142    };
3143    let mut frame = frame_from_console_event(&entry, envelope);
3144    if let Some(redacted) = entry.visibility_policy.redact_payload(&frame) {
3145        frame.payload = redacted;
3146        frame.status = ConsoleFrameStatus::Redacted;
3147    }
3148    let source_cursor = frame
3149        .source_event_id
3150        .clone()
3151        .unwrap_or_else(|| frame.dedupe_key.clone());
3152    let refresh_identity = if console_event_should_refresh_session_history(&frame) {
3153        Some(frame.identity.clone())
3154    } else {
3155        None
3156    };
3157    let opportunistic_refresh_identity = if refresh_identity.is_none()
3158        && console_event_should_start_session_history_backfill(&frame)
3159    {
3160        Some(frame.identity.clone())
3161    } else {
3162        None
3163    };
3164    append_and_emit(&inner, frame).await?;
3165    inner
3166        .store
3167        .record_source_watermark(
3168            &entry.runtime_key,
3169            ConsoleFrameSourceKind::ConsoleEvent,
3170            &source_cursor,
3171        )
3172        .await?;
3173    if let Some(identity) = refresh_identity {
3174        spawn_session_history_backfill_for_identity(inner.clone(), identity, true);
3175    } else if let Some(identity) = opportunistic_refresh_identity {
3176        spawn_opportunistic_session_history_backfill_for_identity(inner.clone(), identity);
3177    }
3178    Ok(())
3179}
3180
3181fn console_event_should_refresh_session_history(frame: &NewConsoleFrame) -> bool {
3182    matches!(
3183        frame.kind.as_str(),
3184        "interaction_complete" | "interaction_failed" | "message_delivery_failed"
3185    ) || frame.session_id.is_some()
3186}
3187
3188fn console_event_should_start_session_history_backfill(frame: &NewConsoleFrame) -> bool {
3189    if frame.identity == SYSTEM_EVENT_IDENTITY {
3190        return false;
3191    }
3192    matches!(
3193        frame.kind.as_str(),
3194        "turn_started"
3195            | "run_started"
3196            | "reasoning_complete"
3197            | "tool_call_requested"
3198            | "tool_call"
3199            | "tool_execution_started"
3200            | "text_delta"
3201            | "system_notice"
3202    )
3203}
3204
3205async fn append_and_emit(
3206    inner: &AggregatorInner,
3207    frame: NewConsoleFrame,
3208) -> ConsoleLogResult<AppendOutcome> {
3209    let outcome = inner.store.append_if_absent(frame).await?;
3210    if outcome.disposition == AppendDisposition::Inserted {
3211        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3212            frame: outcome.frame.clone(),
3213        });
3214    }
3215    Ok(outcome)
3216}
3217
3218async fn update_frame_status_and_emit(
3219    inner: &AggregatorInner,
3220    frame_id: &str,
3221    status: ConsoleFrameStatus,
3222) -> ConsoleLogResult<Option<ConsoleFrame>> {
3223    let Some(updated) = inner.store.update_frame_status(frame_id, status).await? else {
3224        return Ok(None);
3225    };
3226    let update_marker = NewConsoleFrame {
3227        id: None,
3228        dedupe_key: format!("frame-update:{}:{}", updated.id, updated.frame_version),
3229        timestamp_ms: updated.updated_at_ms.unwrap_or_else(current_time_ms),
3230        runtime_key: updated.runtime_key.clone(),
3231        identity: updated.identity.clone(),
3232        conversation_id: updated.conversation_id.clone(),
3233        session_id: updated.session_id.clone(),
3234        kind: "frame_updated".to_string(),
3235        status: updated.status,
3236        payload: json!({ "frame": updated.clone() }),
3237        source: ConsoleFrameSource {
3238            kind: ConsoleFrameSourceKind::Synthetic,
3239            source_cursor: None,
3240        },
3241        source_event_id: None,
3242        interaction_id: updated.interaction_id.clone(),
3243        turn_id: updated.turn_id.clone(),
3244        run_id: updated.run_id.clone(),
3245        parent_frame_id: Some(updated.id.clone()),
3246        caused_by_frame_id: Some(updated.id.clone()),
3247    };
3248    let outcome = inner.store.append_if_absent(update_marker).await?;
3249    if outcome.disposition == AppendDisposition::Inserted {
3250        let _ = inner.event_tx.send(ConsoleTimelineEvent::ConsoleFrame {
3251            frame: outcome.frame,
3252        });
3253    }
3254    Ok(Some(updated))
3255}
3256
3257fn frame_from_console_event(
3258    entry: &RuntimeEntry,
3259    envelope: crate::console_contracts::ConsoleIdentityEventEnvelope,
3260) -> NewConsoleFrame {
3261    let turn_id = envelope
3262        .data
3263        .get("turn_id")
3264        .and_then(Value::as_str)
3265        .map(ToString::to_string);
3266    let run_id = envelope
3267        .data
3268        .get("run_id")
3269        .and_then(Value::as_str)
3270        .map(ToString::to_string);
3271    let status = match envelope.event_type.as_str() {
3272        "interaction_started" => ConsoleFrameStatus::Accepted,
3273        "interaction_failed" | "run_failed" => ConsoleFrameStatus::DeliveryFailed,
3274        "interaction_complete" | "run_completed" => ConsoleFrameStatus::Completed,
3275        _ => ConsoleFrameStatus::Delivered,
3276    };
3277    let identity = apply_namespace(&envelope.identity, &entry.identity_namespace);
3278    NewConsoleFrame {
3279        id: Some(envelope.event_id.clone()),
3280        dedupe_key: format!("console-event:{}:{}", entry.runtime_key, envelope.event_id),
3281        timestamp_ms: envelope.timestamp_ms,
3282        runtime_key: entry.runtime_key.clone(),
3283        identity: identity.clone(),
3284        conversation_id: Some(identity),
3285        session_id: envelope
3286            .data
3287            .get("session_id")
3288            .and_then(Value::as_str)
3289            .map(ToString::to_string),
3290        kind: envelope.event_type,
3291        status,
3292        payload: envelope.data,
3293        source: ConsoleFrameSource {
3294            kind: ConsoleFrameSourceKind::ConsoleEvent,
3295            source_cursor: None,
3296        },
3297        source_event_id: Some(envelope.event_id),
3298        interaction_id: envelope.interaction_id,
3299        turn_id,
3300        run_id,
3301        parent_frame_id: None,
3302        caused_by_frame_id: None,
3303    }
3304}
3305
3306#[cfg(test)]
3307fn frame_from_session_history_message(
3308    runtime_key: &str,
3309    identity: &str,
3310    session_id: &str,
3311    offset: usize,
3312    message: Value,
3313) -> Option<NewConsoleFrame> {
3314    frames_from_session_history_message(runtime_key, identity, session_id, offset, message)
3315        .into_iter()
3316        .next()
3317}
3318
3319#[cfg(test)]
3320fn frames_from_session_history_message(
3321    runtime_key: &str,
3322    identity: &str,
3323    session_id: &str,
3324    offset: usize,
3325    message: Value,
3326) -> Vec<NewConsoleFrame> {
3327    frames_from_session_history_message_with_namespace(
3328        runtime_key,
3329        identity,
3330        "",
3331        session_id,
3332        offset,
3333        message,
3334    )
3335}
3336
3337fn frames_from_session_history_message_with_namespace(
3338    runtime_key: &str,
3339    identity: &str,
3340    identity_namespace: &str,
3341    session_id: &str,
3342    offset: usize,
3343    message: Value,
3344) -> Vec<NewConsoleFrame> {
3345    let payload_hash = hash_short(&serde_json::to_string(&message).unwrap_or_default());
3346    let Some(parsed) = serde_json::from_value::<Message>(message.clone()).ok() else {
3347        return Vec::new();
3348    };
3349    if let Message::ToolResults {
3350        results,
3351        created_at,
3352    } = parsed
3353    {
3354        return results
3355            .into_iter()
3356            .enumerate()
3357            .map(|(idx, result)| {
3358                let content = serde_json::to_value(&result.content).unwrap_or(Value::Null);
3359                let result_text = result.text_content();
3360                let tool_use_id = result.tool_use_id.clone();
3361                NewConsoleFrame {
3362                    id: None,
3363                    dedupe_key: format!(
3364                        "session-history:{runtime_key}:{session_id}:{offset}:{idx}:{payload_hash}"
3365                    ),
3366                    timestamp_ms: created_at.timestamp_millis().max(0) as u64,
3367                    runtime_key: runtime_key.to_string(),
3368                    identity: identity.to_string(),
3369                    conversation_id: Some(identity.to_string()),
3370                    session_id: Some(session_id.to_string()),
3371                    kind: "tool_execution_completed".to_string(),
3372                    status: ConsoleFrameStatus::Completed,
3373                    payload: json!({
3374                        "id": tool_use_id,
3375                        "tool_call_id": tool_use_id,
3376                        "result": result_text,
3377                        "content": content,
3378                        "is_error": result.is_error,
3379                        "source_event_type": "session_history",
3380                        "type": "session_history",
3381                    }),
3382                    source: ConsoleFrameSource {
3383                        kind: ConsoleFrameSourceKind::SessionHistory,
3384                        source_cursor: Some(format!("{session_id}:{offset}:{idx}")),
3385                    },
3386                    source_event_id: None,
3387                    interaction_id: None,
3388                    turn_id: None,
3389                    run_id: None,
3390                    parent_frame_id: None,
3391                    caused_by_frame_id: None,
3392                }
3393            })
3394            .collect();
3395    }
3396    let (kind, timestamp_ms, payload) = match &parsed {
3397        Message::User(user) => {
3398            if session_history_user_message_is_scaffold(&message) {
3399                return Vec::new();
3400            }
3401            (
3402                "user_input",
3403                user.created_at.timestamp_millis().max(0) as u64,
3404                json!({
3405                    "content": user.content,
3406                    "message": message,
3407                }),
3408            )
3409        }
3410        Message::Assistant(assistant) => (
3411            "interaction_complete",
3412            assistant.created_at.timestamp_millis().max(0) as u64,
3413            json!({
3414                "result": assistant.content,
3415                "text": assistant.content,
3416                "message": message,
3417                "source_event_type": "session_history",
3418                "type": "session_history",
3419            }),
3420        ),
3421        Message::BlockAssistant(assistant) => {
3422            let text = assistant.text_blocks().collect::<Vec<_>>().join("");
3423            (
3424                "interaction_complete",
3425                assistant.created_at.timestamp_millis().max(0) as u64,
3426                json!({
3427                    "result": text,
3428                    "text": text,
3429                    "message": message,
3430                    "source_event_type": "session_history",
3431                    "type": "session_history",
3432                }),
3433            )
3434        }
3435        Message::SystemNotice(notice) => (
3436            "system_notice",
3437            notice.created_at.timestamp_millis().max(0) as u64,
3438            json!({
3439                "message": message,
3440                "kind": notice.kind,
3441                "render_class": notice.kind.render_class(),
3442                "body": notice.body,
3443                "blocks": notice.blocks,
3444                "source_event_type": "session_history",
3445                "type": "session_history",
3446            }),
3447        ),
3448        Message::System(_) | Message::ToolResults { .. } => return Vec::new(),
3449    };
3450    let mut frames = vec![NewConsoleFrame {
3451        id: None,
3452        dedupe_key: format!("session-history:{runtime_key}:{session_id}:{offset}:{payload_hash}"),
3453        timestamp_ms,
3454        runtime_key: runtime_key.to_string(),
3455        identity: identity.to_string(),
3456        conversation_id: Some(identity.to_string()),
3457        session_id: Some(session_id.to_string()),
3458        kind: kind.to_string(),
3459        status: ConsoleFrameStatus::Completed,
3460        payload,
3461        source: ConsoleFrameSource {
3462            kind: ConsoleFrameSourceKind::SessionHistory,
3463            source_cursor: Some(format!("{session_id}:{offset}")),
3464        },
3465        source_event_id: None,
3466        interaction_id: None,
3467        turn_id: None,
3468        run_id: None,
3469        parent_frame_id: None,
3470        caused_by_frame_id: None,
3471    }];
3472    if let Message::BlockAssistant(assistant) = &parsed {
3473        frames.extend(spawn_initial_message_frames_from_assistant(
3474            runtime_key,
3475            identity,
3476            identity_namespace,
3477            session_id,
3478            offset,
3479            assistant,
3480            &payload_hash,
3481        ));
3482    }
3483    frames
3484}
3485
3486fn spawn_initial_message_frames_from_assistant(
3487    runtime_key: &str,
3488    parent_identity: &str,
3489    identity_namespace: &str,
3490    session_id: &str,
3491    offset: usize,
3492    assistant: &meerkat_core::types::BlockAssistantMessage,
3493    payload_hash: &str,
3494) -> Vec<NewConsoleFrame> {
3495    let mut frames = Vec::new();
3496    for (tool_idx, tool) in assistant.tool_calls().enumerate() {
3497        if tool.name != "mob_spawn_member" && tool.name != "spawn_member" {
3498            continue;
3499        }
3500        let Ok(args) = serde_json::from_str::<Value>(tool.args.get()) else {
3501            continue;
3502        };
3503        for (spawn_idx, (target_identity, initial_message)) in
3504            spawn_initial_messages_from_tool_args(&args)
3505                .into_iter()
3506                .enumerate()
3507        {
3508            let target_identity = apply_namespace(&target_identity, identity_namespace);
3509            let message_content = match &initial_message {
3510                Value::String(text) => json!([
3511                    {
3512                        "type": "text",
3513                        "text": text,
3514                    }
3515                ]),
3516                other => other.clone(),
3517            };
3518            let message = match &initial_message {
3519                Value::String(text) => json!({
3520                    "role": "user",
3521                    "content": text,
3522                    "created_at": assistant.created_at,
3523                }),
3524                other => json!({
3525                    "role": "user",
3526                    "content": other,
3527                    "created_at": assistant.created_at,
3528                }),
3529            };
3530            frames.push(NewConsoleFrame {
3531                id: None,
3532                dedupe_key: format!(
3533                    "session-history:{runtime_key}:{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}:{payload_hash}"
3534                ),
3535                timestamp_ms: assistant.created_at.timestamp_millis().max(0) as u64,
3536                runtime_key: runtime_key.to_string(),
3537                identity: target_identity.clone(),
3538                conversation_id: Some(target_identity),
3539                session_id: None,
3540                kind: "user_input".to_string(),
3541                status: ConsoleFrameStatus::Completed,
3542                payload: json!({
3543                    "content": message_content,
3544                    "message": message,
3545                    "source_event_type": "session_history_spawn_initial_message",
3546                    "type": "session_history",
3547                    "tool_call_id": tool.id,
3548                    "parent_identity": parent_identity,
3549                    "via_tool": tool.name,
3550                }),
3551                source: ConsoleFrameSource {
3552                    kind: ConsoleFrameSourceKind::SessionHistory,
3553                    source_cursor: Some(format!(
3554                        "{session_id}:{offset}:spawn-initial:{tool_idx}:{spawn_idx}"
3555                    )),
3556                },
3557                source_event_id: None,
3558                interaction_id: None,
3559                turn_id: None,
3560                run_id: None,
3561                parent_frame_id: None,
3562                caused_by_frame_id: None,
3563            });
3564        }
3565    }
3566    frames
3567}
3568
3569fn spawn_initial_messages_from_tool_args(args: &Value) -> Vec<(String, Value)> {
3570    let mut messages = Vec::new();
3571    if let Some(message) = spawn_initial_message_from_record(args) {
3572        messages.push(message);
3573    }
3574    if let Some(specs) = args.get("specs").and_then(Value::as_array) {
3575        for spec in specs {
3576            if let Some(message) = spawn_initial_message_from_record(spec) {
3577                messages.push(message);
3578            }
3579        }
3580    }
3581    messages
3582}
3583
3584fn spawn_initial_message_from_record(record: &Value) -> Option<(String, Value)> {
3585    let target_identity = record
3586        .get("member_id")
3587        .or_else(|| record.get("agent_identity"))
3588        .and_then(Value::as_str)?
3589        .trim();
3590    if target_identity.is_empty() {
3591        return None;
3592    }
3593    let initial_message = record
3594        .get("initial_message")
3595        .or_else(|| record.get("task"))?
3596        .clone();
3597    Some((target_identity.to_string(), initial_message))
3598}
3599
3600fn session_history_user_message_is_scaffold(message: &Value) -> bool {
3601    message
3602        .get("content")
3603        .is_some_and(scaffold_message_content_is_noise)
3604}
3605
3606fn scaffold_message_content_is_noise(value: &Value) -> bool {
3607    match value {
3608        Value::String(text) => scaffold_message_text_is_noise(text),
3609        Value::Array(items) => items.iter().any(scaffold_message_content_is_noise),
3610        Value::Object(map) => ["text", "content", "message"]
3611            .iter()
3612            .filter_map(|key| map.get(*key))
3613            .any(scaffold_message_content_is_noise),
3614        _ => false,
3615    }
3616}
3617
3618fn scaffold_message_text_is_noise(text: &str) -> bool {
3619    let trimmed = text.trim_start();
3620    trimmed.starts_with("[PEER UPDATE]")
3621        || trimmed
3622            .to_ascii_lowercase()
3623            .starts_with("you have been spawned")
3624}
3625
3626fn parse_session_history_watermark(watermark: &str, session_id: &str) -> Option<usize> {
3627    let rest = watermark.strip_prefix(session_id)?.strip_prefix(':')?;
3628    rest.split(':').next()?.parse().ok()
3629}
3630
3631fn format_session_history_watermark(session_id: &str, offset: usize, checked_at_ms: u64) -> String {
3632    format!("{session_id}:{offset}:{checked_at_ms}")
3633}
3634
3635fn session_history_watermark_is_fresh(watermark: &str, session_id: &str, now_ms: u64) -> bool {
3636    let Some(checked_at_ms) = watermark
3637        .rsplit_once(':')
3638        .and_then(|(_, checked_at_ms)| checked_at_ms.parse::<u64>().ok())
3639    else {
3640        return false;
3641    };
3642    let offset = parse_session_history_watermark(watermark, session_id).unwrap_or(0);
3643    let ttl_ms = if offset > 0 {
3644        SESSION_HISTORY_GROWING_REFRESH_TTL_MS
3645    } else {
3646        SESSION_HISTORY_REFRESH_TTL_MS
3647    };
3648    now_ms.saturating_sub(checked_at_ms) < ttl_ms
3649}
3650
3651async fn history_frame_has_existing_counterpart(
3652    inner: &AggregatorInner,
3653    frame: &NewConsoleFrame,
3654) -> ConsoleLogResult<bool> {
3655    let fingerprint = transcript_fingerprint(&frame.kind, &frame.payload);
3656    let Some(fingerprint) = fingerprint else {
3657        return Ok(false);
3658    };
3659    let assistant_terminal = assistant_terminal_fingerprint(&frame.kind, &frame.payload).is_some();
3660    let mut delta_text_by_turn = BTreeMap::<String, String>::new();
3661    let mut after = None;
3662    loop {
3663        let page = inner
3664            .store
3665            .query_frames(ConsoleTimelineQuery {
3666                identity: Some(frame.identity.clone()),
3667                conversation_id: frame.conversation_id.clone(),
3668                after,
3669                limit: 1_000,
3670            })
3671            .await?;
3672        for existing in &page.frames {
3673            let same_session = existing.session_id == frame.session_id
3674                || existing.session_id.is_none()
3675                || frame.session_id.is_none();
3676            if existing.source.kind == ConsoleFrameSourceKind::SessionHistory || !same_session {
3677                continue;
3678            }
3679            if transcript_fingerprint(&existing.kind, &existing.payload).as_ref()
3680                == Some(&fingerprint)
3681            {
3682                return Ok(true);
3683            }
3684            if assistant_terminal
3685                && let Some(delta) = text_delta_payload_text(&existing.kind, &existing.payload)
3686            {
3687                let turn_key = existing
3688                    .interaction_id
3689                    .as_deref()
3690                    .or(existing.turn_id.as_deref())
3691                    .or(existing.run_id.as_deref())
3692                    .unwrap_or("session");
3693                let aggregated = delta_text_by_turn.entry(turn_key.to_string()).or_default();
3694                aggregated.push_str(delta);
3695                if normalize_transcript_fingerprint_text(aggregated) == fingerprint {
3696                    return Ok(true);
3697                }
3698            }
3699        }
3700        if page.frames.is_empty() || page.next_cursor.is_none() {
3701            return Ok(false);
3702        }
3703        after = page.next_cursor;
3704    }
3705}
3706
3707fn session_history_watermark_runtime_key(runtime_key: &str, session_id: &str) -> String {
3708    format!("{runtime_key}:session-history:{session_id}")
3709}
3710
3711fn transcript_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3712    match kind {
3713        "user_input" | "interaction_started" => payload
3714            .get("content")
3715            .map(stable_value_fingerprint)
3716            .or_else(|| payload.get("message").map(stable_value_fingerprint)),
3717        "tool_execution_completed" => tool_result_fingerprint(payload),
3718        "text_delta" => {
3719            text_delta_payload_text(kind, payload).map(normalize_transcript_fingerprint_text)
3720        }
3721        "text_complete" | "interaction_complete" | "run_completed" => {
3722            assistant_terminal_fingerprint(kind, payload)
3723        }
3724        _ => None,
3725    }
3726}
3727
3728fn tool_result_fingerprint(payload: &Value) -> Option<String> {
3729    let id = payload
3730        .get("tool_call_id")
3731        .or_else(|| payload.get("id"))
3732        .and_then(Value::as_str)
3733        .unwrap_or("");
3734    let result = payload
3735        .get("result")
3736        .or_else(|| payload.get("content"))
3737        .map(stable_value_fingerprint)?;
3738    Some(format!("{id}:{result}"))
3739}
3740
3741fn assistant_terminal_fingerprint(kind: &str, payload: &Value) -> Option<String> {
3742    match kind {
3743        "text_complete" | "interaction_complete" | "run_completed" => payload
3744            .get("text")
3745            .or_else(|| payload.get("result"))
3746            .or_else(|| payload.get("content"))
3747            .map(stable_value_fingerprint),
3748        _ => None,
3749    }
3750}
3751
3752fn text_delta_payload_text<'a>(kind: &str, payload: &'a Value) -> Option<&'a str> {
3753    if kind != "text_delta" {
3754        return None;
3755    }
3756    payload
3757        .get("delta")
3758        .or_else(|| payload.get("text"))
3759        .or_else(|| payload.get("content"))
3760        .and_then(Value::as_str)
3761        .or_else(|| payload.as_str())
3762}
3763
3764fn stable_value_fingerprint(value: &Value) -> String {
3765    if let Some(text) = content_value_text(value) {
3766        return normalize_transcript_fingerprint_text(&text);
3767    }
3768    match value {
3769        Value::String(text) => normalize_transcript_fingerprint_text(text),
3770        other => serde_json::to_string(other).unwrap_or_default(),
3771    }
3772}
3773
3774fn content_value_text(value: &Value) -> Option<String> {
3775    match value {
3776        Value::String(text) => Some(text.clone()),
3777        Value::Array(items) => {
3778            let text = items
3779                .iter()
3780                .filter_map(content_value_text)
3781                .collect::<String>();
3782            (!text.is_empty()).then_some(text)
3783        }
3784        Value::Object(map) => ["text", "content", "message", "blocks"]
3785            .iter()
3786            .filter_map(|key| map.get(*key))
3787            .find_map(content_value_text),
3788        _ => None,
3789    }
3790}
3791
3792fn normalize_transcript_fingerprint_text(text: &str) -> String {
3793    let trimmed = text.trim();
3794    trimmed
3795        .strip_prefix("[EVENT via rpc] ")
3796        .unwrap_or(trimmed)
3797        .trim()
3798        .to_string()
3799}
3800
3801#[derive(Debug, Clone, Copy)]
3802enum CachedIdentityVisibility {
3803    Visible,
3804    Hidden,
3805    Missing,
3806}
3807
3808async fn frame_is_visible(
3809    inner: &AggregatorInner,
3810    frame: &ConsoleFrame,
3811    allow_historical_identity: bool,
3812    identity_records: &[ConsoleIdentityRecord],
3813) -> ConsoleLogResult<bool> {
3814    let mut identity_visibility_cache = HashMap::new();
3815    frame_is_visible_cached(
3816        inner,
3817        frame,
3818        allow_historical_identity,
3819        &mut identity_visibility_cache,
3820        identity_records,
3821    )
3822    .await
3823}
3824
3825async fn frame_is_visible_cached(
3826    inner: &AggregatorInner,
3827    frame: &ConsoleFrame,
3828    allow_historical_identity: bool,
3829    identity_visibility_cache: &mut HashMap<(String, String), CachedIdentityVisibility>,
3830    identity_records: &[ConsoleIdentityRecord],
3831) -> ConsoleLogResult<bool> {
3832    let entry = {
3833        let entries = inner
3834            .runtimes
3835            .read()
3836            .map_err(|_| runtime_registry_lock_error())?;
3837        if entries.is_empty() {
3838            return Ok(true);
3839        }
3840        let Some(entry) = entries.get(&frame.runtime_key) else {
3841            return Ok(false);
3842        };
3843        entry.clone()
3844    };
3845    if frame.identity != "__console__" {
3846        let cache_key = (frame.runtime_key.clone(), frame.identity.clone());
3847        let identity_visibility =
3848            if let Some(cached) = identity_visibility_cache.get(&cache_key).copied() {
3849                cached
3850            } else {
3851                let runtime_member_id = strip_namespace(&frame.identity, &entry.identity_namespace)
3852                    .unwrap_or_else(|| frame.identity.clone());
3853                let visibility = match identity_records.iter().find(|record| {
3854                    record.runtime_key == frame.runtime_key
3855                        && (record.identity == frame.identity
3856                            || record.runtime_member_id == runtime_member_id)
3857                }) {
3858                    Some(record) => {
3859                        if console_identity_record_visible(&entry, record).await {
3860                            CachedIdentityVisibility::Visible
3861                        } else {
3862                            CachedIdentityVisibility::Hidden
3863                        }
3864                    }
3865                    None => CachedIdentityVisibility::Missing,
3866                };
3867                identity_visibility_cache.insert(cache_key, visibility);
3868                visibility
3869            };
3870        match identity_visibility {
3871            CachedIdentityVisibility::Visible => {
3872                if frame_matches_hidden_member(&entry, frame).await {
3873                    return Ok(false);
3874                }
3875            }
3876            CachedIdentityVisibility::Hidden => return Ok(false),
3877            CachedIdentityVisibility::Missing => {
3878                if frame_matches_hidden_member(&entry, frame).await {
3879                    return Ok(false);
3880                }
3881                return Ok(
3882                    allow_historical_identity && entry.visibility_policy.frame_visible(frame)
3883                );
3884            }
3885        }
3886    }
3887    Ok(entry.visibility_policy.frame_visible(frame))
3888}
3889
3890async fn identity_record_for_member(
3891    entry: &RuntimeEntry,
3892    handle: &MobHandle,
3893    member: &MobMemberListEntry,
3894) -> Option<ConsoleIdentityRecord> {
3895    let runtime_member_id = member.agent_identity.to_string();
3896    let durable_identity = member
3897        .labels
3898        .get("agent_identity")
3899        .filter(|value| !value.trim().is_empty())
3900        .map_or(runtime_member_id.as_str(), String::as_str);
3901    let identity = apply_namespace(durable_identity, &entry.identity_namespace);
3902    let addressable = member_is_addressable(member);
3903    let visibility = if member.state == meerkat_mob::MemberState::Retiring {
3904        ConsoleVisibility::RetiredReadable
3905    } else if addressable {
3906        ConsoleVisibility::Addressable
3907    } else {
3908        ConsoleVisibility::Hidden
3909    };
3910    let session_id = handle
3911        .resolve_bridge_session_id_observation(&member.agent_identity)
3912        .await
3913        .map(|sid| sid.to_string());
3914    let display_name = member
3915        .labels
3916        .get("display_name")
3917        .cloned()
3918        .unwrap_or_else(|| runtime_member_id.clone());
3919    let mut labels = member.labels.clone();
3920    labels
3921        .entry("role".to_string())
3922        .or_insert_with(|| member.role.to_string());
3923    Some(ConsoleIdentityRecord {
3924        identity,
3925        display_name,
3926        runtime_key: entry.runtime_key.clone(),
3927        runtime_member_id,
3928        session_id,
3929        visibility,
3930        addressable,
3931        health: if addressable {
3932            "ready"
3933        } else {
3934            "hidden_by_policy"
3935        }
3936        .to_string(),
3937        topology_peers: member.wired_to.iter().map(ToString::to_string).collect(),
3938        labels,
3939    })
3940}
3941
3942async fn identity_record_for_resolved_member(
3943    resolved: &ResolvedConsoleMember,
3944) -> Option<ConsoleIdentityRecord> {
3945    let mut record =
3946        identity_record_for_member(&resolved.entry, &resolved.handle, &resolved.member).await?;
3947    if *resolved.entry.runtime.handle().mob_id() != resolved.source_mob_id {
3948        record
3949            .labels
3950            .entry("source_mob_id".to_string())
3951            .or_insert_with(|| resolved.source_mob_id.clone());
3952    }
3953    Some(record)
3954}
3955
3956fn identity_record_for_status(
3957    entry: &RuntimeEntry,
3958    status: &crate::identity_first::IdentityStatus,
3959) -> ConsoleIdentityRecord {
3960    let identity = apply_namespace(status.identity.as_str(), &entry.identity_namespace);
3961    let runtime_member_id = status
3962        .agent_runtime_id
3963        .as_ref()
3964        .map(crate::identity_first::AgentRuntimeId::as_str)
3965        .unwrap_or_else(|| status.identity.as_str())
3966        .to_string();
3967    let addressable = status.addressability
3968        == crate::identity_first::AgentAddressability::Addressable
3969        && matches!(
3970            status.state,
3971            crate::identity_first::IdentityLifecycleState::Active
3972                | crate::identity_first::IdentityLifecycleState::Dormant
3973                | crate::identity_first::IdentityLifecycleState::Uninitialized
3974        );
3975    let visibility = match status.state {
3976        crate::identity_first::IdentityLifecycleState::Retiring => {
3977            ConsoleVisibility::RetiredReadable
3978        }
3979        crate::identity_first::IdentityLifecycleState::Broken
3980        | crate::identity_first::IdentityLifecycleState::Suspended => {
3981            ConsoleVisibility::Unreachable
3982        }
3983        _ if addressable => ConsoleVisibility::Addressable,
3984        _ => ConsoleVisibility::Hidden,
3985    };
3986    let health = match status.state {
3987        crate::identity_first::IdentityLifecycleState::Active => "ready",
3988        crate::identity_first::IdentityLifecycleState::Dormant => "dormant",
3989        crate::identity_first::IdentityLifecycleState::Uninitialized => "uninitialized",
3990        crate::identity_first::IdentityLifecycleState::Broken => "broken",
3991        crate::identity_first::IdentityLifecycleState::Suspended => "suspended",
3992        crate::identity_first::IdentityLifecycleState::Retiring => "retired",
3993    }
3994    .to_string();
3995    let display_name = status
3996        .display_name
3997        .as_ref()
3998        .map(crate::identity_first::DisplayName::as_str)
3999        .unwrap_or_else(|| status.identity.as_str())
4000        .to_string();
4001    ConsoleIdentityRecord {
4002        identity,
4003        display_name,
4004        runtime_key: entry.runtime_key.clone(),
4005        runtime_member_id,
4006        session_id: status.session_id.as_ref().map(ToString::to_string),
4007        visibility,
4008        addressable,
4009        health,
4010        topology_peers: Vec::new(),
4011        labels: status.labels.clone(),
4012    }
4013}
4014
4015pub(crate) fn is_implicit_delegate_member(
4016    role: &str,
4017    labels: &std::collections::BTreeMap<String, String>,
4018) -> bool {
4019    role.eq_ignore_ascii_case("delegate") && labels.contains_key("source_mob_id")
4020}
4021
4022fn member_is_addressable(member: &MobMemberListEntry) -> bool {
4023    member
4024        .labels
4025        .get("addressable")
4026        .map(|value| !value.eq_ignore_ascii_case("false"))
4027        .unwrap_or(true)
4028}
4029
4030fn apply_namespace(identity: &str, namespace: &str) -> String {
4031    let namespace = namespace.trim().trim_matches('/');
4032    if namespace.is_empty() || identity.starts_with(&format!("{namespace}/")) {
4033        identity.to_string()
4034    } else {
4035        format!("{namespace}/{identity}")
4036    }
4037}
4038
4039fn strip_namespace(identity: &str, namespace: &str) -> Option<String> {
4040    let namespace = namespace.trim().trim_matches('/');
4041    if namespace.is_empty() {
4042        return Some(identity.to_string());
4043    }
4044    identity
4045        .strip_prefix(namespace)
4046        .and_then(|rest| rest.strip_prefix('/'))
4047        .map(ToString::to_string)
4048}
4049
4050fn namespace_match_candidates(identity: &str, namespace: &str) -> Vec<String> {
4051    let namespace = namespace.trim().trim_matches('/');
4052    if namespace.is_empty() {
4053        return vec![identity.to_string()];
4054    }
4055    let mut candidates = Vec::new();
4056    if let Some(stripped) = strip_namespace(identity, namespace) {
4057        candidates.push(stripped);
4058        candidates.push(identity.to_string());
4059    }
4060    candidates.sort();
4061    candidates.dedup();
4062    candidates
4063}
4064
4065fn requested_identity_is_runtime_member_alias(identity: &str, namespace: &str) -> bool {
4066    namespace_match_candidates(identity, namespace)
4067        .iter()
4068        .any(|candidate| candidate.starts_with("rt:"))
4069}
4070
4071fn requested_identity_has_runtime_member_alias(identity: &str, entries: &[RuntimeEntry]) -> bool {
4072    entries.iter().any(|entry| {
4073        requested_identity_is_runtime_member_alias(identity, &entry.identity_namespace)
4074    })
4075}
4076
4077fn validate_send_request(request: &ConsoleSendRequest) -> Result<(), ConsoleSendError> {
4078    if request.identity.trim().is_empty() {
4079        return Err(ConsoleSendError::InvalidRequest(
4080            "identity must be non-empty".to_string(),
4081        ));
4082    }
4083    if request.origin.trim().is_empty() {
4084        return Err(ConsoleSendError::InvalidRequest(
4085            "origin must be non-empty".to_string(),
4086        ));
4087    }
4088    if request.idempotency_key.trim().is_empty() {
4089        return Err(ConsoleSendError::InvalidRequest(
4090            "idempotency_key must be non-empty".to_string(),
4091        ));
4092    }
4093    Ok(())
4094}
4095
4096fn content_input_from_value(value: &Value) -> Result<ContentInput, ConsoleSendError> {
4097    let content: ContentInput = serde_json::from_value(value.clone())
4098        .map_err(|err| ConsoleSendError::InvalidContent(err.to_string()))?;
4099    match &content {
4100        ContentInput::Text(text) if text.trim().is_empty() => Err(
4101            ConsoleSendError::InvalidContent("content must be non-empty".to_string()),
4102        ),
4103        ContentInput::Blocks(blocks) if blocks.is_empty() => Err(ConsoleSendError::InvalidContent(
4104            "content blocks must be non-empty".to_string(),
4105        )),
4106        _ => Ok(content),
4107    }
4108}
4109
4110fn parse_handling_mode(
4111    value: Option<&str>,
4112) -> Result<meerkat_core::types::HandlingMode, ConsoleSendError> {
4113    match value.unwrap_or("queue") {
4114        "queue" => Ok(meerkat_core::types::HandlingMode::Queue),
4115        "steer" => Ok(meerkat_core::types::HandlingMode::Steer),
4116        other => Err(ConsoleSendError::InvalidHandlingMode(other.to_string())),
4117    }
4118}
4119
4120fn accepted_from_frame(frame: &ConsoleFrame) -> ConsoleInteractionAccepted {
4121    ConsoleInteractionAccepted {
4122        interaction_id: frame
4123            .interaction_id
4124            .clone()
4125            .unwrap_or_else(|| format!("console-interaction-{}", hash_short(&frame.dedupe_key))),
4126        identity: frame.identity.clone(),
4127        conversation_id: frame.conversation_id.clone(),
4128        session_id: frame.session_id.clone(),
4129        input_frame_id: frame.id.clone(),
4130        cursor: frame.cursor.clone(),
4131        status: frame.status,
4132    }
4133}
4134
4135fn send_dedupe_key(
4136    runtime_key: &str,
4137    identity: &str,
4138    origin: &str,
4139    idempotency_key: &str,
4140) -> String {
4141    format!("send:{runtime_key}:{identity}:{origin}:{idempotency_key}")
4142}
4143
4144fn send_request_fingerprint(origin: &str, content: &Value, handling_mode: &str) -> String {
4145    let content_json = serde_json::to_string(content).unwrap_or_default();
4146    hash_short(&format!("{origin}\n{handling_mode}\n{content_json}"))
4147}
4148
4149fn hash_short(value: &str) -> String {
4150    let mut hasher = Sha256::new();
4151    hasher.update(value.as_bytes());
4152    let digest = hasher.finalize();
4153    to_hex(&digest[..8])
4154}
4155
4156fn to_hex(bytes: &[u8]) -> String {
4157    const HEX: &[u8; 16] = b"0123456789abcdef";
4158    let mut out = String::with_capacity(bytes.len() * 2);
4159    for byte in bytes {
4160        out.push(HEX[(byte >> 4) as usize] as char);
4161        out.push(HEX[(byte & 0x0f) as usize] as char);
4162    }
4163    out
4164}
4165
4166fn current_time_ms() -> u64 {
4167    match SystemTime::now().duration_since(UNIX_EPOCH) {
4168        Ok(duration) => duration.as_millis() as u64,
4169        Err(_) => 0,
4170    }
4171}
4172
4173fn runtime_registry_lock_error() -> ConsoleLogError {
4174    Box::new(std::io::Error::other(
4175        "console runtime registry lock poisoned",
4176    ))
4177}
4178
4179#[cfg(test)]
4180#[allow(clippy::expect_used, clippy::large_futures, clippy::panic)]
4181mod tests {
4182    use std::sync::Arc;
4183    use std::sync::atomic::{AtomicUsize, Ordering};
4184    use std::time::Duration;
4185    use std::time::Instant;
4186
4187    use futures::StreamExt;
4188    use meerkat::{AgentFactory, Config, build_ephemeral_service};
4189    use meerkat_client::types::LlmStream;
4190    use meerkat_client::{LlmClient, LlmDoneOutcome, LlmError, LlmEvent, LlmRequest, TestClient};
4191    use meerkat_core::{
4192        AppendSystemContextRequest, AppendSystemContextResult, CommsRuntime, EventStream,
4193        RunResult, SessionControlError, SessionError, SessionHistoryPage, SessionHistoryQuery,
4194        SessionId, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
4195        SessionServiceHistoryExt, SessionSummary, SessionView, StartTurnRequest, StopReason,
4196        StreamError,
4197    };
4198    use meerkat_mob::{MobDefinition, MobSessionService, MobStorage, SpawnMemberSpec};
4199    use serde_json::json;
4200
4201    use super::*;
4202    use crate::mob_handle_runtime::MobBootstrapSpec;
4203
4204    #[derive(Debug)]
4205    struct HideHiddenNoiseFrames;
4206
4207    impl ConsoleVisibilityPolicy for HideHiddenNoiseFrames {
4208        fn frame_visible(&self, frame: &ConsoleFrame) -> bool {
4209            frame.kind != "hidden_noise"
4210        }
4211    }
4212
4213    #[derive(Debug)]
4214    struct HideRuntimeMemberIdentity(&'static str);
4215
4216    impl ConsoleVisibilityPolicy for HideRuntimeMemberIdentity {
4217        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4218            record.runtime_member_id != self.0
4219        }
4220    }
4221
4222    #[derive(Debug)]
4223    struct HideRuntimeMemberOnly(&'static str);
4224
4225    impl ConsoleVisibilityPolicy for HideRuntimeMemberOnly {
4226        fn member_visible(&self, member: &ConsoleMember) -> bool {
4227            member.agent_identity != self.0
4228        }
4229    }
4230
4231    #[derive(Debug)]
4232    struct HideConsoleIdentity(&'static str);
4233
4234    impl ConsoleVisibilityPolicy for HideConsoleIdentity {
4235        fn identity_visible(&self, record: &ConsoleIdentityRecord) -> bool {
4236            record.identity != self.0
4237        }
4238    }
4239
4240    struct CountingConsoleLogStore {
4241        inner: InMemoryConsoleLogStore,
4242        source_watermark_calls: AtomicUsize,
4243        record_watermark_calls: AtomicUsize,
4244    }
4245
4246    impl CountingConsoleLogStore {
4247        fn new() -> Self {
4248            Self {
4249                inner: InMemoryConsoleLogStore::new(),
4250                source_watermark_calls: AtomicUsize::new(0),
4251                record_watermark_calls: AtomicUsize::new(0),
4252            }
4253        }
4254
4255        fn source_watermark_calls(&self) -> usize {
4256            self.source_watermark_calls.load(Ordering::SeqCst)
4257        }
4258    }
4259
4260    struct SlowTestClient {
4261        delay: Duration,
4262    }
4263
4264    #[async_trait::async_trait]
4265    impl LlmClient for SlowTestClient {
4266        fn project_replay_messages(&self, messages: &[Message]) -> Result<Vec<Message>, LlmError> {
4267            Ok(messages.to_vec())
4268        }
4269
4270        fn stream<'a>(&'a self, _request: &'a LlmRequest) -> LlmStream<'a> {
4271            let delay = self.delay;
4272            let delayed_text = futures::stream::once(async move {
4273                tokio::time::sleep(delay).await;
4274                Ok(LlmEvent::TextDelta {
4275                    delta: "slow ok".to_string(),
4276                    meta: None,
4277                })
4278            });
4279            let done = futures::stream::once(async {
4280                Ok(LlmEvent::Done {
4281                    outcome: LlmDoneOutcome::Success {
4282                        stop_reason: StopReason::EndTurn,
4283                    },
4284                })
4285            });
4286            Box::pin(delayed_text.chain(done))
4287        }
4288
4289        fn provider(&self) -> &'static str {
4290            "slow-test"
4291        }
4292
4293        async fn health_check(&self) -> Result<(), LlmError> {
4294            Ok(())
4295        }
4296    }
4297
4298    #[derive(Clone)]
4299    struct DelayedHistorySessionService {
4300        inner: Arc<dyn MobSessionService>,
4301        delay: Duration,
4302        read_calls: Arc<AtomicUsize>,
4303        active_reads: Arc<AtomicUsize>,
4304        max_active_reads: Arc<AtomicUsize>,
4305    }
4306
4307    impl DelayedHistorySessionService {
4308        fn new(inner: Arc<dyn MobSessionService>, delay: Duration) -> Self {
4309            Self {
4310                inner,
4311                delay,
4312                read_calls: Arc::new(AtomicUsize::new(0)),
4313                active_reads: Arc::new(AtomicUsize::new(0)),
4314                max_active_reads: Arc::new(AtomicUsize::new(0)),
4315            }
4316        }
4317
4318        fn read_calls(&self) -> usize {
4319            self.read_calls.load(Ordering::SeqCst)
4320        }
4321
4322        fn max_active_reads(&self) -> usize {
4323            self.max_active_reads.load(Ordering::SeqCst)
4324        }
4325    }
4326
4327    #[async_trait::async_trait]
4328    impl SessionService for DelayedHistorySessionService {
4329        async fn create_session(
4330            &self,
4331            req: meerkat_core::CreateSessionRequest,
4332        ) -> Result<RunResult, SessionError> {
4333            self.inner.create_session(req).await
4334        }
4335
4336        async fn start_turn(
4337            &self,
4338            id: &SessionId,
4339            req: StartTurnRequest,
4340        ) -> Result<RunResult, SessionError> {
4341            self.inner.start_turn(id, req).await
4342        }
4343
4344        async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
4345            self.inner.interrupt(id).await
4346        }
4347
4348        async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
4349            self.inner.read(id).await
4350        }
4351
4352        async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
4353            self.inner.list(query).await
4354        }
4355
4356        async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
4357            self.inner.archive(id).await
4358        }
4359
4360        async fn subscribe_session_events(
4361            &self,
4362            id: &SessionId,
4363        ) -> Result<EventStream, StreamError> {
4364            SessionService::subscribe_session_events(self.inner.as_ref(), id).await
4365        }
4366    }
4367
4368    #[async_trait::async_trait]
4369    impl SessionServiceCommsExt for DelayedHistorySessionService {
4370        async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CommsRuntime>> {
4371            self.inner.comms_runtime(session_id).await
4372        }
4373    }
4374
4375    #[async_trait::async_trait]
4376    impl SessionServiceControlExt for DelayedHistorySessionService {
4377        async fn append_system_context(
4378            &self,
4379            id: &SessionId,
4380            req: AppendSystemContextRequest,
4381        ) -> Result<AppendSystemContextResult, SessionControlError> {
4382            self.inner.append_system_context(id, req).await
4383        }
4384    }
4385
4386    #[async_trait::async_trait]
4387    impl SessionServiceHistoryExt for DelayedHistorySessionService {
4388        async fn read_history(
4389            &self,
4390            id: &SessionId,
4391            query: SessionHistoryQuery,
4392        ) -> Result<SessionHistoryPage, SessionError> {
4393            self.read_calls.fetch_add(1, Ordering::SeqCst);
4394            let active = self.active_reads.fetch_add(1, Ordering::SeqCst) + 1;
4395            self.max_active_reads.fetch_max(active, Ordering::SeqCst);
4396            tokio::time::sleep(self.delay).await;
4397            let result = self.inner.read_history(id, query).await;
4398            self.active_reads.fetch_sub(1, Ordering::SeqCst);
4399            result
4400        }
4401    }
4402
4403    #[async_trait::async_trait]
4404    impl MobSessionService for DelayedHistorySessionService {
4405        fn supports_persistent_sessions(&self) -> bool {
4406            self.inner.supports_persistent_sessions()
4407        }
4408
4409        fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
4410            self.inner.runtime_adapter()
4411        }
4412
4413        async fn session_belongs_to_mob(
4414            &self,
4415            session_id: &SessionId,
4416            mob_id: &meerkat_mob::MobId,
4417        ) -> bool {
4418            self.inner.session_belongs_to_mob(session_id, mob_id).await
4419        }
4420
4421        async fn cancel_all_checkpointers(&self) {
4422            self.inner.cancel_all_checkpointers().await;
4423        }
4424
4425        async fn rearm_all_checkpointers(&self) {
4426            self.inner.rearm_all_checkpointers().await;
4427        }
4428    }
4429
4430    #[async_trait::async_trait]
4431    impl ConsoleLogStore for CountingConsoleLogStore {
4432        async fn append_if_absent(
4433            &self,
4434            frame: NewConsoleFrame,
4435        ) -> ConsoleLogResult<AppendOutcome> {
4436            self.inner.append_if_absent(frame).await
4437        }
4438
4439        async fn update_frame_status(
4440            &self,
4441            frame_id: &str,
4442            status: ConsoleFrameStatus,
4443        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4444            self.inner.update_frame_status(frame_id, status).await
4445        }
4446
4447        async fn query_frames(
4448            &self,
4449            query: ConsoleTimelineQuery,
4450        ) -> ConsoleLogResult<ConsoleTimelinePage> {
4451            self.inner.query_frames(query).await
4452        }
4453
4454        async fn query_windowed_frames(
4455            &self,
4456            query: ConsoleTimelineWindowQuery,
4457        ) -> ConsoleLogResult<ConsoleTimelineWindowPage> {
4458            self.inner.query_windowed_frames(query).await
4459        }
4460
4461        async fn frame_by_dedupe_key(
4462            &self,
4463            dedupe_key: &str,
4464        ) -> ConsoleLogResult<Option<ConsoleFrame>> {
4465            self.inner.frame_by_dedupe_key(dedupe_key).await
4466        }
4467
4468        async fn latest_cursor(&self) -> ConsoleLogResult<Option<ConsoleCursor>> {
4469            self.inner.latest_cursor().await
4470        }
4471
4472        async fn clear_frames(&self) -> ConsoleLogResult<()> {
4473            self.inner.clear_frames().await
4474        }
4475
4476        async fn record_source_watermark(
4477            &self,
4478            runtime_key: &str,
4479            source_kind: ConsoleFrameSourceKind,
4480            source_cursor: &str,
4481        ) -> ConsoleLogResult<()> {
4482            self.record_watermark_calls.fetch_add(1, Ordering::SeqCst);
4483            self.inner
4484                .record_source_watermark(runtime_key, source_kind, source_cursor)
4485                .await
4486        }
4487
4488        async fn source_watermark(
4489            &self,
4490            runtime_key: &str,
4491            source_kind: ConsoleFrameSourceKind,
4492        ) -> ConsoleLogResult<Option<String>> {
4493            self.source_watermark_calls.fetch_add(1, Ordering::SeqCst);
4494            self.inner.source_watermark(runtime_key, source_kind).await
4495        }
4496    }
4497
4498    async fn build_single_member_runtime() -> UnifiedRuntime {
4499        build_single_member_runtime_with_client(Arc::new(TestClient::default())).await
4500    }
4501
4502    async fn build_single_member_runtime_with_client(client: Arc<dyn LlmClient>) -> UnifiedRuntime {
4503        let definition = MobDefinition::from_toml(
4504            r#"
4505[mob]
4506id = "console-aggregator-perf-test"
4507
4508[profiles.worker]
4509model = "gpt-5.5"
4510external_addressable = true
4511
4512[profiles.worker.tools]
4513comms = true
4514"#,
4515        )
4516        .expect("definition parses");
4517        let runtime = UnifiedRuntime::builder()
4518            .definition(definition)
4519            .default_llm_client(client)
4520            .build()
4521            .await
4522            .expect("runtime builds");
4523        runtime
4524            .spawn(SpawnMemberSpec::from_wire(
4525                "worker".to_string(),
4526                "agent-a".to_string(),
4527                Some("You are agent-a.".into()),
4528                None,
4529                None,
4530            ))
4531            .await
4532            .expect("member spawns");
4533        runtime
4534    }
4535
4536    async fn build_empty_runtime(mob_id: &str) -> UnifiedRuntime {
4537        let definition = MobDefinition::from_toml(&format!(
4538            r#"
4539[mob]
4540id = "{mob_id}"
4541
4542[profiles.worker]
4543model = "gpt-5.5"
4544external_addressable = true
4545
4546[profiles.worker.tools]
4547comms = true
4548"#
4549        ))
4550        .expect("definition parses");
4551        UnifiedRuntime::builder()
4552            .definition(definition)
4553            .default_llm_client(Arc::new(TestClient::default()))
4554            .build()
4555            .await
4556            .expect("runtime builds")
4557    }
4558
4559    async fn build_stress_runtime(
4560        member_count: usize,
4561        history_delay: Duration,
4562    ) -> (
4563        tempfile::TempDir,
4564        Arc<UnifiedRuntime>,
4565        DelayedHistorySessionService,
4566    ) {
4567        let temp_dir = tempfile::tempdir().expect("temp dir");
4568        let session_path = temp_dir.path().join("sessions");
4569        std::fs::create_dir_all(&session_path).expect("session path");
4570        let factory = AgentFactory::new(&session_path).comms(true);
4571        let base_service = Arc::new(build_ephemeral_service(
4572            factory,
4573            Config::default(),
4574            member_count + 8,
4575        ));
4576        let delayed_service = DelayedHistorySessionService::new(base_service, history_delay);
4577        let session_service: Arc<dyn MobSessionService> = Arc::new(delayed_service.clone());
4578        let definition = MobDefinition::from_toml(
4579            r#"
4580[mob]
4581id = "console-aggregator-stress-test"
4582
4583[profiles.worker]
4584model = "gpt-5.5"
4585external_addressable = true
4586
4587[profiles.worker.tools]
4588comms = true
4589"#,
4590        )
4591        .expect("definition parses");
4592        let spec = MobBootstrapSpec::new(definition, MobStorage::in_memory(), session_service)
4593            .with_options(crate::mob_handle_runtime::MobBootstrapOptions {
4594                allow_ephemeral_sessions: true,
4595                notify_orchestrator_on_resume: true,
4596                default_llm_client: Some(Arc::new(TestClient::default())),
4597            });
4598        let runtime = Arc::new(
4599            UnifiedRuntime::bootstrap(
4600                spec,
4601                crate::types::MobKitConfig {
4602                    modules: Vec::new(),
4603                    discovery: crate::types::DiscoverySpec {
4604                        namespace: "stress".to_string(),
4605                        modules: Vec::new(),
4606                    },
4607                    pre_spawn: Vec::new(),
4608                },
4609                Duration::from_secs(5),
4610            )
4611            .await
4612            .expect("runtime boots"),
4613        );
4614        for idx in 0..member_count {
4615            runtime
4616                .spawn(SpawnMemberSpec::from_wire(
4617                    "worker".to_string(),
4618                    format!("agent-{idx}"),
4619                    Some(format!("You are agent-{idx}.").into()),
4620                    None,
4621                    None,
4622                ))
4623                .await
4624                .expect("member spawns");
4625        }
4626        (temp_dir, runtime, delayed_service)
4627    }
4628
4629    fn runtime_entry_for_test(runtime_key: &str, runtime: &UnifiedRuntime) -> RuntimeEntry {
4630        RuntimeEntry {
4631            runtime_key: runtime_key.to_string(),
4632            identity_namespace: "test".to_string(),
4633            runtime: runtime.mob_runtime().clone(),
4634            identity_runtime: runtime.identity_runtime().cloned(),
4635            console_events: runtime.console_events(),
4636            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4637        }
4638    }
4639
4640    fn identity_record_for_test(identity: &str) -> ConsoleIdentityRecord {
4641        ConsoleIdentityRecord {
4642            identity: identity.to_string(),
4643            display_name: identity.to_string(),
4644            runtime_key: "runtime-cache".to_string(),
4645            runtime_member_id: identity.to_string(),
4646            session_id: Some(format!("session-{identity}")),
4647            visibility: ConsoleVisibility::Addressable,
4648            addressable: true,
4649            health: "ready".to_string(),
4650            topology_peers: Vec::new(),
4651            labels: BTreeMap::new(),
4652        }
4653    }
4654
4655    #[test]
4656    fn dedupe_identity_records_preserves_stale_durable_and_live_alias_bindings() {
4657        let mut durable = identity_record_for_test("test/review:singleton");
4658        durable.runtime_member_id = "rt:review:singleton:1".to_string();
4659        durable.session_id = Some("session-stale".to_string());
4660        durable
4661            .labels
4662            .insert("agent_identity".to_string(), "review:singleton".to_string());
4663
4664        let mut live = identity_record_for_test("test/review:singleton");
4665        live.runtime_member_id = "rt:review:singleton:0".to_string();
4666        live.session_id = Some("session-live".to_string());
4667        live.labels
4668            .insert("agent_identity".to_string(), "review:singleton".to_string());
4669
4670        let records = dedupe_identity_records(vec![durable, live]);
4671
4672        assert_eq!(
4673            records.len(),
4674            2,
4675            "list projection must expose stale durable/live alias split-brain instead of picking one record"
4676        );
4677        assert!(
4678            records
4679                .iter()
4680                .any(|record| record.runtime_member_id == "rt:review:singleton:0")
4681        );
4682        assert!(
4683            records
4684                .iter()
4685                .any(|record| record.runtime_member_id == "rt:review:singleton:1")
4686        );
4687    }
4688
4689    #[test]
4690    fn dedupe_identity_records_preserves_duplicate_live_aliases() {
4691        let mut first = identity_record_for_test("test/review:singleton");
4692        first.runtime_member_id = "rt:review:singleton:0".to_string();
4693        first
4694            .labels
4695            .insert("agent_identity".to_string(), "review:singleton".to_string());
4696
4697        let mut second = identity_record_for_test("test/review:singleton");
4698        second.runtime_member_id = "rt:review:singleton:1".to_string();
4699        second
4700            .labels
4701            .insert("agent_identity".to_string(), "review:singleton".to_string());
4702
4703        let records = dedupe_identity_records(vec![first, second]);
4704
4705        assert_eq!(
4706            records.len(),
4707            2,
4708            "duplicate live aliases must remain visible so controls and roster agree on ambiguity"
4709        );
4710    }
4711
4712    #[test]
4713    fn dedupe_identity_records_preserves_same_member_ids_from_different_source_mobs() {
4714        let mut first = identity_record_for_test("test/review:singleton");
4715        first.runtime_member_id = "rt:review:singleton:0".to_string();
4716        first.session_id = None;
4717        first
4718            .labels
4719            .insert("agent_identity".to_string(), "review:singleton".to_string());
4720        first
4721            .labels
4722            .insert("source_mob_id".to_string(), "mob-a".to_string());
4723
4724        let mut second = first.clone();
4725        second
4726            .labels
4727            .insert("source_mob_id".to_string(), "mob-b".to_string());
4728
4729        let records = dedupe_identity_records(vec![first, second]);
4730
4731        assert_eq!(
4732            records.len(),
4733            2,
4734            "same member/session aliases from different source mobs must remain visible"
4735        );
4736    }
4737
4738    #[test]
4739    fn dedupe_identity_records_merges_matching_durable_and_primary_live_rows() {
4740        let durable = identity_record_for_test("test/review:singleton");
4741        let live = durable.clone();
4742
4743        let records = dedupe_identity_records(vec![durable, live]);
4744
4745        assert_eq!(
4746            records.len(),
4747            1,
4748            "healthy durable and primary live projections should not duplicate the roster"
4749        );
4750    }
4751
4752    #[test]
4753    fn stale_durable_record_error_detects_session_split_brain() {
4754        let mut durable = identity_record_for_test("test/review:singleton");
4755        durable.runtime_member_id = "rt:review:singleton:0".to_string();
4756        durable.session_id = Some("session-durable".to_string());
4757
4758        let mut live = durable.clone();
4759        live.session_id = Some("session-live".to_string());
4760
4761        let err = stale_durable_record_error("review:singleton", &durable, &[live])
4762            .expect("session mismatch must be stale split-brain");
4763
4764        assert!(
4765            err.contains("stale durable identity alias"),
4766            "unexpected error: {err}"
4767        );
4768        assert!(
4769            err.contains("session-durable") && err.contains("session-live"),
4770            "error should name both sessions: {err}"
4771        );
4772    }
4773
4774    async fn identity_runtime_for_test(
4775        identities: &[&str],
4776    ) -> Result<Arc<crate::identity_first::IdentityRuntime>, Box<dyn std::error::Error + Send + Sync>>
4777    {
4778        let runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
4779            crate::identity_first::IdentityRuntimeConfig {
4780                continuity_store: Arc::new(
4781                    crate::identity_first::LocalContinuityStore::in_memory()?
4782                ),
4783                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
4784                runtime_instance_id: "console-aggregator-identity-test".to_string(),
4785                has_runtime_store: true,
4786                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
4787                bridge: None,
4788                default_timeout: None,
4789            },
4790        ));
4791        for identity in identities {
4792            let identity = crate::identity_first::AgentIdentity::parse(identity)?;
4793            let record = crate::identity_first::ContinuityRecord {
4794                identity: identity.clone(),
4795                agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(&format!(
4796                    "rt:{}:0",
4797                    identity.as_str()
4798                ))?,
4799                session_id: SessionId::new(),
4800                generation: crate::identity_first::ContinuityGeneration::new(0),
4801                checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
4802            };
4803            runtime
4804                .register(
4805                    crate::identity_first::DurableAgentSpec {
4806                        identity: identity.clone(),
4807                        profile: meerkat_mob::ProfileName::from("worker"),
4808                        addressability: crate::identity_first::AgentAddressability::Addressable,
4809                        display_name: None,
4810                        labels: BTreeMap::new(),
4811                        context: None,
4812                        additional_instructions: Vec::new(),
4813                        initial_message: None,
4814                        runtime_mode_override: None,
4815                    },
4816                    crate::identity_first::IdentityLifecycleState::Active,
4817                    Some(record),
4818                    Some(crate::identity_first::LeaseGrant {
4819                        identity,
4820                        fencing_token: crate::identity_first::FencingToken::new(1),
4821                        ttl: Duration::from_mins(5),
4822                    }),
4823                )
4824                .await;
4825        }
4826        Ok(runtime)
4827    }
4828
4829    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4830    async fn runtime_id_alias_does_not_fall_back_to_durable_rt_identity()
4831    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4832        let runtime = build_empty_runtime("runtime-id-durable-shadow-test").await;
4833        let identity_runtime = identity_runtime_for_test(&["rt:review:singleton:0"]).await?;
4834        let aggregator = MobKitConsoleAggregator::in_memory();
4835        aggregator.register_runtime_handles_with_policy(
4836            "runtime-a",
4837            "",
4838            runtime.mob_runtime().clone(),
4839            Some(identity_runtime),
4840            runtime.console_events(),
4841            Arc::new(AllowAllConsoleVisibilityPolicy),
4842        );
4843
4844        assert!(
4845            aggregator
4846                .inspect_identity("rt:review:singleton:0")
4847                .await?
4848                .is_none(),
4849            "runtime-member-id shaped requests must not inspect a durable rt:* identity"
4850        );
4851        assert!(
4852            !aggregator.retire_identity("rt:review:singleton:0").await?,
4853            "runtime-member-id shaped requests must not retire a durable rt:* identity"
4854        );
4855        let reserve_err = aggregator
4856            .reserve_identity_first_interaction(
4857                ConsoleSendRequest {
4858                    identity: "rt:review:singleton:0".to_string(),
4859                    content: json!("hello"),
4860                    origin: "console:test".to_string(),
4861                    idempotency_key: "rt-shadow-reserve".to_string(),
4862                    handling_mode: Some("queue".to_string()),
4863                },
4864                None,
4865            )
4866            .await
4867            .expect_err(
4868                "runtime-member-id shaped reserve must not fall back to a durable rt:* identity",
4869            );
4870        assert!(
4871            reserve_err.to_string().contains("unknown identity"),
4872            "unexpected reserve error: {reserve_err}"
4873        );
4874        let send_err = aggregator
4875            .send(ConsoleSendRequest {
4876                identity: "rt:review:singleton:0".to_string(),
4877                content: json!("hello"),
4878                origin: "console:test".to_string(),
4879                idempotency_key: "rt-shadow-send".to_string(),
4880                handling_mode: Some("queue".to_string()),
4881            })
4882            .await
4883            .expect_err(
4884                "runtime-member-id shaped send must not be blocked by a durable rt:* identity",
4885            );
4886        assert!(
4887            send_err.to_string().contains("unknown identity"),
4888            "unexpected send error: {send_err}"
4889        );
4890        let blob_err = match aggregator
4891            .binary_blob_store_for_identity("rt:review:singleton:0")
4892            .await
4893        {
4894            Ok(_) => panic!(
4895                "runtime-member-id shaped blob lookup must not fall back to durable rt:* identity"
4896            ),
4897            Err(err) => err,
4898        };
4899        assert!(
4900            blob_err.to_string().contains("unknown identity"),
4901            "unexpected blob error: {blob_err}"
4902        );
4903
4904        let _ = runtime.mob_handle().stop().await;
4905        Ok(())
4906    }
4907
4908    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4909    async fn identity_record_uses_durable_agent_identity_label_for_identity_first_members()
4910    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4911        let runtime = build_empty_runtime("identity-label-test").await;
4912        let mut labels = BTreeMap::new();
4913        labels.insert(
4914            "agent_identity".to_string(),
4915            "channel:C0SMOKEOB3".to_string(),
4916        );
4917        labels.insert("display_name".to_string(), "C0SMOKEOB3".to_string());
4918        runtime
4919            .spawn(
4920                SpawnMemberSpec::from_wire(
4921                    "worker".to_string(),
4922                    "rt:channel:C0SMOKEOB3:0".to_string(),
4923                    Some("You are C0SMOKEOB3.".into()),
4924                    None,
4925                    None,
4926                )
4927                .with_labels(labels),
4928            )
4929            .await
4930            .expect("member spawns");
4931
4932        let entry = RuntimeEntry {
4933            runtime_key: "runtime-a".to_string(),
4934            identity_namespace: String::new(),
4935            runtime: runtime.mob_runtime().clone(),
4936            identity_runtime: runtime.identity_runtime().cloned(),
4937            console_events: runtime.console_events(),
4938            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
4939        };
4940        let aggregator = MobKitConsoleAggregator::in_memory();
4941        aggregator
4942            .inner
4943            .runtimes
4944            .write()
4945            .expect("runtime registry")
4946            .insert("runtime-a".to_string(), entry);
4947
4948        let records = aggregator.list_identities_fresh().await?;
4949        let record = records
4950            .iter()
4951            .find(|record| record.identity == "channel:C0SMOKEOB3")
4952            .expect("durable identity is exposed");
4953        assert_eq!(record.runtime_member_id, "rt:channel:C0SMOKEOB3:0");
4954
4955        let inspection = aggregator
4956            .inspect_identity("channel:C0SMOKEOB3")
4957            .await?
4958            .expect("durable identity resolves back to runtime member");
4959        assert_eq!(inspection.identity.identity, "channel:C0SMOKEOB3");
4960        assert_eq!(
4961            inspection.identity.runtime_member_id,
4962            "rt:channel:C0SMOKEOB3:0"
4963        );
4964
4965        let _ = runtime.mob_handle().stop().await;
4966        Ok(())
4967    }
4968
4969    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4970    async fn member_visibility_policy_hides_live_alias_from_aggregator_controls()
4971    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
4972        let runtime = build_empty_runtime("identity-member-hidden-policy-test").await;
4973        let mut labels = BTreeMap::new();
4974        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4975        runtime
4976            .spawn(
4977                SpawnMemberSpec::from_wire(
4978                    "worker".to_string(),
4979                    "rt:review:singleton:0".to_string(),
4980                    Some("You are the Review Agent.".into()),
4981                    None,
4982                    None,
4983                )
4984                .with_labels(labels),
4985            )
4986            .await?;
4987        let hidden_session_id = runtime
4988            .mob_handle()
4989            .resolve_bridge_session_id_observation(&meerkat_mob::ids::MeerkatId::from(
4990                "rt:review:singleton:0",
4991            ))
4992            .await
4993            .map(|sid| sid.to_string());
4994        let mut duplicate_labels = BTreeMap::new();
4995        duplicate_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
4996        runtime
4997            .spawn(
4998                SpawnMemberSpec::from_wire(
4999                    "worker".to_string(),
5000                    "rt:review:singleton:1".to_string(),
5001                    Some("You are a visible duplicate Review Agent.".into()),
5002                    None,
5003                    None,
5004                )
5005                .with_labels(duplicate_labels),
5006            )
5007            .await?;
5008
5009        let identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
5010        let aggregator = MobKitConsoleAggregator::in_memory();
5011        aggregator.register_runtime_handles_with_policy(
5012            "runtime-a",
5013            "",
5014            runtime.mob_runtime().clone(),
5015            Some(identity_runtime),
5016            runtime.console_events(),
5017            Arc::new(HideRuntimeMemberOnly("rt:review:singleton:0")),
5018        );
5019
5020        assert!(
5021            aggregator
5022                .inspect_identity("review:singleton")
5023                .await?
5024                .is_none(),
5025            "member-hidden live alias must not inspect through aggregator"
5026        );
5027        let records = aggregator.list_identities_fresh().await?;
5028        assert!(
5029            records
5030                .iter()
5031                .all(|record| record.identity != "review:singleton"),
5032            "member-hidden durable/live alias must not appear in identity list: {records:#?}"
5033        );
5034        let appended = aggregator
5035            .store()
5036            .append_if_absent(NewConsoleFrame {
5037                id: None,
5038                dedupe_key: "member-hidden-frame".to_string(),
5039                timestamp_ms: 1,
5040                runtime_key: "runtime-a".to_string(),
5041                identity: "review:singleton".to_string(),
5042                conversation_id: Some("review:singleton".to_string()),
5043                session_id: hidden_session_id,
5044                kind: "text_delta".to_string(),
5045                status: ConsoleFrameStatus::Delivered,
5046                payload: json!({ "delta": "hidden" }),
5047                source: ConsoleFrameSource {
5048                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5049                    source_cursor: None,
5050                },
5051                source_event_id: Some("member-hidden-frame".to_string()),
5052                interaction_id: None,
5053                turn_id: None,
5054                run_id: None,
5055                parent_frame_id: None,
5056                caused_by_frame_id: None,
5057            })
5058            .await?;
5059        assert!(
5060            !aggregator
5061                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5062                    frame: appended.frame.clone()
5063                })
5064                .await,
5065            "member-hidden live alias must not pass timeline event visibility"
5066        );
5067        let identity_only = aggregator
5068            .store()
5069            .append_if_absent(NewConsoleFrame {
5070                id: None,
5071                dedupe_key: "member-hidden-identity-only-frame".to_string(),
5072                timestamp_ms: 2,
5073                runtime_key: "runtime-a".to_string(),
5074                identity: "review:singleton".to_string(),
5075                conversation_id: Some("review:singleton".to_string()),
5076                session_id: None,
5077                kind: "text_delta".to_string(),
5078                status: ConsoleFrameStatus::Delivered,
5079                payload: json!({ "delta": "identity-only-hidden" }),
5080                source: ConsoleFrameSource {
5081                    kind: ConsoleFrameSourceKind::ConsoleEvent,
5082                    source_cursor: None,
5083                },
5084                source_event_id: Some("member-hidden-identity-only-frame".to_string()),
5085                interaction_id: None,
5086                turn_id: None,
5087                run_id: None,
5088                parent_frame_id: None,
5089                caused_by_frame_id: None,
5090            })
5091            .await?;
5092        assert!(
5093            !aggregator
5094                .timeline_event_visible(&ConsoleTimelineEvent::ConsoleFrame {
5095                    frame: identity_only.frame.clone()
5096                })
5097                .await,
5098            "member-hidden durable alias must not leak identity-only timeline frames"
5099        );
5100        let page = aggregator
5101            .query_timeline(ConsoleTimelineQuery {
5102                identity: Some("review:singleton".to_string()),
5103                limit: 10,
5104                ..ConsoleTimelineQuery::default()
5105            })
5106            .await?;
5107        assert!(
5108            page.frames.is_empty(),
5109            "member-hidden live alias must not leak through explicit timeline query: {page:#?}"
5110        );
5111        assert!(
5112            !aggregator.retire_identity("review:singleton").await?,
5113            "member-hidden live alias must not retire through aggregator"
5114        );
5115        let reserve_err = aggregator
5116            .reserve_identity_first_interaction(
5117                ConsoleSendRequest {
5118                    identity: "review:singleton".to_string(),
5119                    content: json!("hello"),
5120                    origin: "console:test".to_string(),
5121                    idempotency_key: "member-hidden-reserve".to_string(),
5122                    handling_mode: Some("queue".to_string()),
5123                },
5124                None,
5125            )
5126            .await
5127            .expect_err("member-hidden durable alias must not reserve identity-first interaction");
5128        assert!(
5129            reserve_err.to_string().contains("unknown identity"),
5130            "unexpected reserve error: {reserve_err}"
5131        );
5132        let send_err = aggregator
5133            .send(ConsoleSendRequest {
5134                identity: "review:singleton".to_string(),
5135                content: json!("hello"),
5136                origin: "console:test".to_string(),
5137                idempotency_key: "member-hidden-send".to_string(),
5138                handling_mode: Some("queue".to_string()),
5139            })
5140            .await
5141            .expect_err("member-hidden live alias must not receive sends");
5142        assert!(
5143            send_err.to_string().contains("unknown identity"),
5144            "unexpected send error: {send_err}"
5145        );
5146        let blob_result = aggregator
5147            .binary_blob_store_for_identity("review:singleton")
5148            .await;
5149        let Err(blob_err) = blob_result else {
5150            panic!("member-hidden live alias must not expose blob store");
5151        };
5152        assert!(
5153            blob_err.to_string().contains("unknown identity"),
5154            "unexpected blob error: {blob_err}"
5155        );
5156
5157        let _ = runtime.mob_handle().stop().await;
5158        Ok(())
5159    }
5160
5161    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5162    async fn explicit_session_backfill_target_resolves_durable_identity_label()
5163    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5164        let runtime = build_empty_runtime("identity-label-backfill-test").await;
5165        let mut labels = BTreeMap::new();
5166        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5167        runtime
5168            .spawn(
5169                SpawnMemberSpec::from_wire(
5170                    "worker".to_string(),
5171                    "rt:review:singleton:0".to_string(),
5172                    Some("You are the Review Agent.".into()),
5173                    None,
5174                    None,
5175                )
5176                .with_labels(labels),
5177            )
5178            .await
5179            .expect("member spawns");
5180
5181        let entry = RuntimeEntry {
5182            runtime_key: "runtime-a".to_string(),
5183            identity_namespace: String::new(),
5184            runtime: runtime.mob_runtime().clone(),
5185            identity_runtime: runtime.identity_runtime().cloned(),
5186            console_events: runtime.console_events(),
5187            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5188        };
5189        let aggregator = MobKitConsoleAggregator::in_memory();
5190        aggregator
5191            .inner
5192            .runtimes
5193            .write()
5194            .expect("runtime registry")
5195            .insert("runtime-a".to_string(), entry);
5196
5197        let target = session_backfill_target_for_identity(&aggregator.inner, "review:singleton")
5198            .await
5199            .expect("durable label should resolve targeted session backfill");
5200        assert_eq!(target.record.identity, "review:singleton");
5201        assert_eq!(target.record.runtime_member_id, "rt:review:singleton:0");
5202
5203        let _ = runtime.mob_handle().stop().await;
5204        Ok(())
5205    }
5206
5207    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5208    async fn inspect_identity_rejects_stale_durable_binding_when_live_alias_disagrees()
5209    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5210        let runtime = build_empty_runtime("identity-stale-durable-test").await;
5211        let mut labels = BTreeMap::new();
5212        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5213        runtime
5214            .spawn(
5215                SpawnMemberSpec::from_wire(
5216                    "worker".to_string(),
5217                    "rt:review:singleton:0".to_string(),
5218                    Some("You are the live Review Agent.".into()),
5219                    None,
5220                    None,
5221                )
5222                .with_labels(labels),
5223            )
5224            .await
5225            .expect("member spawns");
5226
5227        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5228            crate::identity_first::IdentityRuntimeConfig {
5229                continuity_store: Arc::new(
5230                    crate::identity_first::LocalContinuityStore::in_memory()?
5231                ),
5232                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5233                runtime_instance_id: "console-aggregator-stale-durable-test".to_string(),
5234                has_runtime_store: true,
5235                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5236                bridge: None,
5237                default_timeout: None,
5238            },
5239        ));
5240        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5241        let record = crate::identity_first::ContinuityRecord {
5242            identity: identity.clone(),
5243            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5244                "rt:review:singleton:1",
5245            )?,
5246            session_id: SessionId::new(),
5247            generation: crate::identity_first::ContinuityGeneration::new(1),
5248            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5249        };
5250        identity_runtime
5251            .register(
5252                crate::identity_first::DurableAgentSpec {
5253                    identity: identity.clone(),
5254                    profile: meerkat_mob::ProfileName::from("worker"),
5255                    addressability: crate::identity_first::AgentAddressability::Addressable,
5256                    display_name: None,
5257                    labels: BTreeMap::new(),
5258                    context: None,
5259                    additional_instructions: Vec::new(),
5260                    initial_message: None,
5261                    runtime_mode_override: None,
5262                },
5263                crate::identity_first::IdentityLifecycleState::Active,
5264                Some(record),
5265                Some(crate::identity_first::LeaseGrant {
5266                    identity,
5267                    fencing_token: crate::identity_first::FencingToken::new(7),
5268                    ttl: Duration::from_mins(5),
5269                }),
5270            )
5271            .await;
5272
5273        let aggregator = MobKitConsoleAggregator::in_memory();
5274        aggregator
5275            .inner
5276            .runtimes
5277            .write()
5278            .expect("runtime registry")
5279            .insert(
5280                "runtime-a".to_string(),
5281                RuntimeEntry {
5282                    runtime_key: "runtime-a".to_string(),
5283                    identity_namespace: String::new(),
5284                    runtime: runtime.mob_runtime().clone(),
5285                    identity_runtime: Some(identity_runtime),
5286                    console_events: runtime.console_events(),
5287                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5288                },
5289            );
5290
5291        let err = aggregator
5292            .inspect_identity("review:singleton")
5293            .await
5294            .expect_err("stale durable binding must not inspect as healthy");
5295        assert!(
5296            err.to_string().contains("stale durable identity alias"),
5297            "unexpected error: {err}"
5298        );
5299        let retire_err = aggregator
5300            .retire_identity("review:singleton")
5301            .await
5302            .expect_err("stale durable binding must not retire as healthy");
5303        assert!(
5304            retire_err
5305                .to_string()
5306                .contains("stale durable identity alias"),
5307            "unexpected retire error: {retire_err}"
5308        );
5309        let send_err = aggregator
5310            .send(ConsoleSendRequest {
5311                identity: "review:singleton".to_string(),
5312                content: json!("hello"),
5313                origin: "console:test".to_string(),
5314                idempotency_key: "stale-durable-send".to_string(),
5315                handling_mode: Some("queue".to_string()),
5316            })
5317            .await
5318            .expect_err("stale durable binding must not send to the wrong live member");
5319        assert!(
5320            send_err
5321                .to_string()
5322                .contains("stale durable identity alias"),
5323            "unexpected send error: {send_err}"
5324        );
5325        let reserve_err = aggregator
5326            .reserve_identity_first_interaction(
5327                ConsoleSendRequest {
5328                    identity: "review:singleton".to_string(),
5329                    content: json!("hello"),
5330                    origin: "console:test".to_string(),
5331                    idempotency_key: "stale-durable-reserve".to_string(),
5332                    handling_mode: Some("queue".to_string()),
5333                },
5334                None,
5335            )
5336            .await
5337            .expect_err("identity-first reserve must not bypass stale durable binding");
5338        assert!(
5339            reserve_err
5340                .to_string()
5341                .contains("stale durable identity alias"),
5342            "unexpected reserve error: {reserve_err}"
5343        );
5344        let blob_err = match aggregator
5345            .binary_blob_store_for_identity("review:singleton")
5346            .await
5347        {
5348            Ok(_) => panic!("stale durable binding must not blob-resolve as healthy"),
5349            Err(err) => err,
5350        };
5351        assert!(
5352            blob_err
5353                .to_string()
5354                .contains("stale durable identity alias"),
5355            "unexpected blob error: {blob_err}"
5356        );
5357        let records = aggregator.list_identities_fresh().await?;
5358        assert!(
5359            records
5360                .iter()
5361                .any(|record| record.identity == "review:singleton"
5362                    && record.runtime_member_id == "rt:review:singleton:0"),
5363            "live alias should still be listed; records: {records:#?}"
5364        );
5365        assert!(
5366            records
5367                .iter()
5368                .any(|record| record.identity == "review:singleton"
5369                    && record.runtime_member_id == "rt:review:singleton:1"),
5370            "stale durable binding should remain visible as split-brain evidence; records: {records:#?}"
5371        );
5372
5373        let _ = runtime.mob_handle().stop().await;
5374        Ok(())
5375    }
5376
5377    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5378    async fn inspect_identity_rejects_durable_binding_when_runtime_projects_wrong_identity()
5379    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5380        let runtime = build_empty_runtime("identity-wrong-projection-test").await;
5381        let mut labels = BTreeMap::new();
5382        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5383        runtime
5384            .spawn(
5385                SpawnMemberSpec::from_wire(
5386                    "worker".to_string(),
5387                    "rt:review:singleton:0".to_string(),
5388                    Some("You are the mislabeled Review Agent.".into()),
5389                    None,
5390                    None,
5391                )
5392                .with_labels(labels),
5393            )
5394            .await
5395            .expect("member spawns");
5396
5397        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5398            crate::identity_first::IdentityRuntimeConfig {
5399                continuity_store: Arc::new(
5400                    crate::identity_first::LocalContinuityStore::in_memory()?
5401                ),
5402                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5403                runtime_instance_id: "console-aggregator-wrong-projection-test".to_string(),
5404                has_runtime_store: true,
5405                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5406                bridge: None,
5407                default_timeout: None,
5408            },
5409        ));
5410        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5411        let record = crate::identity_first::ContinuityRecord {
5412            identity: identity.clone(),
5413            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5414                "rt:review:singleton:0",
5415            )?,
5416            session_id: SessionId::new(),
5417            generation: crate::identity_first::ContinuityGeneration::new(1),
5418            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5419        };
5420        identity_runtime
5421            .register(
5422                crate::identity_first::DurableAgentSpec {
5423                    identity: identity.clone(),
5424                    profile: meerkat_mob::ProfileName::from("worker"),
5425                    addressability: crate::identity_first::AgentAddressability::Addressable,
5426                    display_name: None,
5427                    labels: BTreeMap::new(),
5428                    context: None,
5429                    additional_instructions: Vec::new(),
5430                    initial_message: None,
5431                    runtime_mode_override: None,
5432                },
5433                crate::identity_first::IdentityLifecycleState::Active,
5434                Some(record),
5435                Some(crate::identity_first::LeaseGrant {
5436                    identity,
5437                    fencing_token: crate::identity_first::FencingToken::new(7),
5438                    ttl: Duration::from_mins(5),
5439                }),
5440            )
5441            .await;
5442
5443        let aggregator = MobKitConsoleAggregator::in_memory();
5444        aggregator
5445            .inner
5446            .runtimes
5447            .write()
5448            .expect("runtime registry")
5449            .insert(
5450                "runtime-a".to_string(),
5451                RuntimeEntry {
5452                    runtime_key: "runtime-a".to_string(),
5453                    identity_namespace: String::new(),
5454                    runtime: runtime.mob_runtime().clone(),
5455                    identity_runtime: Some(identity_runtime),
5456                    console_events: runtime.console_events(),
5457                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5458                },
5459            );
5460
5461        let err = aggregator
5462            .inspect_identity("review:singleton")
5463            .await
5464            .expect_err("wrong projected live identity must not inspect as healthy");
5465        assert!(
5466            err.to_string()
5467                .contains("projects identity other:singleton"),
5468            "unexpected error: {err}"
5469        );
5470        let retire_err = aggregator
5471            .retire_identity("review:singleton")
5472            .await
5473            .expect_err("wrong projected live identity must not retire as healthy");
5474        assert!(
5475            retire_err
5476                .to_string()
5477                .contains("projects identity other:singleton"),
5478            "unexpected retire error: {retire_err}"
5479        );
5480        let send_err = aggregator
5481            .send(ConsoleSendRequest {
5482                identity: "review:singleton".to_string(),
5483                content: json!("hello"),
5484                origin: "console:test".to_string(),
5485                idempotency_key: "wrong-projection-send".to_string(),
5486                handling_mode: Some("queue".to_string()),
5487            })
5488            .await
5489            .expect_err("wrong projected live identity must not send as unknown");
5490        assert!(
5491            send_err
5492                .to_string()
5493                .contains("projects identity other:singleton"),
5494            "unexpected send error: {send_err}"
5495        );
5496        let reserve_err = aggregator
5497            .reserve_identity_first_interaction(
5498                ConsoleSendRequest {
5499                    identity: "review:singleton".to_string(),
5500                    content: json!("hello"),
5501                    origin: "console:test".to_string(),
5502                    idempotency_key: "wrong-projection-reserve".to_string(),
5503                    handling_mode: Some("queue".to_string()),
5504                },
5505                None,
5506            )
5507            .await
5508            .expect_err("identity-first reserve must not bypass wrong live projection");
5509        assert!(
5510            reserve_err
5511                .to_string()
5512                .contains("projects identity other:singleton"),
5513            "unexpected reserve error: {reserve_err}"
5514        );
5515        let blob_err = match aggregator
5516            .binary_blob_store_for_identity("review:singleton")
5517            .await
5518        {
5519            Ok(_) => panic!("wrong projected live identity must not blob-resolve as healthy"),
5520            Err(err) => err,
5521        };
5522        assert!(
5523            blob_err
5524                .to_string()
5525                .contains("projects identity other:singleton"),
5526            "unexpected blob error: {blob_err}"
5527        );
5528        let projected_err = aggregator
5529            .inspect_identity("other:singleton")
5530            .await
5531            .expect_err("wrong projected alias must not inspect through live-only fallback");
5532        assert!(
5533            projected_err
5534                .to_string()
5535                .contains("identity runtime binding for review:singleton"),
5536            "unexpected projected-alias error: {projected_err}"
5537        );
5538        let projected_retire_err = aggregator
5539            .retire_identity("other:singleton")
5540            .await
5541            .expect_err("wrong projected alias must not retire through live-only fallback");
5542        assert!(
5543            projected_retire_err
5544                .to_string()
5545                .contains("identity runtime binding for review:singleton"),
5546            "unexpected projected-alias retire error: {projected_retire_err}"
5547        );
5548
5549        let _ = runtime.mob_handle().stop().await;
5550        Ok(())
5551    }
5552
5553    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5554    async fn inspect_identity_rejects_hidden_wrong_projection_for_durable_binding()
5555    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5556        let runtime = build_empty_runtime("identity-hidden-wrong-projection-test").await;
5557        let mut labels = BTreeMap::new();
5558        labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5559        runtime
5560            .spawn(
5561                SpawnMemberSpec::from_wire(
5562                    "worker".to_string(),
5563                    "rt:review:singleton:0".to_string(),
5564                    Some("You are a hidden wrong projection.".into()),
5565                    None,
5566                    None,
5567                )
5568                .with_labels(labels),
5569            )
5570            .await
5571            .expect("member spawns");
5572
5573        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5574            crate::identity_first::IdentityRuntimeConfig {
5575                continuity_store: Arc::new(
5576                    crate::identity_first::LocalContinuityStore::in_memory()?
5577                ),
5578                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5579                runtime_instance_id: "console-aggregator-hidden-wrong-projection-test".to_string(),
5580                has_runtime_store: true,
5581                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5582                bridge: None,
5583                default_timeout: None,
5584            },
5585        ));
5586        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5587        identity_runtime
5588            .register(
5589                crate::identity_first::DurableAgentSpec {
5590                    identity: identity.clone(),
5591                    profile: meerkat_mob::ProfileName::from("worker"),
5592                    addressability: crate::identity_first::AgentAddressability::Addressable,
5593                    display_name: None,
5594                    labels: BTreeMap::new(),
5595                    context: None,
5596                    additional_instructions: Vec::new(),
5597                    initial_message: None,
5598                    runtime_mode_override: None,
5599                },
5600                crate::identity_first::IdentityLifecycleState::Active,
5601                Some(crate::identity_first::ContinuityRecord {
5602                    identity: identity.clone(),
5603                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5604                        "rt:review:singleton:0",
5605                    )?,
5606                    session_id: SessionId::new(),
5607                    generation: crate::identity_first::ContinuityGeneration::new(1),
5608                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5609                }),
5610                Some(crate::identity_first::LeaseGrant {
5611                    identity,
5612                    fencing_token: crate::identity_first::FencingToken::new(7),
5613                    ttl: Duration::from_mins(5),
5614                }),
5615            )
5616            .await;
5617
5618        let aggregator = MobKitConsoleAggregator::in_memory();
5619        aggregator
5620            .inner
5621            .runtimes
5622            .write()
5623            .expect("runtime registry")
5624            .insert(
5625                "runtime-a".to_string(),
5626                RuntimeEntry {
5627                    runtime_key: "runtime-a".to_string(),
5628                    identity_namespace: String::new(),
5629                    runtime: runtime.mob_runtime().clone(),
5630                    identity_runtime: Some(identity_runtime),
5631                    console_events: runtime.console_events(),
5632                    visibility_policy: Arc::new(HideConsoleIdentity("other:singleton")),
5633                },
5634            );
5635
5636        let err = aggregator
5637            .inspect_identity("review:singleton")
5638            .await
5639            .expect_err("hidden wrong projection must still fail durable binding validation");
5640        assert!(
5641            err.to_string()
5642                .contains("projects identity other:singleton"),
5643            "unexpected hidden wrong-projection error: {err}"
5644        );
5645        let reserve_err = aggregator
5646            .reserve_identity_first_interaction(
5647                ConsoleSendRequest {
5648                    identity: "review:singleton".to_string(),
5649                    content: json!("hello"),
5650                    origin: "console:test".to_string(),
5651                    idempotency_key: "hidden-wrong-projection-reserve".to_string(),
5652                    handling_mode: Some("queue".to_string()),
5653                },
5654                None,
5655            )
5656            .await
5657            .expect_err("reserve must not hide wrong projection from durable validation");
5658        assert!(
5659            reserve_err
5660                .to_string()
5661                .contains("projects identity other:singleton"),
5662            "unexpected reserve error: {reserve_err}"
5663        );
5664
5665        let _ = runtime.mob_handle().stop().await;
5666        Ok(())
5667    }
5668
5669    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5670    async fn runtime_id_inspect_skips_hidden_match_and_uses_visible_runtime()
5671    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5672        let hidden_runtime = build_empty_runtime("runtime-id-hidden-match-test").await;
5673        let visible_runtime = build_empty_runtime("runtime-id-visible-match-test").await;
5674        let hidden_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5675            crate::identity_first::IdentityRuntimeConfig {
5676                continuity_store: Arc::new(
5677                    crate::identity_first::LocalContinuityStore::in_memory()?
5678                ),
5679                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5680                runtime_instance_id: "runtime-id-hidden-match-test".to_string(),
5681                has_runtime_store: true,
5682                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5683                bridge: None,
5684                default_timeout: None,
5685            },
5686        ));
5687        let visible_identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5688            crate::identity_first::IdentityRuntimeConfig {
5689                continuity_store: Arc::new(
5690                    crate::identity_first::LocalContinuityStore::in_memory()?
5691                ),
5692                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5693                runtime_instance_id: "runtime-id-visible-match-test".to_string(),
5694                has_runtime_store: true,
5695                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5696                bridge: None,
5697                default_timeout: None,
5698            },
5699        ));
5700        for identity_runtime in [&hidden_identity_runtime, &visible_identity_runtime] {
5701            let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5702            identity_runtime
5703                .register(
5704                    crate::identity_first::DurableAgentSpec {
5705                        identity: identity.clone(),
5706                        profile: meerkat_mob::ProfileName::from("worker"),
5707                        addressability: crate::identity_first::AgentAddressability::Addressable,
5708                        display_name: None,
5709                        labels: BTreeMap::new(),
5710                        context: None,
5711                        additional_instructions: Vec::new(),
5712                        initial_message: None,
5713                        runtime_mode_override: None,
5714                    },
5715                    crate::identity_first::IdentityLifecycleState::Active,
5716                    Some(crate::identity_first::ContinuityRecord {
5717                        identity: identity.clone(),
5718                        agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5719                            "rt:review:singleton:0",
5720                        )?,
5721                        session_id: SessionId::new(),
5722                        generation: crate::identity_first::ContinuityGeneration::new(1),
5723                        checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5724                    }),
5725                    Some(crate::identity_first::LeaseGrant {
5726                        identity,
5727                        fencing_token: crate::identity_first::FencingToken::new(7),
5728                        ttl: Duration::from_mins(5),
5729                    }),
5730                )
5731                .await;
5732        }
5733
5734        let aggregator = MobKitConsoleAggregator::in_memory();
5735        {
5736            let mut runtimes = aggregator.inner.runtimes.write().expect("runtime registry");
5737            runtimes.insert(
5738                "a-hidden".to_string(),
5739                RuntimeEntry {
5740                    runtime_key: "a-hidden".to_string(),
5741                    identity_namespace: String::new(),
5742                    runtime: hidden_runtime.mob_runtime().clone(),
5743                    identity_runtime: Some(hidden_identity_runtime),
5744                    console_events: hidden_runtime.console_events(),
5745                    visibility_policy: Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
5746                },
5747            );
5748            runtimes.insert(
5749                "b-visible".to_string(),
5750                RuntimeEntry {
5751                    runtime_key: "b-visible".to_string(),
5752                    identity_namespace: String::new(),
5753                    runtime: visible_runtime.mob_runtime().clone(),
5754                    identity_runtime: Some(visible_identity_runtime),
5755                    console_events: visible_runtime.console_events(),
5756                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5757                },
5758            );
5759        }
5760
5761        let inspection = aggregator
5762            .inspect_identity("rt:review:singleton:0")
5763            .await?
5764            .expect("visible runtime-id match should be returned");
5765        assert_eq!(inspection.identity.runtime_key, "b-visible");
5766
5767        let _ = hidden_runtime.mob_handle().stop().await;
5768        let _ = visible_runtime.mob_handle().stop().await;
5769        Ok(())
5770    }
5771
5772    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5773    async fn inspect_identity_rejects_cross_source_wrong_projection_for_runtime_member()
5774    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5775        let runtime = build_empty_runtime("identity-cross-source-primary-test").await;
5776        let mut primary_labels = BTreeMap::new();
5777        primary_labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5778        runtime
5779            .spawn(
5780                SpawnMemberSpec::from_wire(
5781                    "worker".to_string(),
5782                    "rt:review:singleton:0".to_string(),
5783                    Some("You are the primary Review Agent.".into()),
5784                    None,
5785                    None,
5786                )
5787                .with_labels(primary_labels),
5788            )
5789            .await
5790            .expect("primary member spawns");
5791
5792        let delegate_runtime = build_empty_runtime("identity-cross-source-delegate-test").await;
5793        let mut delegate_labels = BTreeMap::new();
5794        delegate_labels.insert("agent_identity".to_string(), "other:singleton".to_string());
5795        delegate_runtime
5796            .spawn(
5797                SpawnMemberSpec::from_wire(
5798                    "worker".to_string(),
5799                    "rt:review:singleton:0".to_string(),
5800                    Some("You are the wrong projected Review Agent.".into()),
5801                    None,
5802                    None,
5803                )
5804                .with_labels(delegate_labels),
5805            )
5806            .await
5807            .expect("delegate member spawns");
5808        runtime
5809            .mob_runtime()
5810            .agent_mob_mcp_state()
5811            .expect("runtime should expose mob MCP state")
5812            .mob_insert_handle(
5813                meerkat_mob::ids::MobId::from("identity-cross-source-delegate-test"),
5814                delegate_runtime.mob_handle().clone(),
5815            )
5816            .await;
5817
5818        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5819            crate::identity_first::IdentityRuntimeConfig {
5820                continuity_store: Arc::new(
5821                    crate::identity_first::LocalContinuityStore::in_memory()?
5822                ),
5823                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5824                runtime_instance_id: "console-aggregator-cross-source-test".to_string(),
5825                has_runtime_store: true,
5826                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5827                bridge: None,
5828                default_timeout: None,
5829            },
5830        ));
5831        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5832        let record = crate::identity_first::ContinuityRecord {
5833            identity: identity.clone(),
5834            agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5835                "rt:review:singleton:0",
5836            )?,
5837            session_id: SessionId::new(),
5838            generation: crate::identity_first::ContinuityGeneration::new(1),
5839            checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5840        };
5841        identity_runtime
5842            .register(
5843                crate::identity_first::DurableAgentSpec {
5844                    identity: identity.clone(),
5845                    profile: meerkat_mob::ProfileName::from("worker"),
5846                    addressability: crate::identity_first::AgentAddressability::Addressable,
5847                    display_name: None,
5848                    labels: BTreeMap::new(),
5849                    context: None,
5850                    additional_instructions: Vec::new(),
5851                    initial_message: None,
5852                    runtime_mode_override: None,
5853                },
5854                crate::identity_first::IdentityLifecycleState::Active,
5855                Some(record),
5856                Some(crate::identity_first::LeaseGrant {
5857                    identity,
5858                    fencing_token: crate::identity_first::FencingToken::new(7),
5859                    ttl: Duration::from_mins(5),
5860                }),
5861            )
5862            .await;
5863
5864        let aggregator = MobKitConsoleAggregator::in_memory();
5865        aggregator
5866            .inner
5867            .runtimes
5868            .write()
5869            .expect("runtime registry")
5870            .insert(
5871                "runtime-a".to_string(),
5872                RuntimeEntry {
5873                    runtime_key: "runtime-a".to_string(),
5874                    identity_namespace: String::new(),
5875                    runtime: runtime.mob_runtime().clone(),
5876                    identity_runtime: Some(identity_runtime),
5877                    console_events: runtime.console_events(),
5878                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5879                },
5880            );
5881
5882        let err = aggregator
5883            .inspect_identity("review:singleton")
5884            .await
5885            .expect_err("cross-source wrong projection must not be hidden by primary source");
5886        assert!(
5887            err.to_string()
5888                .contains("projects identity other:singleton")
5889                || err.to_string().contains("ambiguous live identity alias"),
5890            "unexpected error: {err}"
5891        );
5892
5893        let _ = delegate_runtime.mob_handle().stop().await;
5894        let _ = runtime.mob_handle().stop().await;
5895        Ok(())
5896    }
5897
5898    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5899    async fn inspect_identity_rejects_duplicate_live_alias_even_when_durable_matches_one()
5900    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
5901        let runtime = build_empty_runtime("identity-ambiguous-durable-test").await;
5902        let mut labels = BTreeMap::new();
5903        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
5904        runtime
5905            .spawn(
5906                SpawnMemberSpec::from_wire(
5907                    "worker".to_string(),
5908                    "rt:review:singleton:0".to_string(),
5909                    Some("You are the first Review Agent.".into()),
5910                    None,
5911                    None,
5912                )
5913                .with_labels(labels.clone()),
5914            )
5915            .await
5916            .expect("first member spawns");
5917        runtime
5918            .spawn(
5919                SpawnMemberSpec::from_wire(
5920                    "worker".to_string(),
5921                    "rt:review:singleton:1".to_string(),
5922                    Some("You are the second Review Agent.".into()),
5923                    None,
5924                    None,
5925                )
5926                .with_labels(labels),
5927            )
5928            .await
5929            .expect("second member spawns");
5930
5931        let identity_runtime = Arc::new(crate::identity_first::IdentityRuntime::new(
5932            crate::identity_first::IdentityRuntimeConfig {
5933                continuity_store: Arc::new(
5934                    crate::identity_first::LocalContinuityStore::in_memory()?
5935                ),
5936                lease_provider: Arc::new(crate::identity_first::LocalLeaseProvider::new()),
5937                runtime_instance_id: "console-aggregator-ambiguous-durable-test".to_string(),
5938                has_runtime_store: true,
5939                durability_policy: crate::identity_first::DurabilityPolicy::SyncWriteThrough,
5940                bridge: None,
5941                default_timeout: None,
5942            },
5943        ));
5944        let identity = crate::identity_first::AgentIdentity::parse("review:singleton")?;
5945        identity_runtime
5946            .register(
5947                crate::identity_first::DurableAgentSpec {
5948                    identity: identity.clone(),
5949                    profile: meerkat_mob::ProfileName::from("worker"),
5950                    addressability: crate::identity_first::AgentAddressability::Addressable,
5951                    display_name: None,
5952                    labels: BTreeMap::new(),
5953                    context: None,
5954                    additional_instructions: Vec::new(),
5955                    initial_message: None,
5956                    runtime_mode_override: None,
5957                },
5958                crate::identity_first::IdentityLifecycleState::Active,
5959                Some(crate::identity_first::ContinuityRecord {
5960                    identity: identity.clone(),
5961                    agent_runtime_id: crate::identity_first::AgentRuntimeId::parse(
5962                        "rt:review:singleton:0",
5963                    )?,
5964                    session_id: SessionId::new(),
5965                    generation: crate::identity_first::ContinuityGeneration::new(1),
5966                    checkpoint_version: crate::identity_first::CheckpointVersion::new(0),
5967                }),
5968                Some(crate::identity_first::LeaseGrant {
5969                    identity,
5970                    fencing_token: crate::identity_first::FencingToken::new(8),
5971                    ttl: Duration::from_mins(5),
5972                }),
5973            )
5974            .await;
5975
5976        let aggregator = MobKitConsoleAggregator::in_memory();
5977        aggregator
5978            .inner
5979            .runtimes
5980            .write()
5981            .expect("runtime registry")
5982            .insert(
5983                "runtime-a".to_string(),
5984                RuntimeEntry {
5985                    runtime_key: "runtime-a".to_string(),
5986                    identity_namespace: String::new(),
5987                    runtime: runtime.mob_runtime().clone(),
5988                    identity_runtime: Some(identity_runtime),
5989                    console_events: runtime.console_events(),
5990                    visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
5991                },
5992            );
5993
5994        let err = aggregator
5995            .inspect_identity("review:singleton")
5996            .await
5997            .expect_err("duplicate live alias must not inspect via durable fast path");
5998        assert!(
5999            err.to_string().contains("ambiguous live identity alias"),
6000            "unexpected error: {err}"
6001        );
6002        let retire_err = aggregator
6003            .retire_identity("review:singleton")
6004            .await
6005            .expect_err("duplicate live alias must not retire via durable fast path");
6006        assert!(
6007            retire_err
6008                .to_string()
6009                .contains("ambiguous live identity alias"),
6010            "unexpected retire error: {retire_err}"
6011        );
6012        let runtime_id_err = aggregator
6013            .inspect_identity("rt:review:singleton:0")
6014            .await
6015            .expect_err("runtime-id lookup must still reject duplicate durable live aliases");
6016        assert!(
6017            runtime_id_err
6018                .to_string()
6019                .contains("ambiguous live identity alias"),
6020            "unexpected runtime-id inspect error: {runtime_id_err}"
6021        );
6022        let runtime_id_retire_err = aggregator
6023            .retire_identity("rt:review:singleton:0")
6024            .await
6025            .expect_err("runtime-id retire must still reject duplicate durable live aliases");
6026        assert!(
6027            runtime_id_retire_err
6028                .to_string()
6029                .contains("ambiguous live identity alias")
6030                || runtime_id_retire_err
6031                    .to_string()
6032                    .contains("stale durable identity alias"),
6033            "unexpected runtime-id retire error: {runtime_id_retire_err}"
6034        );
6035
6036        let _ = runtime.mob_handle().stop().await;
6037        Ok(())
6038    }
6039
6040    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6041    async fn inspect_identity_ignores_hidden_live_alias_candidates()
6042    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6043        let runtime = build_empty_runtime("identity-hidden-ambiguous-live-test").await;
6044        let mut labels = BTreeMap::new();
6045        labels.insert("agent_identity".to_string(), "review:singleton".to_string());
6046        runtime
6047            .spawn(
6048                SpawnMemberSpec::from_wire(
6049                    "worker".to_string(),
6050                    "rt:review:singleton:0".to_string(),
6051                    Some("You are the visible Review Agent.".into()),
6052                    None,
6053                    None,
6054                )
6055                .with_labels(labels.clone()),
6056            )
6057            .await
6058            .expect("visible member spawns");
6059        runtime
6060            .spawn(
6061                SpawnMemberSpec::from_wire(
6062                    "worker".to_string(),
6063                    "rt:review:singleton:1".to_string(),
6064                    Some("You are hidden maintenance noise.".into()),
6065                    None,
6066                    None,
6067                )
6068                .with_labels(labels),
6069            )
6070            .await
6071            .expect("hidden member spawns");
6072
6073        let aggregator = MobKitConsoleAggregator::in_memory();
6074        aggregator.register_runtime_handles_with_policy(
6075            "runtime-a",
6076            "",
6077            runtime.mob_runtime().clone(),
6078            None,
6079            runtime.console_events(),
6080            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:1")),
6081        );
6082
6083        let inspection = aggregator
6084            .inspect_identity("review:singleton")
6085            .await?
6086            .expect("visible live alias should inspect without hidden ambiguity");
6087        assert_eq!(inspection.identity.identity, "review:singleton");
6088        assert_eq!(
6089            inspection.identity.runtime_member_id,
6090            "rt:review:singleton:0"
6091        );
6092
6093        let retired = aggregator.retire_identity("review:singleton").await?;
6094        assert!(
6095            retired,
6096            "visible live alias should retire without hidden ambiguity"
6097        );
6098
6099        let _ = runtime.mob_handle().stop().await;
6100        Ok(())
6101    }
6102
6103    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6104    async fn identity_first_reserve_skips_hidden_runtime_and_uses_visible_match()
6105    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6106        let hidden_runtime = build_empty_runtime("identity-reserve-hidden-first-test").await;
6107        let hidden_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6108        let visible_runtime = build_empty_runtime("identity-reserve-visible-second-test").await;
6109        let visible_identity_runtime = identity_runtime_for_test(&["review:singleton"]).await?;
6110
6111        let aggregator = MobKitConsoleAggregator::in_memory();
6112        aggregator.register_runtime_handles_with_policy(
6113            "a-hidden",
6114            "",
6115            hidden_runtime.mob_runtime().clone(),
6116            Some(hidden_identity_runtime),
6117            hidden_runtime.console_events(),
6118            Arc::new(HideRuntimeMemberIdentity("rt:review:singleton:0")),
6119        );
6120        aggregator.register_runtime_handles_with_policy(
6121            "b-visible",
6122            "",
6123            visible_runtime.mob_runtime().clone(),
6124            Some(visible_identity_runtime),
6125            visible_runtime.console_events(),
6126            Arc::new(AllowAllConsoleVisibilityPolicy),
6127        );
6128
6129        let accepted = aggregator
6130            .reserve_identity_first_interaction(
6131                ConsoleSendRequest {
6132                    identity: "review:singleton".to_string(),
6133                    content: json!("hello"),
6134                    origin: "console:test".to_string(),
6135                    idempotency_key: "visible-after-hidden-reserve".to_string(),
6136                    handling_mode: Some("queue".to_string()),
6137                },
6138                None,
6139            )
6140            .await?;
6141        let page = aggregator
6142            .query_timeline(ConsoleTimelineQuery {
6143                identity: Some("review:singleton".to_string()),
6144                ..ConsoleTimelineQuery::default()
6145            })
6146            .await?;
6147        let frame = page
6148            .frames
6149            .iter()
6150            .find(|frame| frame.id == accepted.input_frame_id)
6151            .ok_or("accepted frame missing from timeline")?;
6152        assert_eq!(frame.runtime_key, "b-visible");
6153
6154        let _ = hidden_runtime.mob_handle().stop().await;
6155        let _ = visible_runtime.mob_handle().stop().await;
6156        Ok(())
6157    }
6158
6159    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6160    async fn identity_first_list_identities_projects_cached_topology_peers()
6161    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6162        let runtime = build_empty_runtime("identity-topology-cache-test").await;
6163        let identity_runtime = identity_runtime_for_test(&["agent:alpha", "agent:beta"]).await?;
6164        identity_runtime
6165            .set_desired_peer_edges(vec![crate::identity_first::ManagedPeerEdge::new(
6166                crate::identity_first::AgentIdentity::parse("agent:alpha")?,
6167                crate::identity_first::AgentIdentity::parse("agent:beta")?,
6168            )?])
6169            .await;
6170
6171        let aggregator = MobKitConsoleAggregator::in_memory();
6172        aggregator.register_runtime_handles_with_policy(
6173            "identity-first",
6174            "",
6175            runtime.mob_runtime().clone(),
6176            Some(identity_runtime),
6177            runtime.console_events(),
6178            Arc::new(AllowAllConsoleVisibilityPolicy),
6179        );
6180
6181        let identities = aggregator.list_identities().await?;
6182        let alpha = identities
6183            .iter()
6184            .find(|record| record.identity == "agent:alpha")
6185            .ok_or("agent:alpha identity missing")?;
6186        assert_eq!(alpha.topology_peers, vec!["agent:beta".to_string()]);
6187        let inspection = aggregator
6188            .inspect_identity("agent:alpha")
6189            .await?
6190            .ok_or("agent:alpha inspection missing")?;
6191        assert_eq!(inspection.peers, vec!["agent:beta".to_string()]);
6192
6193        let _ = runtime.mob_handle().stop().await;
6194        Ok(())
6195    }
6196
6197    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6198    async fn identity_first_list_identities_includes_member_only_spawned_workers()
6199    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
6200        let runtime = build_empty_runtime("identity-member-only-test").await;
6201        let identity_runtime = identity_runtime_for_test(&["agent:alpha"]).await?;
6202        let aggregator = MobKitConsoleAggregator::in_memory();
6203        aggregator.register_runtime_handles_with_policy(
6204            "identity-first",
6205            "",
6206            runtime.mob_runtime().clone(),
6207            Some(identity_runtime),
6208            runtime.console_events(),
6209            Arc::new(AllowAllConsoleVisibilityPolicy),
6210        );
6211
6212        runtime
6213            .spawn(SpawnMemberSpec::from_wire(
6214                "worker".to_string(),
6215                "agent:beta".to_string(),
6216                Some("You are beta.".into()),
6217                None,
6218                None,
6219            ))
6220            .await?;
6221
6222        let identities = aggregator.list_identities().await?;
6223        assert!(
6224            identities
6225                .iter()
6226                .any(|record| record.identity == "agent:beta"),
6227            "member-only spawned workers should remain visible in identity-first listings: {identities:#?}"
6228        );
6229
6230        let _ = runtime.mob_handle().stop().await;
6231        Ok(())
6232    }
6233
6234    #[tokio::test]
6235    async fn list_identities_serves_hot_cache_while_identity_refresh_is_in_flight() {
6236        let aggregator = MobKitConsoleAggregator::in_memory();
6237        let record = identity_record_for_test("agent-cached");
6238        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6239        aggregator
6240            .inner
6241            .identity_read_model
6242            .primed
6243            .store(true, Ordering::Release);
6244        let _guard = aggregator
6245            .inner
6246            .identity_read_model
6247            .refresh_lock
6248            .clone()
6249            .lock_owned()
6250            .await;
6251
6252        let identities =
6253            tokio::time::timeout(Duration::from_millis(50), aggregator.list_identities())
6254                .await
6255                .expect("hot identity list should not wait for refresh lock")
6256                .expect("identity list succeeds");
6257
6258        assert_eq!(identities, vec![record]);
6259    }
6260
6261    #[tokio::test]
6262    async fn list_identities_waits_for_inflight_identity_refresh_on_cold_cache() {
6263        let aggregator = MobKitConsoleAggregator::in_memory();
6264        let guard = aggregator
6265            .inner
6266            .identity_read_model
6267            .refresh_lock
6268            .clone()
6269            .lock_owned()
6270            .await;
6271        let waiter = tokio::spawn({
6272            let aggregator = aggregator.clone();
6273            async move { aggregator.list_identities().await }
6274        });
6275
6276        tokio::time::sleep(Duration::from_millis(20)).await;
6277        assert!(
6278            !waiter.is_finished(),
6279            "cold identity list should wait for the in-flight refresh to finish"
6280        );
6281
6282        let record = identity_record_for_test("agent-primed");
6283        *aggregator.inner.identity_read_model.inner.write().await = vec![record.clone()];
6284        aggregator
6285            .inner
6286            .identity_read_model
6287            .primed
6288            .store(true, Ordering::Release);
6289        drop(guard);
6290
6291        let identities = tokio::time::timeout(Duration::from_secs(1), waiter)
6292            .await
6293            .expect("cold identity list waiter should resume")
6294            .expect("waiter joins")
6295            .expect("identity list succeeds");
6296        assert_eq!(identities, vec![record]);
6297    }
6298
6299    #[tokio::test]
6300    async fn query_timeline_reads_from_aggregate_store() {
6301        let aggregator = MobKitConsoleAggregator::in_memory();
6302        let frame = NewConsoleFrame {
6303            id: None,
6304            dedupe_key: "event-1".to_string(),
6305            timestamp_ms: 1,
6306            runtime_key: "runtime-a".to_string(),
6307            identity: "agent-a".to_string(),
6308            conversation_id: Some("agent-a".to_string()),
6309            session_id: None,
6310            kind: "text_delta".to_string(),
6311            status: ConsoleFrameStatus::Delivered,
6312            payload: json!({ "delta": "hello" }),
6313            source: ConsoleFrameSource {
6314                kind: ConsoleFrameSourceKind::ConsoleEvent,
6315                source_cursor: None,
6316            },
6317            source_event_id: Some("event-1".to_string()),
6318            interaction_id: None,
6319            turn_id: None,
6320            run_id: None,
6321            parent_frame_id: None,
6322            caused_by_frame_id: None,
6323        };
6324        aggregator
6325            .store()
6326            .append_if_absent(frame)
6327            .await
6328            .expect("append frame");
6329
6330        let page = aggregator
6331            .query_timeline(ConsoleTimelineQuery {
6332                identity: Some("agent-a".to_string()),
6333                limit: 10,
6334                ..ConsoleTimelineQuery::default()
6335            })
6336            .await
6337            .expect("query timeline");
6338        assert_eq!(page.frames.len(), 1);
6339        assert_eq!(page.frames[0].kind, "text_delta");
6340    }
6341
6342    #[tokio::test]
6343    async fn identity_recent_anchor_respects_query_limit() {
6344        let aggregator = MobKitConsoleAggregator::in_memory();
6345        aggregator
6346            .store()
6347            .append_if_absent(NewConsoleFrame {
6348                id: None,
6349                dedupe_key: "anchored-user-input".to_string(),
6350                timestamp_ms: 1,
6351                runtime_key: "runtime-a".to_string(),
6352                identity: "agent-a".to_string(),
6353                conversation_id: Some("agent-a".to_string()),
6354                session_id: None,
6355                kind: "user_input".to_string(),
6356                status: ConsoleFrameStatus::Delivered,
6357                payload: json!({ "content": "anchor me" }),
6358                source: ConsoleFrameSource {
6359                    kind: ConsoleFrameSourceKind::Synthetic,
6360                    source_cursor: None,
6361                },
6362                source_event_id: Some("anchored-user-input".to_string()),
6363                interaction_id: Some("turn-a".to_string()),
6364                turn_id: None,
6365                run_id: None,
6366                parent_frame_id: None,
6367                caused_by_frame_id: None,
6368            })
6369            .await
6370            .expect("append anchor");
6371        for idx in 2..=40 {
6372            aggregator
6373                .store()
6374                .append_if_absent(NewConsoleFrame {
6375                    id: None,
6376                    dedupe_key: format!("noisy-tail-{idx}"),
6377                    timestamp_ms: idx,
6378                    runtime_key: "runtime-a".to_string(),
6379                    identity: "agent-a".to_string(),
6380                    conversation_id: Some("agent-a".to_string()),
6381                    session_id: None,
6382                    kind: "reasoning_delta".to_string(),
6383                    status: ConsoleFrameStatus::Delivered,
6384                    payload: json!({ "delta": idx }),
6385                    source: ConsoleFrameSource {
6386                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6387                        source_cursor: None,
6388                    },
6389                    source_event_id: Some(format!("noisy-tail-{idx}")),
6390                    interaction_id: Some("turn-a".to_string()),
6391                    turn_id: None,
6392                    run_id: None,
6393                    parent_frame_id: None,
6394                    caused_by_frame_id: None,
6395                })
6396                .await
6397                .expect("append noisy tail");
6398        }
6399
6400        let page = aggregator
6401            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6402                identity: Some("agent-a".to_string()),
6403                mode: ConsoleTimelineMode::Recent,
6404                limit: 5,
6405                ..ConsoleTimelineWindowQuery::default()
6406            })
6407            .await
6408            .expect("query recent identity timeline");
6409
6410        assert_eq!(
6411            page.frames.len(),
6412            5,
6413            "identity anchor merge must not exceed the requested limit"
6414        );
6415        assert!(
6416            page.frames
6417                .iter()
6418                .any(|frame| frame.dedupe_key == "anchored-user-input"),
6419            "the bounded result should still retain the useful turn anchor: {:#?}",
6420            page.frames
6421        );
6422        assert_eq!(
6423            page.frames.last().and_then(|frame| frame.cursor.seq()),
6424            Some(40)
6425        );
6426    }
6427
6428    #[tokio::test]
6429    async fn query_timeline_since_skips_hidden_raw_gaps_with_bounded_paging() {
6430        let aggregator = MobKitConsoleAggregator::in_memory();
6431        let runtime = build_single_member_runtime().await;
6432        let mut entry = runtime_entry_for_test("runtime-a", &runtime);
6433        entry.visibility_policy = Arc::new(HideHiddenNoiseFrames);
6434        aggregator
6435            .inner
6436            .runtimes
6437            .write()
6438            .expect("runtime registry")
6439            .insert("runtime-a".to_string(), entry);
6440
6441        for idx in 0..1_500 {
6442            aggregator
6443                .store()
6444                .append_if_absent(NewConsoleFrame {
6445                    id: None,
6446                    dedupe_key: format!("hidden-gap-{idx}"),
6447                    timestamp_ms: idx,
6448                    runtime_key: "runtime-a".to_string(),
6449                    identity: "agent-a".to_string(),
6450                    conversation_id: Some("agent-a".to_string()),
6451                    session_id: None,
6452                    kind: "hidden_noise".to_string(),
6453                    status: ConsoleFrameStatus::Delivered,
6454                    payload: json!({ "delta": idx }),
6455                    source: ConsoleFrameSource {
6456                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6457                        source_cursor: None,
6458                    },
6459                    source_event_id: Some(format!("hidden-gap-{idx}")),
6460                    interaction_id: None,
6461                    turn_id: None,
6462                    run_id: None,
6463                    parent_frame_id: None,
6464                    caused_by_frame_id: None,
6465                })
6466                .await
6467                .expect("append hidden frame");
6468        }
6469        aggregator
6470            .store()
6471            .append_if_absent(NewConsoleFrame {
6472                id: None,
6473                dedupe_key: "visible-after-gap".to_string(),
6474                timestamp_ms: 1_501,
6475                runtime_key: "runtime-a".to_string(),
6476                identity: "agent-a".to_string(),
6477                conversation_id: Some("agent-a".to_string()),
6478                session_id: None,
6479                kind: "interaction_complete".to_string(),
6480                status: ConsoleFrameStatus::Delivered,
6481                payload: json!({ "text": "visible after hidden gap" }),
6482                source: ConsoleFrameSource {
6483                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6484                    source_cursor: None,
6485                },
6486                source_event_id: Some("visible-after-gap".to_string()),
6487                interaction_id: Some("turn-visible".to_string()),
6488                turn_id: None,
6489                run_id: None,
6490                parent_frame_id: None,
6491                caused_by_frame_id: None,
6492            })
6493            .await
6494            .expect("append visible frame");
6495
6496        let page = aggregator
6497            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6498                identity: Some("agent-a".to_string()),
6499                mode: ConsoleTimelineMode::Since,
6500                limit: 1,
6501                ..ConsoleTimelineWindowQuery::default()
6502            })
6503            .await
6504            .expect("query timeline");
6505        assert_eq!(page.frames.len(), 1);
6506        assert_eq!(page.frames[0].dedupe_key, "visible-after-gap");
6507        assert_eq!(
6508            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6509            Some(1_501)
6510        );
6511        let _ = runtime.mob_handle().stop().await;
6512    }
6513
6514    #[tokio::test]
6515    async fn query_timeline_since_cursor_stops_at_last_visible_returned_frame() {
6516        let aggregator = MobKitConsoleAggregator::in_memory();
6517        for idx in 1..=300 {
6518            aggregator
6519                .store()
6520                .append_if_absent(NewConsoleFrame {
6521                    id: None,
6522                    dedupe_key: format!("visible-{idx}"),
6523                    timestamp_ms: idx,
6524                    runtime_key: "runtime-a".to_string(),
6525                    identity: "agent-a".to_string(),
6526                    conversation_id: Some("agent-a".to_string()),
6527                    session_id: None,
6528                    kind: "interaction_complete".to_string(),
6529                    status: ConsoleFrameStatus::Delivered,
6530                    payload: json!({ "text": format!("visible {idx}") }),
6531                    source: ConsoleFrameSource {
6532                        kind: ConsoleFrameSourceKind::ConsoleEvent,
6533                        source_cursor: None,
6534                    },
6535                    source_event_id: Some(format!("visible-{idx}")),
6536                    interaction_id: Some(format!("turn-{idx}")),
6537                    turn_id: None,
6538                    run_id: None,
6539                    parent_frame_id: None,
6540                    caused_by_frame_id: None,
6541                })
6542                .await
6543                .expect("append visible frame");
6544        }
6545
6546        let first = aggregator
6547            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6548                identity: Some("agent-a".to_string()),
6549                mode: ConsoleTimelineMode::Since,
6550                limit: 10,
6551                ..ConsoleTimelineWindowQuery::default()
6552            })
6553            .await
6554            .expect("query first page");
6555        assert_eq!(first.frames.len(), 10);
6556        assert_eq!(
6557            first.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6558            Some(10)
6559        );
6560
6561        let second = aggregator
6562            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6563                identity: Some("agent-a".to_string()),
6564                mode: ConsoleTimelineMode::Since,
6565                after: first.next_cursor,
6566                limit: 10,
6567                ..ConsoleTimelineWindowQuery::default()
6568            })
6569            .await
6570            .expect("query second page");
6571        assert_eq!(second.frames.len(), 10);
6572        assert_eq!(second.frames[0].dedupe_key, "visible-11");
6573        assert_eq!(
6574            second.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6575            Some(20)
6576        );
6577    }
6578
6579    #[tokio::test]
6580    async fn query_timeline_since_empty_continuation_does_not_force_backfill() {
6581        let store = Arc::new(CountingConsoleLogStore::new());
6582        let aggregator = MobKitConsoleAggregator::new(store.clone());
6583        let runtime = build_single_member_runtime().await;
6584        aggregator
6585            .inner
6586            .runtimes
6587            .write()
6588            .expect("runtime registry")
6589            .insert(
6590                "runtime-a".to_string(),
6591                runtime_entry_for_test("runtime-a", &runtime),
6592            );
6593        let inserted = store
6594            .append_if_absent(NewConsoleFrame {
6595                id: None,
6596                dedupe_key: "event-1".to_string(),
6597                timestamp_ms: 1,
6598                runtime_key: "runtime-a".to_string(),
6599                identity: "agent-a".to_string(),
6600                conversation_id: Some("agent-a".to_string()),
6601                session_id: None,
6602                kind: "text_delta".to_string(),
6603                status: ConsoleFrameStatus::Delivered,
6604                payload: json!({ "delta": "hello" }),
6605                source: ConsoleFrameSource {
6606                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6607                    source_cursor: None,
6608                },
6609                source_event_id: Some("event-1".to_string()),
6610                interaction_id: None,
6611                turn_id: None,
6612                run_id: None,
6613                parent_frame_id: None,
6614                caused_by_frame_id: None,
6615            })
6616            .await
6617            .expect("append frame");
6618
6619        let page = aggregator
6620            .query_timeline_windowed(ConsoleTimelineWindowQuery {
6621                identity: Some("agent-a".to_string()),
6622                mode: ConsoleTimelineMode::Since,
6623                after: Some(inserted.frame.cursor),
6624                limit: 10,
6625                ..ConsoleTimelineWindowQuery::default()
6626            })
6627            .await
6628            .expect("query continuation");
6629
6630        assert!(page.frames.is_empty());
6631        assert_eq!(
6632            store.source_watermark_calls(),
6633            0,
6634            "empty since continuation must not synchronously force session-history backfill"
6635        );
6636        let _ = runtime.mob_handle().stop().await;
6637    }
6638
6639    #[test]
6640    fn legacy_timeline_struct_literals_remain_source_compatible() {
6641        let query = ConsoleTimelineQuery {
6642            identity: Some("agent-a".to_string()),
6643            conversation_id: None,
6644            after: Some(ConsoleCursor::from("console:1")),
6645            limit: 10,
6646        };
6647        let page = ConsoleTimelinePage {
6648            frames: Vec::new(),
6649            next_cursor: query.after.clone(),
6650        };
6651
6652        assert_eq!(query.limit, 10);
6653        assert_eq!(
6654            page.next_cursor.as_ref().and_then(ConsoleCursor::seq),
6655            Some(1)
6656        );
6657    }
6658
6659    #[tokio::test]
6660    async fn query_timeline_is_store_local_for_registered_runtimes() {
6661        let store = Arc::new(CountingConsoleLogStore::new());
6662        let aggregator = MobKitConsoleAggregator::new(store.clone());
6663        let runtime = build_single_member_runtime().await;
6664        aggregator
6665            .inner
6666            .runtimes
6667            .write()
6668            .expect("runtime registry")
6669            .insert(
6670                "runtime-a".to_string(),
6671                runtime_entry_for_test("runtime-a", &runtime),
6672            );
6673        store
6674            .append_if_absent(NewConsoleFrame {
6675                id: None,
6676                dedupe_key: "event-1".to_string(),
6677                timestamp_ms: 1,
6678                runtime_key: "runtime-a".to_string(),
6679                identity: "agent-a".to_string(),
6680                conversation_id: Some("agent-a".to_string()),
6681                session_id: None,
6682                kind: "text_delta".to_string(),
6683                status: ConsoleFrameStatus::Delivered,
6684                payload: json!({ "delta": "hello" }),
6685                source: ConsoleFrameSource {
6686                    kind: ConsoleFrameSourceKind::ConsoleEvent,
6687                    source_cursor: None,
6688                },
6689                source_event_id: Some("event-1".to_string()),
6690                interaction_id: None,
6691                turn_id: None,
6692                run_id: None,
6693                parent_frame_id: None,
6694                caused_by_frame_id: None,
6695            })
6696            .await
6697            .expect("append frame");
6698
6699        let page = tokio::time::timeout(
6700            Duration::from_millis(250),
6701            aggregator.query_timeline(ConsoleTimelineQuery {
6702                identity: Some("agent-a".to_string()),
6703                limit: 10,
6704                ..ConsoleTimelineQuery::default()
6705            }),
6706        )
6707        .await
6708        .expect("timeline query should not wait for session history")
6709        .expect("timeline query succeeds");
6710
6711        assert_eq!(page.frames.len(), 1);
6712        assert_eq!(
6713            store.source_watermark_calls(),
6714            0,
6715            "query_timeline must not synchronously touch session-history watermarks"
6716        );
6717        let _ = runtime.mob_handle().stop().await;
6718    }
6719
6720    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6721    async fn console_send_returns_after_acceptance_without_waiting_for_turn_completion() {
6722        let runtime = build_single_member_runtime_with_client(Arc::new(SlowTestClient {
6723            delay: Duration::from_secs(2),
6724        }))
6725        .await;
6726        let aggregator = MobKitConsoleAggregator::in_memory();
6727        aggregator
6728            .inner
6729            .runtimes
6730            .write()
6731            .expect("runtime registry")
6732            .insert(
6733                "runtime-a".to_string(),
6734                runtime_entry_for_test("runtime-a", &runtime),
6735            );
6736
6737        let start = Instant::now();
6738        let accepted = tokio::time::timeout(
6739            Duration::from_millis(300),
6740            aggregator.send(ConsoleSendRequest {
6741                identity: "test/agent-a".to_string(),
6742                content: json!("hello slow agent"),
6743                origin: "console:test".to_string(),
6744                idempotency_key: "nonblocking-send".to_string(),
6745                handling_mode: Some("queue".to_string()),
6746            }),
6747        )
6748        .await
6749        .expect("console send should return once the input is accepted")
6750        .expect("send succeeds");
6751
6752        assert_eq!(accepted.status, ConsoleFrameStatus::Accepted);
6753        assert!(
6754            start.elapsed() < Duration::from_secs(1),
6755            "console send should not wait for the delayed LLM turn"
6756        );
6757
6758        wait_for_session_history_text(
6759            &aggregator,
6760            "test/agent-a",
6761            "slow ok",
6762            Duration::from_secs(5),
6763        )
6764        .await
6765        .expect("background dispatch should still complete and project history");
6766        let _ = runtime.mob_handle().stop().await;
6767    }
6768
6769    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6770    async fn discovered_late_member_session_backfills_without_manual_refresh() -> Result<(), String>
6771    {
6772        let runtime = Arc::new(build_empty_runtime("console-aggregator-late-member-test").await);
6773        let aggregator = MobKitConsoleAggregator::in_memory();
6774        aggregator.register_runtime(ConsoleRuntimeRegistration {
6775            runtime_key: "runtime-late".to_string(),
6776            runtime: runtime.clone(),
6777            identity_namespace: "late".to_string(),
6778            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
6779        });
6780
6781        runtime
6782            .spawn(SpawnMemberSpec::from_wire(
6783                "worker".to_string(),
6784                "agent-late".to_string(),
6785                Some("You are agent-late.".into()),
6786                None,
6787                None,
6788            ))
6789            .await
6790            .expect("late member spawns");
6791        let session_id = send_message_on_mob_with_mode(
6792            &runtime.mob_handle(),
6793            "agent-late",
6794            ContentInput::Text("hello after registration".to_string()),
6795            meerkat_core::types::HandlingMode::Queue,
6796        )
6797        .await
6798        .expect("direct member send succeeds");
6799        wait_for_identity_record(
6800            &aggregator,
6801            "late/agent-late",
6802            Some(session_id.as_str()),
6803            Duration::from_secs(5),
6804        )
6805        .await?;
6806
6807        wait_for_session_history_text(
6808            &aggregator,
6809            "late/agent-late",
6810            "You are agent-late.",
6811            Duration::from_secs(5),
6812        )
6813        .await?;
6814        let _ = runtime.mob_handle().stop().await;
6815        Ok(())
6816    }
6817
6818    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6819    async fn explicit_empty_identity_query_force_refreshes_past_fresh_watermark()
6820    -> Result<(), String> {
6821        let runtime = build_single_member_runtime().await;
6822        let entry = runtime_entry_for_test("runtime-a", &runtime);
6823        let resolved = member_sources_for_entry(&entry)
6824            .await
6825            .into_iter()
6826            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6827            .expect("agent-a member exists");
6828        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6829            .await
6830            .expect("identity record exists");
6831        let session_id = record.session_id.expect("agent-a has a session");
6832        wait_for_runtime_session_history_text(
6833            &runtime,
6834            &session_id,
6835            "You are agent-a.",
6836            Duration::from_secs(5),
6837        )
6838        .await?;
6839
6840        let aggregator = MobKitConsoleAggregator::in_memory();
6841        aggregator
6842            .inner
6843            .runtimes
6844            .write()
6845            .expect("runtime registry")
6846            .insert("runtime-a".to_string(), entry);
6847        let watermark_key = session_history_watermark_runtime_key("runtime-a", &session_id);
6848        aggregator
6849            .store()
6850            .record_source_watermark(
6851                &watermark_key,
6852                ConsoleFrameSourceKind::SessionHistory,
6853                &format_session_history_watermark(&session_id, 0, current_time_ms()),
6854            )
6855            .await
6856            .expect("record fresh empty watermark");
6857
6858        let page = tokio::time::timeout(
6859            Duration::from_secs(2),
6860            aggregator.query_timeline(ConsoleTimelineQuery {
6861                identity: Some("test/agent-a".to_string()),
6862                limit: 20,
6863                ..ConsoleTimelineQuery::default()
6864            }),
6865        )
6866        .await
6867        .expect("explicit identity query should not stall")
6868        .expect("query succeeds");
6869
6870        assert!(
6871            page.frames.is_empty(),
6872            "empty fresh-watermark query should return promptly before async backfill; frames: {:#?}",
6873            page.frames
6874        );
6875        wait_for_session_history_text(
6876            &aggregator,
6877            "test/agent-a",
6878            "You are agent-a.",
6879            Duration::from_secs(5),
6880        )
6881        .await?;
6882        let _ = runtime.mob_handle().stop().await;
6883        Ok(())
6884    }
6885
6886    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887    async fn explicit_identity_query_refreshes_stale_existing_session_history() -> Result<(), String>
6888    {
6889        let store = Arc::new(CountingConsoleLogStore::new());
6890        let aggregator = MobKitConsoleAggregator::new(store.clone());
6891        let runtime = build_single_member_runtime().await;
6892        let entry = runtime_entry_for_test("runtime-a", &runtime);
6893        let resolved = member_sources_for_entry(&entry)
6894            .await
6895            .into_iter()
6896            .find(|candidate| candidate.member.agent_identity.as_str() == "agent-a")
6897            .expect("agent-a member exists");
6898        let record = identity_record_for_member(&entry, &resolved.handle, &resolved.member)
6899            .await
6900            .expect("identity record exists");
6901        let session_id = record.session_id.expect("agent-a has a session");
6902        wait_for_runtime_session_history_text(
6903            &runtime,
6904            &session_id,
6905            "You are agent-a.",
6906            Duration::from_secs(5),
6907        )
6908        .await?;
6909        aggregator
6910            .inner
6911            .runtimes
6912            .write()
6913            .expect("runtime registry")
6914            .insert("runtime-a".to_string(), entry);
6915
6916        store
6917            .append_if_absent(NewConsoleFrame {
6918                id: None,
6919                dedupe_key: "stale-session-history-agent-a".to_string(),
6920                timestamp_ms: 10,
6921                runtime_key: "runtime-a".to_string(),
6922                identity: "test/agent-a".to_string(),
6923                conversation_id: Some("test/agent-a".to_string()),
6924                session_id: Some(session_id.clone()),
6925                kind: "user_input".to_string(),
6926                status: ConsoleFrameStatus::Delivered,
6927                payload: json!({
6928                    "text": "stale projected history",
6929                    "type": "user_input",
6930                }),
6931                source: ConsoleFrameSource {
6932                    kind: ConsoleFrameSourceKind::SessionHistory,
6933                    source_cursor: Some("stale-session-history-agent-a".to_string()),
6934                },
6935                source_event_id: Some("stale-session-history-agent-a".to_string()),
6936                interaction_id: None,
6937                turn_id: None,
6938                run_id: None,
6939                parent_frame_id: None,
6940                caused_by_frame_id: None,
6941            })
6942            .await
6943            .expect("append stale history frame");
6944
6945        let fresh_prompt = "fresh prompt after stale history";
6946        let sent_session_id = send_message_on_mob_with_mode(
6947            &runtime.mob_handle(),
6948            "agent-a",
6949            ContentInput::Text(fresh_prompt.to_string()),
6950            meerkat_core::types::HandlingMode::Queue,
6951        )
6952        .await
6953        .expect("direct member send succeeds");
6954        assert_eq!(sent_session_id, session_id);
6955        wait_for_runtime_session_history_text(
6956            &runtime,
6957            &session_id,
6958            fresh_prompt,
6959            Duration::from_secs(5),
6960        )
6961        .await?;
6962
6963        let page = aggregator
6964            .query_timeline(ConsoleTimelineQuery {
6965                identity: Some("test/agent-a".to_string()),
6966                limit: 50,
6967                ..ConsoleTimelineQuery::default()
6968            })
6969            .await
6970            .expect("query child timeline");
6971
6972        assert!(
6973            page.frames.iter().any(|frame| {
6974                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
6975                    && session_history_content_text(frame).as_deref()
6976                        == Some("stale projected history")
6977            }),
6978            "explicit identity query should return existing history before async refresh; frames: {:#?}",
6979            page.frames
6980        );
6981        wait_for_session_history_text(
6982            &aggregator,
6983            "test/agent-a",
6984            fresh_prompt,
6985            Duration::from_secs(5),
6986        )
6987        .await?;
6988        assert!(
6989            store.source_watermark_calls() > 0,
6990            "stale existing session history should force a targeted source refresh"
6991        );
6992        let _ = runtime.mob_handle().stop().await;
6993        Ok(())
6994    }
6995
6996    async fn wait_for_session_history_text(
6997        aggregator: &MobKitConsoleAggregator,
6998        identity: &str,
6999        expected: &str,
7000        timeout: Duration,
7001    ) -> Result<(), String> {
7002        let deadline = Instant::now() + timeout;
7003        let mut observed = Vec::new();
7004        while Instant::now() < deadline {
7005            let _ = aggregator.list_identities().await;
7006            let page = aggregator
7007                .query_timeline(ConsoleTimelineQuery {
7008                    identity: Some(identity.to_string()),
7009                    limit: 20,
7010                    ..ConsoleTimelineQuery::default()
7011                })
7012                .await
7013                .expect("query timeline");
7014            observed = page.frames;
7015            if observed.iter().any(|frame| {
7016                frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7017                    && session_history_content_text(frame).as_deref() == Some(expected)
7018            }) {
7019                return Ok(());
7020            }
7021            tokio::time::sleep(Duration::from_millis(25)).await;
7022        }
7023
7024        Err(format!(
7025            "session history text {expected:?} was not backfilled; observed frames: {observed:#?}",
7026        ))
7027    }
7028
7029    async fn wait_for_session_backfill_target(
7030        aggregator: &MobKitConsoleAggregator,
7031        identity: &str,
7032        timeout: Duration,
7033    ) -> Result<String, String> {
7034        let deadline = Instant::now() + timeout;
7035        while Instant::now() < deadline {
7036            if let Some(target) =
7037                session_backfill_target_for_identity(&aggregator.inner, identity).await
7038            {
7039                return Ok(target.session_id);
7040            }
7041            tokio::time::sleep(Duration::from_millis(25)).await;
7042        }
7043        Err(format!(
7044            "session backfill target for {identity:?} was not resolvable"
7045        ))
7046    }
7047
7048    async fn wait_for_runtime_session_history_text(
7049        runtime: &UnifiedRuntime,
7050        session_id: &str,
7051        expected: &str,
7052        timeout: Duration,
7053    ) -> Result<(), String> {
7054        let deadline = Instant::now() + timeout;
7055        let mut observed = Vec::new();
7056        while Instant::now() < deadline {
7057            let page = runtime
7058                .mob_runtime()
7059                .read_session_history(session_id, 0, Some(20))
7060                .await
7061                .map_err(|err| err.to_string())?;
7062            observed = page.messages;
7063            if observed.iter().enumerate().any(|(idx, message)| {
7064                let Some(message) = serde_json::to_value(message).ok() else {
7065                    return false;
7066                };
7067                frames_from_session_history_message(
7068                    "runtime-a",
7069                    "test/agent-a",
7070                    session_id,
7071                    idx,
7072                    message,
7073                )
7074                .iter()
7075                .any(|frame| {
7076                    matches!(
7077                        frame.kind.as_str(),
7078                        "user_input" | "system_notice" | "interaction_complete"
7079                    ) && session_history_frame_content_text(frame).as_deref() == Some(expected)
7080                })
7081            }) {
7082                return Ok(());
7083            }
7084            tokio::time::sleep(Duration::from_millis(25)).await;
7085        }
7086
7087        Err(format!(
7088            "runtime session history text {expected:?} was not readable before watermark setup; observed messages: {observed:#?}",
7089        ))
7090    }
7091
7092    async fn wait_for_identity_record(
7093        aggregator: &MobKitConsoleAggregator,
7094        identity: &str,
7095        session_id: Option<&str>,
7096        timeout: Duration,
7097    ) -> Result<(), String> {
7098        let deadline = Instant::now() + timeout;
7099        let mut observed = Vec::new();
7100        while Instant::now() < deadline {
7101            observed = aggregator
7102                .list_identities()
7103                .await
7104                .map_err(|err| err.to_string())?;
7105            if observed.iter().any(|record| {
7106                record.identity == identity && record.session_id.as_deref() == session_id
7107            }) {
7108                return Ok(());
7109            }
7110            tokio::time::sleep(Duration::from_millis(25)).await;
7111        }
7112
7113        Err(format!(
7114            "identity {identity:?} with session {session_id:?} was not projected; observed identities: {observed:#?}",
7115        ))
7116    }
7117
7118    fn session_history_content_text(frame: &ConsoleFrame) -> Option<String> {
7119        session_history_payload_text(&frame.payload)
7120    }
7121
7122    fn session_history_frame_content_text(frame: &NewConsoleFrame) -> Option<String> {
7123        session_history_payload_text(&frame.payload)
7124    }
7125
7126    fn session_history_payload_text(payload: &Value) -> Option<String> {
7127        if let Some(text) = payload.get("text").and_then(Value::as_str) {
7128            return Some(text.to_string());
7129        }
7130        if let Some(text) = payload.get("result").and_then(Value::as_str) {
7131            return Some(text.to_string());
7132        }
7133        if let Some(text) = payload.get("body").and_then(Value::as_str) {
7134            return Some(text.to_string());
7135        }
7136        match payload.get("content")? {
7137            Value::String(text) => Some(text.clone()),
7138            Value::Array(blocks) => Some(
7139                blocks
7140                    .iter()
7141                    .filter_map(|block| block.get("text").and_then(Value::as_str))
7142                    .collect::<Vec<_>>()
7143                    .join(""),
7144            ),
7145            _ => None,
7146        }
7147    }
7148
7149    #[tokio::test]
7150    async fn query_timeline_handles_large_store_without_backfill_calls() {
7151        let store = Arc::new(CountingConsoleLogStore::new());
7152        let aggregator = MobKitConsoleAggregator::new(store.clone());
7153        for idx in 0..5_000 {
7154            store
7155                .append_if_absent(NewConsoleFrame {
7156                    id: None,
7157                    dedupe_key: format!("event-{idx}"),
7158                    timestamp_ms: idx,
7159                    runtime_key: "runtime-a".to_string(),
7160                    identity: "agent-a".to_string(),
7161                    conversation_id: Some("agent-a".to_string()),
7162                    session_id: Some("session-a".to_string()),
7163                    kind: "text_delta".to_string(),
7164                    status: ConsoleFrameStatus::Delivered,
7165                    payload: json!({ "delta": idx }),
7166                    source: ConsoleFrameSource {
7167                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7168                        source_cursor: None,
7169                    },
7170                    source_event_id: Some(format!("event-{idx}")),
7171                    interaction_id: None,
7172                    turn_id: None,
7173                    run_id: None,
7174                    parent_frame_id: None,
7175                    caused_by_frame_id: None,
7176                })
7177                .await
7178                .expect("append frame");
7179        }
7180
7181        let page = aggregator
7182            .query_timeline(ConsoleTimelineQuery {
7183                identity: Some("agent-a".to_string()),
7184                limit: 1_000,
7185                ..ConsoleTimelineQuery::default()
7186            })
7187            .await
7188            .expect("large query");
7189
7190        assert_eq!(page.frames.len(), 1_000);
7191        assert_eq!(store.source_watermark_calls(), 0);
7192    }
7193
7194    #[tokio::test]
7195    async fn query_timeline_rejects_future_cursor_after_store_reset() {
7196        let aggregator = MobKitConsoleAggregator::in_memory();
7197        let err = aggregator
7198            .query_timeline_windowed(ConsoleTimelineWindowQuery {
7199                after: Some(ConsoleCursor::from("console:99")),
7200                limit: 10,
7201                ..ConsoleTimelineWindowQuery::default()
7202            })
7203            .await
7204            .expect_err("future cursor on empty/reset store must be replay-unavailable");
7205
7206        assert!(
7207            err.to_string()
7208                .contains("beyond the current store frontier"),
7209            "unexpected error: {err}"
7210        );
7211    }
7212
7213    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7214    async fn explicit_identity_timeline_backfills_kickoff_when_live_tool_frames_arrive_first() {
7215        let store = Arc::new(CountingConsoleLogStore::new());
7216        let aggregator = MobKitConsoleAggregator::new(store.clone());
7217        let runtime = build_single_member_runtime().await;
7218        aggregator
7219            .inner
7220            .runtimes
7221            .write()
7222            .expect("runtime registry")
7223            .insert(
7224                "runtime-a".to_string(),
7225                runtime_entry_for_test("runtime-a", &runtime),
7226            );
7227        let session_id =
7228            wait_for_session_backfill_target(&aggregator, "test/agent-a", Duration::from_secs(5))
7229                .await
7230                .expect("spawned member is resolvable for targeted backfill");
7231        wait_for_runtime_session_history_text(
7232            &runtime,
7233            &session_id,
7234            "You are agent-a.",
7235            Duration::from_secs(5),
7236        )
7237        .await
7238        .expect("spawned member kickoff is readable before live-frame race");
7239        store
7240            .append_if_absent(NewConsoleFrame {
7241                id: None,
7242                dedupe_key: "live-tool-before-history".to_string(),
7243                timestamp_ms: 10,
7244                runtime_key: "runtime-a".to_string(),
7245                identity: "test/agent-a".to_string(),
7246                conversation_id: Some("test/agent-a".to_string()),
7247                session_id: None,
7248                kind: "tool_execution_started".to_string(),
7249                status: ConsoleFrameStatus::Delivered,
7250                payload: json!({
7251                    "id": "call-live",
7252                    "name": "king_search",
7253                    "source_event_type": "tool_execution_started",
7254                    "type": "tool_execution_started",
7255                }),
7256                source: ConsoleFrameSource {
7257                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7258                    source_cursor: Some("live-tool-before-history".to_string()),
7259                },
7260                source_event_id: Some("live-tool-before-history".to_string()),
7261                interaction_id: None,
7262                turn_id: None,
7263                run_id: None,
7264                parent_frame_id: None,
7265                caused_by_frame_id: None,
7266            })
7267            .await
7268            .expect("append live tool frame");
7269
7270        let page = aggregator
7271            .query_timeline(ConsoleTimelineQuery {
7272                identity: Some("test/agent-a".to_string()),
7273                limit: 20,
7274                ..ConsoleTimelineQuery::default()
7275            })
7276            .await
7277            .expect("query child timeline");
7278
7279        assert!(
7280            page.frames
7281                .iter()
7282                .any(|frame| frame.kind == "tool_execution_started"),
7283            "initial explicit child timeline should return existing store frames without waiting for session history"
7284        );
7285
7286        let mut observed_backfill = false;
7287        for _ in 0..80 {
7288            let page = aggregator
7289                .query_timeline(ConsoleTimelineQuery {
7290                    identity: Some("test/agent-a".to_string()),
7291                    limit: 20,
7292                    ..ConsoleTimelineQuery::default()
7293                })
7294                .await
7295                .expect("query child timeline after scheduled backfill");
7296            if page.frames.iter().any(|frame| {
7297                frame.kind == "user_input"
7298                    && frame.source.kind == ConsoleFrameSourceKind::SessionHistory
7299                    && frame.payload.to_string().contains("You are agent-a.")
7300            }) {
7301                observed_backfill = true;
7302                break;
7303            }
7304            tokio::time::sleep(Duration::from_millis(25)).await;
7305        }
7306        assert!(
7307            observed_backfill,
7308            "scheduled explicit child timeline backfill should eventually include the kickoff prompt"
7309        );
7310        let _ = runtime.mob_handle().stop().await;
7311    }
7312
7313    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7314    async fn explicit_identity_timeline_query_does_not_resolve_roster_per_frame() {
7315        let store = Arc::new(CountingConsoleLogStore::new());
7316        let aggregator = MobKitConsoleAggregator::new(store.clone());
7317        let (_temp, runtime, _delayed_service) =
7318            build_stress_runtime(64, Duration::from_millis(0)).await;
7319        aggregator
7320            .inner
7321            .runtimes
7322            .write()
7323            .expect("runtime registry")
7324            .insert(
7325                "runtime-a".to_string(),
7326                runtime_entry_for_test("runtime-a", runtime.as_ref()),
7327            );
7328        for idx in 0..1_000 {
7329            store
7330                .append_if_absent(NewConsoleFrame {
7331                    id: None,
7332                    dedupe_key: format!("event-{idx}"),
7333                    timestamp_ms: idx,
7334                    runtime_key: "runtime-a".to_string(),
7335                    identity: "test/agent-0".to_string(),
7336                    conversation_id: Some("test/agent-0".to_string()),
7337                    session_id: Some("session-a".to_string()),
7338                    kind: "text_delta".to_string(),
7339                    status: ConsoleFrameStatus::Delivered,
7340                    payload: json!({ "delta": idx }),
7341                    source: ConsoleFrameSource {
7342                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7343                        source_cursor: None,
7344                    },
7345                    source_event_id: Some(format!("event-{idx}")),
7346                    interaction_id: None,
7347                    turn_id: None,
7348                    run_id: None,
7349                    parent_frame_id: None,
7350                    caused_by_frame_id: None,
7351                })
7352                .await
7353                .expect("append frame");
7354        }
7355
7356        let page = tokio::time::timeout(
7357            Duration::from_secs(2),
7358            aggregator.query_timeline(ConsoleTimelineQuery {
7359                identity: Some("test/agent-0".to_string()),
7360                limit: 1_000,
7361                ..ConsoleTimelineQuery::default()
7362            }),
7363        )
7364        .await
7365        .expect("identity timeline query should not rediscover the roster per frame")
7366        .expect("timeline query succeeds");
7367
7368        assert_eq!(page.frames.len(), 1_000);
7369        assert_eq!(store.source_watermark_calls(), 0);
7370        let _ = runtime.mob_handle().stop().await;
7371    }
7372
7373    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7374    async fn refresh_session_history_parallelizes_slow_member_backfills_at_scale() {
7375        const MEMBER_COUNT: usize = 32;
7376        let (_temp, runtime, delayed_service) =
7377            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(40)).await;
7378        let aggregator = MobKitConsoleAggregator::in_memory();
7379        aggregator
7380            .inner
7381            .runtimes
7382            .write()
7383            .expect("runtime registry")
7384            .insert(
7385                "runtime-stress".to_string(),
7386                runtime_entry_for_test("runtime-stress", &runtime),
7387            );
7388
7389        let started = Instant::now();
7390        aggregator
7391            .refresh_session_history()
7392            .await
7393            .expect("stress refresh");
7394        let elapsed = started.elapsed();
7395
7396        assert!(
7397            delayed_service.read_calls() >= MEMBER_COUNT,
7398            "expected at least one history read per member, saw {}",
7399            delayed_service.read_calls()
7400        );
7401        assert!(
7402            delayed_service.max_active_reads() > 1,
7403            "session history backfill should fan out instead of reading members serially"
7404        );
7405        assert!(
7406            delayed_service.max_active_reads()
7407                <= ConsoleAggregatorOptions::default().max_concurrent_session_backfills,
7408            "session history backfill should respect the default concurrency limit"
7409        );
7410        assert!(
7411            elapsed < Duration::from_millis(600),
7412            "parallel backfill should be far below serial {}ms path, elapsed: {elapsed:?}",
7413            MEMBER_COUNT * 40
7414        );
7415        let _ = runtime.mob_handle().stop().await;
7416    }
7417
7418    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7419    async fn session_history_backfill_respects_configured_concurrency_limit() {
7420        const MEMBER_COUNT: usize = 16;
7421        let (_temp, runtime, delayed_service) =
7422            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(30)).await;
7423        let aggregator =
7424            MobKitConsoleAggregator::in_memory_with_options(ConsoleAggregatorOptions {
7425                max_concurrent_session_backfills: 4,
7426                ..ConsoleAggregatorOptions::default()
7427            });
7428        aggregator
7429            .inner
7430            .runtimes
7431            .write()
7432            .expect("runtime registry")
7433            .insert(
7434                "runtime-stress".to_string(),
7435                runtime_entry_for_test("runtime-stress", &runtime),
7436            );
7437
7438        aggregator
7439            .refresh_session_history()
7440            .await
7441            .expect("stress refresh");
7442
7443        assert!(
7444            delayed_service.max_active_reads() <= 4,
7445            "configured concurrency limit should bound session history reads, saw {}",
7446            delayed_service.max_active_reads()
7447        );
7448        let _ = runtime.mob_handle().stop().await;
7449    }
7450
7451    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
7452    async fn live_event_burst_reaches_store_while_slow_backfill_is_running() {
7453        const MEMBER_COUNT: usize = 24;
7454        const LIVE_EVENT_COUNT: usize = 2_048;
7455        let (_temp, runtime, _delayed_service) =
7456            build_stress_runtime(MEMBER_COUNT, Duration::from_millis(200)).await;
7457        let aggregator = MobKitConsoleAggregator::in_memory();
7458        aggregator.register_runtime(ConsoleRuntimeRegistration {
7459            runtime_key: "runtime-burst".to_string(),
7460            runtime: runtime.clone(),
7461            identity_namespace: "stress".to_string(),
7462            visibility_policy: Arc::new(AllowAllConsoleVisibilityPolicy),
7463        });
7464        let console_events = runtime.console_events();
7465        for idx in 0..LIVE_EVENT_COUNT {
7466            console_events
7467                .append(
7468                    "agent-0",
7469                    Some("burst-turn".to_string()),
7470                    "text_delta",
7471                    json!({ "delta": format!("frame-{idx}") }),
7472                )
7473                .await;
7474        }
7475
7476        let deadline = Instant::now() + Duration::from_secs(5);
7477        let mut observed = 0;
7478        while Instant::now() < deadline {
7479            observed = count_console_event_frames(&aggregator, "stress/agent-0").await;
7480            if observed >= LIVE_EVENT_COUNT {
7481                break;
7482            }
7483            tokio::time::sleep(Duration::from_millis(25)).await;
7484        }
7485
7486        assert_eq!(
7487            observed, LIVE_EVENT_COUNT,
7488            "live pump should not drop frames while slow background backfill is running"
7489        );
7490        let _ = runtime.mob_handle().stop().await;
7491    }
7492
7493    async fn count_console_event_frames(
7494        aggregator: &MobKitConsoleAggregator,
7495        identity: &str,
7496    ) -> usize {
7497        let mut after = None;
7498        let mut count = 0;
7499        loop {
7500            let page = aggregator
7501                .query_timeline(ConsoleTimelineQuery {
7502                    identity: Some(identity.to_string()),
7503                    after,
7504                    limit: 1_000,
7505                    ..ConsoleTimelineQuery::default()
7506                })
7507                .await
7508                .expect("query burst timeline");
7509            if page.frames.is_empty() {
7510                break;
7511            }
7512            count += page
7513                .frames
7514                .iter()
7515                .filter(|frame| frame.source.kind == ConsoleFrameSourceKind::ConsoleEvent)
7516                .count();
7517            after = page.next_cursor;
7518            if after.is_none() {
7519                break;
7520            }
7521        }
7522        count
7523    }
7524
7525    #[tokio::test]
7526    async fn status_updates_get_replayable_aggregate_cursors() {
7527        let aggregator = MobKitConsoleAggregator::in_memory();
7528        let frame = NewConsoleFrame {
7529            id: None,
7530            dedupe_key: "send-1".to_string(),
7531            timestamp_ms: 1,
7532            runtime_key: "runtime-a".to_string(),
7533            identity: "agent-a".to_string(),
7534            conversation_id: Some("agent-a".to_string()),
7535            session_id: Some("session-1".to_string()),
7536            kind: "user_input".to_string(),
7537            status: ConsoleFrameStatus::Accepted,
7538            payload: json!({ "content": "hello" }),
7539            source: ConsoleFrameSource {
7540                kind: ConsoleFrameSourceKind::Send,
7541                source_cursor: None,
7542            },
7543            source_event_id: None,
7544            interaction_id: Some("interaction-1".to_string()),
7545            turn_id: None,
7546            run_id: None,
7547            parent_frame_id: None,
7548            caused_by_frame_id: None,
7549        };
7550        let inserted = aggregator
7551            .store()
7552            .append_if_absent(frame)
7553            .await
7554            .expect("append frame");
7555
7556        update_frame_status_and_emit(
7557            &aggregator.inner,
7558            &inserted.frame.id,
7559            ConsoleFrameStatus::Delivered,
7560        )
7561        .await
7562        .expect("update status");
7563
7564        let page = aggregator
7565            .query_timeline(ConsoleTimelineQuery {
7566                identity: Some("agent-a".to_string()),
7567                after: Some(inserted.frame.cursor.clone()),
7568                limit: 10,
7569                ..ConsoleTimelineQuery::default()
7570            })
7571            .await
7572            .expect("query timeline");
7573        assert_eq!(page.frames.len(), 1);
7574        assert_eq!(page.frames[0].kind, "frame_updated");
7575        assert_eq!(page.frames[0].parent_frame_id, Some(inserted.frame.id));
7576        assert_eq!(
7577            page.frames[0]
7578                .payload
7579                .get("frame")
7580                .and_then(|frame| frame.get("status"))
7581                .and_then(Value::as_str),
7582            Some("delivered")
7583        );
7584    }
7585
7586    #[tokio::test]
7587    async fn steer_delivery_appends_terminal_control_frame() {
7588        let aggregator = MobKitConsoleAggregator::in_memory();
7589        let frame = NewConsoleFrame {
7590            id: None,
7591            dedupe_key: "send-steer-1".to_string(),
7592            timestamp_ms: 1,
7593            runtime_key: "runtime-a".to_string(),
7594            identity: "agent-a".to_string(),
7595            conversation_id: Some("agent-a".to_string()),
7596            session_id: Some("session-1".to_string()),
7597            kind: "user_input".to_string(),
7598            status: ConsoleFrameStatus::Delivered,
7599            payload: json!({
7600                "content": "operator steer",
7601                "handling_mode": "steer",
7602            }),
7603            source: ConsoleFrameSource {
7604                kind: ConsoleFrameSourceKind::Send,
7605                source_cursor: None,
7606            },
7607            source_event_id: None,
7608            interaction_id: Some("interaction-steer-1".to_string()),
7609            turn_id: None,
7610            run_id: None,
7611            parent_frame_id: None,
7612            caused_by_frame_id: None,
7613        };
7614        let inserted = aggregator
7615            .store()
7616            .append_if_absent(frame)
7617            .await
7618            .expect("append steer input");
7619
7620        append_steer_delivery_terminal(&aggregator.inner, &inserted.frame, "interaction-steer-1")
7621            .await
7622            .expect("append steer terminal");
7623
7624        let page = aggregator
7625            .query_timeline(ConsoleTimelineQuery {
7626                identity: Some("agent-a".to_string()),
7627                after: Some(inserted.frame.cursor.clone()),
7628                limit: 10,
7629                ..ConsoleTimelineQuery::default()
7630            })
7631            .await
7632            .expect("query timeline");
7633
7634        assert_eq!(page.frames.len(), 1);
7635        let terminal = &page.frames[0];
7636        assert_eq!(terminal.kind, "interaction_complete");
7637        assert_eq!(terminal.status, ConsoleFrameStatus::Completed);
7638        assert_eq!(
7639            terminal.interaction_id.as_deref(),
7640            Some("interaction-steer-1")
7641        );
7642        assert_eq!(
7643            terminal.parent_frame_id.as_deref(),
7644            Some(inserted.frame.id.as_str())
7645        );
7646        assert_eq!(
7647            terminal.payload.get("reason").and_then(Value::as_str),
7648            Some("steer_delivered")
7649        );
7650    }
7651
7652    #[tokio::test]
7653    async fn history_counterpart_scan_is_not_capped_to_one_page() {
7654        let aggregator = MobKitConsoleAggregator::in_memory();
7655        for idx in 0..1_005 {
7656            aggregator
7657                .store()
7658                .append_if_absent(NewConsoleFrame {
7659                    id: None,
7660                    dedupe_key: format!("filler-{idx}"),
7661                    timestamp_ms: idx,
7662                    runtime_key: "runtime-a".to_string(),
7663                    identity: "agent-a".to_string(),
7664                    conversation_id: Some("agent-a".to_string()),
7665                    session_id: Some("session-a".to_string()),
7666                    kind: "text_delta".to_string(),
7667                    status: ConsoleFrameStatus::Completed,
7668                    payload: json!({ "delta": idx }),
7669                    source: ConsoleFrameSource {
7670                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7671                        source_cursor: None,
7672                    },
7673                    source_event_id: Some(format!("filler-{idx}")),
7674                    interaction_id: None,
7675                    turn_id: None,
7676                    run_id: None,
7677                    parent_frame_id: None,
7678                    caused_by_frame_id: None,
7679                })
7680                .await
7681                .expect("append filler");
7682        }
7683        aggregator
7684            .store()
7685            .append_if_absent(NewConsoleFrame {
7686                id: None,
7687                dedupe_key: "live-user-input".to_string(),
7688                timestamp_ms: 2_000,
7689                runtime_key: "runtime-a".to_string(),
7690                identity: "agent-a".to_string(),
7691                conversation_id: Some("agent-a".to_string()),
7692                session_id: Some("session-a".to_string()),
7693                kind: "user_input".to_string(),
7694                status: ConsoleFrameStatus::Delivered,
7695                payload: json!({ "content": "already here" }),
7696                source: ConsoleFrameSource {
7697                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7698                    source_cursor: None,
7699                },
7700                source_event_id: Some("live-user-input".to_string()),
7701                interaction_id: None,
7702                turn_id: None,
7703                run_id: None,
7704                parent_frame_id: None,
7705                caused_by_frame_id: None,
7706            })
7707            .await
7708            .expect("append live input");
7709
7710        let history = NewConsoleFrame {
7711            id: None,
7712            dedupe_key: "history-user-input".to_string(),
7713            timestamp_ms: 3_000,
7714            runtime_key: "runtime-a".to_string(),
7715            identity: "agent-a".to_string(),
7716            conversation_id: Some("agent-a".to_string()),
7717            session_id: Some("session-a".to_string()),
7718            kind: "user_input".to_string(),
7719            status: ConsoleFrameStatus::Completed,
7720            payload: json!({ "content": "already here" }),
7721            source: ConsoleFrameSource {
7722                kind: ConsoleFrameSourceKind::SessionHistory,
7723                source_cursor: Some("session-a:1006".to_string()),
7724            },
7725            source_event_id: None,
7726            interaction_id: None,
7727            turn_id: None,
7728            run_id: None,
7729            parent_frame_id: None,
7730            caused_by_frame_id: None,
7731        };
7732
7733        assert!(
7734            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7735                .await
7736                .expect("counterpart scan")
7737        );
7738    }
7739
7740    #[tokio::test]
7741    async fn history_counterpart_scan_matches_rpc_wrapped_user_prompts() {
7742        let aggregator = MobKitConsoleAggregator::in_memory();
7743        aggregator
7744            .store()
7745            .append_if_absent(NewConsoleFrame {
7746                id: None,
7747                dedupe_key: "live-user-input".to_string(),
7748                timestamp_ms: 2_000,
7749                runtime_key: "runtime-a".to_string(),
7750                identity: "agent-a".to_string(),
7751                conversation_id: Some("agent-a".to_string()),
7752                session_id: Some("session-a".to_string()),
7753                kind: "user_input".to_string(),
7754                status: ConsoleFrameStatus::Delivered,
7755                payload: json!({ "content": "hello from operator" }),
7756                source: ConsoleFrameSource {
7757                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7758                    source_cursor: None,
7759                },
7760                source_event_id: Some("live-user-input".to_string()),
7761                interaction_id: None,
7762                turn_id: None,
7763                run_id: None,
7764                parent_frame_id: None,
7765                caused_by_frame_id: None,
7766            })
7767            .await
7768            .expect("append live input");
7769
7770        let history = NewConsoleFrame {
7771            id: None,
7772            dedupe_key: "history-user-input".to_string(),
7773            timestamp_ms: 3_000,
7774            runtime_key: "runtime-a".to_string(),
7775            identity: "agent-a".to_string(),
7776            conversation_id: Some("agent-a".to_string()),
7777            session_id: Some("session-a".to_string()),
7778            kind: "user_input".to_string(),
7779            status: ConsoleFrameStatus::Completed,
7780            payload: json!({ "content": "[EVENT via rpc] hello from operator" }),
7781            source: ConsoleFrameSource {
7782                kind: ConsoleFrameSourceKind::SessionHistory,
7783                source_cursor: Some("session-a:2".to_string()),
7784            },
7785            source_event_id: None,
7786            interaction_id: None,
7787            turn_id: None,
7788            run_id: None,
7789            parent_frame_id: None,
7790            caused_by_frame_id: None,
7791        };
7792
7793        assert!(
7794            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7795                .await
7796                .expect("counterpart scan")
7797        );
7798    }
7799
7800    #[tokio::test]
7801    async fn history_counterpart_scan_matches_content_block_user_prompts() {
7802        let aggregator = MobKitConsoleAggregator::in_memory();
7803        aggregator
7804            .store()
7805            .append_if_absent(NewConsoleFrame {
7806                id: None,
7807                dedupe_key: "live-user-input".to_string(),
7808                timestamp_ms: 2_000,
7809                runtime_key: "runtime-a".to_string(),
7810                identity: "agent-a".to_string(),
7811                conversation_id: Some("agent-a".to_string()),
7812                session_id: Some("session-a".to_string()),
7813                kind: "user_input".to_string(),
7814                status: ConsoleFrameStatus::Delivered,
7815                payload: json!({ "content": "hello from operator" }),
7816                source: ConsoleFrameSource {
7817                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7818                    source_cursor: None,
7819                },
7820                source_event_id: Some("live-user-input".to_string()),
7821                interaction_id: None,
7822                turn_id: None,
7823                run_id: None,
7824                parent_frame_id: None,
7825                caused_by_frame_id: None,
7826            })
7827            .await
7828            .expect("append live input");
7829
7830        let history = NewConsoleFrame {
7831            id: None,
7832            dedupe_key: "history-user-input".to_string(),
7833            timestamp_ms: 3_000,
7834            runtime_key: "runtime-a".to_string(),
7835            identity: "agent-a".to_string(),
7836            conversation_id: Some("agent-a".to_string()),
7837            session_id: Some("session-a".to_string()),
7838            kind: "user_input".to_string(),
7839            status: ConsoleFrameStatus::Completed,
7840            payload: json!({
7841                "content": [{ "type": "text", "text": "hello from operator" }]
7842            }),
7843            source: ConsoleFrameSource {
7844                kind: ConsoleFrameSourceKind::SessionHistory,
7845                source_cursor: Some("session-a:2".to_string()),
7846            },
7847            source_event_id: None,
7848            interaction_id: None,
7849            turn_id: None,
7850            run_id: None,
7851            parent_frame_id: None,
7852            caused_by_frame_id: None,
7853        };
7854
7855        assert!(
7856            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7857                .await
7858                .expect("counterpart scan")
7859        );
7860    }
7861
7862    #[tokio::test]
7863    async fn history_counterpart_scan_matches_live_tool_results() {
7864        let aggregator = MobKitConsoleAggregator::in_memory();
7865        aggregator
7866            .store()
7867            .append_if_absent(NewConsoleFrame {
7868                id: None,
7869                dedupe_key: "live-tool-result".to_string(),
7870                timestamp_ms: 2_000,
7871                runtime_key: "runtime-a".to_string(),
7872                identity: "agent-a".to_string(),
7873                conversation_id: Some("agent-a".to_string()),
7874                session_id: Some("session-a".to_string()),
7875                kind: "tool_execution_completed".to_string(),
7876                status: ConsoleFrameStatus::Delivered,
7877                payload: json!({
7878                    "id": "call-1",
7879                    "tool_call_id": "call-1",
7880                    "result": "{ \"count\": 70 }"
7881                }),
7882                source: ConsoleFrameSource {
7883                    kind: ConsoleFrameSourceKind::ConsoleEvent,
7884                    source_cursor: None,
7885                },
7886                source_event_id: Some("live-tool-result".to_string()),
7887                interaction_id: None,
7888                turn_id: None,
7889                run_id: None,
7890                parent_frame_id: None,
7891                caused_by_frame_id: None,
7892            })
7893            .await
7894            .expect("append live tool result");
7895
7896        let history = NewConsoleFrame {
7897            id: None,
7898            dedupe_key: "history-tool-result".to_string(),
7899            timestamp_ms: 3_000,
7900            runtime_key: "runtime-a".to_string(),
7901            identity: "agent-a".to_string(),
7902            conversation_id: Some("agent-a".to_string()),
7903            session_id: Some("session-a".to_string()),
7904            kind: "tool_execution_completed".to_string(),
7905            status: ConsoleFrameStatus::Completed,
7906            payload: json!({
7907                "id": "call-1",
7908                "tool_call_id": "call-1",
7909                "result": "{ \"count\": 70 }",
7910                "content": [{ "type": "text", "text": "{ \"count\": 70 }" }]
7911            }),
7912            source: ConsoleFrameSource {
7913                kind: ConsoleFrameSourceKind::SessionHistory,
7914                source_cursor: Some("session-a:3:0".to_string()),
7915            },
7916            source_event_id: None,
7917            interaction_id: None,
7918            turn_id: None,
7919            run_id: None,
7920            parent_frame_id: None,
7921            caused_by_frame_id: None,
7922        };
7923
7924        assert!(
7925            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7926                .await
7927                .expect("counterpart scan")
7928        );
7929    }
7930
7931    #[tokio::test]
7932    async fn history_counterpart_scan_matches_streamed_text_delta_completion() {
7933        let aggregator = MobKitConsoleAggregator::in_memory();
7934        for (idx, delta) in ["Ready ", "and standing by."].iter().enumerate() {
7935            aggregator
7936                .store()
7937                .append_if_absent(NewConsoleFrame {
7938                    id: None,
7939                    dedupe_key: format!("live-delta-{idx}"),
7940                    timestamp_ms: 2_000 + idx as u64,
7941                    runtime_key: "runtime-a".to_string(),
7942                    identity: "agent-a".to_string(),
7943                    conversation_id: Some("agent-a".to_string()),
7944                    session_id: Some("session-a".to_string()),
7945                    kind: "text_delta".to_string(),
7946                    status: ConsoleFrameStatus::Delivered,
7947                    payload: json!({ "delta": delta }),
7948                    source: ConsoleFrameSource {
7949                        kind: ConsoleFrameSourceKind::ConsoleEvent,
7950                        source_cursor: None,
7951                    },
7952                    source_event_id: Some(format!("live-delta-{idx}")),
7953                    interaction_id: Some("turn-a".to_string()),
7954                    turn_id: None,
7955                    run_id: None,
7956                    parent_frame_id: None,
7957                    caused_by_frame_id: None,
7958                })
7959                .await
7960                .expect("append live delta");
7961        }
7962
7963        let history = NewConsoleFrame {
7964            id: None,
7965            dedupe_key: "history-assistant-complete".to_string(),
7966            timestamp_ms: 3_000,
7967            runtime_key: "runtime-a".to_string(),
7968            identity: "agent-a".to_string(),
7969            conversation_id: Some("agent-a".to_string()),
7970            session_id: Some("session-a".to_string()),
7971            kind: "interaction_complete".to_string(),
7972            status: ConsoleFrameStatus::Completed,
7973            payload: json!({ "result": "Ready and standing by." }),
7974            source: ConsoleFrameSource {
7975                kind: ConsoleFrameSourceKind::SessionHistory,
7976                source_cursor: Some("session-a:3".to_string()),
7977            },
7978            source_event_id: None,
7979            interaction_id: None,
7980            turn_id: None,
7981            run_id: None,
7982            parent_frame_id: None,
7983            caused_by_frame_id: None,
7984        };
7985
7986        assert!(
7987            history_frame_has_existing_counterpart(&aggregator.inner, &history)
7988                .await
7989                .expect("counterpart scan")
7990        );
7991    }
7992
7993    #[test]
7994    fn session_history_watermark_key_is_session_scoped() {
7995        assert_ne!(
7996            session_history_watermark_runtime_key("runtime-a", "session-1"),
7997            session_history_watermark_runtime_key("runtime-a", "session-2")
7998        );
7999    }
8000
8001    #[test]
8002    fn session_history_watermarks_are_cursor_and_ttl_aware() {
8003        let legacy = "session:with:colon:42";
8004        let checked = format_session_history_watermark("session:with:colon", 43, 1_000);
8005        let empty_checked = format_session_history_watermark("session:with:colon", 0, 1_000);
8006
8007        assert_eq!(
8008            parse_session_history_watermark(legacy, "session:with:colon"),
8009            Some(42)
8010        );
8011        assert_eq!(
8012            parse_session_history_watermark(&checked, "session:with:colon"),
8013            Some(43)
8014        );
8015        assert!(session_history_watermark_is_fresh(
8016            &checked,
8017            "session:with:colon",
8018            1_500
8019        ));
8020        assert!(!session_history_watermark_is_fresh(
8021            &checked,
8022            "session:with:colon",
8023            1_000 + SESSION_HISTORY_GROWING_REFRESH_TTL_MS + 1
8024        ));
8025        assert!(session_history_watermark_is_fresh(
8026            &empty_checked,
8027            "session:with:colon",
8028            1_500
8029        ));
8030        assert!(!session_history_watermark_is_fresh(
8031            &empty_checked,
8032            "session:with:colon",
8033            1_000 + SESSION_HISTORY_REFRESH_TTL_MS + 1
8034        ));
8035    }
8036
8037    #[test]
8038    fn session_history_messages_project_to_renderable_frames() {
8039        let user = frame_from_session_history_message(
8040            "runtime-a",
8041            "agent-a",
8042            "session-a",
8043            0,
8044            json!({
8045                "role": "user",
8046                "content": "hello",
8047                "timestamp_ms": 10
8048            }),
8049        )
8050        .expect("user history frame");
8051        let assistant = frame_from_session_history_message(
8052            "runtime-a",
8053            "agent-a",
8054            "session-a",
8055            1,
8056            json!({
8057                "role": "assistant",
8058                "content": "hi there",
8059                "stop_reason": "end_turn",
8060                "usage": { "input_tokens": 1, "output_tokens": 1, "total_tokens": 2 },
8061                "timestamp_ms": 11
8062            }),
8063        )
8064        .expect("assistant history frame");
8065
8066        assert_eq!(user.kind, "user_input");
8067        assert_eq!(user.source.kind, ConsoleFrameSourceKind::SessionHistory);
8068        assert_eq!(
8069            user.payload["content"],
8070            json!([{ "type": "text", "text": "hello" }])
8071        );
8072        assert_eq!(assistant.kind, "interaction_complete");
8073        assert_eq!(assistant.payload["text"], json!("hi there"));
8074        assert!(
8075            assistant
8076                .dedupe_key
8077                .starts_with("session-history:runtime-a:session-a:1:")
8078        );
8079    }
8080
8081    #[test]
8082    fn session_history_projection_filters_scaffold_user_messages() {
8083        let spawn_notice = frame_from_session_history_message(
8084            "runtime-a",
8085            "agent-a",
8086            "session-a",
8087            0,
8088            json!({
8089                "role": "user",
8090                "content": "You have been spawned as 'agent-a' (role: worker) in mob 'mob-a'.",
8091                "timestamp_ms": 10
8092            }),
8093        );
8094        let peer_update = frame_from_session_history_message(
8095            "runtime-a",
8096            "agent-a",
8097            "session-a",
8098            1,
8099            json!({
8100                "role": "user",
8101                "content": [{ "type": "text", "text": "[PEER UPDATE] alpha wired to beta" }],
8102                "timestamp_ms": 11
8103            }),
8104        );
8105        let real_user = frame_from_session_history_message(
8106            "runtime-a",
8107            "agent-a",
8108            "session-a",
8109            2,
8110            json!({
8111                "role": "user",
8112                "content": "Please review the incident notes.",
8113                "timestamp_ms": 12
8114            }),
8115        );
8116
8117        assert!(spawn_notice.is_none());
8118        assert!(peer_update.is_none());
8119        assert!(real_user.is_some());
8120    }
8121
8122    #[test]
8123    fn session_history_projection_skips_non_transcript_messages() {
8124        let skipped = frame_from_session_history_message(
8125            "runtime-a",
8126            "agent-a",
8127            "session-a",
8128            0,
8129            json!({
8130                "content": "internal system prompt"
8131            }),
8132        );
8133        assert!(skipped.is_none());
8134    }
8135
8136    #[test]
8137    fn session_history_projection_extracts_assistant_blocks() {
8138        let frame = frame_from_session_history_message(
8139            "runtime-a",
8140            "agent-a",
8141            "session-a",
8142            0,
8143            json!({
8144                "role": "block_assistant",
8145                "blocks": [
8146                    { "block_type": "text", "data": { "text": "hello " } },
8147                    { "block_type": "text", "data": { "text": "there" } }
8148                ],
8149                "stop_reason": "end_turn"
8150            }),
8151        )
8152        .expect("assistant block history frame");
8153        assert_eq!(frame.payload["text"], json!("hello there"));
8154    }
8155
8156    #[test]
8157    fn session_history_projection_extracts_nested_text_block_data() {
8158        let frame = frame_from_session_history_message(
8159            "runtime-a",
8160            "agent-a",
8161            "session-a",
8162            0,
8163            json!({
8164                "role": "block_assistant",
8165                "blocks": [
8166                    {
8167                        "block_type": "text",
8168                        "data": { "text": "Ready and standing by." }
8169                    }
8170                ],
8171                "stop_reason": "end_turn",
8172                "created_at": "1970-01-01T00:00:00.010Z"
8173            }),
8174        )
8175        .expect("assistant block history frame");
8176
8177        assert_eq!(frame.kind, "interaction_complete");
8178        assert_eq!(frame.payload["result"], json!("Ready and standing by."));
8179    }
8180
8181    #[test]
8182    fn session_history_projection_drops_reasoning_blocks_from_result_text() {
8183        let frame = frame_from_session_history_message(
8184            "runtime-a",
8185            "agent-a",
8186            "session-a",
8187            0,
8188            json!({
8189                "role": "block_assistant",
8190                "blocks": [
8191                    {
8192                        "block_type": "reasoning",
8193                        "data": { "text": "**Planning**\nI should not be rendered." }
8194                    },
8195                    {
8196                        "block_type": "text",
8197                        "data": { "text": "Visible answer." }
8198                    }
8199                ],
8200                "stop_reason": "end_turn",
8201                "created_at": "1970-01-01T00:00:00.010Z"
8202            }),
8203        )
8204        .expect("assistant block history frame");
8205
8206        assert_eq!(frame.kind, "interaction_complete");
8207        assert_eq!(frame.payload["result"], json!("Visible answer."));
8208        assert_eq!(frame.payload["text"], json!("Visible answer."));
8209    }
8210
8211    #[test]
8212    fn session_history_projection_leaves_reasoning_only_result_empty() {
8213        let frame = frame_from_session_history_message(
8214            "runtime-a",
8215            "agent-a",
8216            "session-a",
8217            0,
8218            json!({
8219                "role": "block_assistant",
8220                "blocks": [
8221                    {
8222                        "block_type": "reasoning",
8223                        "data": { "text": "Private planning text." }
8224                    },
8225                    {
8226                        "block_type": "tool_use",
8227                        "data": { "id": "toolu-1", "name": "peers", "args": {} }
8228                    }
8229                ],
8230                "stop_reason": "end_turn",
8231                "created_at": "1970-01-01T00:00:00.010Z"
8232            }),
8233        )
8234        .expect("assistant block history frame");
8235
8236        assert_eq!(frame.kind, "interaction_complete");
8237        assert_eq!(frame.payload["result"], json!(""));
8238        assert_eq!(frame.payload["text"], json!(""));
8239    }
8240
8241    #[test]
8242    fn session_history_projection_preserves_tool_results() {
8243        let frames = frames_from_session_history_message(
8244            "runtime-a",
8245            "agent-a",
8246            "session-a",
8247            5,
8248            json!({
8249                "role": "tool_results",
8250                "results": [
8251                    {
8252                        "tool_use_id": "call-peers",
8253                        "content": "{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}",
8254                        "is_error": false
8255                    }
8256                ],
8257                "created_at": "1970-01-01T00:00:00.050Z"
8258            }),
8259        );
8260
8261        assert_eq!(frames.len(), 1);
8262        let frame = &frames[0];
8263        assert_eq!(frame.kind, "tool_execution_completed");
8264        assert_eq!(frame.payload["tool_call_id"], json!("call-peers"));
8265        assert_eq!(
8266            frame.payload["result"],
8267            json!("{\"peers\":[{\"peer_id\":\"peer-1\",\"name\":\"mob/worker/peer-1\"}]}")
8268        );
8269        assert_eq!(frame.source.kind, ConsoleFrameSourceKind::SessionHistory);
8270        assert_eq!(frame.timestamp_ms, 50);
8271    }
8272
8273    #[test]
8274    fn session_history_projection_surfaces_spawn_initial_message_on_child_timeline() {
8275        let frames = frames_from_session_history_message(
8276            "runtime-a",
8277            "review:singleton",
8278            "session-a",
8279            7,
8280            json!({
8281                "role": "block_assistant",
8282                "blocks": [
8283                    {
8284                        "block_type": "tool_use",
8285                        "data": {
8286                            "id": "call-spawn",
8287                            "name": "mob_spawn_member",
8288                            "args": {
8289                                "member_id": "review-worker-alpha",
8290                                "profile": "review-worker",
8291                                "initial_message": "Review Initiative Alpha and save the result."
8292                            }
8293                        }
8294                    }
8295                ],
8296                "stop_reason": "tool_use",
8297                "created_at": "1970-01-01T00:00:00.070Z"
8298            }),
8299        );
8300
8301        assert_eq!(frames.len(), 2);
8302        let child = frames
8303            .iter()
8304            .find(|frame| frame.identity == "review-worker-alpha")
8305            .expect("spawned member initial message frame");
8306        assert_eq!(child.kind, "user_input");
8307        assert_eq!(child.timestamp_ms, 70);
8308        assert_eq!(
8309            child.payload["message"]["content"],
8310            "Review Initiative Alpha and save the result."
8311        );
8312        assert_eq!(child.payload["parent_identity"], "review:singleton");
8313        assert_eq!(child.payload["via_tool"], "mob_spawn_member");
8314        assert_eq!(child.source.kind, ConsoleFrameSourceKind::SessionHistory);
8315    }
8316
8317    #[test]
8318    fn session_history_projection_surfaces_specs_spawn_initial_messages() {
8319        let frames = frames_from_session_history_message(
8320            "runtime-a",
8321            "review:singleton",
8322            "session-a",
8323            8,
8324            json!({
8325                "role": "block_assistant",
8326                "blocks": [
8327                    {
8328                        "block_type": "tool_use",
8329                        "data": {
8330                            "id": "call-spawn-many",
8331                            "name": "mob_spawn_member",
8332                            "args": {
8333                                "specs": [
8334                                    {
8335                                        "agent_identity": "review-worker-alpha",
8336                                        "profile": "review-worker",
8337                                        "initial_message": "Review Initiative Alpha."
8338                                    },
8339                                    {
8340                                        "agent_identity": "review-worker-beta",
8341                                        "profile": "review-worker",
8342                                        "initial_message": "Review Initiative Beta."
8343                                    }
8344                                ]
8345                            }
8346                        }
8347                    }
8348                ],
8349                "stop_reason": "tool_use",
8350                "created_at": "1970-01-01T00:00:00.080Z"
8351            }),
8352        );
8353
8354        assert_eq!(frames.len(), 3);
8355        let child_messages = frames
8356            .iter()
8357            .filter(|frame| frame.kind == "user_input")
8358            .map(|frame| {
8359                (
8360                    frame.identity.as_str(),
8361                    frame.payload["message"]["content"]
8362                        .as_str()
8363                        .unwrap_or_default(),
8364                )
8365            })
8366            .collect::<Vec<_>>();
8367        assert_eq!(
8368            child_messages,
8369            vec![
8370                ("review-worker-alpha", "Review Initiative Alpha."),
8371                ("review-worker-beta", "Review Initiative Beta."),
8372            ]
8373        );
8374    }
8375
8376    #[tokio::test]
8377    async fn query_timeline_surfaces_spawn_initial_message_on_child_identity_only() {
8378        let aggregator = MobKitConsoleAggregator::in_memory();
8379        let frames = frames_from_session_history_message(
8380            "runtime-a",
8381            "review:singleton",
8382            "session-a",
8383            9,
8384            json!({
8385                "role": "block_assistant",
8386                "blocks": [
8387                    {
8388                        "block_type": "tool_use",
8389                        "data": {
8390                            "id": "call-spawn-many",
8391                            "name": "mob_spawn_member",
8392                            "args": {
8393                                "specs": [
8394                                    {
8395                                        "agent_identity": "review-worker-alpha",
8396                                        "profile": "review-worker",
8397                                        "initial_message": "Review Initiative Alpha."
8398                                    },
8399                                    {
8400                                        "agent_identity": "review-worker-beta",
8401                                        "profile": "review-worker",
8402                                        "initial_message": "Review Initiative Beta."
8403                                    }
8404                                ]
8405                            }
8406                        }
8407                    }
8408                ],
8409                "stop_reason": "tool_use",
8410                "created_at": "1970-01-01T00:00:00.090Z"
8411            }),
8412        );
8413        for frame in frames {
8414            aggregator
8415                .store()
8416                .append_if_absent(frame)
8417                .await
8418                .expect("append projected session-history frame");
8419        }
8420
8421        let child_page = aggregator
8422            .query_timeline(ConsoleTimelineQuery {
8423                identity: Some("review-worker-alpha".to_string()),
8424                limit: 20,
8425                ..ConsoleTimelineQuery::default()
8426            })
8427            .await
8428            .expect("query child timeline");
8429        assert_eq!(child_page.frames.len(), 1);
8430        assert_eq!(child_page.frames[0].kind, "user_input");
8431        assert_eq!(
8432            child_page.frames[0].payload["message"]["content"],
8433            "Review Initiative Alpha."
8434        );
8435        assert_eq!(
8436            child_page.frames[0].payload["parent_identity"],
8437            "review:singleton"
8438        );
8439
8440        let parent_page = aggregator
8441            .query_timeline(ConsoleTimelineQuery {
8442                identity: Some("review:singleton".to_string()),
8443                limit: 20,
8444                ..ConsoleTimelineQuery::default()
8445            })
8446            .await
8447            .expect("query parent timeline");
8448        assert!(
8449            parent_page
8450                .frames
8451                .iter()
8452                .all(|frame| frame.identity == "review:singleton"),
8453            "child initial messages must not leak into parent timeline: {:#?}",
8454            parent_page.frames
8455        );
8456    }
8457
8458    #[test]
8459    fn session_history_projection_applies_namespace_to_spawned_child_identity() {
8460        let frames = frames_from_session_history_message_with_namespace(
8461            "runtime-a",
8462            "test/review:singleton",
8463            "test",
8464            "session-a",
8465            10,
8466            json!({
8467                "role": "block_assistant",
8468                "blocks": [
8469                    {
8470                        "block_type": "tool_use",
8471                        "data": {
8472                            "id": "call-spawn-many",
8473                            "name": "mob_spawn_member",
8474                            "args": {
8475                                "specs": [
8476                                    {
8477                                        "agent_identity": "review-worker-alpha",
8478                                        "profile": "review-worker",
8479                                        "initial_message": "Review Initiative Alpha."
8480                                    },
8481                                    {
8482                                        "agent_identity": "test/review-worker-beta",
8483                                        "profile": "review-worker",
8484                                        "initial_message": "Review Initiative Beta."
8485                                    }
8486                                ]
8487                            }
8488                        }
8489                    }
8490                ],
8491                "stop_reason": "tool_use",
8492                "created_at": "1970-01-01T00:00:00.100Z"
8493            }),
8494        );
8495
8496        let child_identities = frames
8497            .iter()
8498            .filter(|frame| frame.kind == "user_input")
8499            .map(|frame| frame.identity.as_str())
8500            .collect::<Vec<_>>();
8501        assert_eq!(
8502            child_identities,
8503            vec!["test/review-worker-alpha", "test/review-worker-beta"]
8504        );
8505    }
8506
8507    #[tokio::test]
8508    async fn query_timeline_matches_namespaced_spawn_initial_message_identity() {
8509        let aggregator = MobKitConsoleAggregator::in_memory();
8510        aggregator
8511            .inner
8512            .identity_read_model
8513            .replace(vec![
8514                identity_record_for_test("test/review:singleton"),
8515                identity_record_for_test("test/review-worker-alpha"),
8516            ])
8517            .await;
8518        let frames = frames_from_session_history_message_with_namespace(
8519            "runtime-a",
8520            "test/review:singleton",
8521            "test",
8522            "session-a",
8523            11,
8524            json!({
8525                "role": "block_assistant",
8526                "blocks": [
8527                    {
8528                        "block_type": "tool_use",
8529                        "data": {
8530                            "id": "call-spawn",
8531                            "name": "mob_spawn_member",
8532                            "args": {
8533                                "agent_identity": "review-worker-alpha",
8534                                "profile": "review-worker",
8535                                "initial_message": "Review Initiative Alpha."
8536                            }
8537                        }
8538                    }
8539                ],
8540                "stop_reason": "tool_use",
8541                "created_at": "1970-01-01T00:00:00.110Z"
8542            }),
8543        );
8544        for frame in frames {
8545            aggregator
8546                .store()
8547                .append_if_absent(frame)
8548                .await
8549                .expect("append projected namespaced session-history frame");
8550        }
8551
8552        let namespaced_child_page = aggregator
8553            .query_timeline(ConsoleTimelineQuery {
8554                identity: Some("test/review-worker-alpha".to_string()),
8555                limit: 20,
8556                ..ConsoleTimelineQuery::default()
8557            })
8558            .await
8559            .expect("query namespaced child timeline");
8560        assert_eq!(namespaced_child_page.frames.len(), 1);
8561        assert_eq!(
8562            namespaced_child_page.frames[0].identity,
8563            "test/review-worker-alpha"
8564        );
8565        assert_eq!(
8566            namespaced_child_page.frames[0].payload["message"]["content"],
8567            "Review Initiative Alpha."
8568        );
8569
8570        let raw_child_page = aggregator
8571            .query_timeline(ConsoleTimelineQuery {
8572                identity: Some("review-worker-alpha".to_string()),
8573                limit: 20,
8574                ..ConsoleTimelineQuery::default()
8575            })
8576            .await
8577            .expect("query raw child timeline");
8578        assert!(
8579            raw_child_page.frames.is_empty(),
8580            "raw unnamespaced query must not see namespaced synthetic frames: {:#?}",
8581            raw_child_page.frames
8582        );
8583    }
8584
8585    #[test]
8586    fn namespace_helpers_preserve_namespace_named_member_identity() {
8587        assert_eq!(apply_namespace("test", "test"), "test/test");
8588        assert_eq!(
8589            strip_namespace("test/test", "test").as_deref(),
8590            Some("test")
8591        );
8592        assert_eq!(apply_namespace("test/worker", "test"), "test/worker");
8593        assert_eq!(
8594            strip_namespace("test/worker", "test").as_deref(),
8595            Some("worker")
8596        );
8597        assert_eq!(strip_namespace("test", "test"), None);
8598    }
8599
8600    #[test]
8601    fn generated_runtime_ids_do_not_match_sibling_colon_identities() {
8602        assert!(!member_id_matches_durable_identity(
8603            "rt:review:singleton:0",
8604            "review:singleton"
8605        ));
8606        assert!(!member_id_matches_durable_identity(
8607            "review:singleton:gen1",
8608            "review:singleton"
8609        ));
8610        assert!(!member_id_matches_durable_identity(
8611            "review:singleton:1",
8612            "review:singleton"
8613        ));
8614        assert!(!member_id_matches_durable_identity(
8615            "rt:review:singleton:qa:0",
8616            "review:singleton"
8617        ));
8618        assert!(!member_id_matches_durable_identity(
8619            "review:singleton:qa",
8620            "review:singleton"
8621        ));
8622    }
8623
8624    #[test]
8625    fn session_history_projection_uses_rfc3339_created_at_timestamp() {
8626        let frame = frame_from_session_history_message(
8627            "runtime-a",
8628            "agent-a",
8629            "session-a",
8630            0,
8631            json!({
8632                "role": "user",
8633                "content": "hello",
8634                "created_at": "2026-05-12T05:00:06.564227Z"
8635            }),
8636        )
8637        .expect("user history frame");
8638
8639        assert_eq!(frame.timestamp_ms, 1_778_562_006_564);
8640    }
8641}