Skip to main content

meerkat_mobkit/identity_first/
runtime.rs

1//! Identity-first runtime: delivery, status, lifecycle, and ownership enforcement.
2//!
3//! This module implements the behavioral core of identity-first continuity:
4//! - Delivery: `send()` and `dispatch()` with addressability and lease enforcement
5//! - Status: `status()` returning `IdentityStatus`
6//! - Lifecycle: `retire()`, `respawn()`, `reset()`, `delete_identity()`
7//! - Ownership: lease tracking, fencing, and invariant enforcement
8
9use std::collections::{BTreeMap, BTreeSet, HashMap};
10use std::future::Future;
11use std::sync::atomic::{AtomicBool, Ordering};
12use std::sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock, Weak};
13use std::time::{Duration, Instant};
14
15use futures::stream::{self, StreamExt};
16use meerkat_core::types::{HandlingMode, SessionId};
17use tokio::sync::{Mutex, Notify, RwLock, broadcast, oneshot, watch};
18use tokio::task::{JoinHandle, JoinSet};
19
20use super::agent_memory::{
21    AgentMemoryError, AgentMemoryForgetResult, AgentMemoryRecallRequest, AgentMemoryRecord,
22    AgentMemoryRuntimeInjector, NewAgentMemory,
23};
24use super::bridge::{
25    BridgeError, CommittedBoundaryRepair, ResumeRejectionKind, SessionBridge,
26    archived_not_revivable_park_reason,
27};
28use super::contracts::{
29    AgentCustomizer, ContinuityStore, LeaseProvider, RosterProvider, TopologyProvider,
30};
31use super::types::{
32    AgentAddressability, AgentBuildContext, AgentIdentity, AgentRuntimeId, AgentRuntimeServices,
33    CheckpointVersion, CompletionCursor, CompletionProgress, ContinuityGeneration,
34    ContinuityHealth, ContinuityRecord, ContinuityStoreError, ContinuityUnrecoverable,
35    DispatchAdmission, DispatchInput, DurabilityPolicy, DurableAgentSpec, FencingToken,
36    HostRejectedBuildPark, IdentityBootstrapEntry, IdentityBootstrapMode, IdentityBootstrapState,
37    IdentityBootstrapStatus, IdentityLifecycleState, IdentityStatus, LeaseGrant, LeaseInfo,
38    ManagedPeerEdge, NotAddressable, RosterContext, SendAdmission, SessionSnapshot,
39    TopologyContext,
40};
41use crate::memory::records::{
42    ManifestTier, MemoryId, MemoryKind, MemoryScope, NewMemoryRecord, RecordMeta, UsageEvent,
43};
44
45const MANAGED_PEER_RECONCILE_CONCURRENCY: usize = 64;
46/// Poll cadence for [`IdentityRuntime::wait_for_completion`]. The cursor is
47/// advanced by an event, so this only bounds observation latency.
48const COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(100);
49const MATERIALIZATION_FAILURE_BACKOFF: Duration = Duration::from_secs(30);
50const RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN: usize = 256;
51const BACKGROUND_WARM_CANCELLED: &str =
52    "identity background warm cancelled before session installation";
53fn durable_spec_uses_external_binding(spec: &DurableAgentSpec) -> bool {
54    matches!(spec.backend, Some(meerkat_mob::MobBackendKind::External))
55        || matches!(
56            spec.binding.as_ref(),
57            Some(meerkat_contracts::WireRuntimeBinding::External { .. })
58        )
59}
60
61fn interaction_id_for_delivery<'a>(
62    spec: &DurableAgentSpec,
63    interaction_id: Option<&'a str>,
64) -> Option<&'a str> {
65    // Meerkat 0.8.2 deliberately rejects transcript interaction ids on
66    // remotely hosted / peer-only member turns: that metadata carrier is not
67    // representable on the wire path. MobKit still correlates the response
68    // through its pending-interaction ledger and the peer terminal event.
69    (!durable_spec_uses_external_binding(spec))
70        .then_some(interaction_id)
71        .flatten()
72}
73
74// ---------------------------------------------------------------------------
75// Error types
76// ---------------------------------------------------------------------------
77
78/// Errors from identity-first runtime operations.
79#[derive(Debug)]
80pub enum IdentityRuntimeError {
81    /// Target identity is not registered/active.
82    UnknownIdentity(AgentIdentity),
83    /// send() rejected: target is InternalOnly.
84    NotAddressable(NotAddressable),
85    /// Operation rejected: no active lease for this identity.
86    NoActiveLease(AgentIdentity),
87    /// Fail-closed single-embodiment guard: the identity's durable lease is
88    /// held by another live runtime instance — a second live embodiment is
89    /// refused, loudly and with the holder named. This is a POLICY point,
90    /// not a structural assumption: multi-bind / forked-session semantics
91    /// (future work) relax exactly this arm.
92    AlreadyEmbodied {
93        identity: AgentIdentity,
94        holder: String,
95    },
96    /// Operation rejected: lease was lost.
97    LeaseLost(AgentIdentity),
98    /// Operation rejected: identity is not in a state that permits this operation.
99    InvalidState {
100        identity: AgentIdentity,
101        state: IdentityLifecycleState,
102        operation: &'static str,
103    },
104    /// Dispatch rejected BEFORE bridge admission: the delivery identity is
105    /// half-formed (idempotency key without a correlation id, or vice
106    /// versa) or fails upstream canonical validation (non-nil canonical
107    /// UUID correlation). Fail-closed on purpose: a degraded delivery under
108    /// a broken identity is a silent dedup hole - the caller gets dedup or
109    /// this refusal, never at-least-once. NO delivery occurs.
110    InvalidDeliveryIdentity {
111        identity: AgentIdentity,
112        detail: String,
113    },
114    /// Continuity store error.
115    Store(ContinuityStoreError),
116    /// Lease provider error.
117    Lease(super::types::LeaseError),
118    /// Duplicate identities in roster.
119    DuplicateIdentity(AgentIdentity),
120    /// Stale fencing token on checkpoint.
121    StaleFencingToken {
122        identity: AgentIdentity,
123        presented: FencingToken,
124        current: FencingToken,
125    },
126    /// Stale checkpoint version.
127    StaleCheckpointVersion {
128        identity: AgentIdentity,
129        presented: CheckpointVersion,
130        current: CheckpointVersion,
131    },
132    StaleContinuityGeneration {
133        identity: AgentIdentity,
134        presented: ContinuityGeneration,
135        current: ContinuityGeneration,
136    },
137    /// A lifecycle request named an old generated runtime alias after the
138    /// durable identity had already advanced to another generation.
139    StaleRuntimeAlias {
140        identity: AgentIdentity,
141        requested: String,
142        current: Option<AgentRuntimeId>,
143    },
144    /// A completion wait presented a baseline from a superseded runtime
145    /// incarnation. Turn counts do not carry across incarnations, so this is
146    /// reported rather than resolved either way — the caller must capture a
147    /// fresh baseline instead of inferring that its turn did or did not run.
148    CompletionIncarnationChanged {
149        identity: AgentIdentity,
150        baseline: CompletionCursor,
151        observed: CompletionCursor,
152    },
153    /// The identity is parked because the HOST deterministically rejected
154    /// its build (the candidate-mode effect gate class): the app-side
155    /// `callback/build_agent` round trip completed and the host answered
156    /// with an error. No automatic retry — a roster/policy (spec) change
157    /// clears the park; `reason` carries the host's rejection for operators.
158    HostRejectedBuild {
159        identity: AgentIdentity,
160        reason: String,
161    },
162    /// Generic I/O or internal error.
163    Internal(String),
164}
165
166impl std::fmt::Display for IdentityRuntimeError {
167    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168        match self {
169            Self::UnknownIdentity(id) => write!(f, "unknown identity: {id}"),
170            Self::NotAddressable(err) => write!(f, "{err}"),
171            Self::NoActiveLease(id) => write!(f, "no active lease for {id}"),
172            Self::AlreadyEmbodied { identity, holder } => write!(
173                f,
174                "identity {identity} is already embodied by runtime instance '{holder}' \
175                 (single-embodiment guard: refusing a second live bind)"
176            ),
177            Self::LeaseLost(id) => write!(f, "lease lost for {id}"),
178            Self::InvalidState {
179                identity,
180                state,
181                operation,
182            } => write!(
183                f,
184                "cannot {operation} identity {identity} in state {state:?}"
185            ),
186            Self::InvalidDeliveryIdentity { identity, detail } => write!(
187                f,
188                "dispatch to {identity} refused before bridge admission: invalid delivery \
189                 identity ({detail}); dedup or typed refusal, never a degraded delivery"
190            ),
191            Self::Store(err) => write!(f, "continuity store: {err}"),
192            Self::Lease(err) => write!(f, "lease provider: {err}"),
193            Self::DuplicateIdentity(id) => write!(f, "duplicate identity in roster: {id}"),
194            Self::StaleFencingToken {
195                identity,
196                presented,
197                current,
198            } => write!(
199                f,
200                "stale fencing token for {identity}: presented {presented}, current {current}"
201            ),
202            Self::StaleCheckpointVersion {
203                identity,
204                presented,
205                current,
206            } => write!(
207                f,
208                "stale checkpoint version for {identity}: presented {presented}, current {current}"
209            ),
210            Self::StaleContinuityGeneration {
211                identity,
212                presented,
213                current,
214            } => write!(
215                f,
216                "stale continuity generation for {identity}: presented {presented}, current {current}"
217            ),
218            Self::StaleRuntimeAlias {
219                identity,
220                requested,
221                current,
222            } => write!(
223                f,
224                "stale runtime alias for {identity}: requested {requested}, current {}",
225                current
226                    .as_ref()
227                    .map(AgentRuntimeId::as_str)
228                    .unwrap_or("<none>")
229            ),
230            Self::CompletionIncarnationChanged {
231                identity,
232                baseline,
233                observed,
234            } => write!(
235                f,
236                "completion baseline {baseline} for {identity} belongs to a superseded runtime \
237                 incarnation (now {observed}); capture a fresh baseline"
238            ),
239            Self::HostRejectedBuild { identity, reason } => write!(
240                f,
241                "identity {identity} is parked: the host deterministically rejected its build \
242                 ({reason}); a roster/policy (spec) change clears the park"
243            ),
244            Self::Internal(msg) => write!(f, "internal: {msg}"),
245        }
246    }
247}
248
249/// Digest of the exact roster spec, for [`HostRejectedBuildPark`] scoping.
250/// Process-local (std hasher over the canonical JSON form): the park itself
251/// is in-memory, so cross-process stability is not required.
252pub(crate) fn durable_spec_digest(spec: &DurableAgentSpec) -> u64 {
253    use std::hash::{Hash, Hasher};
254    let mut hasher = std::collections::hash_map::DefaultHasher::new();
255    match serde_json::to_string(spec) {
256        Ok(json) => json.hash(&mut hasher),
257        // A roster spec is plain data; serialization cannot realistically
258        // fail. Degrade to "spec never changes" rather than panicking.
259        Err(_) => spec.identity.as_str().hash(&mut hasher),
260    }
261    hasher.finish()
262}
263
264/// A build failure whose root cause is the HOST's own answer: the
265/// `callback/build_agent` round trip COMPLETED and the app returned an error
266/// (rpc_gateway mints `callback/build_agent failed: callback error: <host
267/// message>` for exactly this case — see `StdioCallbackBridge::call`).
268/// Transport-tier callback failures ("callback transport closed", timeouts,
269/// dropped channels) deliberately do NOT match: those are retryable and stay
270/// on the existing reconcile/repair lanes (their backoff is the upstream
271/// meerkat fix, not this park).
272pub(crate) fn is_host_rejected_build_error(detail: &str) -> bool {
273    detail.contains("callback/build_agent failed: callback error:")
274}
275
276impl std::error::Error for IdentityRuntimeError {}
277
278impl From<ContinuityStoreError> for IdentityRuntimeError {
279    fn from(err: ContinuityStoreError) -> Self {
280        match err {
281            ContinuityStoreError::StaleFencingToken {
282                identity,
283                presented,
284                current,
285            } => Self::StaleFencingToken {
286                identity,
287                presented,
288                current,
289            },
290            ContinuityStoreError::StaleCheckpointVersion {
291                identity,
292                presented,
293                current,
294            } => Self::StaleCheckpointVersion {
295                identity,
296                presented,
297                current,
298            },
299            ContinuityStoreError::StaleContinuityGeneration {
300                identity,
301                presented,
302                current,
303            } => Self::StaleContinuityGeneration {
304                identity,
305                presented,
306                current,
307            },
308            other => Self::Store(other),
309        }
310    }
311}
312
313// ---------------------------------------------------------------------------
314// Per-identity runtime state
315// ---------------------------------------------------------------------------
316
317/// Tracks the live state for a single identity within the runtime.
318#[derive(Debug, Clone)]
319pub(crate) struct IdentityEntry {
320    pub spec: DurableAgentSpec,
321    /// Bootstrap pass that last accepted `spec` as this entry's desired
322    /// roster projection. Foreground materialization binds readiness writes
323    /// to this entry-owned generation only after acquiring the lifecycle
324    /// lock, so a pass that wins that lock cannot be mistaken for either its
325    /// predecessor or successor merely because the global status epoch moved.
326    pub bootstrap_generation: u64,
327    pub state: IdentityLifecycleState,
328    pub continuity: Option<ContinuityRecord>,
329    pub lease: Option<LeaseEntry>,
330    /// Exact authority that a completed lower-plane transition tried and
331    /// failed to release. This is deliberately separate from `lease`: Broken
332    /// identities must not advertise or renew active authority, but repair
333    /// still needs the exact fencing token to retry the provider release
334    /// before any reacquire attempt.
335    pub pending_lease_release: Option<LeaseGrant>,
336    pub checkpoint_version: CheckpointVersion,
337    /// Whether a durable runtime_store is available (affects dispatch ack semantics).
338    pub has_runtime_store: bool,
339    /// Terminal heal verdict from the bridge's heal authority. While set,
340    /// the continuity repair supervisor must not re-attempt this identity and
341    /// reconcile must not cosmetically reset it to Dormant — either would
342    /// restart the 2026-07-29 heal/re-Break loop. Cleared by any non-Broken
343    /// lifecycle projection (a real recovery or an operator reset).
344    pub continuity_unrecoverable: Option<ContinuityUnrecoverable>,
345    /// Typed park for a build the host deterministically rejected (the
346    /// candidate-mode effect gate class). While set AND the recorded spec
347    /// digest still matches `spec`, materialization fails fast typed (no
348    /// bridge/callback churn) and the repair supervisor skips the identity.
349    /// A changed spec clears it — the retry is then permitted.
350    pub host_rejected_build_park: Option<HostRejectedBuildPark>,
351}
352
353/// Tracks a held lease for an identity.
354#[derive(Debug, Clone)]
355pub(crate) struct LeaseEntry {
356    pub fencing_token: FencingToken,
357    pub ttl: Duration,
358    pub acquired_at: Instant,
359}
360
361impl LeaseEntry {
362    pub fn is_expired(&self) -> bool {
363        self.acquired_at.elapsed() > self.ttl
364    }
365
366    pub fn ttl_remaining(&self) -> Duration {
367        self.ttl.saturating_sub(self.acquired_at.elapsed())
368    }
369
370    pub fn is_healthy(&self) -> bool {
371        // Healthy if more than 20% TTL remains
372        let remaining = self.ttl_remaining();
373        remaining > self.ttl / 5
374    }
375}
376
377// ---------------------------------------------------------------------------
378// Identity-scoped events
379// ---------------------------------------------------------------------------
380
381/// Events emitted for a specific identity, used by `subscribe()`.
382#[derive(Debug, Clone)]
383pub enum IdentityEvent {
384    /// Lifecycle state changed.
385    StateChanged {
386        identity: AgentIdentity,
387        new_state: IdentityLifecycleState,
388    },
389    /// Lease acquired or renewed.
390    LeaseUpdated {
391        identity: AgentIdentity,
392        fencing_token: FencingToken,
393    },
394    /// Lease lost.
395    LeaseLost { identity: AgentIdentity },
396    /// Checkpoint completed.
397    CheckpointCompleted {
398        identity: AgentIdentity,
399        version: CheckpointVersion,
400    },
401    /// Resume could not reuse a persisted runtime binding and materialization
402    /// fresh-spawned a member instead.
403    ResumeFallback {
404        identity: AgentIdentity,
405        reason: super::bridge::ResumeFallbackReason,
406    },
407}
408
409/// Per-identity event channel capacity.
410const IDENTITY_EVENT_CHANNEL_CAPACITY: usize = 64;
411const DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL: Duration = Duration::from_mins(1);
412const DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL: Duration = Duration::from_millis(10);
413/// Base delay for the lease-renewal failure backoff. The renewal tick's
414/// normal cadence is TTL-derived (down to a 10ms floor), so a lease provider
415/// that errors persistently would otherwise retry — and warn — at that floor
416/// rate. On failure we back off from this base, doubling toward the max poll
417/// interval, so a backend outage can't spin the renewal task.
418const LEASE_RENEWAL_FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(1);
419
420fn lease_renewal_failure_backoff(
421    consecutive_failures: u32,
422    max_poll_interval: Duration,
423) -> Duration {
424    LEASE_RENEWAL_FAILURE_BACKOFF_BASE
425        .saturating_mul(1u32 << consecutive_failures.min(6))
426        .min(max_poll_interval)
427}
428
429// ---------------------------------------------------------------------------
430// IdentityRuntime
431// ---------------------------------------------------------------------------
432
433/// Configuration for the identity-first runtime.
434pub struct IdentityRuntimeConfig {
435    pub continuity_store: Arc<dyn ContinuityStore>,
436    pub lease_provider: Arc<dyn LeaseProvider>,
437    pub runtime_instance_id: String,
438    pub has_runtime_store: bool,
439    pub durability_policy: DurabilityPolicy,
440    /// Optional session bridge for real session delivery. When `None`,
441    /// delivery operations validate invariants but do not forward to
442    /// the Meerkat session pipeline (useful for tests).
443    pub bridge: Option<Arc<dyn SessionBridge>>,
444    /// Default timeout for wait_for_output / wait_for_output_containing.
445    /// Defaults to 90 seconds if not set.
446    pub default_timeout: Option<Duration>,
447}
448
449#[derive(Clone)]
450pub struct IdentityFirstRuntimeContext {
451    pub runtime: Arc<IdentityRuntime>,
452    pub roster_provider: Arc<dyn RosterProvider>,
453    pub topology_provider: Option<Arc<dyn TopologyProvider>>,
454    pub customizer: Option<Arc<dyn AgentCustomizer>>,
455    mob_definition: Option<meerkat_mob::MobDefinition>,
456    bootstrap_mode: IdentityBootstrapMode,
457}
458
459impl IdentityFirstRuntimeContext {
460    pub(crate) async fn topology_snapshot_inputs(
461        &self,
462    ) -> Result<(Vec<DurableAgentSpec>, Vec<ManagedPeerEdge>), IdentityRuntimeError> {
463        let previous_identities = self.runtime.registered_identities().await;
464        let roster = self
465            .roster_provider
466            .roster(&RosterContext {
467                mob_definition: self.mob_definition.clone(),
468                previous_identities,
469            })
470            .await
471            .map_err(|error| IdentityRuntimeError::Internal(format!("roster provider: {error}")))?;
472        let identities = roster
473            .iter()
474            .map(|spec| spec.identity.clone())
475            .collect::<Vec<_>>();
476        let declared = match self.topology_provider.as_deref() {
477            Some(provider) => provider
478                .compute_edges(
479                    &identities,
480                    &TopologyContext {
481                        roster: roster.clone(),
482                    },
483                )
484                .await
485                .map_err(|error| {
486                    IdentityRuntimeError::Internal(format!("topology provider: {error}"))
487                })?,
488            None => self.runtime.desired_peer_edges.read().await.clone(),
489        };
490        Ok((roster, declared))
491    }
492
493    pub fn new(
494        runtime: Arc<IdentityRuntime>,
495        roster_provider: Arc<dyn RosterProvider>,
496        topology_provider: Option<Arc<dyn TopologyProvider>>,
497        customizer: Option<Arc<dyn AgentCustomizer>>,
498        mob_definition: Option<meerkat_mob::MobDefinition>,
499    ) -> Self {
500        Self::new_with_lazy_materialization(
501            runtime,
502            roster_provider,
503            topology_provider,
504            customizer,
505            mob_definition,
506            false,
507        )
508    }
509
510    pub fn new_with_lazy_materialization(
511        runtime: Arc<IdentityRuntime>,
512        roster_provider: Arc<dyn RosterProvider>,
513        topology_provider: Option<Arc<dyn TopologyProvider>>,
514        customizer: Option<Arc<dyn AgentCustomizer>>,
515        mob_definition: Option<meerkat_mob::MobDefinition>,
516        lazy_materialization: bool,
517    ) -> Self {
518        Self::new_with_bootstrap_mode(
519            runtime,
520            roster_provider,
521            topology_provider,
522            customizer,
523            mob_definition,
524            if lazy_materialization {
525                IdentityBootstrapMode::LazyMaterialize
526            } else {
527                IdentityBootstrapMode::EagerMaterialize
528            },
529        )
530    }
531
532    /// Construct a context that preserves the complete startup policy across
533    /// later roster reconciliation.
534    pub fn new_with_bootstrap_mode(
535        runtime: Arc<IdentityRuntime>,
536        roster_provider: Arc<dyn RosterProvider>,
537        topology_provider: Option<Arc<dyn TopologyProvider>>,
538        customizer: Option<Arc<dyn AgentCustomizer>>,
539        mob_definition: Option<meerkat_mob::MobDefinition>,
540        bootstrap_mode: IdentityBootstrapMode,
541    ) -> Self {
542        runtime.set_reset_roster_provider_context(
543            Some(roster_provider.clone()),
544            mob_definition.clone(),
545        );
546        Self {
547            runtime,
548            roster_provider,
549            topology_provider,
550            customizer,
551            mob_definition,
552            bootstrap_mode,
553        }
554    }
555
556    pub fn bootstrap_mode(&self) -> &IdentityBootstrapMode {
557        &self.bootstrap_mode
558    }
559
560    /// Apply the configured bootstrap policy to an already-resolved roster.
561    /// The same helper is used at startup and during reconcile so a lazy
562    /// deployment can never accidentally hydrate the full fleet.
563    pub async fn bootstrap_roster(
564        &self,
565        roster: &[DurableAgentSpec],
566    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
567        let _controller = self.runtime.bootstrap_controller.lock().await;
568        let generation = self
569            .runtime
570            .begin_identity_bootstrap_pending(self.bootstrap_mode.clone());
571        if let Err(error) = self.prepare_controlled_bootstrap().await {
572            self.runtime.fail_identity_bootstrap(generation, &error);
573            return Err(error);
574        }
575        self.apply_roster_controlled(generation, roster, true).await
576    }
577
578    /// Apply a roster under the runtime's single bootstrap controller.
579    ///
580    /// Startup callers may skip snapshot payloads for bridges that explicitly
581    /// opt out because they discard [`RestoreOutcome`] after registration.
582    /// The existing public refresh API must preserve its historical payload,
583    /// so it always selects the full restore path.
584    async fn prepare_controlled_bootstrap(&self) -> Result<(), IdentityRuntimeError> {
585        if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
586            return Err(IdentityRuntimeError::Internal(
587                "identity bootstrap is shutting down".to_string(),
588            ));
589        }
590        // Stop admitting new warm items and let any materialization already in
591        // flight reach a transaction boundary before applying the next roster.
592        self.runtime.request_identity_bootstrap_stop();
593        self.runtime.join_identity_bootstrap_task().await;
594        if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
595            return Err(IdentityRuntimeError::Internal(
596                "identity bootstrap is shutting down".to_string(),
597            ));
598        }
599        Ok(())
600    }
601
602    async fn apply_roster_controlled(
603        &self,
604        generation: u64,
605        roster: &[DurableAgentSpec],
606        optimize_startup_snapshot_load: bool,
607    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
608        if let Err(message) = self.bootstrap_mode.validate() {
609            let error = IdentityRuntimeError::Internal(message);
610            self.runtime
611                .begin_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster)
612                .await;
613            self.runtime.fail_identity_bootstrap(generation, &error);
614            return Err(error);
615        }
616        // Publish the new pass before any provider/restore await. RPC dispatch
617        // is concurrent, so retaining the previous complete/ready snapshot here
618        // would let a readiness waiter falsely pass while reconcile is active.
619        self.runtime
620            .begin_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster)
621            .await;
622        // Converge the physical runtime before either restore policy publishes
623        // the new roster. The restore flows only iterate desired identities;
624        // without this boundary a removed member can remain live (and leased)
625        // while the reduced bootstrap snapshot reports ready. Likewise, an
626        // already-active member must not keep running an old build after its
627        // desired spec changes.
628        if let Err(error) = self
629            .runtime
630            .reconcile_roster_members(roster, generation)
631            .await
632        {
633            self.runtime.fail_identity_bootstrap(generation, &error);
634            return Err(error);
635        }
636        let result = match (&self.bootstrap_mode, optimize_startup_snapshot_load) {
637            (IdentityBootstrapMode::EagerMaterialize, true) => {
638                super::orchestrator::restore_flow_for_bootstrap(
639                    &self.runtime,
640                    roster,
641                    self.topology_provider.as_deref(),
642                    self.customizer.as_deref(),
643                )
644                .await
645            }
646            (IdentityBootstrapMode::EagerMaterialize, false) => {
647                super::orchestrator::restore_flow(
648                    &self.runtime,
649                    roster,
650                    self.topology_provider.as_deref(),
651                    self.customizer.as_deref(),
652                )
653                .await
654            }
655            (
656                IdentityBootstrapMode::LazyMaterialize
657                | IdentityBootstrapMode::LazyWithBackgroundWarm { .. },
658                _,
659            ) => {
660                super::orchestrator::lazy_register_flow(
661                    &self.runtime,
662                    roster,
663                    self.topology_provider.as_deref(),
664                )
665                .await
666            }
667        };
668        let result = match result {
669            Ok(result) => result,
670            Err(error) => {
671                self.runtime.fail_identity_bootstrap(generation, &error);
672                return Err(error);
673            }
674        };
675        if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
676            let error =
677                IdentityRuntimeError::Internal("identity bootstrap is shutting down".to_string());
678            self.runtime.fail_identity_bootstrap(generation, &error);
679            return Err(error);
680        }
681        self.runtime
682            .install_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster, &result)
683            .await;
684        Ok(result)
685    }
686
687    pub async fn refresh_desired_topology(
688        &self,
689    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
690        // One runtime owns one controller from roster discovery through task
691        // installation. Provider calls are user code and can be slow; publish
692        // the in-flight pass before awaiting them so concurrent readiness RPCs
693        // cannot observe the previous terminal snapshot.
694        let _controller = self.runtime.bootstrap_controller.lock().await;
695        let generation = self
696            .runtime
697            .begin_identity_bootstrap_pending(self.bootstrap_mode.clone());
698        if let Err(error) = self.prepare_controlled_bootstrap().await {
699            self.runtime.fail_identity_bootstrap(generation, &error);
700            return Err(error);
701        }
702        let roster = match self
703            .roster_provider
704            .roster(&RosterContext {
705                mob_definition: self.mob_definition.clone(),
706                previous_identities: Vec::new(),
707            })
708            .await
709        {
710            Ok(roster) => roster,
711            Err(err) => {
712                let error = IdentityRuntimeError::Internal(format!("roster provider: {err}"));
713                self.runtime.fail_identity_bootstrap(generation, &error);
714                return Err(error);
715            }
716        };
717
718        self.apply_roster_controlled(generation, &roster, false)
719            .await
720    }
721
722    /// Cancellation-safe reconcile for RPC/host request boundaries.
723    /// Dropping the caller only drops its result receiver; the runtime owns
724    /// the pass until every acquired lease and bridge mutation reaches an
725    /// explicit commit or rollback boundary.
726    pub async fn refresh_desired_topology_tracked(
727        self: &Arc<Self>,
728    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
729        let context = Arc::clone(self);
730        let runtime = Arc::clone(&self.runtime);
731        runtime
732            .run_tracked_foreground(async move { context.refresh_desired_topology().await })
733            .await
734    }
735
736    /// Background repair for Broken identities. A rejected resume degrades the
737    /// identity to Broken while preserving the durable session; the documented
738    /// contract is "the next reconcile retries the resume" — but without this
739    /// task, nothing runs that reconcile: delivery refuses Broken identities
740    /// (REQ-13 fails loudly), `materialize` refuses the Broken state, and the
741    /// only retries were a manual `mobkit/reconcile_identity` RPC or a process
742    /// restart. HomeCore 0.7.23 sat with 14 preserved-but-parked identities
743    /// because of exactly that gap.
744    ///
745    /// The loop sleeps, and only when at least one identity is Broken re-runs
746    /// [`Self::refresh_desired_topology`] — the same idempotent flow the
747    /// reconcile RPC runs (eager: `restore_flow` retries the resume; lazy:
748    /// `lazy_register_flow` re-registers a store-Ready identity as Dormant so
749    /// on-demand materialization retries). Backoff doubles while identities
750    /// stay Broken — persistent causes (e.g. an upstream store regression)
751    /// produce bounded log noise, and transient causes (disk full, lock
752    /// contention) heal without a restart.
753    pub fn spawn_broken_identity_repair_task(
754        self: Arc<Self>,
755        policy: ContinuityRepairPolicy,
756    ) -> JoinHandle<()> {
757        tokio::spawn(self.run_broken_identity_repair_loop(policy, None))
758    }
759
760    pub(crate) fn spawn_tracked_broken_identity_repair_task(
761        self: Arc<Self>,
762        policy: ContinuityRepairPolicy,
763    ) -> TrackedContinuityRepairTask {
764        let (cancel, receiver) = watch::channel(false);
765        let join = tokio::spawn(self.run_broken_identity_repair_loop(policy, Some(receiver)));
766        TrackedContinuityRepairTask { cancel, join }
767    }
768
769    async fn run_broken_identity_repair_loop(
770        self: Arc<Self>,
771        policy: ContinuityRepairPolicy,
772        mut cancellation: Option<watch::Receiver<bool>>,
773    ) {
774        let mut backoff = policy.initial_backoff;
775        // Bounded non-identical retries (OB3 0.8.12-era evidence): a repair
776        // pass whose failure comes back byte-identical N times in a row is a
777        // deterministic wall, and each blind retry re-executes the pass's
778        // DESTRUCTIVE dispose steps against the same blocking precondition.
779        // Track consecutive per-identity failure signatures and park typed
780        // after [`REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS`].
781        let mut identical_failure_streaks: HashMap<AgentIdentity, (String, u32)> = HashMap::new();
782        loop {
783            if let Some(cancellation) = cancellation.as_mut() {
784                if *cancellation.borrow() {
785                    return;
786                }
787                tokio::select! {
788                    () = tokio::time::sleep(backoff) => {}
789                    changed = cancellation.changed() => {
790                        match changed {
791                            Ok(()) if *cancellation.borrow() => return,
792                            Ok(()) => continue,
793                            // Dropping the runtime-owned supervisor drops the
794                            // only sender. Treat that as cancellation; looping
795                            // on the permanently closed receiver would spin a
796                            // detached task at 100% CPU.
797                            Err(_) => return,
798                        }
799                    }
800                }
801            } else {
802                tokio::time::sleep(backoff).await;
803            }
804            let broken = self.runtime.broken_identities().await;
805            if broken.is_empty() {
806                backoff = policy.initial_backoff;
807                continue;
808            }
809            // Heal must be REAL before reconcile runs: reconcile alone only
810            // resets the runtime entry (lazy mode re-registers Dormant) while
811            // the durable head can stay an intra-turn projection that the
812            // next materialization re-Breaks — the measured 2026-07-29
813            // production heal/re-Break loop. Drive the bridge's heal
814            // authority FIRST; only identities whose durable head is (now)
815            // committed — or whose bridge has no heal seam — proceed.
816            let mut repairable = Vec::new();
817            let mut recovery_failures = 0usize;
818            for identity in &broken {
819                if self
820                    .runtime
821                    .continuity_unrecoverable(identity)
822                    .await
823                    .is_some()
824                {
825                    // Terminal typed verdict already recorded: stable across
826                    // calls, so re-healing every cycle is exactly the loop
827                    // this replaces. Operators act on the surfaced reason.
828                    continue;
829                }
830                if self
831                    .runtime
832                    .host_rejected_build_park(identity)
833                    .await
834                    .is_some()
835                {
836                    // The host's gate rejects this exact spec
837                    // deterministically: reattempting re-asks the same
838                    // question and burns a build + callback round trip per
839                    // cycle. A spec change clears the park (checked inside
840                    // the accessor); until then this identity is parked, not
841                    // repaired.
842                    continue;
843                }
844                match self.attempt_committed_boundary_recovery(identity).await {
845                    BrokenRepairDisposition::Repairable => repairable.push(identity.clone()),
846                    BrokenRepairDisposition::Unprovable => {}
847                    BrokenRepairDisposition::RetryLater => recovery_failures += 1,
848                }
849            }
850            if repairable.is_empty() {
851                // Nothing eligible this pass: either every Broken identity
852                // carries a terminal verdict (idle at base cadence — a cheap
853                // read, no reconcile churn) or recovery itself failed
854                // transiently (back off before retrying recovery).
855                backoff = if recovery_failures > 0 {
856                    (backoff * 2).min(policy.max_backoff)
857                } else {
858                    policy.initial_backoff
859                };
860                if cancellation
861                    .as_ref()
862                    .is_some_and(|cancellation| *cancellation.borrow())
863                {
864                    return;
865                }
866                continue;
867            }
868            tracing::info!(
869                broken = repairable.len(),
870                "continuity repair: retrying restore for Broken identities"
871            );
872            let pass = match self.refresh_desired_topology().await {
873                Ok(pass) => pass,
874                Err(err) => {
875                    tracing::warn!(
876                        error = %err,
877                        "continuity repair reconcile failed; backing off"
878                    );
879                    backoff = (backoff * 2).min(policy.max_backoff);
880                    if cancellation
881                        .as_ref()
882                        .is_some_and(|cancellation| *cancellation.borrow())
883                    {
884                        return;
885                    }
886                    continue;
887                }
888            };
889            let still_broken = self.runtime.repairable_broken_identities().await;
890            let healed = repairable
891                .iter()
892                .filter(|id| !still_broken.contains(id))
893                .count();
894            if healed > 0 {
895                tracing::info!(
896                    healed,
897                    still_broken = still_broken.len(),
898                    "continuity repair healed identities"
899                );
900            }
901            for identity in &repairable {
902                if !still_broken.contains(identity) {
903                    identical_failure_streaks.remove(identity);
904                    continue;
905                }
906                let Some(super::orchestrator::RestoreOutcome::Broken(failure)) =
907                    pass.outcomes.get(identity)
908                else {
909                    // No comparable typed failure for this pass (the identity
910                    // broke through a different door); a streak cannot be
911                    // byte-compared across shapes.
912                    identical_failure_streaks.remove(identity);
913                    continue;
914                };
915                let signature = format!("{:?}: {}", failure.kind, failure.detail);
916                let streak = identical_failure_streaks
917                    .entry(identity.clone())
918                    .or_insert_with(|| (signature.clone(), 0));
919                if streak.0 == signature {
920                    streak.1 += 1;
921                } else {
922                    *streak = (signature.clone(), 1);
923                }
924                if streak.1 >= REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS {
925                    identical_failure_streaks.remove(identity);
926                    tracing::error!(
927                        %identity,
928                        attempts = REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS,
929                        blocking_failure = %signature,
930                        "continuity repair failed byte-identically on every attempt; \
931                         parking the identity typed instead of re-executing destructive \
932                         repair steps on a timer. Operator path back after fixing the \
933                         blocking failure: restart the gateway (the park is \
934                         process-local; boot re-attempts repair once) or reset the \
935                         identity via `mobkit/reset` (deliberate fresh start)"
936                    );
937                    if !self
938                        .runtime
939                        .mark_continuity_unrecoverable(
940                            identity,
941                            format!(
942                                "continuity repair parked after \
943                                 {REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS} consecutive \
944                                 byte-identical repair failures; blocking failure: \
945                                 {signature}. After fixing it, restart the gateway \
946                                 (process-local park; boot re-attempts repair) or reset \
947                                 the identity via `mobkit/reset`"
948                            ),
949                        )
950                        .await
951                    {
952                        tracing::debug!(
953                            %identity,
954                            "identity left Broken before the repair park could be recorded"
955                        );
956                    }
957                }
958            }
959            backoff = if still_broken.is_empty() && recovery_failures == 0 {
960                policy.initial_backoff
961            } else {
962                (backoff * 2).min(policy.max_backoff)
963            };
964            if cancellation
965                .as_ref()
966                .is_some_and(|cancellation| *cancellation.borrow())
967            {
968                return;
969            }
970        }
971    }
972
973    /// Ask the bridge's heal authority to drive this Broken identity's
974    /// durable session head to a strict-resume-acceptable committed boundary,
975    /// and translate the verdict into what the repair pass may do next.
976    async fn attempt_committed_boundary_recovery(
977        &self,
978        identity: &AgentIdentity,
979    ) -> BrokenRepairDisposition {
980        let Some(bridge) = self.runtime.bridge() else {
981            // Metadata-only runtime (tests): reconcile owns the retry.
982            return BrokenRepairDisposition::Repairable;
983        };
984        let Some(session_id) = self.runtime.continuity_session_id(identity).await else {
985            // No durable session bound (e.g. the store failed before a record
986            // existed): there is no head to heal; reconcile retries as before.
987            return BrokenRepairDisposition::Repairable;
988        };
989        match bridge.recover_committed_boundary(&session_id).await {
990            Ok(
991                CommittedBoundaryRepair::AlreadyCommitted | CommittedBoundaryRepair::Unsupported,
992            ) => BrokenRepairDisposition::Repairable,
993            Ok(CommittedBoundaryRepair::Recovered) => {
994                tracing::info!(
995                    %identity,
996                    %session_id,
997                    "continuity heal: recovery persisted a committed durable head; \
998                     proceeding to reconcile"
999                );
1000                BrokenRepairDisposition::Repairable
1001            }
1002            Ok(CommittedBoundaryRepair::Unprovable { reason }) => {
1003                tracing::error!(
1004                    %identity,
1005                    %session_id,
1006                    reason = %reason,
1007                    "continuity heal verdict: durable head unprovable; parking the \
1008                     identity as Broken until an operator intervenes"
1009                );
1010                if !self
1011                    .runtime
1012                    .mark_continuity_unrecoverable(identity, reason)
1013                    .await
1014                {
1015                    // The entry left Broken between the read and the mark
1016                    // (an operator reset raced us); nothing to park.
1017                    tracing::debug!(
1018                        %identity,
1019                        "unprovable verdict arrived after the identity left Broken"
1020                    );
1021                }
1022                BrokenRepairDisposition::Unprovable
1023            }
1024            Err(error) => {
1025                // Only the error tier is retryable per the heal contract
1026                // (Busy mid-turn, store I/O, CAS races).
1027                tracing::warn!(
1028                    %identity,
1029                    %session_id,
1030                    error = %error,
1031                    "committed-boundary recovery failed; retrying next repair pass"
1032                );
1033                BrokenRepairDisposition::RetryLater
1034            }
1035        }
1036    }
1037}
1038
1039/// What the repair pass may do with one Broken identity after consulting the
1040/// bridge's heal authority.
1041enum BrokenRepairDisposition {
1042    /// Reconcile may retry this identity now (head committed, recovered, or
1043    /// no heal seam to consult).
1044    Repairable,
1045    /// Terminal typed verdict recorded; excluded until an operator clears it.
1046    Unprovable,
1047    /// The recovery attempt itself failed transiently; retry next pass.
1048    RetryLater,
1049}
1050
1051/// Runtime-owned repair supervisor with cooperative idle cancellation.
1052///
1053/// Cancellation is observed while sleeping or between passes. An active
1054/// restore pass is joined to its explicit commit/rollback boundary instead of
1055/// being raw-aborted after lease acquisition.
1056pub(crate) struct TrackedContinuityRepairTask {
1057    cancel: watch::Sender<bool>,
1058    join: JoinHandle<()>,
1059}
1060
1061impl TrackedContinuityRepairTask {
1062    pub(crate) fn cancel(&self) {
1063        let _ = self.cancel.send(true);
1064    }
1065
1066    pub(crate) async fn cancel_and_join(self) {
1067        self.cancel();
1068        let _ = self.join.await;
1069    }
1070}
1071
1072/// Runtime-owned lease-renewal supervisor with cooperative cancellation.
1073///
1074/// Cancellation is observed while the supervisor is idle or between ticks.
1075/// An in-flight renewal is always joined through publication of the provider's
1076/// returned fencing token so final shutdown releases current authority.
1077pub(crate) struct TrackedLeaseRenewalTask {
1078    cancel: watch::Sender<bool>,
1079    join: JoinHandle<()>,
1080}
1081
1082impl TrackedLeaseRenewalTask {
1083    pub(crate) fn cancel(&self) {
1084        let _ = self.cancel.send(true);
1085    }
1086
1087    pub(crate) async fn cancel_and_join(self) {
1088        self.cancel();
1089        let _ = self.join.await;
1090    }
1091}
1092
1093/// Consecutive byte-identical repair failures tolerated for one identity
1094/// before the repair supervisor parks it typed
1095/// ([`IdentityRuntime::mark_continuity_unrecoverable`]) instead of
1096/// re-executing the pass's destructive dispose steps on a timer.
1097const REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS: u32 = 3;
1098
1099/// Retry cadence for [`IdentityFirstRuntimeContext::spawn_broken_identity_repair_task`].
1100#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1101pub struct ContinuityRepairPolicy {
1102    /// Delay before the first check and after every fully-healed pass.
1103    pub initial_backoff: Duration,
1104    /// Ceiling for the doubling backoff while identities stay Broken.
1105    pub max_backoff: Duration,
1106}
1107
1108impl Default for ContinuityRepairPolicy {
1109    fn default() -> Self {
1110        Self {
1111            initial_backoff: Duration::from_secs(30),
1112            max_backoff: Duration::from_mins(10),
1113        }
1114    }
1115}
1116
1117/// Weak keyed locks serialize claims on the shared raw/durable alias
1118/// namespace without retaining every caller-chosen alias for the lifetime of
1119/// the runtime. Sweeps grow geometrically while many aliases are live (as in
1120/// fleet bootstrap), avoiding a full-map scan for every new roster member.
1121struct RawMemberAliasLockTable {
1122    entries: BTreeMap<String, Weak<Mutex<()>>>,
1123    next_sweep_len: usize,
1124    #[cfg(test)]
1125    sweep_count: usize,
1126}
1127
1128impl Default for RawMemberAliasLockTable {
1129    fn default() -> Self {
1130        Self {
1131            entries: BTreeMap::new(),
1132            next_sweep_len: RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN,
1133            #[cfg(test)]
1134            sweep_count: 0,
1135        }
1136    }
1137}
1138
1139impl RawMemberAliasLockTable {
1140    fn sweep_if_needed(&mut self) {
1141        if self.entries.len() < self.next_sweep_len {
1142            return;
1143        }
1144        self.entries.retain(|_, lock| lock.upgrade().is_some());
1145        self.next_sweep_len = self
1146            .entries
1147            .len()
1148            .saturating_mul(2)
1149            .max(RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN);
1150        #[cfg(test)]
1151        {
1152            self.sweep_count += 1;
1153        }
1154    }
1155}
1156
1157/// The identity-first runtime tracks active identities and enforces delivery,
1158/// ownership, and lifecycle invariants.
1159pub struct IdentityRuntime {
1160    entries: RwLock<BTreeMap<AgentIdentity, IdentityEntry>>,
1161    event_channels: RwLock<BTreeMap<AgentIdentity, broadcast::Sender<IdentityEvent>>>,
1162    continuity_store: Arc<dyn ContinuityStore>,
1163    lease_provider: Arc<dyn LeaseProvider>,
1164    runtime_instance_id: String,
1165    has_runtime_store: bool,
1166    durability_policy: DurabilityPolicy,
1167    bridge: Option<Arc<dyn SessionBridge>>,
1168    reset_roster_source: StdRwLock<Option<ResetRosterSource>>,
1169    runtime_services: AgentRuntimeServices,
1170    managed_peer_edges: RwLock<BTreeSet<(AgentIdentity, AgentIdentity)>>,
1171    managed_peer_reconcile_lock: Mutex<()>,
1172    desired_peer_edges: RwLock<Vec<ManagedPeerEdge>>,
1173    topology_controller: StdRwLock<Option<crate::topology_control::TopologyController>>,
1174    materialization_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1175    best_effort_materialization_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1176    lifecycle_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1177    raw_member_alias_locks: RwLock<RawMemberAliasLockTable>,
1178    customizer: RwLock<Option<Arc<dyn AgentCustomizer>>>,
1179    agent_memory: RwLock<Option<AgentMemoryRuntimeInjector>>,
1180    lease_renewal_notify: Notify,
1181    /// Exact grants whose restore task failed before an IdentityEntry existed.
1182    /// These must outlive the failed task so reconcile or shutdown can retry
1183    /// provider release instead of leaving invisible, non-expiring authority.
1184    pending_unactivated_lease_releases: RwLock<Vec<LeaseGrant>>,
1185    pending_unactivated_lease_release_gate: Mutex<()>,
1186    default_timeout: Duration,
1187    materialization_failure_backoff: RwLock<BTreeMap<AgentIdentity, MaterializationFailureBackoff>>,
1188    error_hook: StdRwLock<Option<crate::unified_runtime::ErrorHook>>,
1189    bootstrap_status: watch::Sender<IdentityBootstrapStatus>,
1190    bootstrap_generation: StdMutex<u64>,
1191    bootstrap_controller: Mutex<()>,
1192    bootstrap_task: Mutex<Option<JoinHandle<()>>>,
1193    bootstrap_cancel: StdRwLock<Option<watch::Sender<bool>>>,
1194    bootstrap_shutdown: AtomicBool,
1195    foreground_operations: Mutex<JoinSet<()>>,
1196    foreground_cancel: watch::Sender<bool>,
1197    foreground_shutdown: AtomicBool,
1198    /// Post-commit cleanup for reset-superseded bridge generations. The debt
1199    /// is recorded before task spawn, survives task failure/timeout, and is
1200    /// retried synchronously before shutdown can attest cleanup or release
1201    /// identity fencing authority.
1202    pending_reset_bridge_cleanups: Arc<RwLock<BTreeMap<String, PendingResetBridgeCleanup>>>,
1203    reset_bridge_cleanup_tasks: Mutex<JoinSet<()>>,
1204    /// Per-identity completed-turn counters ([`CompletionCursor`]).
1205    ///
1206    /// RETAINED for the process lifetime — never pruned on retire, reset, or
1207    /// delete. Dropping an entry would let a re-registered identity republish
1208    /// a cursor it already published, which is the one thing a completion
1209    /// cursor must never do. The map is bounded by the identities this process
1210    /// has seen, i.e. by roster size.
1211    completion_cursors: StdMutex<BTreeMap<AgentIdentity, CompletionCursor>>,
1212}
1213
1214/// One generated member alias plus the lifecycle lock owned by its durable
1215/// identity runtime. Cross-runtime topology code resolves all endpoints first,
1216/// then acquires these targets in one global order before inspecting or
1217/// mutating either side.
1218pub(crate) struct MemberAliasLifecycleTarget {
1219    runtime: Arc<IdentityRuntime>,
1220    identity: AgentIdentity,
1221    alias: String,
1222    lock: Arc<Mutex<()>>,
1223}
1224
1225struct MultiRuntimeForegroundCompletion(watch::Sender<bool>);
1226
1227impl Drop for MultiRuntimeForegroundCompletion {
1228    fn drop(&mut self) {
1229        self.0.send_replace(true);
1230    }
1231}
1232
1233/// Internal result of one dispatch attempt: the caller-facing admission plus
1234/// the concrete bridge session that accepted the work (scheduler delivery
1235/// needs the latter; RPC callers do not).
1236struct DispatchOutcome {
1237    admission: DispatchAdmission,
1238    session_id: Option<SessionId>,
1239}
1240
1241#[derive(Clone)]
1242struct ResetRosterSource {
1243    provider: Arc<dyn RosterProvider>,
1244    mob_definition: Option<meerkat_mob::MobDefinition>,
1245}
1246
1247#[derive(Debug, Clone)]
1248struct MaterializationFailureBackoff {
1249    suppress_until: Instant,
1250    error: String,
1251}
1252
1253#[derive(Clone)]
1254struct PendingResetMemoryCapture {
1255    injector: AgentMemoryRuntimeInjector,
1256    identity: AgentIdentity,
1257    session_key: String,
1258    generation: u64,
1259}
1260
1261impl PendingResetMemoryCapture {
1262    async fn run(&self) {
1263        // Repeat the synchronous marks in the owned task so a raw reset
1264        // future dropped after debt publication cannot let the distiller read
1265        // evidence before the reset quarantine boundary exists.
1266        self.injector.note_reset_boundary(&self.session_key);
1267        self.injector
1268            .note_session_generation(&self.identity, &self.session_key, self.generation);
1269        self.injector
1270            .distill_before_rotation(
1271                &self.identity,
1272                &self.session_key,
1273                crate::memory::distiller::DistillCause::Reset,
1274            )
1275            .await;
1276    }
1277}
1278
1279#[derive(Clone)]
1280struct PendingResetBridgeCleanup {
1281    runtime_id: Option<AgentRuntimeId>,
1282    session_id: Option<SessionId>,
1283    memory_capture: Option<PendingResetMemoryCapture>,
1284}
1285
1286impl PartialEq for PendingResetBridgeCleanup {
1287    fn eq(&self, other: &Self) -> bool {
1288        self.runtime_id == other.runtime_id && self.session_id == other.session_id
1289    }
1290}
1291
1292impl Eq for PendingResetBridgeCleanup {}
1293
1294impl PendingResetBridgeCleanup {
1295    fn key(&self) -> String {
1296        format!(
1297            "{}|{}",
1298            self.runtime_id
1299                .as_ref()
1300                .map(AgentRuntimeId::as_str)
1301                .unwrap_or("-"),
1302            self.session_id
1303                .as_ref()
1304                .map(std::string::ToString::to_string)
1305                .as_deref()
1306                .unwrap_or("-")
1307        )
1308    }
1309}
1310
1311impl IdentityRuntime {
1312    /// Create a new identity runtime with the given configuration.
1313    pub fn new(config: IdentityRuntimeConfig) -> Self {
1314        let (bootstrap_status, _) = watch::channel(IdentityBootstrapStatus::empty(
1315            IdentityBootstrapMode::EagerMaterialize,
1316        ));
1317        let (foreground_cancel, _) = watch::channel(false);
1318        Self {
1319            entries: RwLock::new(BTreeMap::new()),
1320            event_channels: RwLock::new(BTreeMap::new()),
1321            continuity_store: config.continuity_store,
1322            lease_provider: config.lease_provider,
1323            runtime_instance_id: config.runtime_instance_id,
1324            has_runtime_store: config.has_runtime_store,
1325            durability_policy: config.durability_policy,
1326            bridge: config.bridge,
1327            reset_roster_source: StdRwLock::new(None),
1328            runtime_services: AgentRuntimeServices::empty(),
1329            managed_peer_edges: RwLock::new(BTreeSet::new()),
1330            managed_peer_reconcile_lock: Mutex::new(()),
1331            desired_peer_edges: RwLock::new(Vec::new()),
1332            topology_controller: StdRwLock::new(None),
1333            materialization_locks: RwLock::new(BTreeMap::new()),
1334            best_effort_materialization_locks: RwLock::new(BTreeMap::new()),
1335            lifecycle_locks: RwLock::new(BTreeMap::new()),
1336            raw_member_alias_locks: RwLock::new(RawMemberAliasLockTable::default()),
1337            customizer: RwLock::new(None),
1338            agent_memory: RwLock::new(None),
1339            lease_renewal_notify: Notify::new(),
1340            pending_unactivated_lease_releases: RwLock::new(Vec::new()),
1341            pending_unactivated_lease_release_gate: Mutex::new(()),
1342            default_timeout: config.default_timeout.unwrap_or(Duration::from_secs(90)),
1343            materialization_failure_backoff: RwLock::new(BTreeMap::new()),
1344            error_hook: StdRwLock::new(None),
1345            bootstrap_status,
1346            bootstrap_generation: StdMutex::new(0),
1347            bootstrap_controller: Mutex::new(()),
1348            bootstrap_task: Mutex::new(None),
1349            bootstrap_cancel: StdRwLock::new(None),
1350            bootstrap_shutdown: AtomicBool::new(false),
1351            foreground_operations: Mutex::new(JoinSet::new()),
1352            foreground_cancel,
1353            foreground_shutdown: AtomicBool::new(false),
1354            pending_reset_bridge_cleanups: Arc::new(RwLock::new(BTreeMap::new())),
1355            reset_bridge_cleanup_tasks: Mutex::new(JoinSet::new()),
1356            completion_cursors: StdMutex::new(BTreeMap::new()),
1357        }
1358    }
1359
1360    pub fn with_runtime_services(mut self, runtime_services: AgentRuntimeServices) -> Self {
1361        self.runtime_services = runtime_services;
1362        self
1363    }
1364
1365    pub fn with_reset_roster_provider(self, provider: Arc<dyn RosterProvider>) -> Self {
1366        self.set_reset_roster_provider(Some(provider));
1367        self
1368    }
1369
1370    pub fn with_reset_roster_provider_context(
1371        self,
1372        provider: Arc<dyn RosterProvider>,
1373        mob_definition: Option<meerkat_mob::MobDefinition>,
1374    ) -> Self {
1375        self.set_reset_roster_provider_context(Some(provider), mob_definition);
1376        self
1377    }
1378
1379    pub(crate) fn runtime_services(&self) -> AgentRuntimeServices {
1380        self.runtime_services.clone()
1381    }
1382
1383    /// Current typed bootstrap snapshot. Reading it never waits on an
1384    /// in-flight materialization.
1385    pub fn identity_bootstrap_status(&self) -> IdentityBootstrapStatus {
1386        self.identity_bootstrap_status_with_generation().1
1387    }
1388
1389    pub(crate) fn identity_bootstrap_status_with_generation(
1390        &self,
1391    ) -> (u64, IdentityBootstrapStatus) {
1392        let generation = self
1393            .bootstrap_generation
1394            .lock()
1395            .unwrap_or_else(std::sync::PoisonError::into_inner);
1396        (*generation, self.bootstrap_status.borrow().clone())
1397    }
1398
1399    pub(crate) fn subscribe_identity_bootstrap_status(
1400        &self,
1401    ) -> watch::Receiver<IdentityBootstrapStatus> {
1402        self.bootstrap_status.subscribe()
1403    }
1404
1405    /// Wait until every tracked identity has reached Active or Broken.
1406    /// Broken is terminal (and `ready == false`), so callers receive a useful
1407    /// failure snapshot instead of hanging forever.
1408    pub async fn wait_identity_bootstrap_terminal(
1409        &self,
1410        timeout: Duration,
1411    ) -> (IdentityBootstrapStatus, bool) {
1412        let (status, timed_out, _) = self
1413            .wait_identity_bootstrap_terminal_with_generation(timeout)
1414            .await;
1415        (status, timed_out)
1416    }
1417
1418    pub(crate) async fn wait_identity_bootstrap_terminal_with_generation(
1419        &self,
1420        timeout: Duration,
1421    ) -> (IdentityBootstrapStatus, bool, u64) {
1422        let mut receiver = self.subscribe_identity_bootstrap_status();
1423        let wait = async {
1424            loop {
1425                let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1426                if snapshot.complete && snapshot.materialization_terminal() {
1427                    return (snapshot, generation);
1428                }
1429                if receiver.changed().await.is_err() {
1430                    let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1431                    return (snapshot, generation);
1432                }
1433            }
1434        };
1435        match tokio::time::timeout(timeout, wait).await {
1436            Ok((snapshot, generation)) => (snapshot, false, generation),
1437            Err(_) => {
1438                let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1439                (snapshot, true, generation)
1440            }
1441        }
1442    }
1443
1444    fn request_identity_bootstrap_stop(&self) {
1445        if let Some(cancel) = self
1446            .bootstrap_cancel
1447            .read()
1448            .unwrap_or_else(std::sync::PoisonError::into_inner)
1449            .clone()
1450        {
1451            let _ = cancel.send(true);
1452        }
1453    }
1454
1455    async fn join_identity_bootstrap_task(&self) {
1456        if let Some(task) = self.bootstrap_task.lock().await.take() {
1457            let _ = task.await;
1458        }
1459        *self
1460            .bootstrap_cancel
1461            .write()
1462            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
1463    }
1464
1465    /// Serialize status mutation with pass supersession. The generation lock
1466    /// closes the check-then-write race that an atomic epoch alone would leave
1467    /// between a retiring warm task and a newly-published reconcile barrier.
1468    fn modify_bootstrap_status<F>(&self, expected_generation: Option<u64>, modify: F) -> bool
1469    where
1470        F: FnOnce(&mut IdentityBootstrapStatus),
1471    {
1472        let current_generation = self
1473            .bootstrap_generation
1474            .lock()
1475            .unwrap_or_else(std::sync::PoisonError::into_inner);
1476        if expected_generation.is_some_and(|expected| expected != *current_generation) {
1477            return false;
1478        }
1479        self.bootstrap_status.send_modify(modify);
1480        true
1481    }
1482
1483    fn replace_bootstrap_status(
1484        &self,
1485        expected_generation: u64,
1486        status: IdentityBootstrapStatus,
1487    ) -> bool {
1488        let current_generation = self
1489            .bootstrap_generation
1490            .lock()
1491            .unwrap_or_else(std::sync::PoisonError::into_inner);
1492        if expected_generation != *current_generation {
1493            return false;
1494        }
1495        self.bootstrap_status.send_replace(status);
1496        true
1497    }
1498
1499    /// Close the controller, cooperatively cancel warm operations before
1500    /// bridge/member installation, and join the tracked task. An acquired but
1501    /// uninstalled lease is released explicitly; operations past that boundary
1502    /// reach their explicit commit/rollback path so no external lease or
1503    /// bridge session can be leaked by a raw task abort.
1504    pub(crate) async fn cancel_identity_bootstrap(&self) {
1505        self.bootstrap_shutdown.store(true, Ordering::Release);
1506        self.request_identity_bootstrap_stop();
1507        let _controller = self.bootstrap_controller.lock().await;
1508        self.request_identity_bootstrap_stop();
1509        self.join_identity_bootstrap_task().await;
1510        self.modify_bootstrap_status(None, |snapshot| {
1511            for entry in snapshot.identities.values_mut() {
1512                if entry.state == IdentityBootstrapState::Warming {
1513                    entry.state = IdentityBootstrapState::Dormant;
1514                }
1515            }
1516            snapshot.complete = true;
1517            snapshot.refresh_aggregates();
1518        });
1519    }
1520
1521    async fn begin_identity_bootstrap(
1522        &self,
1523        generation: u64,
1524        mode: IdentityBootstrapMode,
1525        roster: &[DurableAgentSpec],
1526    ) {
1527        let entries = self.entries.read().await;
1528        let identities = roster
1529            .iter()
1530            .map(|spec| {
1531                let lifecycle = entries.get(&spec.identity).map(|entry| entry.state);
1532                let state = match lifecycle {
1533                    Some(IdentityLifecycleState::Active) => IdentityBootstrapState::Active,
1534                    _ if matches!(&mode, IdentityBootstrapMode::EagerMaterialize) => {
1535                        IdentityBootstrapState::Warming
1536                    }
1537                    _ => IdentityBootstrapState::Dormant,
1538                };
1539                (
1540                    spec.identity.clone(),
1541                    IdentityBootstrapEntry { state, error: None },
1542                )
1543            })
1544            .collect();
1545        drop(entries);
1546        let mut status = IdentityBootstrapStatus {
1547            mode,
1548            complete: false,
1549            ready: false,
1550            error: None,
1551            counts: Default::default(),
1552            identities,
1553        };
1554        status.refresh_aggregates();
1555        // A roster consisting only of already-active identities still has a
1556        // reconcile pass in flight. `complete` is the barrier guard even when
1557        // the aggregate states themselves happen to look ready.
1558        status.complete = false;
1559        status.ready = false;
1560        self.replace_bootstrap_status(generation, status);
1561    }
1562
1563    fn begin_identity_bootstrap_pending(&self, mode: IdentityBootstrapMode) -> u64 {
1564        let mut generation = self
1565            .bootstrap_generation
1566            .lock()
1567            .unwrap_or_else(std::sync::PoisonError::into_inner);
1568        *generation = generation.wrapping_add(1);
1569        if *generation == 0 {
1570            *generation = 1;
1571        }
1572        let current_generation = *generation;
1573        self.bootstrap_status.send_modify(|snapshot| {
1574            snapshot.mode = mode;
1575            snapshot.complete = false;
1576            snapshot.ready = false;
1577            snapshot.error = None;
1578        });
1579        current_generation
1580    }
1581
1582    fn fail_identity_bootstrap(&self, generation: u64, error: &IdentityRuntimeError) {
1583        self.modify_bootstrap_status(Some(generation), |snapshot| {
1584            let detail = error.to_string();
1585            snapshot.complete = true;
1586            snapshot.error = Some(detail.clone());
1587            for entry in snapshot.identities.values_mut() {
1588                if entry.state != IdentityBootstrapState::Active {
1589                    entry.state = IdentityBootstrapState::Broken;
1590                    entry.error = Some(detail.clone());
1591                }
1592            }
1593            snapshot.refresh_aggregates();
1594        });
1595    }
1596
1597    #[cfg(test)]
1598    pub(crate) fn test_supersede_identity_bootstrap_ready(&self) {
1599        let generation =
1600            self.begin_identity_bootstrap_pending(IdentityBootstrapMode::EagerMaterialize);
1601        self.modify_bootstrap_status(Some(generation), |snapshot| {
1602            snapshot.identities.clear();
1603            snapshot.complete = true;
1604            snapshot.error = None;
1605            snapshot.refresh_aggregates();
1606        });
1607    }
1608
1609    #[cfg(test)]
1610    pub(crate) fn test_fail_identity_bootstrap(&self, detail: &str) {
1611        let generation =
1612            self.begin_identity_bootstrap_pending(IdentityBootstrapMode::EagerMaterialize);
1613        self.modify_bootstrap_status(Some(generation), |snapshot| {
1614            snapshot.identities.clear();
1615            if let Ok(identity) = AgentIdentity::parse("agent:test-bootstrap-failure") {
1616                snapshot.identities.insert(
1617                    identity,
1618                    IdentityBootstrapEntry {
1619                        state: IdentityBootstrapState::Dormant,
1620                        error: None,
1621                    },
1622                );
1623            }
1624            snapshot.refresh_aggregates();
1625        });
1626        self.fail_identity_bootstrap(
1627            generation,
1628            &IdentityRuntimeError::Internal(detail.to_string()),
1629        );
1630    }
1631
1632    async fn install_identity_bootstrap(
1633        self: &Arc<Self>,
1634        generation: u64,
1635        mode: IdentityBootstrapMode,
1636        roster: &[DurableAgentSpec],
1637        result: &super::orchestrator::RestoreFlowResult,
1638    ) {
1639        let background_concurrency = match mode {
1640            IdentityBootstrapMode::LazyWithBackgroundWarm { concurrency } => Some(concurrency),
1641            _ => None,
1642        };
1643        let mut status = IdentityBootstrapStatus {
1644            mode: mode.clone(),
1645            complete: background_concurrency.is_none(),
1646            ready: false,
1647            error: None,
1648            counts: Default::default(),
1649            identities: BTreeMap::new(),
1650        };
1651        for spec in roster {
1652            let broken_outcome = result.outcomes.get(&spec.identity).and_then(|outcome| {
1653                if let super::orchestrator::RestoreOutcome::Broken(failure) = outcome {
1654                    Some(failure.detail.clone())
1655                } else {
1656                    None
1657                }
1658            });
1659            let lifecycle = self
1660                .status(&spec.identity)
1661                .await
1662                .ok()
1663                .map(|item| item.state);
1664            let (state, error) = match (broken_outcome, lifecycle) {
1665                // A store-projected Broken result is itself authoritative.
1666                // restore_flow intentionally does not fabricate a lifecycle
1667                // entry for it, so consulting lifecycle alone would default
1668                // to Dormant and make a terminal wait barrier time out.
1669                (Some(detail), _) => (IdentityBootstrapState::Broken, Some(detail)),
1670                (None, Some(IdentityLifecycleState::Active)) => {
1671                    (IdentityBootstrapState::Active, None)
1672                }
1673                (None, Some(IdentityLifecycleState::Broken)) => {
1674                    (IdentityBootstrapState::Broken, None)
1675                }
1676                (None, _) => (IdentityBootstrapState::Dormant, None),
1677            };
1678            status.identities.insert(
1679                spec.identity.clone(),
1680                IdentityBootstrapEntry { state, error },
1681            );
1682        }
1683        status.refresh_aggregates();
1684        if !self.replace_bootstrap_status(generation, status.clone()) {
1685            return;
1686        }
1687
1688        let Some(concurrency) = background_concurrency else {
1689            return;
1690        };
1691        let identities = status
1692            .identities
1693            .iter()
1694            .filter(|(_, entry)| entry.state == IdentityBootstrapState::Dormant)
1695            .map(|(identity, _)| identity.clone())
1696            .collect::<Vec<_>>();
1697        if identities.is_empty() {
1698            self.modify_bootstrap_status(Some(generation), |snapshot| {
1699                snapshot.complete = true;
1700                snapshot.refresh_aggregates();
1701            });
1702            return;
1703        }
1704
1705        let runtime = Arc::clone(self);
1706        let (cancel, task_cancel) = watch::channel(false);
1707        let task = tokio::spawn(async move {
1708            stream::iter(identities.into_iter().map(|identity| {
1709                let runtime = Arc::clone(&runtime);
1710                let mut cancel = task_cancel.clone();
1711                async move {
1712                    if *cancel.borrow() {
1713                        return;
1714                    }
1715                    let Some(result) = runtime
1716                        .materialize_for_background(&identity, &mut cancel, generation)
1717                        .await
1718                    else {
1719                        return;
1720                    };
1721                    if let Err(error) = result {
1722                        tracing::warn!(
1723                            %identity,
1724                            error = %error,
1725                            "identity background warm failed"
1726                        );
1727                        runtime
1728                            .record_best_effort_materialization_failure(
1729                                &identity,
1730                                None,
1731                                "background_warm",
1732                                &error,
1733                            )
1734                            .await;
1735                    }
1736                }
1737            }))
1738            .buffer_unordered(concurrency)
1739            .collect::<Vec<_>>()
1740            .await;
1741            if !*task_cancel.borrow() {
1742                runtime.modify_bootstrap_status(Some(generation), |snapshot| {
1743                    snapshot.complete = true;
1744                    snapshot.refresh_aggregates();
1745                });
1746            }
1747        });
1748        *self
1749            .bootstrap_cancel
1750            .write()
1751            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cancel);
1752        *self.bootstrap_task.lock().await = Some(task);
1753    }
1754
1755    fn mark_bootstrap_materialization_started(
1756        &self,
1757        identity: &AgentIdentity,
1758        generation: Option<u64>,
1759    ) {
1760        self.modify_bootstrap_status(generation, |snapshot| {
1761            let Some(entry) = snapshot.identities.get_mut(identity) else {
1762                return;
1763            };
1764            entry.state = IdentityBootstrapState::Warming;
1765            entry.error = None;
1766            snapshot.refresh_aggregates();
1767        });
1768    }
1769
1770    fn mark_bootstrap_from_lifecycle(
1771        &self,
1772        identity: &AgentIdentity,
1773        lifecycle: IdentityLifecycleState,
1774        error: Option<String>,
1775    ) {
1776        self.modify_bootstrap_status(None, |snapshot| {
1777            let Some(entry) = snapshot.identities.get_mut(identity) else {
1778                return;
1779            };
1780            entry.state = match lifecycle {
1781                IdentityLifecycleState::Active => IdentityBootstrapState::Active,
1782                IdentityLifecycleState::Broken => IdentityBootstrapState::Broken,
1783                IdentityLifecycleState::Dormant
1784                | IdentityLifecycleState::Retiring
1785                | IdentityLifecycleState::Suspended
1786                | IdentityLifecycleState::Uninitialized => IdentityBootstrapState::Dormant,
1787            };
1788            entry.error = if entry.state == IdentityBootstrapState::Broken {
1789                error
1790            } else {
1791                None
1792            };
1793            snapshot.refresh_aggregates();
1794        });
1795    }
1796
1797    fn mark_bootstrap_materialization_finished(
1798        &self,
1799        identity: &AgentIdentity,
1800        result: &Result<ContinuityRecord, IdentityRuntimeError>,
1801        generation: Option<u64>,
1802    ) {
1803        self.modify_bootstrap_status(generation, |snapshot| {
1804            let Some(entry) = snapshot.identities.get_mut(identity) else {
1805                return;
1806            };
1807            match result {
1808                Ok(_) => {
1809                    entry.state = IdentityBootstrapState::Active;
1810                    entry.error = None;
1811                }
1812                Err(error) => {
1813                    entry.state = IdentityBootstrapState::Broken;
1814                    entry.error = Some(error.to_string());
1815                }
1816            }
1817            snapshot.refresh_aggregates();
1818        });
1819    }
1820
1821    fn mark_bootstrap_materialization_cancelled(
1822        &self,
1823        identity: &AgentIdentity,
1824        generation: Option<u64>,
1825    ) {
1826        self.modify_bootstrap_status(generation, |snapshot| {
1827            if let Some(entry) = snapshot.identities.get_mut(identity) {
1828                entry.state = IdentityBootstrapState::Dormant;
1829                entry.error = None;
1830                snapshot.refresh_aggregates();
1831            }
1832        });
1833    }
1834
1835    /// Exact concrete member ids represented by the tracked bootstrap roster.
1836    /// Used only after `ready == true`, preventing a false-ready snapshot of a
1837    /// partially warmed mob.
1838    pub async fn identity_bootstrap_member_ids(&self) -> Vec<meerkat_mob::ids::AgentIdentity> {
1839        let tracked = self.identity_bootstrap_status();
1840        self.identity_bootstrap_member_ids_for_status(&tracked)
1841            .await
1842    }
1843
1844    pub(crate) async fn identity_bootstrap_member_ids_for_status(
1845        &self,
1846        tracked: &IdentityBootstrapStatus,
1847    ) -> Vec<meerkat_mob::ids::AgentIdentity> {
1848        let entries = self.entries.read().await;
1849        tracked
1850            .identities
1851            .keys()
1852            .filter_map(|identity| {
1853                let entry = entries.get(identity)?;
1854                let runtime_id = entry.continuity.as_ref()?.agent_runtime_id.as_str();
1855                let alias = if durable_spec_uses_external_binding(&entry.spec) {
1856                    identity.as_str()
1857                } else {
1858                    runtime_id
1859                };
1860                Some(crate::member_comms_id::mob_member_id(alias))
1861            })
1862            .collect()
1863    }
1864
1865    pub async fn set_agent_customizer(&self, customizer: Option<Arc<dyn AgentCustomizer>>) {
1866        *self.customizer.write().await = customizer;
1867    }
1868
1869    pub async fn set_agent_memory(&self, injector: Option<AgentMemoryRuntimeInjector>) {
1870        *self.agent_memory.write().await = injector;
1871    }
1872
1873    pub async fn agent_memory_supports_recall(&self) -> bool {
1874        self.agent_memory.read().await.is_some()
1875    }
1876
1877    pub async fn agent_memory_supports_remember(&self) -> bool {
1878        self.agent_memory
1879            .read()
1880            .await
1881            .as_ref()
1882            .is_some_and(|injector| injector.provider().supports_remember())
1883    }
1884
1885    pub async fn agent_memory_supports_forget(&self) -> bool {
1886        self.agent_memory
1887            .read()
1888            .await
1889            .as_ref()
1890            .is_some_and(|injector| injector.provider().supports_forget())
1891    }
1892
1893    pub async fn agent_memory_supports_update(&self) -> bool {
1894        self.agent_memory
1895            .read()
1896            .await
1897            .as_ref()
1898            .is_some_and(|injector| injector.provider().supports_supersede())
1899    }
1900
1901    pub async fn agent_memory_supports_manifest(&self) -> bool {
1902        self.agent_memory
1903            .read()
1904            .await
1905            .as_ref()
1906            .is_some_and(|injector| injector.provider().supports_manifest())
1907    }
1908
1909    pub async fn remember_agent_memory(
1910        &self,
1911        realm: &str,
1912        identity: &AgentIdentity,
1913        memory: NewAgentMemory,
1914    ) -> Result<AgentMemoryRecord, AgentMemoryError> {
1915        self.status(identity)
1916            .await
1917            .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1918        let provider = self
1919            .agent_memory
1920            .read()
1921            .await
1922            .as_ref()
1923            .map(AgentMemoryRuntimeInjector::provider)
1924            .ok_or_else(|| {
1925                AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1926            })?;
1927        provider.remember(realm, identity, memory).await
1928    }
1929
1930    pub async fn forget_agent_memory(
1931        &self,
1932        realm: &str,
1933        identity: &AgentIdentity,
1934        memory_id: &str,
1935    ) -> Result<AgentMemoryForgetResult, AgentMemoryError> {
1936        self.status(identity)
1937            .await
1938            .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1939        let provider = self
1940            .agent_memory
1941            .read()
1942            .await
1943            .as_ref()
1944            .map(AgentMemoryRuntimeInjector::provider)
1945            .ok_or_else(|| {
1946                AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1947            })?;
1948        provider.forget(realm, identity, memory_id).await
1949    }
1950
1951    /// Supersede `memory_id` within its lineage (the D4 fix): the new
1952    /// title/body/tags become the active record; the prior stays
1953    /// retrievable with provenance.
1954    pub async fn update_agent_memory(
1955        &self,
1956        realm: &str,
1957        identity: &AgentIdentity,
1958        memory_id: &str,
1959        memory: NewAgentMemory,
1960    ) -> Result<MemoryId, AgentMemoryError> {
1961        self.status(identity)
1962            .await
1963            .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1964        let provider = self
1965            .agent_memory
1966            .read()
1967            .await
1968            .as_ref()
1969            .map(AgentMemoryRuntimeInjector::provider)
1970            .ok_or_else(|| {
1971                AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1972            })?;
1973        let scope = MemoryScope::Identity {
1974            realm: realm.to_string(),
1975            identity: identity.as_str().to_string(),
1976        };
1977        let record = NewMemoryRecord {
1978            kind: MemoryKind::Fact,
1979            title: memory.title,
1980            description: String::new(),
1981            body: memory.body,
1982            tags: memory.tags,
1983            evidence: Vec::new(),
1984            verification: None,
1985        };
1986        provider.supersede(&scope, memory_id, record).await
1987    }
1988
1989    /// Tiered metadata manifest for the identity's own scope (§8.3).
1990    pub async fn manifest_agent_memory(
1991        &self,
1992        realm: &str,
1993        identity: &AgentIdentity,
1994        tier: ManifestTier,
1995    ) -> Result<Vec<RecordMeta>, AgentMemoryError> {
1996        self.status(identity)
1997            .await
1998            .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1999        let provider = self
2000            .agent_memory
2001            .read()
2002            .await
2003            .as_ref()
2004            .map(AgentMemoryRuntimeInjector::provider)
2005            .ok_or_else(|| {
2006                AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
2007            })?;
2008        let scope = MemoryScope::Identity {
2009            realm: realm.to_string(),
2010            identity: identity.as_str().to_string(),
2011        };
2012        provider.manifest(&[scope], tier).await
2013    }
2014
2015    pub async fn recall_agent_memory(
2016        &self,
2017        request: AgentMemoryRecallRequest,
2018    ) -> Result<Vec<AgentMemoryRecord>, AgentMemoryError> {
2019        self.status(&request.identity)
2020            .await
2021            .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
2022        let provider = self
2023            .agent_memory
2024            .read()
2025            .await
2026            .as_ref()
2027            .map(AgentMemoryRuntimeInjector::provider)
2028            .ok_or_else(|| {
2029                AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
2030            })?;
2031        let records = provider.recall(request).await?;
2032        // §9.2: explicit recall reads mark usage mechanically. Telemetry
2033        // never fails the read — providers without usage support
2034        // (markdown) return Unsupported, which is downgraded here.
2035        if !records.is_empty() {
2036            let ids: Vec<MemoryId> = records
2037                .iter()
2038                .map(|record| record.memory_id.clone())
2039                .collect();
2040            if let Err(err) = provider.mark_usage(&ids, UsageEvent::ExplicitRecall).await {
2041                tracing::debug!(error = %err, "agent memory explicit-recall usage marking skipped");
2042            }
2043        }
2044        Ok(records)
2045    }
2046
2047    /// Attach the roster provider reset should consult for current specs.
2048    pub fn set_reset_roster_provider(&self, provider: Option<Arc<dyn RosterProvider>>) {
2049        self.set_reset_roster_provider_context(provider, None);
2050    }
2051
2052    /// Attach the roster provider and context reset should consult for current specs.
2053    pub fn set_reset_roster_provider_context(
2054        &self,
2055        provider: Option<Arc<dyn RosterProvider>>,
2056        mob_definition: Option<meerkat_mob::MobDefinition>,
2057    ) {
2058        let source = provider.map(|provider| ResetRosterSource {
2059            provider,
2060            mob_definition,
2061        });
2062        match self.reset_roster_source.write() {
2063            Ok(mut stored_source) => *stored_source = source,
2064            Err(err) => {
2065                tracing::warn!(
2066                    error = %err,
2067                    "identity runtime reset roster source lock poisoned; dropping provider update"
2068                );
2069            }
2070        }
2071    }
2072
2073    async fn adopt_current_roster_spec_for_reset(&self, identity: &AgentIdentity) {
2074        let source = match self.reset_roster_source.read() {
2075            Ok(stored_source) => stored_source.clone(),
2076            Err(err) => {
2077                tracing::warn!(
2078                    error = %err,
2079                    "reset: roster source lock poisoned; rebuilding on stored spec"
2080                );
2081                None
2082            }
2083        };
2084        if let Some(source) = source {
2085            self.adopt_roster_spec_with_context(
2086                &source.provider,
2087                identity,
2088                source.mob_definition.clone(),
2089            )
2090            .await;
2091        }
2092    }
2093
2094    /// Attach a best-effort operational error hook used for alerting.
2095    pub fn set_error_hook(&self, hook: Option<crate::unified_runtime::ErrorHook>) {
2096        match self.error_hook.write() {
2097            Ok(mut stored_hook) => *stored_hook = hook,
2098            Err(err) => {
2099                tracing::warn!(
2100                    error = %err,
2101                    "identity runtime error hook lock poisoned; dropping hook update"
2102                );
2103            }
2104        }
2105    }
2106
2107    /// Spawn a background supervisor that renews active identity leases before
2108    /// they reach their TTL deadline.
2109    pub fn spawn_lease_renewal_task(self: Arc<Self>) -> JoinHandle<()> {
2110        self.spawn_lease_renewal_task_with_poll_interval(DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL)
2111    }
2112
2113    /// Spawn the runtime-owned renewal supervisor. Unlike the public
2114    /// fire-and-forget helper, this handle cooperatively cancels while idle
2115    /// and joins any in-flight renewal through provider, continuity-store,
2116    /// bridge, and local-publication commit boundaries.
2117    pub(crate) fn spawn_tracked_lease_renewal_task(self: Arc<Self>) -> TrackedLeaseRenewalTask {
2118        self.spawn_tracked_lease_renewal_task_with_poll_interval(
2119            DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL,
2120        )
2121    }
2122
2123    pub(crate) fn spawn_tracked_lease_renewal_task_with_poll_interval(
2124        self: Arc<Self>,
2125        max_poll_interval: Duration,
2126    ) -> TrackedLeaseRenewalTask {
2127        let (cancel, receiver) = watch::channel(false);
2128        let join = tokio::spawn(self.run_lease_renewal_loop(max_poll_interval, Some(receiver)));
2129        TrackedLeaseRenewalTask { cancel, join }
2130    }
2131
2132    /// Spawn a lease renewal supervisor with a caller-provided maximum poll
2133    /// interval. Embedders can use this for shorter external lease TTLs; tests
2134    /// use it to exercise renewal without waiting on wall-clock TTLs.
2135    pub fn spawn_lease_renewal_task_with_poll_interval(
2136        self: Arc<Self>,
2137        max_poll_interval: Duration,
2138    ) -> JoinHandle<()> {
2139        tokio::spawn(self.run_lease_renewal_loop(max_poll_interval, None))
2140    }
2141
2142    async fn run_lease_renewal_loop(
2143        self: Arc<Self>,
2144        max_poll_interval: Duration,
2145        mut cancellation: Option<watch::Receiver<bool>>,
2146    ) {
2147        let max_poll_interval = max_poll_interval.max(DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL);
2148        let mut consecutive_failures: u32 = 0;
2149        loop {
2150            if cancellation
2151                .as_ref()
2152                .is_some_and(|cancellation| *cancellation.borrow())
2153            {
2154                return;
2155            }
2156            let base = self.lease_renewal_sleep_interval(max_poll_interval).await;
2157            // While the provider is failing, hold off at least the backoff
2158            // delay so a persistent outage retries at a bounded rate instead
2159            // of the TTL-derived floor (down to 10ms).
2160            let sleep = if consecutive_failures > 0 {
2161                base.max(lease_renewal_failure_backoff(
2162                    consecutive_failures,
2163                    max_poll_interval,
2164                ))
2165            } else {
2166                base
2167            };
2168            if let Some(cancellation) = cancellation.as_mut() {
2169                tokio::select! {
2170                    () = tokio::time::sleep(sleep) => {}
2171                    () = self.lease_renewal_notify.notified() => {}
2172                    changed = cancellation.changed() => {
2173                        match changed {
2174                            Ok(()) if *cancellation.borrow() => return,
2175                            Ok(()) => continue,
2176                            Err(_) => return,
2177                        }
2178                    }
2179                }
2180            } else {
2181                tokio::select! {
2182                    () = tokio::time::sleep(sleep) => {}
2183                    () = self.lease_renewal_notify.notified() => {}
2184                }
2185            }
2186
2187            // Do not select cancellation against this future. Once renewal
2188            // has entered the provider, the returned token may already be
2189            // authoritative; the runtime must publish that exact grant before
2190            // shutdown performs its final release.
2191            match self.renew_due_leases_once().await {
2192                Ok(_) => consecutive_failures = 0,
2193                Err(err) => {
2194                    // Warn once, then debounce to debug so a backend outage
2195                    // can't flood the log at the renewal cadence.
2196                    if consecutive_failures == 0 {
2197                        tracing::warn!(
2198                            error = %err,
2199                            "identity-first proactive lease renewal tick failed; backing off"
2200                        );
2201                    } else {
2202                        tracing::debug!(
2203                            error = %err,
2204                            consecutive_failures,
2205                            "identity-first lease renewal still failing; backing off"
2206                        );
2207                    }
2208                    consecutive_failures = consecutive_failures.saturating_add(1);
2209                }
2210            }
2211            if cancellation
2212                .as_ref()
2213                .is_some_and(|cancellation| *cancellation.borrow())
2214            {
2215                return;
2216            }
2217        }
2218    }
2219
2220    async fn lease_renewal_sleep_interval(&self, max_poll_interval: Duration) -> Duration {
2221        let entries = self.entries.read().await;
2222        entries
2223            .values()
2224            .filter(|entry| entry.state == IdentityLifecycleState::Active)
2225            .filter_map(|entry| entry.lease.as_ref())
2226            .map(|lease| (lease.ttl / 10).max(DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL))
2227            .min()
2228            .unwrap_or(max_poll_interval)
2229            .min(max_poll_interval)
2230    }
2231
2232    /// Renew every active lease that has entered the runtime's renewal window.
2233    pub async fn renew_due_leases_once(&self) -> Result<usize, IdentityRuntimeError> {
2234        let due = {
2235            let entries = self.entries.read().await;
2236            entries
2237                .iter()
2238                .filter(|(_, entry)| entry.state == IdentityLifecycleState::Active)
2239                .filter_map(|(identity, entry)| {
2240                    entry
2241                        .lease
2242                        .as_ref()
2243                        .filter(|lease| !lease.is_healthy())
2244                        .map(|_| identity.clone())
2245                })
2246                .collect::<Vec<_>>()
2247        };
2248
2249        let mut renewed = 0;
2250        let mut first_error = None;
2251        for identity in due {
2252            let lifecycle_lock = self.lifecycle_lock_for(&identity).await;
2253            let _lifecycle_guard = lifecycle_lock.lock().await;
2254            match self.ensure_active_lease(&identity).await {
2255                Ok(_) => renewed += 1,
2256                Err(err) => {
2257                    if first_error.is_none() {
2258                        first_error = Some(err);
2259                    }
2260                }
2261            }
2262        }
2263        if let Some(err) = first_error {
2264            Err(err)
2265        } else {
2266            Ok(renewed)
2267        }
2268    }
2269
2270    async fn release_uninstalled_materialize_lease(&self, grant: &LeaseGrant) -> Option<String> {
2271        self.release_or_park_untracked_leases(std::slice::from_ref(grant))
2272            .await
2273            .err()
2274            .map(|err| err.to_string())
2275    }
2276
2277    async fn cancel_uninstalled_background_materialization(
2278        &self,
2279        grant: &LeaseGrant,
2280    ) -> IdentityRuntimeError {
2281        let cleanup_error = self.release_uninstalled_materialize_lease(grant).await;
2282        IdentityRuntimeError::Internal(
2283            cleanup_error
2284                .map(|error| format!("{BACKGROUND_WARM_CANCELLED}; lease cleanup failed: {error}"))
2285                .unwrap_or_else(|| BACKGROUND_WARM_CANCELLED.to_string()),
2286        )
2287    }
2288
2289    pub async fn set_desired_peer_edges(&self, edges: Vec<ManagedPeerEdge>) {
2290        *self.desired_peer_edges.write().await = edges;
2291    }
2292
2293    pub(crate) fn set_topology_controller(
2294        &self,
2295        controller: crate::topology_control::TopologyController,
2296    ) {
2297        *self
2298            .topology_controller
2299            .write()
2300            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controller);
2301    }
2302
2303    fn topology_controller(&self) -> Option<crate::topology_control::TopologyController> {
2304        self.topology_controller
2305            .read()
2306            .unwrap_or_else(std::sync::PoisonError::into_inner)
2307            .clone()
2308    }
2309
2310    pub async fn desired_peer_edges(&self) -> Vec<ManagedPeerEdge> {
2311        let declared = self.desired_peer_edges.read().await.clone();
2312        match self.topology_controller() {
2313            Some(controller) => {
2314                // Materialization/console reads must not observe the target
2315                // intent while a topology transaction is between WAL and
2316                // terminal commit/rollback.
2317                let _admission = controller.mutation_guard().await;
2318                match controller.compose_managed_peer_edges(&declared).await {
2319                    Ok(edges) => edges,
2320                    Err(error) => {
2321                        tracing::error!(error = %error, "failed to compose identity topology overlay");
2322                        Vec::new()
2323                    }
2324                }
2325            }
2326            None => declared,
2327        }
2328    }
2329
2330    async fn registered_identities(&self) -> Vec<AgentIdentity> {
2331        self.entries.read().await.keys().cloned().collect()
2332    }
2333
2334    async fn reachable_peer_identities(&self, identity: &AgentIdentity) -> Vec<AgentIdentity> {
2335        self.desired_peer_edges()
2336            .await
2337            .iter()
2338            .filter_map(|edge| {
2339                if edge.a() == identity {
2340                    Some(edge.b().clone())
2341                } else if edge.b() == identity {
2342                    Some(edge.a().clone())
2343                } else {
2344                    None
2345                }
2346            })
2347            .collect::<BTreeSet<_>>()
2348            .into_iter()
2349            .collect()
2350    }
2351
2352    #[must_use]
2353    pub fn has_session_bridge(&self) -> bool {
2354        self.bridge.is_some()
2355    }
2356
2357    pub(crate) async fn logical_peer_edges(
2358        &self,
2359    ) -> Result<Vec<ManagedPeerEdge>, IdentityRuntimeError> {
2360        let Some(bridge) = self.bridge.as_ref() else {
2361            return Ok(Vec::new());
2362        };
2363        let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = self
2364            .entries
2365            .read()
2366            .await
2367            .iter()
2368            .filter_map(|(identity, entry)| {
2369                entry
2370                    .continuity
2371                    .as_ref()
2372                    .map(|record| (record.agent_runtime_id.clone(), identity.clone()))
2373            })
2374            .collect();
2375        let runtime_edges = bridge.current_member_wires().await.map_err(|error| {
2376            IdentityRuntimeError::Internal(format!("bridge current_member_wires: {error}"))
2377        })?;
2378        Ok(runtime_edges
2379            .into_iter()
2380            .filter_map(|(runtime_a, runtime_b)| {
2381                let a = runtime_identities.get(&runtime_a)?.clone();
2382                let b = runtime_identities.get(&runtime_b)?.clone();
2383                ManagedPeerEdge::new(a, b).ok()
2384            })
2385            .collect())
2386    }
2387
2388    pub(crate) async fn logical_peer_edges_any_half(
2389        &self,
2390    ) -> Result<Vec<ManagedPeerEdge>, IdentityRuntimeError> {
2391        let Some(bridge) = self.bridge.as_ref() else {
2392            return Ok(Vec::new());
2393        };
2394        let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = self
2395            .entries
2396            .read()
2397            .await
2398            .iter()
2399            .filter_map(|(identity, entry)| {
2400                entry
2401                    .continuity
2402                    .as_ref()
2403                    .map(|record| (record.agent_runtime_id.clone(), identity.clone()))
2404            })
2405            .collect();
2406        let runtime_edges = bridge
2407            .current_member_wires_any_half()
2408            .await
2409            .map_err(|error| {
2410                IdentityRuntimeError::Internal(format!(
2411                    "bridge current_member_wires_any_half: {error}"
2412                ))
2413            })?;
2414        Ok(runtime_edges
2415            .into_iter()
2416            .filter_map(|(runtime_a, runtime_b)| {
2417                let a = runtime_identities.get(&runtime_a)?.clone();
2418                let b = runtime_identities.get(&runtime_b)?.clone();
2419                ManagedPeerEdge::new(a, b).ok()
2420            })
2421            .collect())
2422    }
2423
2424    pub(crate) async fn managed_peer_edges_snapshot(
2425        &self,
2426    ) -> BTreeSet<(AgentIdentity, AgentIdentity)> {
2427        self.managed_peer_edges.read().await.clone()
2428    }
2429
2430    pub(crate) async fn retain_managed_peer_edges(
2431        &self,
2432        edges: &BTreeSet<(AgentIdentity, AgentIdentity)>,
2433    ) {
2434        self.managed_peer_edges
2435            .write()
2436            .await
2437            .extend(edges.iter().cloned());
2438    }
2439
2440    /// Logical identity actuator used only while the shared topology
2441    /// controller's admission lock is already held by TopologyRuntimeHandle.
2442    pub(crate) async fn mutate_managed_peer_edge_admitted(
2443        &self,
2444        action: crate::topology_control::TopologyAction,
2445        edge: &ManagedPeerEdge,
2446    ) -> Result<(), IdentityRuntimeError> {
2447        let _guard = self.managed_peer_reconcile_lock.lock().await;
2448        let _lifecycle_guards = self
2449            .lifecycle_guards_for([edge.a().clone(), edge.b().clone()])
2450            .await;
2451        let Some(bridge) = self.bridge.clone() else {
2452            return Err(IdentityRuntimeError::Internal(
2453                "topology mutation requires a session bridge".to_string(),
2454            ));
2455        };
2456        let (runtime_a, runtime_b) = {
2457            let entries = self.entries.read().await;
2458            let resolve = |identity: &AgentIdentity| {
2459                entries
2460                    .get(identity)
2461                    .filter(|entry| entry.state == IdentityLifecycleState::Active)
2462                    .and_then(|entry| entry.continuity.as_ref())
2463                    .map(|record| record.agent_runtime_id.clone())
2464                    .ok_or_else(|| {
2465                        IdentityRuntimeError::Internal(format!(
2466                            "topology endpoint is not active: {identity}"
2467                        ))
2468                    })
2469            };
2470            (resolve(edge.a())?, resolve(edge.b())?)
2471        };
2472        let key = (edge.a().clone(), edge.b().clone());
2473        let current = bridge.current_member_wires().await.map_err(|error| {
2474            IdentityRuntimeError::Internal(format!("bridge current_member_wires: {error}"))
2475        })?;
2476        let actual = current.iter().any(|(a, b)| {
2477            (a == &runtime_a && b == &runtime_b) || (a == &runtime_b && b == &runtime_a)
2478        });
2479        let any_half = bridge
2480            .current_member_wires_any_half()
2481            .await
2482            .map_err(|error| {
2483                IdentityRuntimeError::Internal(format!(
2484                    "bridge current_member_wires_any_half: {error}"
2485                ))
2486            })?
2487            .iter()
2488            .any(|(a, b)| {
2489                (a == &runtime_a && b == &runtime_b) || (a == &runtime_b && b == &runtime_a)
2490            });
2491        match action {
2492            crate::topology_control::TopologyAction::Connect if !actual => bridge
2493                .wire_peers_batch(&[(runtime_a, runtime_b)])
2494                .await
2495                .map_err(|error| {
2496                    IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {error}"))
2497                })?,
2498            crate::topology_control::TopologyAction::Reconnect => {
2499                if any_half {
2500                    bridge
2501                        .unwire_peer(&runtime_a, &runtime_b)
2502                        .await
2503                        .map_err(|error| {
2504                            IdentityRuntimeError::Internal(format!("bridge unwire_peer: {error}"))
2505                        })?;
2506                }
2507                bridge
2508                    .wire_peers_batch(&[(runtime_a, runtime_b)])
2509                    .await
2510                    .map_err(|error| {
2511                        IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {error}"))
2512                    })?;
2513            }
2514            crate::topology_control::TopologyAction::Disconnect if any_half => bridge
2515                .unwire_peer(&runtime_a, &runtime_b)
2516                .await
2517                .map_err(|error| {
2518                    IdentityRuntimeError::Internal(format!("bridge unwire_peer: {error}"))
2519                })?,
2520            _ => {}
2521        }
2522        let mut managed = self.managed_peer_edges.write().await;
2523        if matches!(action, crate::topology_control::TopologyAction::Disconnect) {
2524            managed.remove(&key);
2525        } else {
2526            managed.insert(key);
2527        }
2528        Ok(())
2529    }
2530
2531    /// Apply identity-first managed topology to the concrete mob graph.
2532    ///
2533    /// Topology providers return stable logical identities. The mob comms graph
2534    /// is keyed by active runtime member IDs, so this resolves each endpoint
2535    /// through continuity records before calling the same-mob bridge wire APIs.
2536    pub async fn reconcile_managed_peer_edges(
2537        &self,
2538        desired_edges: &[ManagedPeerEdge],
2539    ) -> Result<(), IdentityRuntimeError> {
2540        let topology_controller = self.topology_controller();
2541        let _topology_guard = match topology_controller.as_ref() {
2542            Some(controller) => Some(controller.mutation_guard().await),
2543            None => None,
2544        };
2545        if let Some(controller) = topology_controller.as_ref() {
2546            controller
2547                .prepare_pending_recovery()
2548                .await
2549                .map_err(|error| {
2550                    IdentityRuntimeError::Internal(format!("topology recovery journal: {error}"))
2551                })?;
2552        }
2553        let pending_recovery_edges = if let Some(controller) = topology_controller.as_ref() {
2554            controller
2555                .pending_local_recovery_edges()
2556                .await
2557                .map_err(|error| {
2558                    IdentityRuntimeError::Internal(format!("topology recovery ownership: {error}"))
2559                })?
2560        } else {
2561            BTreeSet::new()
2562        };
2563        if !pending_recovery_edges.is_empty() {
2564            self.retain_managed_peer_edges(&pending_recovery_edges)
2565                .await;
2566        }
2567        let composed_edges;
2568        let desired_edges = if let Some(controller) = topology_controller.as_ref() {
2569            composed_edges = controller
2570                .compose_managed_peer_edges(desired_edges)
2571                .await
2572                .map_err(|error| {
2573                    IdentityRuntimeError::Internal(format!("topology overlay: {error}"))
2574                })?;
2575            composed_edges.as_slice()
2576        } else {
2577            desired_edges
2578        };
2579        let result = self
2580            .reconcile_managed_peer_edges_admitted(desired_edges)
2581            .await;
2582        let mut recovery_inspection_error = None;
2583        if let Some(controller) = topology_controller.as_ref() {
2584            let recovery_complete = if result.is_ok() && !pending_recovery_edges.is_empty() {
2585                match self
2586                    .pending_recovery_is_physically_complete(desired_edges, &pending_recovery_edges)
2587                    .await
2588                {
2589                    Ok(complete) => complete,
2590                    Err(error) => {
2591                        recovery_inspection_error = Some(error);
2592                        false
2593                    }
2594                }
2595            } else {
2596                false
2597            };
2598            controller
2599                .finalize_recovered_pending(result.is_ok() && recovery_complete)
2600                .await
2601                .map_err(|error| {
2602                    IdentityRuntimeError::Internal(format!("topology recovery receipt: {error}"))
2603                })?;
2604        }
2605        result?;
2606        if let Some(error) = recovery_inspection_error {
2607            return Err(error);
2608        }
2609        Ok(())
2610    }
2611
2612    pub(crate) async fn pending_recovery_is_physically_complete(
2613        &self,
2614        desired_edges: &[ManagedPeerEdge],
2615        pending_recovery_edges: &BTreeSet<(AgentIdentity, AgentIdentity)>,
2616    ) -> Result<bool, IdentityRuntimeError> {
2617        if self.bridge.is_none() {
2618            return Ok(false);
2619        }
2620        let active_identities = self
2621            .entries
2622            .read()
2623            .await
2624            .iter()
2625            .filter_map(|(identity, entry)| {
2626                (entry.state == IdentityLifecycleState::Active && entry.continuity.is_some())
2627                    .then_some(identity.clone())
2628            })
2629            .collect::<BTreeSet<_>>();
2630        if pending_recovery_edges
2631            .iter()
2632            .any(|(a, b)| !active_identities.contains(a) || !active_identities.contains(b))
2633        {
2634            return Ok(false);
2635        }
2636        let desired = desired_edges
2637            .iter()
2638            .map(|edge| (edge.a().clone(), edge.b().clone()))
2639            .collect::<BTreeSet<_>>();
2640        let actual = self
2641            .logical_peer_edges()
2642            .await?
2643            .into_iter()
2644            .map(|edge| (edge.a().clone(), edge.b().clone()))
2645            .collect::<BTreeSet<_>>();
2646        let actual_any_half = self
2647            .logical_peer_edges_any_half()
2648            .await?
2649            .into_iter()
2650            .map(|edge| (edge.a().clone(), edge.b().clone()))
2651            .collect::<BTreeSet<_>>();
2652
2653        Ok(desired.is_subset(&actual)
2654            && pending_recovery_edges
2655                .difference(&desired)
2656                .all(|edge| !actual_any_half.contains(edge)))
2657    }
2658
2659    /// Reconcile an already-composed desired topology while the caller holds
2660    /// the shared topology-controller admission guard.
2661    pub(crate) async fn reconcile_managed_peer_edges_admitted(
2662        &self,
2663        desired_edges: &[ManagedPeerEdge],
2664    ) -> Result<(), IdentityRuntimeError> {
2665        let _guard = self.managed_peer_reconcile_lock.lock().await;
2666        let Some(bridge) = self.bridge.clone() else {
2667            return Ok(());
2668        };
2669
2670        let managed_snapshot = self.managed_peer_edges.read().await.clone();
2671        let topology_identities = desired_edges
2672            .iter()
2673            .flat_map(|edge| [edge.a().clone(), edge.b().clone()])
2674            .chain(
2675                managed_snapshot
2676                    .iter()
2677                    .flat_map(|(a, b)| [a.clone(), b.clone()]),
2678            );
2679        let _lifecycle_guards = self.lifecycle_guards_for(topology_identities).await;
2680
2681        let (known_runtimes, active_runtimes): (
2682            BTreeMap<AgentIdentity, AgentRuntimeId>,
2683            BTreeMap<AgentIdentity, AgentRuntimeId>,
2684        ) = {
2685            let entries = self.entries.read().await;
2686            let known = entries
2687                .iter()
2688                .filter_map(|(identity, entry)| {
2689                    entry
2690                        .continuity
2691                        .as_ref()
2692                        .map(|record| (identity.clone(), record.agent_runtime_id.clone()))
2693                })
2694                .collect::<BTreeMap<_, _>>();
2695            let active = entries
2696                .iter()
2697                .filter_map(|(identity, entry)| {
2698                    if entry.state != IdentityLifecycleState::Active {
2699                        return None;
2700                    }
2701                    entry
2702                        .continuity
2703                        .as_ref()
2704                        .map(|record| (identity.clone(), record.agent_runtime_id.clone()))
2705                })
2706                .collect();
2707            (known, active)
2708        };
2709        let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = known_runtimes
2710            .iter()
2711            .map(|(identity, runtime_id)| (runtime_id.clone(), identity.clone()))
2712            .collect();
2713        let current_logical_edges: Option<BTreeSet<(AgentIdentity, AgentIdentity)>> =
2714            match bridge.current_member_wires().await {
2715                Ok(current_runtime_edges) => Some(
2716                    current_runtime_edges
2717                        .iter()
2718                        .filter_map(|(runtime_a, runtime_b)| {
2719                            let a = runtime_identities.get(runtime_a)?;
2720                            let b = runtime_identities.get(runtime_b)?;
2721                            if a <= b {
2722                                Some((a.clone(), b.clone()))
2723                            } else {
2724                                Some((b.clone(), a.clone()))
2725                            }
2726                        })
2727                        .collect(),
2728                ),
2729                Err(err) => {
2730                    tracing::debug!(
2731                        error = %err,
2732                        "identity-first topology reconcile could not inspect current member wires"
2733                    );
2734                    None
2735                }
2736            };
2737        let current_any_half_edges: Option<BTreeSet<(AgentIdentity, AgentIdentity)>> =
2738            match bridge.current_member_wires_any_half().await {
2739                Ok(current_runtime_edges) => Some(
2740                    current_runtime_edges
2741                        .iter()
2742                        .filter_map(|(runtime_a, runtime_b)| {
2743                            let a = runtime_identities.get(runtime_a)?;
2744                            let b = runtime_identities.get(runtime_b)?;
2745                            if a <= b {
2746                                Some((a.clone(), b.clone()))
2747                            } else {
2748                                Some((b.clone(), a.clone()))
2749                            }
2750                        })
2751                        .collect(),
2752                ),
2753                Err(error) => {
2754                    tracing::debug!(
2755                        %error,
2756                        "identity-first topology reconcile could not inspect orphan wire halves"
2757                    );
2758                    None
2759                }
2760            };
2761
2762        let desired: BTreeSet<(AgentIdentity, AgentIdentity)> = desired_edges
2763            .iter()
2764            .map(|edge| (edge.a().clone(), edge.b().clone()))
2765            .collect();
2766
2767        let edge_is_managed_and_live = |edge: &(AgentIdentity, AgentIdentity)| {
2768            // Managed-but-missing live edges are retried deliberately so tolerant topology restores self-heal.
2769            managed_snapshot.contains(edge)
2770                && current_logical_edges
2771                    .as_ref()
2772                    .is_none_or(|edges| edges.contains(edge))
2773        };
2774        let retained_logical_edges: Vec<(AgentIdentity, AgentIdentity)> = desired
2775            .iter()
2776            .filter(|edge| !edge_is_managed_and_live(edge))
2777            .filter(|edge| {
2778                current_logical_edges
2779                    .as_ref()
2780                    .is_some_and(|edges| edges.contains(*edge))
2781            })
2782            .filter(|(a, b)| active_runtimes.contains_key(a) && active_runtimes.contains_key(b))
2783            .cloned()
2784            .collect();
2785        let to_wire: Vec<(AgentIdentity, AgentIdentity, AgentRuntimeId, AgentRuntimeId)> = desired
2786            .iter()
2787            .filter(|edge| !edge_is_managed_and_live(edge))
2788            .filter(|edge| {
2789                current_logical_edges
2790                    .as_ref()
2791                    .is_none_or(|edges| !edges.contains(*edge))
2792            })
2793            .filter_map(|(a, b)| {
2794                let runtime_a = active_runtimes.get(a)?;
2795                let runtime_b = active_runtimes.get(b)?;
2796                Some((a.clone(), b.clone(), runtime_a.clone(), runtime_b.clone()))
2797            })
2798            .collect();
2799
2800        let stale: Vec<(AgentIdentity, AgentIdentity)> = managed_snapshot
2801            .iter()
2802            .filter(|edge| !desired.contains(*edge))
2803            .cloned()
2804            .collect();
2805        let to_unwire: Vec<(AgentIdentity, AgentIdentity, AgentRuntimeId, AgentRuntimeId)> = stale
2806            .iter()
2807            .filter_map(|(a, b)| {
2808                let runtime_a = active_runtimes.get(a)?;
2809                let runtime_b = active_runtimes.get(b)?;
2810                if current_any_half_edges
2811                    .as_ref()
2812                    .is_some_and(|edges| !edges.contains(&(a.clone(), b.clone())))
2813                {
2814                    return None;
2815                }
2816                Some((a.clone(), b.clone(), runtime_a.clone(), runtime_b.clone()))
2817            })
2818            .collect();
2819
2820        let wire_logical_edges = to_wire
2821            .iter()
2822            .map(|(a, b, _, _)| (a.clone(), b.clone()))
2823            .collect::<Vec<_>>();
2824        let wire_runtime_edges = to_wire
2825            .iter()
2826            .map(|(_, _, runtime_a, runtime_b)| (runtime_a.clone(), runtime_b.clone()))
2827            .collect::<Vec<_>>();
2828        if !wire_runtime_edges.is_empty() {
2829            bridge
2830                .wire_peers_batch(&wire_runtime_edges)
2831                .await
2832                .map_err(|e| {
2833                    IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {e}"))
2834                })?;
2835        }
2836
2837        let unwire_results =
2838            stream::iter(to_unwire.into_iter().map(|(a, b, runtime_a, runtime_b)| {
2839                let bridge = bridge.clone();
2840                async move {
2841                    let result = bridge
2842                        .unwire_peer(&runtime_a, &runtime_b)
2843                        .await
2844                        .map_err(|e| format!("{e}"));
2845                    (a, b, result)
2846                }
2847            }))
2848            .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
2849            .collect::<Vec<_>>()
2850            .await;
2851
2852        let mut managed = self.managed_peer_edges.write().await;
2853        for (a, b) in retained_logical_edges {
2854            managed.insert((a, b));
2855        }
2856        for (a, b) in wire_logical_edges {
2857            managed.insert((a, b));
2858        }
2859
2860        for (a, b) in stale {
2861            let key = (a.clone(), b.clone());
2862            if current_any_half_edges
2863                .as_ref()
2864                .is_some_and(|edges| !edges.contains(&key))
2865            {
2866                managed.remove(&key);
2867            }
2868        }
2869        for (a, b, result) in unwire_results {
2870            result
2871                .map_err(|e| IdentityRuntimeError::Internal(format!("bridge unwire_peer: {e}")))?;
2872            managed.remove(&(a, b));
2873        }
2874
2875        Ok(())
2876    }
2877
2878    /// Emit an event for the given identity. Best-effort — no error if no subscribers.
2879    async fn emit_event(&self, identity: &AgentIdentity, event: IdentityEvent) {
2880        let channels = self.event_channels.read().await;
2881        if let Some(tx) = channels.get(identity) {
2882            let _ = tx.send(event);
2883        }
2884    }
2885
2886    fn emit_error(&self, event: crate::unified_runtime::types::ErrorEvent) {
2887        let hook = match self.error_hook.read() {
2888            Ok(stored_hook) => stored_hook.clone(),
2889            Err(err) => {
2890                tracing::warn!(
2891                    error = %err,
2892                    "identity runtime error hook lock poisoned; dropping error event"
2893                );
2894                None
2895            }
2896        };
2897        if let Some(hook) = hook {
2898            tokio::spawn(async move {
2899                let () = hook(event).await;
2900            });
2901        }
2902    }
2903
2904    async fn materialization_backoff_error(&self, identity: &AgentIdentity) -> Option<String> {
2905        let backoffs = self.materialization_failure_backoff.read().await;
2906        let backoff = backoffs.get(identity)?;
2907        if Instant::now() < backoff.suppress_until {
2908            Some(backoff.error.clone())
2909        } else {
2910            None
2911        }
2912    }
2913
2914    async fn clear_materialization_backoff(&self, identity: &AgentIdentity) {
2915        self.materialization_failure_backoff
2916            .write()
2917            .await
2918            .remove(identity);
2919    }
2920
2921    async fn record_best_effort_materialization_failure(
2922        &self,
2923        identity: &AgentIdentity,
2924        initiator: Option<&AgentIdentity>,
2925        operation: &'static str,
2926        err: &IdentityRuntimeError,
2927    ) {
2928        let error = err.to_string();
2929        let suppress_until = Instant::now() + MATERIALIZATION_FAILURE_BACKOFF;
2930        self.materialization_failure_backoff.write().await.insert(
2931            identity.clone(),
2932            MaterializationFailureBackoff {
2933                suppress_until,
2934                error: error.clone(),
2935            },
2936        );
2937        self.emit_error(
2938            crate::unified_runtime::types::ErrorEvent::IdentityMaterializationFailure {
2939                identity: identity.to_string(),
2940                initiator: initiator.map(ToString::to_string),
2941                operation: operation.to_string(),
2942                error,
2943            },
2944        );
2945    }
2946
2947    // -----------------------------------------------------------------------
2948    // Registration / activation
2949    // -----------------------------------------------------------------------
2950
2951    /// Register an identity entry in the runtime (called during restore flow).
2952    pub async fn register(
2953        &self,
2954        spec: DurableAgentSpec,
2955        state: IdentityLifecycleState,
2956        continuity: Option<ContinuityRecord>,
2957        lease: Option<LeaseGrant>,
2958    ) {
2959        let identity = spec.identity.clone();
2960        let bootstrap_generation = self.identity_bootstrap_status_with_generation().0;
2961        let cpv = continuity
2962            .as_ref()
2963            .map(|r| r.checkpoint_version)
2964            .unwrap_or(CheckpointVersion::new(0));
2965        let lease_entry = lease.map(|g| LeaseEntry {
2966            fencing_token: g.fencing_token,
2967            ttl: g.ttl,
2968            acquired_at: Instant::now(),
2969        });
2970        let has_active_lease = state == IdentityLifecycleState::Active && lease_entry.is_some();
2971        {
2972            let mut entries = self.entries.write().await;
2973            // A terminal heal verdict is about the durable head, not this
2974            // entry instance: re-projecting Broken (a repair retry, an eager
2975            // reconcile) must not silently forget it, while any non-Broken
2976            // projection is a real lifecycle transition that supersedes it
2977            // (2026-07-29 heal/re-Break incident).
2978            let continuity_unrecoverable = if state == IdentityLifecycleState::Broken {
2979                entries
2980                    .get(&identity)
2981                    .and_then(|existing| existing.continuity_unrecoverable.clone())
2982            } else {
2983                None
2984            };
2985            // A host-rejected-build park is about the SPEC, not this entry
2986            // instance: re-registration (a reconcile pass, a repair retry)
2987            // with the same spec must not forget it — retrying an unchanged
2988            // spec against a deterministic gate re-burns a build + callback
2989            // round trip for the same answer. A changed spec clears it.
2990            let host_rejected_build_park = entries.get(&identity).and_then(|existing| {
2991                existing
2992                    .host_rejected_build_park
2993                    .clone()
2994                    .filter(|park| park.spec_digest == durable_spec_digest(&spec))
2995            });
2996            let entry = IdentityEntry {
2997                spec,
2998                bootstrap_generation,
2999                state,
3000                continuity,
3001                lease: lease_entry,
3002                pending_lease_release: None,
3003                checkpoint_version: cpv,
3004                has_runtime_store: self.has_runtime_store,
3005                continuity_unrecoverable,
3006                host_rejected_build_park,
3007            };
3008            entries.insert(identity.clone(), entry);
3009        }
3010
3011        // Create event channel for this identity
3012        let (tx, _) = broadcast::channel(IDENTITY_EVENT_CHANNEL_CAPACITY);
3013        self.event_channels
3014            .write()
3015            .await
3016            .insert(identity.clone(), tx);
3017        if has_active_lease {
3018            self.lease_renewal_notify.notify_one();
3019        }
3020        self.mark_bootstrap_from_lifecycle(&identity, state, None);
3021    }
3022
3023    async fn materialization_lock_for(&self, identity: &AgentIdentity) -> Arc<Mutex<()>> {
3024        if let Some(lock) = self.materialization_locks.read().await.get(identity) {
3025            return lock.clone();
3026        }
3027        let mut locks = self.materialization_locks.write().await;
3028        locks
3029            .entry(identity.clone())
3030            .or_insert_with(|| Arc::new(Mutex::new(())))
3031            .clone()
3032    }
3033
3034    async fn best_effort_materialization_lock_for(
3035        &self,
3036        identity: &AgentIdentity,
3037    ) -> Arc<Mutex<()>> {
3038        if let Some(lock) = self
3039            .best_effort_materialization_locks
3040            .read()
3041            .await
3042            .get(identity)
3043        {
3044            return lock.clone();
3045        }
3046        let mut locks = self.best_effort_materialization_locks.write().await;
3047        locks
3048            .entry(identity.clone())
3049            .or_insert_with(|| Arc::new(Mutex::new(())))
3050            .clone()
3051    }
3052
3053    pub(crate) async fn lifecycle_lock_for(&self, identity: &AgentIdentity) -> Arc<Mutex<()>> {
3054        if let Some(lock) = self.lifecycle_locks.read().await.get(identity) {
3055            return lock.clone();
3056        }
3057        let mut locks = self.lifecycle_locks.write().await;
3058        locks
3059            .entry(identity.clone())
3060            .or_insert_with(|| Arc::new(Mutex::new(())))
3061            .clone()
3062    }
3063
3064    pub(crate) async fn raw_member_alias_lock(&self, alias: &str) -> Arc<Mutex<()>> {
3065        let alias = crate::member_comms_id::runtime_alias_str(alias.trim()).into_owned();
3066        if let Some(lock) = self
3067            .raw_member_alias_locks
3068            .read()
3069            .await
3070            .entries
3071            .get(&alias)
3072            .and_then(Weak::upgrade)
3073        {
3074            return lock;
3075        }
3076        let mut table = self.raw_member_alias_locks.write().await;
3077        table.sweep_if_needed();
3078        // Another caller may have installed the key after our read miss. The
3079        // write-side upgrade is the authority boundary that prevents two live
3080        // mutexes from serializing the same alias independently.
3081        if let Some(lock) = table.entries.get(&alias).and_then(Weak::upgrade) {
3082            return lock;
3083        }
3084        let lock = Arc::new(Mutex::new(()));
3085        table.entries.insert(alias, Arc::downgrade(&lock));
3086        lock
3087    }
3088
3089    #[cfg(test)]
3090    pub(crate) async fn raw_member_alias_lock_metrics(&self) -> (usize, usize, usize) {
3091        let table = self.raw_member_alias_locks.read().await;
3092        (table.entries.len(), table.next_sweep_len, table.sweep_count)
3093    }
3094
3095    pub(crate) async fn ensure_raw_member_alias_available(
3096        &self,
3097        identity: &AgentIdentity,
3098    ) -> Result<(), IdentityRuntimeError> {
3099        if let Some(bridge) = self.bridge.as_ref()
3100            && bridge
3101                .raw_member_alias_exists(identity.as_str())
3102                .await
3103                .map_err(|error| {
3104                    IdentityRuntimeError::Internal(format!(
3105                        "raw member namespace inspection for {identity}: {error}"
3106                    ))
3107                })?
3108        {
3109            return Err(IdentityRuntimeError::Internal(format!(
3110                "durable identity '{identity}' collides with an existing raw member alias"
3111            )));
3112        }
3113        Ok(())
3114    }
3115
3116    /// Resolve an alias into a lifecycle-lock target. Classic members return
3117    /// `None`; generated aliases always resolve to an authority target even
3118    /// after deletion so validation under the lock fails closed.
3119    pub(crate) async fn member_alias_lifecycle_target(
3120        self: &Arc<Self>,
3121        alias: &str,
3122    ) -> Result<Option<MemberAliasLifecycleTarget>, IdentityRuntimeError> {
3123        let alias = crate::member_comms_id::runtime_alias_str(alias).into_owned();
3124        match self.identity_for_member_mutation(&alias).await {
3125            Some(identity) => {
3126                // Fail stale/deleted generated aliases before unrelated
3127                // operation prerequisites (for example cross-mob directory
3128                // lookup). The tracked operation validates again after taking
3129                // the lifecycle lock, so this preflight does not become the
3130                // authority boundary or introduce a TOCTOU gap.
3131                self.ensure_expected_member_alias_current(&identity, &alias)
3132                    .await?;
3133                Ok(Some(MemberAliasLifecycleTarget {
3134                    runtime: Arc::clone(self),
3135                    lock: self.lifecycle_lock_for(&identity).await,
3136                    identity,
3137                    alias,
3138                }))
3139            }
3140            None if crate::member_comms_id::is_reserved_generated_alias(&alias) => {
3141                Err(IdentityRuntimeError::Internal(format!(
3142                    "generated member alias requires identity authority: {alias}"
3143                )))
3144            }
3145            None => Ok(None),
3146        }
3147    }
3148
3149    /// Acquire member-alias lifecycle targets across one or more identity
3150    /// runtimes in a deterministic process-global order, then validate every
3151    /// alias while its owning lock is held. Duplicate identities acquire one
3152    /// lock but still validate every spelling/generation.
3153    pub(crate) async fn acquire_member_alias_lifecycle_targets(
3154        mut targets: Vec<MemberAliasLifecycleTarget>,
3155    ) -> Result<Vec<tokio::sync::OwnedMutexGuard<()>>, IdentityRuntimeError> {
3156        targets.sort_by(|a, b| {
3157            a.runtime
3158                .runtime_instance_id
3159                .cmp(&b.runtime.runtime_instance_id)
3160                .then_with(|| {
3161                    (Arc::as_ptr(&a.runtime) as usize).cmp(&(Arc::as_ptr(&b.runtime) as usize))
3162                })
3163                .then_with(|| a.identity.cmp(&b.identity))
3164                .then_with(|| a.alias.cmp(&b.alias))
3165        });
3166
3167        let mut guards = Vec::with_capacity(targets.len());
3168        let mut held: Option<(usize, AgentIdentity)> = None;
3169        for target in targets {
3170            let key = (
3171                Arc::as_ptr(&target.runtime) as usize,
3172                target.identity.clone(),
3173            );
3174            if held.as_ref() != Some(&key) {
3175                guards.push(target.lock.lock_owned().await);
3176                held = Some(key);
3177            }
3178            target
3179                .runtime
3180                .ensure_expected_member_alias_current(&target.identity, &target.alias)
3181                .await?;
3182            let state = target
3183                .runtime
3184                .entries
3185                .read()
3186                .await
3187                .get(&target.identity)
3188                .map(|entry| entry.state)
3189                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(target.identity.clone()))?;
3190            if state != IdentityLifecycleState::Active {
3191                return Err(IdentityRuntimeError::InvalidState {
3192                    identity: target.identity,
3193                    state,
3194                    operation: "mutate member alias",
3195                });
3196            }
3197        }
3198        Ok(guards)
3199    }
3200
3201    /// Cancellation-safe operation spanning one or more alias targets. The
3202    /// first target in global order supervises the transaction to its explicit
3203    /// commit/rollback boundary if the request future is dropped.
3204    pub(crate) async fn run_member_alias_targets_operation_tracked<T, F, Fut>(
3205        targets: Vec<MemberAliasLifecycleTarget>,
3206        operation: F,
3207    ) -> Result<T, IdentityRuntimeError>
3208    where
3209        T: Send + 'static,
3210        F: FnOnce() -> Fut + Send + 'static,
3211        Fut: Future<Output = Result<T, String>> + Send + 'static,
3212    {
3213        let mut runtimes = targets
3214            .iter()
3215            .map(|target| Arc::clone(&target.runtime))
3216            .collect::<Vec<_>>();
3217        runtimes.sort_by(|a, b| {
3218            a.runtime_instance_id
3219                .cmp(&b.runtime_instance_id)
3220                .then_with(|| (Arc::as_ptr(a) as usize).cmp(&(Arc::as_ptr(b) as usize)))
3221        });
3222        runtimes.dedup_by(|a, b| Arc::ptr_eq(a, b));
3223        let transaction = async move {
3224            let _guards = Self::acquire_member_alias_lifecycle_targets(targets).await?;
3225            operation().await.map_err(IdentityRuntimeError::Internal)
3226        };
3227        if !runtimes.is_empty() {
3228            return Self::run_tracked_foreground_multi(runtimes, transaction).await;
3229        }
3230        transaction.await
3231    }
3232
3233    /// Register one compound transaction with every participating runtime.
3234    /// Non-owner shutdowns wait on a completion task in their own JoinSet,
3235    /// while the globally first runtime owns the actual operation and result.
3236    async fn run_tracked_foreground_multi<T, F>(
3237        runtimes: Vec<Arc<Self>>,
3238        operation: F,
3239    ) -> Result<T, IdentityRuntimeError>
3240    where
3241        T: Send + 'static,
3242        F: Future<Output = Result<T, IdentityRuntimeError>> + Send + 'static,
3243    {
3244        let Some((supervisor, participants)) = runtimes.split_first() else {
3245            return operation.await;
3246        };
3247        if runtimes
3248            .iter()
3249            .any(|runtime| runtime.foreground_shutdown.load(Ordering::Acquire))
3250        {
3251            return Err(IdentityRuntimeError::Internal(
3252                "identity runtime is shutting down".to_string(),
3253            ));
3254        }
3255
3256        let (completion, _) = watch::channel(false);
3257        for runtime in participants {
3258            let mut operations = runtime.foreground_operations.lock().await;
3259            if runtime.foreground_shutdown.load(Ordering::Acquire) {
3260                completion.send_replace(true);
3261                return Err(IdentityRuntimeError::Internal(
3262                    "identity runtime is shutting down".to_string(),
3263                ));
3264            }
3265            while let Some(result) = operations.try_join_next() {
3266                if let Err(error) = result {
3267                    tracing::error!(
3268                        error = %error,
3269                        "tracked foreground identity operation panicked"
3270                    );
3271                }
3272            }
3273            let mut completed = completion.subscribe();
3274            operations.spawn(async move {
3275                while !*completed.borrow() && completed.changed().await.is_ok() {}
3276            });
3277        }
3278
3279        let (sender, receiver) = oneshot::channel();
3280        {
3281            let mut operations = supervisor.foreground_operations.lock().await;
3282            if supervisor.foreground_shutdown.load(Ordering::Acquire) {
3283                completion.send_replace(true);
3284                return Err(IdentityRuntimeError::Internal(
3285                    "identity runtime is shutting down".to_string(),
3286                ));
3287            }
3288            while let Some(result) = operations.try_join_next() {
3289                if let Err(error) = result {
3290                    tracing::error!(
3291                        error = %error,
3292                        "tracked foreground identity operation panicked"
3293                    );
3294                }
3295            }
3296            operations.spawn(async move {
3297                let _completion = MultiRuntimeForegroundCompletion(completion);
3298                let _ = sender.send(operation.await);
3299            });
3300        }
3301        receiver.await.map_err(|_| {
3302            IdentityRuntimeError::Internal(
3303                "tracked foreground identity operation terminated without a result".to_string(),
3304            )
3305        })?
3306    }
3307
3308    /// Acquire several identity lifecycle locks in stable identity order.
3309    /// Topology operations span two or more generated aliases; global ordering
3310    /// prevents opposite-direction edge requests from deadlocking.
3311    async fn lifecycle_guards_for(
3312        &self,
3313        identities: impl IntoIterator<Item = AgentIdentity>,
3314    ) -> Vec<tokio::sync::OwnedMutexGuard<()>> {
3315        let identities = identities.into_iter().collect::<BTreeSet<_>>();
3316        let mut guards = Vec::with_capacity(identities.len());
3317        for identity in identities {
3318            guards.push(self.lifecycle_lock_for(&identity).await.lock_owned().await);
3319        }
3320        guards
3321    }
3322
3323    /// Run an externally-cancellable operation under runtime ownership.
3324    ///
3325    /// Dropping the caller only drops the result receiver; the transaction
3326    /// remains in the runtime's join set and reaches its explicit
3327    /// commit/rollback boundary. Graceful shutdown closes admission and joins
3328    /// every such task before lease renewal or the mob actor is stopped.
3329    async fn run_tracked_foreground<T, F>(
3330        self: &Arc<Self>,
3331        operation: F,
3332    ) -> Result<T, IdentityRuntimeError>
3333    where
3334        T: Send + 'static,
3335        F: Future<Output = Result<T, IdentityRuntimeError>> + Send + 'static,
3336    {
3337        if self.foreground_shutdown.load(Ordering::Acquire) {
3338            return Err(IdentityRuntimeError::Internal(
3339                "identity runtime is shutting down".to_string(),
3340            ));
3341        }
3342        let (sender, receiver) = oneshot::channel();
3343        {
3344            let mut operations = self.foreground_operations.lock().await;
3345            if self.foreground_shutdown.load(Ordering::Acquire) {
3346                return Err(IdentityRuntimeError::Internal(
3347                    "identity runtime is shutting down".to_string(),
3348                ));
3349            }
3350            while let Some(result) = operations.try_join_next() {
3351                if let Err(error) = result {
3352                    tracing::error!(
3353                        error = %error,
3354                        "tracked foreground identity operation panicked"
3355                    );
3356                }
3357            }
3358            operations.spawn(async move {
3359                let _ = sender.send(operation.await);
3360            });
3361        }
3362        receiver.await.map_err(|_| {
3363            IdentityRuntimeError::Internal(
3364                "tracked foreground identity operation terminated without a result".to_string(),
3365            )
3366        })?
3367    }
3368
3369    pub(crate) fn close_foreground_operations(&self) {
3370        self.foreground_shutdown.store(true, Ordering::Release);
3371        // `watch::Sender::send` discards the value when no receiver exists.
3372        // A just-admitted JoinSet task may not have subscribed yet, so retain
3373        // shutdown truth for future receivers explicitly.
3374        self.foreground_cancel.send_replace(true);
3375    }
3376
3377    pub(crate) async fn join_foreground_operations(&self) {
3378        let mut operations = self.foreground_operations.lock().await;
3379        while let Some(result) = operations.join_next().await {
3380            if let Err(error) = result {
3381                tracing::error!(
3382                    error = %error,
3383                    "tracked foreground identity operation panicked during shutdown"
3384                );
3385            }
3386        }
3387    }
3388
3389    /// Preserve exact grants acquired by a restore task that failed before it
3390    /// could publish an [`IdentityEntry`]. Entry-scoped pending-release state
3391    /// cannot represent this phase, so the runtime owns a small orphan-grant
3392    /// ledger until reconcile or shutdown successfully releases it.
3393    pub(crate) async fn park_unactivated_lease_releases(&self, grants: &[LeaseGrant]) {
3394        let mut pending = self.pending_unactivated_lease_releases.write().await;
3395        for grant in grants {
3396            if !pending.iter().any(|parked| {
3397                parked.identity == grant.identity && parked.fencing_token == grant.fencing_token
3398            }) {
3399                pending.push(grant.clone());
3400            }
3401        }
3402    }
3403
3404    /// Release grants that do not yet have an IdentityEntry capable of
3405    /// retaining retry state. A provider failure atomically transfers their
3406    /// exact fencing tokens into the runtime-owned orphan ledger.
3407    pub(crate) async fn release_or_park_untracked_leases(
3408        &self,
3409        grants: &[LeaseGrant],
3410    ) -> Result<(), super::types::LeaseError> {
3411        let _release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3412        match self.lease_provider.release_leases(grants).await {
3413            Ok(()) => Ok(()),
3414            Err(error) => {
3415                self.park_unactivated_lease_releases(grants).await;
3416                Err(error)
3417            }
3418        }
3419    }
3420
3421    /// Retry every grant parked before lifecycle publication. This runs under
3422    /// the restore controller before any new batch acquisition, and again from
3423    /// shutdown after all identity work has joined.
3424    pub(crate) async fn release_parked_unactivated_leases(
3425        &self,
3426    ) -> Result<usize, IdentityRuntimeError> {
3427        self.release_parked_unactivated_leases_matching(None).await
3428    }
3429
3430    /// Retry parked pre-publication grants for one identity before a direct
3431    /// lazy materialization attempts to acquire fresh authority. The caller
3432    /// holds that identity's lifecycle lock, so another same-identity path
3433    /// cannot park a replacement grant between this drain and acquisition.
3434    async fn release_parked_unactivated_leases_for_identity(
3435        &self,
3436        identity: &AgentIdentity,
3437    ) -> Result<usize, IdentityRuntimeError> {
3438        self.release_parked_unactivated_leases_matching(Some(identity))
3439            .await
3440    }
3441
3442    async fn release_parked_unactivated_leases_matching(
3443        &self,
3444        identity: Option<&AgentIdentity>,
3445    ) -> Result<usize, IdentityRuntimeError> {
3446        let _release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3447        let grants = self
3448            .pending_unactivated_lease_releases
3449            .read()
3450            .await
3451            .iter()
3452            .filter(|grant| identity.is_none_or(|identity| grant.identity == *identity))
3453            .cloned()
3454            .collect::<Vec<_>>();
3455        if grants.is_empty() {
3456            return Ok(0);
3457        }
3458        self.lease_provider
3459            .release_leases(&grants)
3460            .await
3461            .map_err(IdentityRuntimeError::Lease)?;
3462        let mut pending = self.pending_unactivated_lease_releases.write().await;
3463        pending.retain(|parked| {
3464            !grants.iter().any(|released| {
3465                released.identity == parked.identity
3466                    && released.fencing_token == parked.fencing_token
3467            })
3468        });
3469        Ok(grants.len())
3470    }
3471
3472    /// Release every durable identity grant after the lower mob plane has
3473    /// quiesced. Successful release clears the in-memory authority so the
3474    /// method is idempotent; failures retain the exact grants for inspection
3475    /// or a subsequent retry.
3476    pub(crate) async fn release_all_leases_for_shutdown(
3477        &self,
3478    ) -> Result<usize, IdentityRuntimeError> {
3479        let _parked_release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3480        let mut grants = self
3481            .entries
3482            .read()
3483            .await
3484            .iter()
3485            .filter_map(|(identity, entry)| {
3486                entry.pending_lease_release.clone().or_else(|| {
3487                    entry.lease.as_ref().map(|lease| LeaseGrant {
3488                        identity: identity.clone(),
3489                        fencing_token: lease.fencing_token,
3490                        ttl: lease.ttl,
3491                    })
3492                })
3493            })
3494            .collect::<Vec<_>>();
3495        let parked_grants = self.pending_unactivated_lease_releases.read().await.clone();
3496        for grant in parked_grants {
3497            if !grants.iter().any(|existing| {
3498                existing.identity == grant.identity && existing.fencing_token == grant.fencing_token
3499            }) {
3500                grants.push(grant);
3501            }
3502        }
3503        if grants.is_empty() {
3504            return Ok(0);
3505        }
3506
3507        self.lease_provider
3508            .release_leases(&grants)
3509            .await
3510            .map_err(IdentityRuntimeError::Lease)?;
3511
3512        let mut entries = self.entries.write().await;
3513        let mut lifecycle_updates = Vec::new();
3514        for grant in &grants {
3515            if let Some(entry) = entries.get_mut(&grant.identity) {
3516                let released_active_lease = entry
3517                    .lease
3518                    .as_ref()
3519                    .is_some_and(|lease| lease.fencing_token == grant.fencing_token);
3520                if released_active_lease {
3521                    entry.lease = None;
3522                }
3523                if entry
3524                    .pending_lease_release
3525                    .as_ref()
3526                    .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
3527                {
3528                    entry.pending_lease_release = None;
3529                }
3530                if released_active_lease
3531                    && matches!(
3532                        entry.state,
3533                        IdentityLifecycleState::Active | IdentityLifecycleState::Suspended
3534                    )
3535                {
3536                    entry.state = IdentityLifecycleState::Dormant;
3537                    lifecycle_updates.push(grant.identity.clone());
3538                }
3539            }
3540        }
3541        drop(entries);
3542        let mut pending = self.pending_unactivated_lease_releases.write().await;
3543        pending.retain(|parked| {
3544            !grants.iter().any(|released| {
3545                released.identity == parked.identity
3546                    && released.fencing_token == parked.fencing_token
3547            })
3548        });
3549        drop(pending);
3550        for identity in lifecycle_updates {
3551            self.mark_bootstrap_from_lifecycle(&identity, IdentityLifecycleState::Dormant, None);
3552        }
3553        Ok(grants.len())
3554    }
3555
3556    /// Materialize a dormant identity into a concrete mob member/session.
3557    ///
3558    /// This is the lazy counterpart to eager `restore_flow`: it performs the
3559    /// expensive bridge create/resume and snapshot load only when an identity is
3560    /// actually touched. Parallel calls for one identity coalesce on a
3561    /// per-identity lock and re-check state after acquiring it.
3562    pub async fn materialize(
3563        &self,
3564        identity: &AgentIdentity,
3565    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3566        self.materialize_with_expected_member_alias(identity, None)
3567            .await
3568    }
3569
3570    async fn materialize_with_expected_member_alias(
3571        &self,
3572        identity: &AgentIdentity,
3573        expected_alias: Option<&str>,
3574    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3575        self.materialize_with_expected_member_alias_after_inner(
3576            identity,
3577            expected_alias,
3578            std::future::ready(()),
3579        )
3580        .await
3581    }
3582
3583    /// Complete a foreground materialization with an internal seam after the
3584    /// lifecycle transaction commits. Production callers use an immediately
3585    /// ready future; deterministic concurrency tests pause here to exercise
3586    /// superseding roster passes without adding runtime-visible hooks.
3587    async fn materialize_with_expected_member_alias_after_inner<F>(
3588        &self,
3589        identity: &AgentIdentity,
3590        expected_alias: Option<&str>,
3591        after_inner: F,
3592    ) -> Result<ContinuityRecord, IdentityRuntimeError>
3593    where
3594        F: Future<Output = ()>,
3595    {
3596        // `materialize_inner` binds this attempt to the generation stored on
3597        // the IdentityEntry after it acquires the lifecycle lock. Sampling the
3598        // global generation here is insufficient: a newer reconcile can
3599        // publish G+1 and win that lock before this future reaches the entry,
3600        // in which case the materialized spec belongs to G+1, not G.
3601        let mut bootstrap_generation = None;
3602        let mut shutdown = self.foreground_cancel.subscribe();
3603        let result = self
3604            .materialize_inner(
3605                identity,
3606                expected_alias,
3607                Some(&mut shutdown),
3608                None,
3609                &mut bootstrap_generation,
3610            )
3611            .await;
3612        after_inner.await;
3613        if matches!(
3614            &result,
3615            Err(IdentityRuntimeError::Internal(message)) if message == BACKGROUND_WARM_CANCELLED
3616        ) {
3617            if let Some(generation) = bootstrap_generation {
3618                self.mark_bootstrap_materialization_cancelled(identity, Some(generation));
3619            }
3620            return result;
3621        }
3622        // Alias validation happens under the lifecycle lock before
3623        // `materialize_inner` marks bootstrap work as started. A stale alias
3624        // therefore must leave the replacement generation's exact readiness
3625        // state untouched.
3626        if matches!(&result, Err(IdentityRuntimeError::StaleRuntimeAlias { .. })) {
3627            return result;
3628        }
3629        if result.is_ok() {
3630            let desired_edges = self.desired_peer_edges.read().await.clone();
3631            let has_pending_topology_recovery = match self.topology_controller() {
3632                Some(controller) => controller.has_pending().await,
3633                None => false,
3634            };
3635            if (!desired_edges.is_empty() || has_pending_topology_recovery)
3636                && let Err(error) = self.reconcile_managed_peer_edges(&desired_edges).await
3637            {
3638                tracing::warn!(
3639                    identity = %identity,
3640                    %error,
3641                    "identity materialized with topology reconcile warning"
3642                );
3643            }
3644        }
3645        if let Some(generation) = bootstrap_generation {
3646            self.mark_bootstrap_materialization_finished(identity, &result, Some(generation));
3647        }
3648        result
3649    }
3650
3651    /// Cancellation-safe materialization for request/host boundaries.
3652    pub async fn materialize_tracked(
3653        self: &Arc<Self>,
3654        identity: &AgentIdentity,
3655    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3656        let runtime = Arc::clone(self);
3657        let identity = identity.clone();
3658        self.run_tracked_foreground(async move { runtime.materialize(&identity).await })
3659            .await
3660    }
3661
3662    /// Cancellation-safe compatibility restore used by embedders that pass
3663    /// an RPC identity context without attaching an
3664    /// [`IdentityFirstRuntimeContext`] to the unified runtime.
3665    pub(crate) async fn restore_flow_tracked(
3666        self: &Arc<Self>,
3667        roster: Vec<DurableAgentSpec>,
3668        topology_provider: Option<Arc<dyn TopologyProvider>>,
3669        customizer: Option<Arc<dyn AgentCustomizer>>,
3670    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
3671        let runtime = Arc::clone(self);
3672        self.run_tracked_foreground(async move {
3673            super::orchestrator::restore_flow(
3674                runtime.as_ref(),
3675                &roster,
3676                topology_provider.as_deref(),
3677                customizer.as_deref(),
3678            )
3679            .await
3680        })
3681        .await
3682    }
3683
3684    /// Cancellation-safe retirement for RPC/host request boundaries.
3685    pub async fn retire_tracked(
3686        self: &Arc<Self>,
3687        identity: &AgentIdentity,
3688    ) -> Result<FencingToken, IdentityRuntimeError> {
3689        let runtime = Arc::clone(self);
3690        let identity = identity.clone();
3691        self.run_tracked_foreground(async move { runtime.retire(&identity).await })
3692            .await
3693    }
3694
3695    /// Cancellation-safe retirement that atomically rejects an old generated
3696    /// runtime alias under the same lifecycle lock as the mutation.
3697    pub async fn retire_member_alias_tracked(
3698        self: &Arc<Self>,
3699        identity: &AgentIdentity,
3700        expected_alias: &str,
3701    ) -> Result<FencingToken, IdentityRuntimeError> {
3702        let runtime = Arc::clone(self);
3703        let identity = identity.clone();
3704        let expected_alias = expected_alias.to_string();
3705        self.run_tracked_foreground(async move {
3706            runtime
3707                .retire_with_expected_member_alias(&identity, Some(&expected_alias))
3708                .await
3709        })
3710        .await
3711    }
3712
3713    /// Retire identity authority and its captured lower-plane generations
3714    /// under one lifecycle lock. Enumeration performed by `cleanup` therefore
3715    /// observes the generation actually retired, even when this request had
3716    /// waited behind a concurrent reset.
3717    pub(crate) async fn retire_and_cleanup_live_members_tracked<T, F, Fut>(
3718        self: &Arc<Self>,
3719        identity: &AgentIdentity,
3720        expected_alias: Option<&str>,
3721        cleanup: F,
3722    ) -> Result<(FencingToken, T), IdentityRuntimeError>
3723    where
3724        T: Send + 'static,
3725        F: FnOnce(Option<AgentRuntimeId>) -> Fut + Send + 'static,
3726        Fut: Future<Output = T> + Send + 'static,
3727    {
3728        let runtime = Arc::clone(self);
3729        let identity = identity.clone();
3730        let expected_alias = expected_alias.map(str::to_owned);
3731        self.run_tracked_foreground(async move {
3732            let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3733            let _lifecycle_guard = lifecycle_lock.lock().await;
3734            if let Some(expected_alias) = expected_alias.as_deref() {
3735                runtime
3736                    .ensure_expected_member_alias_current(&identity, expected_alias)
3737                    .await?;
3738            }
3739            let retired_alias = runtime
3740                .entries
3741                .read()
3742                .await
3743                .get(&identity)
3744                .and_then(|entry| entry.continuity.as_ref())
3745                .map(|record| record.agent_runtime_id.clone());
3746            let token = runtime.retire_locked(&identity).await?;
3747            let metadata = cleanup(retired_alias).await;
3748            Ok((token, metadata))
3749        })
3750        .await
3751    }
3752
3753    /// Cancellation-safe respawn for RPC/host request boundaries.
3754    pub async fn respawn_tracked(
3755        self: &Arc<Self>,
3756        identity: &AgentIdentity,
3757    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3758        let runtime = Arc::clone(self);
3759        let identity = identity.clone();
3760        self.run_tracked_foreground(async move { runtime.respawn(&identity).await })
3761            .await
3762    }
3763
3764    /// Cancellation-safe respawn that atomically rejects an old generated
3765    /// runtime alias under the same lifecycle lock as the mutation.
3766    pub async fn respawn_member_alias_tracked(
3767        self: &Arc<Self>,
3768        identity: &AgentIdentity,
3769        expected_alias: &str,
3770    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3771        let runtime = Arc::clone(self);
3772        let identity = identity.clone();
3773        let expected_alias = expected_alias.to_string();
3774        self.run_tracked_foreground(async move {
3775            runtime
3776                .respawn_with_expected_member_alias(&identity, Some(&expected_alias))
3777                .await
3778        })
3779        .await
3780    }
3781
3782    /// Execute a lower member-plane mutation while pinning a generated alias
3783    /// to the durable identity generation that owns it. This is the common
3784    /// authority boundary for legacy member RPCs (for example force-cancel)
3785    /// that cannot otherwise express identity continuity.
3786    pub(crate) async fn run_member_alias_operation_tracked<T, F, Fut>(
3787        self: &Arc<Self>,
3788        identity: &AgentIdentity,
3789        expected_alias: &str,
3790        operation: F,
3791    ) -> Result<T, IdentityRuntimeError>
3792    where
3793        T: Send + 'static,
3794        F: FnOnce() -> Fut + Send + 'static,
3795        Fut: Future<Output = Result<T, String>> + Send + 'static,
3796    {
3797        let runtime = Arc::clone(self);
3798        let identity = identity.clone();
3799        let expected_alias = expected_alias.to_string();
3800        self.run_tracked_foreground(async move {
3801            let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3802            let _lifecycle_guard = lifecycle_lock.lock().await;
3803            runtime
3804                .ensure_expected_member_alias_current(&identity, &expected_alias)
3805                .await?;
3806            operation().await.map_err(IdentityRuntimeError::Internal)
3807        })
3808        .await
3809    }
3810
3811    /// Cancellation-safe identity-first respawn for RPC and console boundaries.
3812    ///
3813    /// A durable identity already owns an authoritative Meerkat session. Its
3814    /// respawn operation therefore fences the old owner, refreshes the persisted
3815    /// runtime state, and reactivates that exact continuity record in place. It
3816    /// must not call the raw member-plane respawn convenience: that operation
3817    /// creates a new session and would silently turn a non-destructive identity
3818    /// recovery into a destructive reset.
3819    pub(crate) async fn respawn_identity_in_place_tracked(
3820        self: &Arc<Self>,
3821        identity: &AgentIdentity,
3822        expected_alias: Option<&str>,
3823    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3824        match expected_alias {
3825            Some(expected_alias) => {
3826                self.respawn_member_alias_tracked(identity, expected_alias)
3827                    .await
3828            }
3829            None => self.respawn_tracked(identity).await,
3830        }
3831    }
3832
3833    /// Cancellation-safe live-session rebind for RPC/host boundaries.
3834    pub async fn rebind_session_after_live_respawn_tracked(
3835        self: &Arc<Self>,
3836        identity: &AgentIdentity,
3837        session_id: SessionId,
3838    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3839        let runtime = Arc::clone(self);
3840        let identity = identity.clone();
3841        self.run_tracked_foreground(async move {
3842            runtime
3843                .rebind_session_after_live_respawn(&identity, session_id)
3844                .await
3845        })
3846        .await
3847    }
3848
3849    /// Cancellation-safe live-session rebind pinned to the generation that
3850    /// initiated the lower-level member respawn.
3851    pub async fn rebind_session_after_live_respawn_member_alias_tracked(
3852        self: &Arc<Self>,
3853        identity: &AgentIdentity,
3854        expected_alias: &str,
3855        session_id: SessionId,
3856    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3857        let runtime = Arc::clone(self);
3858        let identity = identity.clone();
3859        let expected_alias = expected_alias.to_string();
3860        self.run_tracked_foreground(async move {
3861            runtime
3862                .rebind_session_after_live_respawn_with_expected_member_alias(
3863                    &identity,
3864                    Some(&expected_alias),
3865                    session_id,
3866                )
3867                .await
3868        })
3869        .await
3870    }
3871
3872    /// Cancellation-safe destructive reset for RPC/host boundaries.
3873    pub async fn reset_tracked(
3874        self: &Arc<Self>,
3875        identity: &AgentIdentity,
3876    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3877        let runtime = Arc::clone(self);
3878        let identity = identity.clone();
3879        self.run_tracked_foreground(async move { runtime.reset(&identity).await })
3880            .await
3881    }
3882
3883    /// Cancellation-safe reset that atomically rejects an old generated
3884    /// runtime alias under the same lifecycle lock as the mutation.
3885    pub async fn reset_member_alias_tracked(
3886        self: &Arc<Self>,
3887        identity: &AgentIdentity,
3888        expected_alias: &str,
3889    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3890        let runtime = Arc::clone(self);
3891        let identity = identity.clone();
3892        let expected_alias = expected_alias.to_string();
3893        self.run_tracked_foreground(async move {
3894            runtime
3895                .reset_with_expected_member_alias(&identity, Some(&expected_alias))
3896                .await
3897        })
3898        .await
3899    }
3900
3901    /// Cancellation-safe identity deletion for RPC/host boundaries.
3902    pub async fn delete_identity_tracked(
3903        self: &Arc<Self>,
3904        identity: &AgentIdentity,
3905    ) -> Result<(), IdentityRuntimeError> {
3906        let runtime = Arc::clone(self);
3907        let identity = identity.clone();
3908        self.run_tracked_foreground(async move { runtime.delete_identity(&identity).await })
3909            .await
3910    }
3911
3912    /// Cancellation-safe deletion that atomically rejects an old generated
3913    /// runtime alias under the same lifecycle lock as the mutation.
3914    pub async fn delete_identity_member_alias_tracked(
3915        self: &Arc<Self>,
3916        identity: &AgentIdentity,
3917        expected_alias: &str,
3918    ) -> Result<(), IdentityRuntimeError> {
3919        let runtime = Arc::clone(self);
3920        let identity = identity.clone();
3921        let expected_alias = expected_alias.to_string();
3922        self.run_tracked_foreground(async move {
3923            runtime
3924                .delete_identity_with_expected_member_alias(&identity, Some(&expected_alias))
3925                .await
3926        })
3927        .await
3928    }
3929
3930    /// Delete identity authority and clean its captured concrete generations
3931    /// in the same lifecycle transaction.
3932    pub(crate) async fn delete_identity_and_cleanup_live_members_tracked<T, F, Fut>(
3933        self: &Arc<Self>,
3934        identity: &AgentIdentity,
3935        expected_alias: Option<&str>,
3936        cleanup: F,
3937    ) -> Result<T, IdentityRuntimeError>
3938    where
3939        T: Send + 'static,
3940        F: FnOnce(Option<AgentRuntimeId>) -> Fut + Send + 'static,
3941        Fut: Future<Output = T> + Send + 'static,
3942    {
3943        let runtime = Arc::clone(self);
3944        let identity = identity.clone();
3945        let expected_alias = expected_alias.map(str::to_owned);
3946        self.run_tracked_foreground(async move {
3947            let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3948            let _lifecycle_guard = lifecycle_lock.lock().await;
3949            if let Some(expected_alias) = expected_alias.as_deref() {
3950                runtime
3951                    .ensure_expected_member_alias_current(&identity, expected_alias)
3952                    .await?;
3953            }
3954            let deleted_alias = runtime
3955                .entries
3956                .read()
3957                .await
3958                .get(&identity)
3959                .and_then(|entry| entry.continuity.as_ref())
3960                .map(|record| record.agent_runtime_id.clone());
3961            runtime.delete_identity_locked(&identity).await?;
3962            Ok(cleanup(deleted_alias).await)
3963        })
3964        .await
3965    }
3966
3967    /// Background warming observes controller cancellation after lease
3968    /// acquisition bookkeeping but before bridge/member installation. A
3969    /// cancelled grant is explicitly released; after customization completes,
3970    /// the operation is joined to its explicit commit/rollback boundary.
3971    async fn materialize_for_background(
3972        &self,
3973        identity: &AgentIdentity,
3974        cancellation: &mut watch::Receiver<bool>,
3975        generation: u64,
3976    ) -> Option<Result<ContinuityRecord, IdentityRuntimeError>> {
3977        let mut bound_generation = None;
3978        let result = self
3979            .materialize_inner(
3980                identity,
3981                None,
3982                Some(cancellation),
3983                Some(generation),
3984                &mut bound_generation,
3985            )
3986            .await;
3987        if matches!(
3988            &result,
3989            Err(IdentityRuntimeError::Internal(message)) if message == BACKGROUND_WARM_CANCELLED
3990        ) {
3991            self.mark_bootstrap_materialization_cancelled(identity, Some(generation));
3992            return None;
3993        }
3994        if result.is_ok() {
3995            let desired_edges = self.desired_peer_edges.read().await.clone();
3996            if !desired_edges.is_empty()
3997                && let Err(error) = self.reconcile_managed_peer_edges(&desired_edges).await
3998            {
3999                tracing::warn!(
4000                    identity = %identity,
4001                    %error,
4002                    "background identity materialized with topology reconcile warning"
4003                );
4004            }
4005        }
4006        self.mark_bootstrap_materialization_finished(identity, &result, Some(generation));
4007        Some(result)
4008    }
4009
4010    async fn materialize_inner(
4011        &self,
4012        identity: &AgentIdentity,
4013        expected_alias: Option<&str>,
4014        mut cancellation: Option<&mut watch::Receiver<bool>>,
4015        expected_bootstrap_generation: Option<u64>,
4016        bound_bootstrap_generation: &mut Option<u64>,
4017    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
4018        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
4019        let _lifecycle_guard = lifecycle_lock.lock().await;
4020        let bootstrap_generation = {
4021            let entries = self.entries.read().await;
4022            entries
4023                .get(identity)
4024                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?
4025                .bootstrap_generation
4026        };
4027        *bound_bootstrap_generation = Some(bootstrap_generation);
4028        // A background item belongs to one controller generation. If a newer
4029        // roster pass converged the entry before this item acquired lifecycle
4030        // authority, treat the old work as cancelled instead of materializing
4031        // the newer spec on behalf of a superseded pass.
4032        if expected_bootstrap_generation.is_some_and(|expected| expected != bootstrap_generation) {
4033            return Err(IdentityRuntimeError::Internal(
4034                BACKGROUND_WARM_CANCELLED.to_string(),
4035            ));
4036        }
4037        let raw_alias_lock = self.raw_member_alias_lock(identity.as_str()).await;
4038        let _raw_alias_guard = raw_alias_lock.lock().await;
4039        self.ensure_raw_member_alias_available(identity).await?;
4040        if let Some(expected_alias) = expected_alias {
4041            self.ensure_expected_member_alias_current(identity, expected_alias)
4042                .await?;
4043        }
4044        let lock = self.materialization_lock_for(identity).await;
4045        let _guard = lock.lock().await;
4046
4047        let (spec, continuity, state) = {
4048            let entries = self.entries.read().await;
4049            let entry = entries
4050                .get(identity)
4051                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4052            if entry.state == IdentityLifecycleState::Active {
4053                let continuity = entry.continuity.clone().ok_or_else(|| {
4054                    IdentityRuntimeError::Internal(format!(
4055                        "active identity {identity} has no continuity record"
4056                    ))
4057                })?;
4058                drop(entries);
4059                self.clear_materialization_backoff(identity).await;
4060                return Ok(continuity);
4061            }
4062            (entry.spec.clone(), entry.continuity.clone(), entry.state)
4063        };
4064        // Host-rejected-build park: the app-side gate answered this exact
4065        // spec with a deterministic rejection. Fail fast typed — every
4066        // attempt would otherwise burn a full member build plus a callback
4067        // round trip for the same answer. A spec change clears the park
4068        // (checked inside the accessor).
4069        if let Some(park) = self.host_rejected_build_park(identity).await {
4070            return Err(IdentityRuntimeError::HostRejectedBuild {
4071                identity: identity.clone(),
4072                reason: park.reason,
4073            });
4074        }
4075        let original_continuity = continuity.clone();
4076        let continuity = if durable_spec_uses_external_binding(&spec) {
4077            None
4078        } else {
4079            continuity
4080        };
4081
4082        match state {
4083            IdentityLifecycleState::Dormant | IdentityLifecycleState::Uninitialized => {}
4084            IdentityLifecycleState::Broken
4085            | IdentityLifecycleState::Retiring
4086            | IdentityLifecycleState::Suspended => {
4087                return Err(IdentityRuntimeError::InvalidState {
4088                    identity: identity.clone(),
4089                    state,
4090                    operation: "materialize",
4091                });
4092            }
4093            IdentityLifecycleState::Active => unreachable!("active handled above"),
4094        }
4095        // Begin readiness bookkeeping only after ownership validation and
4096        // only for an identity that will actually materialize. An old alias
4097        // can now fail without a transient or terminal mutation of the
4098        // replacement generation's bootstrap entry.
4099        self.mark_bootstrap_materialization_started(identity, Some(bootstrap_generation));
4100
4101        // A previous customization/installation failure can leave its exact
4102        // unactivated grant parked when provider cleanup itself fails. Strict
4103        // providers reject reacquisition while that grant still exists, so a
4104        // direct lazy retry must drain this identity's orphan before acquiring
4105        // fresh authority. Other identities' cleanup failures remain isolated.
4106        self.release_parked_unactivated_leases_for_identity(identity)
4107            .await?;
4108
4109        // Establish single-embodiment ownership before invoking arbitrary
4110        // host customizer code, preserving the public ordering contract. No
4111        // bridge/member state exists yet, so every exit below can still
4112        // release this uninstalled grant explicitly.
4113        let lease_results = self
4114            .lease_provider
4115            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
4116            .await
4117            .map_err(IdentityRuntimeError::Lease)?;
4118        let grant = match lease_results.get(identity) {
4119            Some(super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
4120            Some(super::types::LeaseAcquireResult::AlreadyHeld { holder, .. }) => {
4121                tracing::error!(
4122                    %identity,
4123                    holder = %holder,
4124                    "single-embodiment guard: refusing to materialize an identity whose \
4125                     durable lease is held by another live runtime instance"
4126                );
4127                return Err(IdentityRuntimeError::AlreadyEmbodied {
4128                    identity: identity.clone(),
4129                    holder: holder.clone(),
4130                });
4131            }
4132            None => return Err(IdentityRuntimeError::NoActiveLease(identity.clone())),
4133        };
4134        if let Some(record) = continuity.as_ref()
4135            && let Err(err) = self
4136                .continuity_store
4137                .upsert_continuity_record(record, grant.fencing_token)
4138                .await
4139        {
4140            let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4141            return Err(IdentityRuntimeError::Internal(format!(
4142                "continuity upsert before materialize: {err}{}",
4143                cleanup_error
4144                    .as_ref()
4145                    .map(|e| format!("; lease cleanup failed: {e}"))
4146                    .unwrap_or_default(),
4147            )));
4148        }
4149
4150        if cancellation
4151            .as_ref()
4152            .is_some_and(|cancellation| *cancellation.borrow())
4153        {
4154            return Err(self
4155                .cancel_uninstalled_background_materialization(&grant)
4156                .await);
4157        }
4158
4159        let active_peers = self.entries.read().await.keys().cloned().collect();
4160        let managed_edges = self.desired_peer_edges.read().await.clone();
4161        let build_context = AgentBuildContext {
4162            identity: identity.clone(),
4163            active_peers,
4164            managed_edges,
4165            runtime_services: self.runtime_services(),
4166        };
4167        let mut draft = super::types::AgentBuildDraft {
4168            model: None,
4169            system_prompt: None,
4170            additional_instructions: spec.additional_instructions.clone(),
4171            labels: spec.labels.clone(),
4172            app_context: spec.context.clone(),
4173            external_tools: Vec::new(),
4174            local_external_tools: Default::default(),
4175            provider_params: None,
4176        };
4177        if let Some(customizer) = self.customizer.read().await.clone() {
4178            let customize = customizer.customize_build(&build_context, &spec, &mut draft);
4179            tokio::pin!(customize);
4180            let customize_result = if let Some(cancellation) = cancellation.as_mut() {
4181                let cancellation = &mut **cancellation;
4182                tokio::select! {
4183                    result = &mut customize => result,
4184                    changed = cancellation.changed() => {
4185                        if changed.is_ok() && *cancellation.borrow() {
4186                            return Err(
4187                                self.cancel_uninstalled_background_materialization(&grant)
4188                                    .await,
4189                            );
4190                        }
4191                        customize.await
4192                    }
4193                }
4194            } else {
4195                customize.await
4196            };
4197            if let Err(err) = customize_result {
4198                let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4199                return Err(IdentityRuntimeError::Internal(format!(
4200                    "customizer: {err}{}",
4201                    cleanup_error
4202                        .as_ref()
4203                        .map(|e| format!("; lease cleanup failed: {e}"))
4204                        .unwrap_or_default(),
4205                )));
4206            }
4207        }
4208
4209        let mut abandoned_session_registrations: Vec<SessionId> = Vec::new();
4210        let mut record = if let Some(mut record) = continuity {
4211            let snapshot = if self
4212                .bridge
4213                .as_ref()
4214                .is_none_or(|bridge| bridge.requires_resume_snapshot())
4215            {
4216                match self
4217                    .continuity_store
4218                    .load_session_snapshot(&record.session_id)
4219                    .await
4220                {
4221                    Ok(snapshot) => snapshot,
4222                    Err(err) => {
4223                        let cleanup_error =
4224                            self.release_uninstalled_materialize_lease(&grant).await;
4225                        return Err(IdentityRuntimeError::Internal(format!(
4226                            "load session snapshot before materialize: {err}{}",
4227                            cleanup_error
4228                                .as_ref()
4229                                .map(|e| format!("; lease cleanup failed: {e}"))
4230                                .unwrap_or_default(),
4231                        )));
4232                    }
4233                }
4234            } else {
4235                None
4236            };
4237
4238            if let Some(bridge) = self.bridge.as_ref() {
4239                if let Err(err) = bridge
4240                    .register_session_runtime_state(
4241                        &record.session_id,
4242                        identity,
4243                        record.generation,
4244                        record.checkpoint_version,
4245                        grant.fencing_token,
4246                    )
4247                    .await
4248                {
4249                    let unregister_error = Self::unregister_bridge_session_runtime_states(
4250                        bridge.as_ref(),
4251                        std::slice::from_ref(&record.session_id),
4252                    )
4253                    .await;
4254                    let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4255                    return Err(IdentityRuntimeError::Internal(format!(
4256                        "bridge register_session_runtime_state: {err}{}{}",
4257                        unregister_error
4258                            .as_ref()
4259                            .map(|e| format!("; unregister session failed: {e}"))
4260                            .unwrap_or_default(),
4261                        cleanup_error
4262                            .as_ref()
4263                            .map(|e| format!("; lease cleanup failed: {e}"))
4264                            .unwrap_or_default(),
4265                    )));
4266                }
4267                let registered_session_id = record.session_id.clone();
4268                let snapshot = snapshot.unwrap_or(SessionSnapshot { data: Vec::new() });
4269                let outcome = bridge
4270                    .resume_session(
4271                        identity,
4272                        &record.agent_runtime_id,
4273                        &spec,
4274                        &draft,
4275                        &record.session_id,
4276                        &snapshot,
4277                    )
4278                    .await;
4279                let outcome = match outcome {
4280                    Ok(outcome) => outcome,
4281                    Err(err) => {
4282                        // A rejected resume must NEVER abandon the durable
4283                        // session (the transcript is the only copy). Keep the
4284                        // identity → session binding intact — no retire, no
4285                        // continuity rebind — mark the identity Broken with
4286                        // the error attached so this send fails loudly, and
4287                        // let the next reconcile retry the resume. Unregister
4288                        // only the session-runtime-state bookkeeping (it is
4289                        // re-registered on retry).
4290                        tracing::error!(
4291                            %identity,
4292                            session_id = %registered_session_id,
4293                            error = %err,
4294                            "materialize resume rejected; marking identity Broken and preserving \
4295                             the durable session for reconcile retry"
4296                        );
4297                        let unregister_error = Self::unregister_bridge_session_runtime_states(
4298                            bridge.as_ref(),
4299                            std::slice::from_ref(&registered_session_id),
4300                        )
4301                        .await;
4302                        let lease_cleanup_error =
4303                            self.release_uninstalled_materialize_lease(&grant).await;
4304                        {
4305                            let mut entries = self.entries.write().await;
4306                            if let Some(entry) = entries.get_mut(identity) {
4307                                entry.state = IdentityLifecycleState::Broken;
4308                                entry.lease = None;
4309                            }
4310                        }
4311                        self.emit_event(
4312                            identity,
4313                            IdentityEvent::StateChanged {
4314                                identity: identity.clone(),
4315                                new_state: IdentityLifecycleState::Broken,
4316                            },
4317                        )
4318                        .await;
4319                        {
4320                            let rejection = err.to_string();
4321                            if is_host_rejected_build_error(&rejection) {
4322                                self.mark_host_rejected_build_park(identity, rejection)
4323                                    .await;
4324                            }
4325                        }
4326                        // Repair honesty (OB3 rehearsal): the typed
4327                        // ArchivedNotRevivable refusal is a stable,
4328                        // deterministic wall - record the terminal verdict on
4329                        // this FIRST refusal so the repair supervisor parks
4330                        // instead of heal-looping (the roster heal succeeds,
4331                        // this materialize precondition never does).
4332                        if let BridgeError::ResumeRejected {
4333                            kind: ResumeRejectionKind::ArchivedNotRevivable,
4334                            detail,
4335                        } = &err
4336                            && !self
4337                                .mark_continuity_unrecoverable(
4338                                    identity,
4339                                    archived_not_revivable_park_reason(
4340                                        &registered_session_id,
4341                                        detail,
4342                                    ),
4343                                )
4344                                .await
4345                        {
4346                            tracing::debug!(
4347                                %identity,
4348                                "identity left Broken before the archived-not-revivable \
4349                                 park could be recorded"
4350                            );
4351                        }
4352                        let detail = format!(
4353                            "bridge resume_session rejected (identity degraded, durable session \
4354                             preserved for reconcile retry): {err}{}{}",
4355                            unregister_error
4356                                .as_ref()
4357                                .map(|e| format!("; unregister session failed: {e}"))
4358                                .unwrap_or_default(),
4359                            lease_cleanup_error
4360                                .as_ref()
4361                                .map(|e| format!("; lease cleanup failed: {e}"))
4362                                .unwrap_or_default(),
4363                        );
4364                        return Err(IdentityRuntimeError::Internal(detail));
4365                    }
4366                };
4367                if let Some(reason) = outcome.fallback_reason().cloned() {
4368                    tracing::warn!(
4369                        %identity,
4370                        reason = ?reason,
4371                        "lazy identity materialization fresh-spawned after typed resume fallback"
4372                    );
4373                    self.emit_event(
4374                        identity,
4375                        IdentityEvent::ResumeFallback {
4376                            identity: identity.clone(),
4377                            reason,
4378                        },
4379                    )
4380                    .await;
4381                }
4382                let effective_session_id = outcome.session_id().clone();
4383                if effective_session_id != registered_session_id {
4384                    // §8.4 trigger (b): the resume fallback abandoned the
4385                    // registered session — harvest it detached (materialize
4386                    // is a hot path; the session store read stays valid).
4387                    if let Some(injector) = self.agent_memory.read().await.as_ref() {
4388                        let abandoned_key = registered_session_id.to_string();
4389                        injector.note_session_generation(
4390                            identity,
4391                            &abandoned_key,
4392                            record.generation.get(),
4393                        );
4394                        injector.spawn_rotation_distillation(
4395                            identity,
4396                            &abandoned_key,
4397                            crate::memory::distiller::DistillCause::ResumeFallback,
4398                        );
4399                    }
4400                    abandoned_session_registrations.push(registered_session_id);
4401                }
4402                record.session_id = effective_session_id;
4403            }
4404            record
4405        } else {
4406            let new_runtime_id =
4407                AgentRuntimeId::parse(&format!("rt:{identity}:0")).map_err(|err| {
4408                    IdentityRuntimeError::Internal(format!("failed to mint runtime id: {err}"))
4409                })?;
4410            let mut record = ContinuityRecord {
4411                identity: identity.clone(),
4412                agent_runtime_id: new_runtime_id,
4413                session_id: meerkat_core::types::SessionId::new(),
4414                generation: ContinuityGeneration::new(0),
4415                checkpoint_version: CheckpointVersion::new(0),
4416            };
4417            if let Err(err) = self
4418                .continuity_store
4419                .upsert_continuity_record(&record, grant.fencing_token)
4420                .await
4421            {
4422                let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4423                return Err(IdentityRuntimeError::Internal(format!(
4424                    "continuity upsert before materialize create: {err}{}",
4425                    cleanup_error
4426                        .as_ref()
4427                        .map(|e| format!("; lease cleanup failed: {e}"))
4428                        .unwrap_or_default(),
4429                )));
4430            }
4431            if let Some(bridge) = self.bridge.as_ref() {
4432                let provisional_session_id = record.session_id.clone();
4433                if let Err(err) = bridge
4434                    .register_session_runtime_state(
4435                        &record.session_id,
4436                        identity,
4437                        record.generation,
4438                        record.checkpoint_version,
4439                        grant.fencing_token,
4440                    )
4441                    .await
4442                    .map_err(|err| {
4443                        IdentityRuntimeError::Internal(format!(
4444                            "bridge register_session_runtime_state: {err}"
4445                        ))
4446                    })
4447                {
4448                    let unregister_error = Self::unregister_bridge_session_runtime_states(
4449                        bridge.as_ref(),
4450                        std::slice::from_ref(&provisional_session_id),
4451                    )
4452                    .await;
4453                    let delete_error = self
4454                        .continuity_store
4455                        .delete_continuity_record(identity, grant.fencing_token)
4456                        .await
4457                        .err();
4458                    let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4459                    if let Some(delete_error) = delete_error {
4460                        return Err(IdentityRuntimeError::Internal(format!(
4461                            "{err}{}; tentative continuity cleanup failed: {delete_error}{}",
4462                            unregister_error
4463                                .as_ref()
4464                                .map(|e| format!("; unregister session failed: {e}"))
4465                                .unwrap_or_default(),
4466                            cleanup_error
4467                                .as_ref()
4468                                .map(|e| format!("; lease cleanup failed: {e}"))
4469                                .unwrap_or_default(),
4470                        )));
4471                    }
4472                    if let Some(cleanup_error) = cleanup_error {
4473                        return Err(IdentityRuntimeError::Internal(format!(
4474                            "{err}{}; lease cleanup failed: {cleanup_error}",
4475                            unregister_error
4476                                .as_ref()
4477                                .map(|e| format!("; unregister session failed: {e}"))
4478                                .unwrap_or_default(),
4479                        )));
4480                    }
4481                    if let Some(unregister_error) = unregister_error {
4482                        return Err(IdentityRuntimeError::Internal(format!(
4483                            "{err}; unregister session failed: {unregister_error}"
4484                        )));
4485                    }
4486                    return Err(err);
4487                }
4488                let created_session_id = bridge
4489                    .create_session(
4490                        identity,
4491                        &record.agent_runtime_id,
4492                        &spec,
4493                        &draft,
4494                        &record.session_id,
4495                    )
4496                    .await
4497                    .map_err(|err| {
4498                        IdentityRuntimeError::Internal(format!("bridge create_session: {err}"))
4499                    });
4500                if let Err(err) = &created_session_id {
4501                    let detail = err.to_string();
4502                    if is_host_rejected_build_error(&detail) {
4503                        self.mark_host_rejected_build_park(identity, detail).await;
4504                    }
4505                }
4506                match created_session_id {
4507                    Ok(session_id) => {
4508                        if session_id != provisional_session_id {
4509                            abandoned_session_registrations.push(provisional_session_id);
4510                        }
4511                        record.session_id = session_id;
4512                    }
4513                    Err(err) => {
4514                        let unregister_error = Self::unregister_bridge_session_runtime_states(
4515                            bridge.as_ref(),
4516                            std::slice::from_ref(&provisional_session_id),
4517                        )
4518                        .await;
4519                        let cleanup_error =
4520                            bridge.retire_member(&record.agent_runtime_id).await.err();
4521                        let delete_error = self
4522                            .continuity_store
4523                            .delete_continuity_record(identity, grant.fencing_token)
4524                            .await
4525                            .err();
4526                        let lease_cleanup_error =
4527                            self.release_uninstalled_materialize_lease(&grant).await;
4528                        if unregister_error.is_some()
4529                            || cleanup_error.is_some()
4530                            || delete_error.is_some()
4531                            || lease_cleanup_error.is_some()
4532                        {
4533                            return Err(IdentityRuntimeError::Internal(format!(
4534                                "{err}{}{}{}{}",
4535                                unregister_error
4536                                    .as_ref()
4537                                    .map(|e| format!("; unregister session failed: {e}"))
4538                                    .unwrap_or_default(),
4539                                cleanup_error
4540                                    .as_ref()
4541                                    .map(|e| format!("; cleanup retire failed: {e}"))
4542                                    .unwrap_or_default(),
4543                                delete_error
4544                                    .as_ref()
4545                                    .map(|e| format!("; tentative continuity cleanup failed: {e}"))
4546                                    .unwrap_or_default(),
4547                                lease_cleanup_error
4548                                    .as_ref()
4549                                    .map(|e| format!("; lease cleanup failed: {e}"))
4550                                    .unwrap_or_default(),
4551                            )));
4552                        }
4553                        return Err(err);
4554                    }
4555                }
4556            }
4557            record
4558        };
4559
4560        if let Err(err) = self
4561            .continuity_store
4562            .upsert_continuity_record(&record, grant.fencing_token)
4563            .await
4564        {
4565            let unregister_error = if let Some(bridge) = self.bridge.as_ref() {
4566                let mut sessions_to_unregister = abandoned_session_registrations.clone();
4567                sessions_to_unregister.push(record.session_id.clone());
4568                Self::unregister_bridge_session_runtime_states(
4569                    bridge.as_ref(),
4570                    &sessions_to_unregister,
4571                )
4572                .await
4573            } else {
4574                None
4575            };
4576            let cleanup_error = if let Some(bridge) = self.bridge.as_ref() {
4577                bridge.retire_member(&record.agent_runtime_id).await.err()
4578            } else {
4579                None
4580            };
4581            let restore_error = self
4582                .restore_continuity_after_materialize_failure(
4583                    identity,
4584                    original_continuity.as_ref(),
4585                    &grant,
4586                )
4587                .await;
4588            let lease_cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4589            if unregister_error.is_some()
4590                || cleanup_error.is_some()
4591                || restore_error.is_some()
4592                || lease_cleanup_error.is_some()
4593            {
4594                return Err(IdentityRuntimeError::Internal(format!(
4595                    "continuity upsert after materialize: {err}{}{}{}{}",
4596                    unregister_error
4597                        .as_ref()
4598                        .map(|e| format!("; unregister session failed: {e}"))
4599                        .unwrap_or_default(),
4600                    cleanup_error
4601                        .as_ref()
4602                        .map(|e| format!("; cleanup retire failed: {e}"))
4603                        .unwrap_or_default(),
4604                    restore_error
4605                        .as_ref()
4606                        .map(|e| format!("; continuity rollback failed: {e}"))
4607                        .unwrap_or_default(),
4608                    lease_cleanup_error
4609                        .as_ref()
4610                        .map(|e| format!("; lease cleanup failed: {e}"))
4611                        .unwrap_or_default(),
4612                )));
4613            }
4614            return Err(IdentityRuntimeError::Internal(format!(
4615                "continuity upsert after materialize: {err}"
4616            )));
4617        }
4618        if let Some(bridge) = self.bridge.as_ref() {
4619            let register_result = bridge
4620                .register_session_runtime_state(
4621                    &record.session_id,
4622                    identity,
4623                    record.generation,
4624                    record.checkpoint_version,
4625                    grant.fencing_token,
4626                )
4627                .await;
4628            let effective_checkpoint_version = match register_result {
4629                Ok(version) => version,
4630                Err(err) => {
4631                    let mut sessions_to_unregister = abandoned_session_registrations.clone();
4632                    sessions_to_unregister.push(record.session_id.clone());
4633                    let unregister_error = Self::unregister_bridge_session_runtime_states(
4634                        bridge.as_ref(),
4635                        &sessions_to_unregister,
4636                    )
4637                    .await;
4638                    let cleanup_error = bridge.retire_member(&record.agent_runtime_id).await.err();
4639                    let restore_error = self
4640                        .restore_continuity_after_materialize_failure(
4641                            identity,
4642                            original_continuity.as_ref(),
4643                            &grant,
4644                        )
4645                        .await;
4646                    let lease_cleanup_error =
4647                        self.release_uninstalled_materialize_lease(&grant).await;
4648                    return Err(IdentityRuntimeError::Internal(format!(
4649                        "bridge register actual session runtime state: {err}{}{}{}{}",
4650                        unregister_error
4651                            .as_ref()
4652                            .map(|e| format!("; unregister session failed: {e}"))
4653                            .unwrap_or_default(),
4654                        cleanup_error
4655                            .as_ref()
4656                            .map(|e| format!("; cleanup retire failed: {e}"))
4657                            .unwrap_or_default(),
4658                        restore_error
4659                            .as_ref()
4660                            .map(|e| format!("; continuity rollback failed: {e}"))
4661                            .unwrap_or_default(),
4662                        lease_cleanup_error
4663                            .as_ref()
4664                            .map(|e| format!("; lease cleanup failed: {e}"))
4665                            .unwrap_or_default(),
4666                    )));
4667                }
4668            };
4669            record.checkpoint_version = effective_checkpoint_version;
4670            if let Some(err) = Self::unregister_bridge_session_runtime_states(
4671                bridge.as_ref(),
4672                &abandoned_session_registrations,
4673            )
4674            .await
4675            {
4676                let actual_unregister_error = Self::unregister_bridge_session_runtime_states(
4677                    bridge.as_ref(),
4678                    std::slice::from_ref(&record.session_id),
4679                )
4680                .await;
4681                let unregister_error = actual_unregister_error
4682                    .map(|actual_err| format!("{err}; actual session: {actual_err}"))
4683                    .unwrap_or(err);
4684                let cleanup_error = bridge.retire_member(&record.agent_runtime_id).await.err();
4685                let restore_error = self
4686                    .restore_continuity_after_materialize_failure(
4687                        identity,
4688                        original_continuity.as_ref(),
4689                        &grant,
4690                    )
4691                    .await;
4692                let lease_cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4693                return Err(IdentityRuntimeError::Internal(format!(
4694                    "bridge unregister abandoned session runtime state: {unregister_error}{}{}{}",
4695                    cleanup_error
4696                        .as_ref()
4697                        .map(|e| format!("; cleanup retire failed: {e}"))
4698                        .unwrap_or_default(),
4699                    restore_error
4700                        .as_ref()
4701                        .map(|e| format!("; continuity rollback failed: {e}"))
4702                        .unwrap_or_default(),
4703                    lease_cleanup_error
4704                        .as_ref()
4705                        .map(|e| format!("; lease cleanup failed: {e}"))
4706                        .unwrap_or_default(),
4707                )));
4708            }
4709        }
4710
4711        {
4712            let mut entries = self.entries.write().await;
4713            let entry = entries
4714                .get_mut(identity)
4715                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4716            entry.continuity = Some(record.clone());
4717            entry.lease = Some(Self::lease_entry_from_grant(&grant));
4718            entry.state = IdentityLifecycleState::Active;
4719            entry.checkpoint_version = record.checkpoint_version;
4720        }
4721        self.emit_event(
4722            identity,
4723            IdentityEvent::StateChanged {
4724                identity: identity.clone(),
4725                new_state: IdentityLifecycleState::Active,
4726            },
4727        )
4728        .await;
4729        self.clear_materialization_backoff(identity).await;
4730        Ok(record)
4731    }
4732
4733    async fn best_effort_materialize_identity(
4734        &self,
4735        identity: AgentIdentity,
4736        initiator: Option<&AgentIdentity>,
4737        operation: &'static str,
4738    ) -> Option<ContinuityRecord> {
4739        let attempt_lock = self.best_effort_materialization_lock_for(&identity).await;
4740        let _attempt_guard = attempt_lock.lock().await;
4741
4742        if let Some(error) = self.materialization_backoff_error(&identity).await {
4743            tracing::debug!(
4744                identity = %identity,
4745                initiator = initiator.map(ToString::to_string).as_deref(),
4746                error = %error,
4747                "identity best-effort materialization skipped due to materialization backoff"
4748            );
4749            return None;
4750        }
4751
4752        match self.materialize(&identity).await {
4753            Ok(record) => {
4754                self.clear_materialization_backoff(&identity).await;
4755                Some(record)
4756            }
4757            Err(err) => {
4758                tracing::warn!(
4759                    identity = %identity,
4760                    initiator = initiator.map(ToString::to_string).as_deref(),
4761                    error = %err,
4762                    "identity best-effort materialization skipped identity after materialization failure"
4763                );
4764                self.record_best_effort_materialization_failure(
4765                    &identity, initiator, operation, &err,
4766                )
4767                .await;
4768                None
4769            }
4770        }
4771    }
4772
4773    async fn materialize_all_records(
4774        &self,
4775    ) -> Vec<(
4776        AgentIdentity,
4777        Result<ContinuityRecord, IdentityRuntimeError>,
4778    )> {
4779        let identities = self.registered_identities().await;
4780        stream::iter(identities.into_iter().map(|identity| async move {
4781            let result = self.materialize(&identity).await;
4782            (identity, result)
4783        }))
4784        .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4785        .collect::<Vec<_>>()
4786        .await
4787    }
4788
4789    /// Materialize all identities currently registered with the runtime.
4790    ///
4791    /// Fleet hydration is best-effort: one member that cannot build is skipped
4792    /// and surfaced through logs/error hooks rather than aborting unrelated
4793    /// members.
4794    pub async fn materialize_all(&self) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4795        let identities = self.registered_identities().await;
4796        let records = stream::iter(identities.into_iter().map(|identity| async move {
4797            self.best_effort_materialize_identity(identity, None, "materialize_all")
4798                .await
4799        }))
4800        .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4801        .filter_map(async move |record| record)
4802        .collect::<Vec<_>>()
4803        .await;
4804
4805        let desired_edges = self.desired_peer_edges.read().await.clone();
4806        if !desired_edges.is_empty()
4807            && let Err(err) = self.reconcile_managed_peer_edges(&desired_edges).await
4808        {
4809            tracing::warn!(
4810                error = %err,
4811                "identity materialize_all completed with topology reconcile warning"
4812            );
4813        }
4814
4815        Ok(records)
4816    }
4817
4818    /// Materialize all identities and fail if any registered identity cannot
4819    /// hydrate. Flow admission uses this strict variant so a run is not accepted
4820    /// with a partially materialized identity-first fleet.
4821    pub async fn materialize_all_required(
4822        &self,
4823    ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4824        let results = self.materialize_all_records().await;
4825        let mut records = Vec::with_capacity(results.len());
4826        let mut failures = Vec::new();
4827
4828        for (identity, result) in results {
4829            match result {
4830                Ok(record) => records.push(record),
4831                Err(err) => failures.push(format!("{identity}: {err}")),
4832            }
4833        }
4834
4835        if !failures.is_empty() {
4836            return Err(IdentityRuntimeError::Internal(format!(
4837                "identity-first required materialization failed for {} identities: {}",
4838                failures.len(),
4839                failures.join("; ")
4840            )));
4841        }
4842
4843        let desired_edges = self.desired_peer_edges.read().await.clone();
4844        if !desired_edges.is_empty() {
4845            self.reconcile_managed_peer_edges(&desired_edges).await?;
4846        }
4847
4848        Ok(records)
4849    }
4850
4851    /// Cancellation-safe strict fleet hydration for flow/request boundaries.
4852    pub async fn materialize_all_required_tracked(
4853        self: &Arc<Self>,
4854    ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4855        let runtime = Arc::clone(self);
4856        self.run_tracked_foreground(async move { runtime.materialize_all_required().await })
4857            .await
4858    }
4859
4860    /// Ensure an active identity's desired peer neighborhood exists in the
4861    /// concrete mob graph before ordinary communication starts.
4862    pub async fn materialize_reachable_peers(
4863        &self,
4864        identity: &AgentIdentity,
4865    ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4866        let peers = self.reachable_peer_identities(identity).await;
4867        let records = stream::iter(peers.into_iter().map(|peer| async move {
4868            self.best_effort_materialize_identity(
4869                peer,
4870                Some(identity),
4871                "materialize_reachable_peers",
4872            )
4873            .await
4874        }))
4875        .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4876        .filter_map(async move |record| record)
4877        .collect::<Vec<_>>()
4878        .await;
4879
4880        let desired_edges = self.desired_peer_edges.read().await.clone();
4881        if !desired_edges.is_empty()
4882            && let Err(err) = self.reconcile_managed_peer_edges(&desired_edges).await
4883        {
4884            tracing::warn!(
4885                identity = %identity,
4886                error = %err,
4887                "identity peer materialization completed with topology reconcile warning"
4888            );
4889        }
4890
4891        Ok(records)
4892    }
4893
4894    // -----------------------------------------------------------------------
4895    // Subscribe — REQ-06
4896    // -----------------------------------------------------------------------
4897
4898    /// Subscribe to identity-scoped events.
4899    ///
4900    /// Returns a broadcast receiver that yields `IdentityEvent` items for
4901    /// state changes, lease updates, lease loss, and checkpoint completions.
4902    pub async fn subscribe(
4903        &self,
4904        identity: &AgentIdentity,
4905    ) -> Result<broadcast::Receiver<IdentityEvent>, IdentityRuntimeError> {
4906        let channels = self.event_channels.read().await;
4907        let tx = channels
4908            .get(identity)
4909            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4910        Ok(tx.subscribe())
4911    }
4912
4913    /// Update the spec for an existing identity (used during reconciliation).
4914    pub async fn update_spec(&self, spec: DurableAgentSpec) -> Result<(), IdentityRuntimeError> {
4915        let mut entries = self.entries.write().await;
4916        let entry = entries
4917            .get_mut(&spec.identity)
4918            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
4919        entry.spec = spec;
4920        Ok(())
4921    }
4922
4923    /// Release every exact provider grant retained by a Broken entry. The
4924    /// caller owns this identity's lifecycle lock.
4925    ///
4926    /// Most failed lower-plane transitions already park authority in
4927    /// `pending_lease_release`, but rollback of an originally Active entry can
4928    /// deliberately retain its current grant in `lease` while projecting
4929    /// Broken. Move that live grant into pending state before the provider await
4930    /// so cancellation, a provider failure, and shutdown all retain the exact
4931    /// fencing token. A failed release aborts reconcile before lazy registration
4932    /// can overwrite the entry or restore can reacquire authority.
4933    async fn release_broken_lease_locked(
4934        &self,
4935        identity: &AgentIdentity,
4936    ) -> Result<bool, IdentityRuntimeError> {
4937        let pending = {
4938            let mut entries = self.entries.write().await;
4939            let entry = entries
4940                .get(identity)
4941                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4942            if entry.state != IdentityLifecycleState::Broken {
4943                return Ok(false);
4944            }
4945
4946            let existing_pending = entry.pending_lease_release.clone();
4947            let retained_lease = entry.lease.as_ref().map(|lease| LeaseGrant {
4948                identity: identity.clone(),
4949                fencing_token: lease.fencing_token,
4950                ttl: lease.ttl,
4951            });
4952            if let (Some(pending), Some(retained)) =
4953                (existing_pending.as_ref(), retained_lease.as_ref())
4954                && pending.fencing_token != retained.fencing_token
4955            {
4956                return Err(IdentityRuntimeError::Internal(format!(
4957                    "Broken identity {identity} retained conflicting fencing tokens {} and {}",
4958                    pending.fencing_token, retained.fencing_token
4959                )));
4960            }
4961
4962            let grant = existing_pending.or(retained_lease);
4963            if let Some(grant) = grant.as_ref() {
4964                let entry = entries
4965                    .get_mut(identity)
4966                    .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4967                entry.lease = None;
4968                entry.pending_lease_release = Some(grant.clone());
4969            }
4970            grant
4971        };
4972        let Some(grant) = pending else {
4973            return Ok(false);
4974        };
4975
4976        if let Err(error) = self
4977            .lease_provider
4978            .release_leases(std::slice::from_ref(&grant))
4979            .await
4980        {
4981            self.mark_bootstrap_from_lifecycle(
4982                identity,
4983                IdentityLifecycleState::Broken,
4984                Some(format!("pending lease release retry: {error}")),
4985            );
4986            return Err(IdentityRuntimeError::Lease(error));
4987        }
4988
4989        let mut entries = self.entries.write().await;
4990        let entry = entries
4991            .get_mut(identity)
4992            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4993        match entry.pending_lease_release.as_ref() {
4994            Some(current) if current.fencing_token == grant.fencing_token => {
4995                entry.pending_lease_release = None;
4996            }
4997            Some(current) => {
4998                return Err(IdentityRuntimeError::Internal(format!(
4999                    "pending lease release for {identity} changed from fencing token {} to {} under lifecycle authority",
5000                    grant.fencing_token, current.fencing_token
5001                )));
5002            }
5003            None => {
5004                return Err(IdentityRuntimeError::Internal(format!(
5005                    "pending lease release for {identity} disappeared under lifecycle authority"
5006                )));
5007            }
5008        }
5009        Ok(true)
5010    }
5011
5012    /// Dispose the concrete member and its session-store authority retained by
5013    /// a continuity-backed Broken entry. Lease loss is fail-closed at the
5014    /// identity plane, but it cannot synchronously remove the lower member;
5015    /// every roster reconcile that will reproject the Broken entry must do so
5016    /// before abandoning or reusing the runtime alias. The real bridge treats
5017    /// an already-retired member as success, and session unregister is
5018    /// idempotent, making retries safe when a later exact-grant cleanup step
5019    /// fails.
5020    ///
5021    /// This deliberately does not mutate `lease` or `pending_lease_release`:
5022    /// provider authority is an independent exact-token cleanup obligation.
5023    async fn cleanup_broken_lower_plane_locked(
5024        &self,
5025        identity: &AgentIdentity,
5026    ) -> Result<(), IdentityRuntimeError> {
5027        let continuity = {
5028            let entries = self.entries.read().await;
5029            let entry = entries
5030                .get(identity)
5031                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5032            (entry.state == IdentityLifecycleState::Broken)
5033                .then(|| entry.continuity.clone())
5034                .flatten()
5035        };
5036        let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5037            return Ok(());
5038        };
5039
5040        let retire_error = bridge.retire_member(&record.agent_runtime_id).await.err();
5041        let unregister_error = bridge
5042            .unregister_session_runtime_state(&record.session_id)
5043            .await
5044            .err();
5045        match (retire_error, unregister_error) {
5046            (None, None) => Ok(()),
5047            (retire_error, unregister_error) => Err(IdentityRuntimeError::Internal(format!(
5048                "cleanup Broken lower-plane state for {identity}{}{}",
5049                retire_error
5050                    .map(|error| format!("; retire member: {error}"))
5051                    .unwrap_or_default(),
5052                unregister_error
5053                    .map(|error| format!("; unregister session: {error}"))
5054                    .unwrap_or_default()
5055            ))),
5056        }
5057    }
5058
5059    async fn accept_bootstrap_spec_locked(
5060        &self,
5061        spec: DurableAgentSpec,
5062        generation: u64,
5063    ) -> Result<(), IdentityRuntimeError> {
5064        let mut entries = self.entries.write().await;
5065        let entry = entries
5066            .get_mut(&spec.identity)
5067            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
5068        entry.spec = spec;
5069        entry.bootstrap_generation = generation;
5070        Ok(())
5071    }
5072
5073    /// Converge currently registered embodiments with a newly desired roster.
5074    ///
5075    /// Restore flows only walk the desired roster, so removal and replacement
5076    /// must happen first. Each transition shares the same per-identity lock as
5077    /// foreground materialization and lifecycle RPCs. Same-profile changes are
5078    /// REQ-33 hot reloads: publish the new registry metadata while preserving
5079    /// the exact live session and grant. Profile changes retire to Dormant
5080    /// before the new spec is installed; the selected eager/background/lazy
5081    /// policy then decides when to rebuild the physical member.
5082    async fn reconcile_roster_members(
5083        &self,
5084        roster: &[DurableAgentSpec],
5085        generation: u64,
5086    ) -> Result<(), IdentityRuntimeError> {
5087        self.reconcile_roster_members_after_lifecycle_lock(roster, generation, |_| {
5088            std::future::ready(())
5089        })
5090        .await
5091    }
5092
5093    /// Internal deterministic seam used by concurrency regressions. The hook
5094    /// runs with each identity's lifecycle lock held and before the entry is
5095    /// inspected or mutated.
5096    async fn reconcile_roster_members_after_lifecycle_lock<H, F>(
5097        &self,
5098        roster: &[DurableAgentSpec],
5099        generation: u64,
5100        mut after_lifecycle_lock: H,
5101    ) -> Result<(), IdentityRuntimeError>
5102    where
5103        H: FnMut(&AgentIdentity) -> F,
5104        F: Future<Output = ()>,
5105    {
5106        Self::validate_roster_uniqueness(roster)?;
5107        let desired = roster
5108            .iter()
5109            .map(|spec| (spec.identity.clone(), spec.clone()))
5110            .collect::<BTreeMap<_, _>>();
5111        let registered = self.registered_identities().await;
5112
5113        for identity in registered
5114            .iter()
5115            .filter(|identity| !desired.contains_key(*identity))
5116        {
5117            let lifecycle_lock = self.lifecycle_lock_for(identity).await;
5118            let _lifecycle_guard = lifecycle_lock.lock().await;
5119            after_lifecycle_lock(identity).await;
5120            self.cleanup_broken_lower_plane_locked(identity).await?;
5121            self.release_broken_lease_locked(identity).await?;
5122            let state = self
5123                .entries
5124                .read()
5125                .await
5126                .get(identity)
5127                .map(|entry| entry.state);
5128            let Some(state) = state else {
5129                continue;
5130            };
5131            match state {
5132                IdentityLifecycleState::Active => {
5133                    self.retire_locked(identity).await?;
5134                }
5135                IdentityLifecycleState::Dormant
5136                | IdentityLifecycleState::Broken
5137                | IdentityLifecycleState::Uninitialized => {
5138                    let lease = self
5139                        .entries
5140                        .read()
5141                        .await
5142                        .get(identity)
5143                        .and_then(|entry| entry.lease.as_ref())
5144                        .map(|lease| LeaseGrant {
5145                            identity: identity.clone(),
5146                            fencing_token: lease.fencing_token,
5147                            ttl: lease.ttl,
5148                        });
5149                    if let Some(lease) = lease {
5150                        self.lease_provider
5151                            .release_leases(std::slice::from_ref(&lease))
5152                            .await
5153                            .map_err(IdentityRuntimeError::Lease)?;
5154                    }
5155                }
5156                IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended => {
5157                    return Err(IdentityRuntimeError::InvalidState {
5158                        identity: identity.clone(),
5159                        state,
5160                        operation: "reconcile_roster_remove",
5161                    });
5162                }
5163            }
5164            self.event_channels.write().await.remove(identity);
5165            self.entries.write().await.remove(identity);
5166        }
5167
5168        for (identity, desired_spec) in desired {
5169            let lifecycle_lock = self.lifecycle_lock_for(&identity).await;
5170            let _lifecycle_guard = lifecycle_lock.lock().await;
5171            after_lifecycle_lock(&identity).await;
5172            // Initial bootstrap reaches this loop before lazy/eager restore has
5173            // registered the desired identities. Pending-release recovery only
5174            // applies to an existing lifecycle entry.
5175            if !self.entries.read().await.contains_key(&identity) {
5176                continue;
5177            }
5178            let reconciling_broken_entry = {
5179                let entries = self.entries.read().await;
5180                entries
5181                    .get(&identity)
5182                    .is_some_and(|entry| entry.state == IdentityLifecycleState::Broken)
5183            };
5184            if reconciling_broken_entry {
5185                self.cleanup_broken_lower_plane_locked(&identity).await?;
5186            }
5187            // This runs before the same-spec fast path. A failed retire may
5188            // leave the desired metadata unchanged while the physical member
5189            // is already gone and its exact provider grant is still held.
5190            self.release_broken_lease_locked(&identity).await?;
5191            let current = self
5192                .entries
5193                .read()
5194                .await
5195                .get(&identity)
5196                .map(|entry| (entry.spec.clone(), entry.state));
5197            let Some((current_spec, state)) = current else {
5198                continue;
5199            };
5200            if current_spec == desired_spec {
5201                self.accept_bootstrap_spec_locked(desired_spec, generation)
5202                    .await?;
5203                continue;
5204            }
5205            if matches!(
5206                state,
5207                IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended
5208            ) {
5209                return Err(IdentityRuntimeError::InvalidState {
5210                    identity,
5211                    state,
5212                    operation: "reconcile_roster_replace",
5213                });
5214            }
5215            // REQ-33 and `compute_reconcile_actions` classify every
5216            // same-profile change (addressability, labels, display name,
5217            // context, and instructions) as a metadata hot reload. Retiring
5218            // an Active member here needlessly rotates its exact grant and can
5219            // collide with a still-draining session during eager restore.
5220            if current_spec.profile == desired_spec.profile {
5221                self.accept_bootstrap_spec_locked(desired_spec, generation)
5222                    .await?;
5223                continue;
5224            }
5225            match state {
5226                IdentityLifecycleState::Active => {
5227                    self.retire_locked(&identity).await?;
5228                }
5229                IdentityLifecycleState::Dormant
5230                | IdentityLifecycleState::Broken
5231                | IdentityLifecycleState::Uninitialized => {}
5232                IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended => {
5233                    return Err(IdentityRuntimeError::InvalidState {
5234                        identity,
5235                        state,
5236                        operation: "reconcile_roster_replace",
5237                    });
5238                }
5239            }
5240            self.accept_bootstrap_spec_locked(desired_spec, generation)
5241                .await?;
5242        }
5243
5244        Ok(())
5245    }
5246
5247    /// Adopt the roster's CURRENT spec for `identity` into the in-memory entry,
5248    /// so a subsequent [`reset`](Self::reset) rebuilds the regenerated session
5249    /// on the current profile instead of carrying the stored one forward.
5250    ///
5251    /// Best-effort: logs and leaves the stored spec in place if the roster
5252    /// can't be resolved or no longer lists the identity (reset's primary job —
5253    /// the destructive continuity reset — must not fail because the roster
5254    /// provider hiccuped). The runtime stays roster-agnostic; the provider is
5255    /// supplied by the caller (the reset RPC handler), which owns it.
5256    pub async fn adopt_roster_spec(
5257        &self,
5258        roster_provider: &Arc<dyn RosterProvider>,
5259        identity: &AgentIdentity,
5260    ) {
5261        self.adopt_roster_spec_with_context(roster_provider, identity, None)
5262            .await;
5263    }
5264
5265    async fn adopt_roster_spec_with_context(
5266        &self,
5267        roster_provider: &Arc<dyn RosterProvider>,
5268        identity: &AgentIdentity,
5269        mob_definition: Option<meerkat_mob::MobDefinition>,
5270    ) {
5271        match roster_provider
5272            .roster(&RosterContext {
5273                mob_definition,
5274                previous_identities: Vec::new(),
5275            })
5276            .await
5277        {
5278            Ok(specs) => {
5279                if let Some(spec) = specs.into_iter().find(|s| &s.identity == identity)
5280                    && let Err(err) = self.update_spec(spec).await
5281                {
5282                    tracing::warn!(
5283                        identity = %identity,
5284                        error = %err,
5285                        "reset: failed to adopt current roster spec; rebuilding on stored spec",
5286                    );
5287                }
5288            }
5289            Err(err) => {
5290                tracing::warn!(
5291                    identity = %identity,
5292                    error = %err,
5293                    "reset: roster provider failed; rebuilding on stored spec",
5294                );
5295            }
5296        }
5297    }
5298
5299    /// Update the lease for an identity.
5300    pub async fn update_lease(
5301        &self,
5302        identity: &AgentIdentity,
5303        grant: LeaseGrant,
5304    ) -> Result<(), IdentityRuntimeError> {
5305        let fencing_token = grant.fencing_token;
5306        let mut entries = self.entries.write().await;
5307        let entry = entries
5308            .get_mut(identity)
5309            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5310        entry.lease = Some(LeaseEntry {
5311            fencing_token,
5312            ttl: grant.ttl,
5313            acquired_at: Instant::now(),
5314        });
5315        drop(entries);
5316        self.lease_renewal_notify.notify_one();
5317        self.emit_event(
5318            identity,
5319            IdentityEvent::LeaseUpdated {
5320                identity: identity.clone(),
5321                fencing_token,
5322            },
5323        )
5324        .await;
5325        Ok(())
5326    }
5327
5328    /// Mark a lease as lost for an identity (INV-02).
5329    pub async fn mark_lease_lost(
5330        &self,
5331        identity: &AgentIdentity,
5332    ) -> Result<(), IdentityRuntimeError> {
5333        let mut entries = self.entries.write().await;
5334        let entry = entries
5335            .get_mut(identity)
5336            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5337        entry.state = IdentityLifecycleState::Broken;
5338        entry.lease = None;
5339        drop(entries);
5340        self.mark_bootstrap_from_lifecycle(
5341            identity,
5342            IdentityLifecycleState::Broken,
5343            Some("external lease authority was lost".to_string()),
5344        );
5345        self.emit_event(
5346            identity,
5347            IdentityEvent::LeaseLost {
5348                identity: identity.clone(),
5349            },
5350        )
5351        .await;
5352        Ok(())
5353    }
5354
5355    /// Remove an identity from the runtime.
5356    #[allow(dead_code)]
5357    pub(crate) async fn remove(&self, identity: &AgentIdentity) -> Option<IdentityEntry> {
5358        self.event_channels.write().await.remove(identity);
5359        self.entries.write().await.remove(identity)
5360    }
5361
5362    /// Set the lifecycle state for an identity.
5363    pub async fn set_state(
5364        &self,
5365        identity: &AgentIdentity,
5366        state: IdentityLifecycleState,
5367    ) -> Result<(), IdentityRuntimeError> {
5368        let mut entries = self.entries.write().await;
5369        let entry = entries
5370            .get_mut(identity)
5371            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5372        entry.state = state;
5373        drop(entries);
5374        if state == IdentityLifecycleState::Active {
5375            self.lease_renewal_notify.notify_one();
5376        }
5377        self.emit_event(
5378            identity,
5379            IdentityEvent::StateChanged {
5380                identity: identity.clone(),
5381                new_state: state,
5382            },
5383        )
5384        .await;
5385        Ok(())
5386    }
5387
5388    /// Fail the exact active embodiment when its live session has been torn
5389    /// down underneath the identity layer. A stale close notification from a
5390    /// replaced generation is ignored, while the current embodiment becomes
5391    /// `Broken` so the existing continuity repair supervisor can rebuild it.
5392    pub(crate) async fn mark_active_runtime_broken(
5393        &self,
5394        identity: &AgentIdentity,
5395        expected_runtime_id: &str,
5396        expected_fencing_token: u64,
5397        detail: &str,
5398    ) -> Result<bool, IdentityRuntimeError> {
5399        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
5400        let _lifecycle_guard = lifecycle_lock.lock().await;
5401        let changed = {
5402            let mut entries = self.entries.write().await;
5403            let entry = entries
5404                .get_mut(identity)
5405                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5406            let is_current_active = entry.state == IdentityLifecycleState::Active
5407                && entry
5408                    .continuity
5409                    .as_ref()
5410                    .is_some_and(|record| record.agent_runtime_id.as_str() == expected_runtime_id)
5411                && entry
5412                    .lease
5413                    .as_ref()
5414                    .is_some_and(|lease| lease.fencing_token.get() == expected_fencing_token);
5415            if is_current_active {
5416                // Retain the exact live lease. Broken repair owns lower-plane
5417                // cleanup and releases that grant before rematerializing.
5418                entry.state = IdentityLifecycleState::Broken;
5419            }
5420            is_current_active
5421        };
5422        if changed {
5423            self.mark_bootstrap_from_lifecycle(
5424                identity,
5425                IdentityLifecycleState::Broken,
5426                Some(detail.to_string()),
5427            );
5428            self.emit_event(
5429                identity,
5430                IdentityEvent::StateChanged {
5431                    identity: identity.clone(),
5432                    new_state: IdentityLifecycleState::Broken,
5433                },
5434            )
5435            .await;
5436        }
5437        Ok(changed)
5438    }
5439
5440    // -----------------------------------------------------------------------
5441    // Lease checking (INV-01, INV-02)
5442    // -----------------------------------------------------------------------
5443
5444    /// Check that the identity has an active, non-expired lease.
5445    /// Returns the fencing token if valid.
5446    fn check_lease(entry: &IdentityEntry) -> Result<FencingToken, IdentityRuntimeError> {
5447        match &entry.lease {
5448            Some(lease) if !lease.is_expired() => Ok(lease.fencing_token),
5449            Some(_) => Err(IdentityRuntimeError::LeaseLost(entry.spec.identity.clone())),
5450            None => Err(IdentityRuntimeError::NoActiveLease(
5451                entry.spec.identity.clone(),
5452            )),
5453        }
5454    }
5455
5456    fn lease_entry_from_grant(grant: &LeaseGrant) -> LeaseEntry {
5457        LeaseEntry {
5458            fencing_token: grant.fencing_token,
5459            ttl: grant.ttl,
5460            acquired_at: Instant::now(),
5461        }
5462    }
5463
5464    /// Transfer a provider-returned active grant into runtime ownership before
5465    /// any fallible continuity or bridge projection. The provider call is the
5466    /// authority commit: once it returns, retaining only the previous token is
5467    /// unsafe because exact-token release would become a no-op.
5468    async fn stage_active_grant(
5469        &self,
5470        identity: &AgentIdentity,
5471        expected_previous: Option<FencingToken>,
5472        grant: &LeaseGrant,
5473    ) -> Result<Option<ContinuityRecord>, IdentityRuntimeError> {
5474        let staged = {
5475            let mut entries = self.entries.write().await;
5476            (|| {
5477                if grant.identity != *identity {
5478                    return Err(IdentityRuntimeError::Internal(format!(
5479                        "provider returned grant for {} while publishing active authority for {identity}",
5480                        grant.identity
5481                    )));
5482                }
5483                let entry = entries
5484                    .get_mut(identity)
5485                    .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5486                if entry.state != IdentityLifecycleState::Active {
5487                    return Err(IdentityRuntimeError::InvalidState {
5488                        identity: identity.clone(),
5489                        state: entry.state,
5490                        operation: "stage_active_grant",
5491                    });
5492                }
5493                let current = entry
5494                    .lease
5495                    .as_ref()
5496                    .ok_or_else(|| IdentityRuntimeError::NoActiveLease(identity.clone()))?;
5497                if let Some(expected) = expected_previous
5498                    && current.fencing_token != expected
5499                {
5500                    return Err(IdentityRuntimeError::Internal(format!(
5501                        "active lease for {identity} changed from fencing token {expected} to {} before renewed authority could be staged",
5502                        current.fencing_token
5503                    )));
5504                }
5505                if let Some(pending) = entry.pending_lease_release.as_ref() {
5506                    return Err(IdentityRuntimeError::Internal(format!(
5507                        "active identity {identity} already has pending fencing token {} while staging {}",
5508                        pending.fencing_token, grant.fencing_token
5509                    )));
5510                }
5511
5512                let continuity = entry.continuity.clone();
5513                // Broken + pending is the fail-closed intermediate state.
5514                // The lifecycle lock prevents another operation from
5515                // observing it as a completed transition, while a dropped
5516                // caller still leaves repair/shutdown an exact token.
5517                entry.state = IdentityLifecycleState::Broken;
5518                entry.lease = None;
5519                entry.pending_lease_release = Some(grant.clone());
5520                Ok(continuity)
5521            })()
5522        };
5523
5524        match staged {
5525            Ok(continuity) => Ok(continuity),
5526            Err(error) => {
5527                if let Err(cleanup_error) = self
5528                    .release_or_park_untracked_leases(std::slice::from_ref(grant))
5529                    .await
5530                {
5531                    return Err(IdentityRuntimeError::Internal(format!(
5532                        "{error}; exact grant cleanup failed: {cleanup_error}"
5533                    )));
5534                }
5535                Err(error)
5536            }
5537        }
5538    }
5539
5540    async fn fail_staged_active_grant(
5541        &self,
5542        identity: &AgentIdentity,
5543        grant: &LeaseGrant,
5544        detail: String,
5545    ) {
5546        let retained_by_entry = {
5547            let mut entries = self.entries.write().await;
5548            entries.get_mut(identity).is_some_and(|entry| {
5549                if entry
5550                    .pending_lease_release
5551                    .as_ref()
5552                    .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
5553                {
5554                    entry.state = IdentityLifecycleState::Broken;
5555                    entry.lease = None;
5556                    true
5557                } else {
5558                    false
5559                }
5560            })
5561        };
5562        if !retained_by_entry {
5563            self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5564                .await;
5565        }
5566        self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Broken, Some(detail));
5567        self.emit_event(
5568            identity,
5569            IdentityEvent::StateChanged {
5570                identity: identity.clone(),
5571                new_state: IdentityLifecycleState::Broken,
5572            },
5573        )
5574        .await;
5575    }
5576
5577    async fn suspend_bridge_continuity(
5578        &self,
5579        continuity: Option<&ContinuityRecord>,
5580    ) -> Result<(), IdentityRuntimeError> {
5581        let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5582            return Ok(());
5583        };
5584        bridge
5585            .suspend_session_runtime_state(&record.session_id)
5586            .await
5587            .map_err(|error| {
5588                IdentityRuntimeError::Internal(format!(
5589                    "suspend bridge session runtime state before authority rotation: {error}"
5590                ))
5591            })
5592    }
5593
5594    async fn resume_bridge_continuity(
5595        &self,
5596        identity: &AgentIdentity,
5597        continuity: Option<&ContinuityRecord>,
5598        grant: &LeaseGrant,
5599    ) -> Result<(), IdentityRuntimeError> {
5600        let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5601            return Ok(());
5602        };
5603        bridge
5604            .register_session_runtime_state(
5605                &record.session_id,
5606                identity,
5607                record.generation,
5608                record.checkpoint_version,
5609                grant.fencing_token,
5610            )
5611            .await
5612            .map(|_| ())
5613            .map_err(|error| {
5614                IdentityRuntimeError::Internal(format!(
5615                    "resume bridge session runtime state after pre-commit renewal failure: {error}"
5616                ))
5617            })
5618    }
5619
5620    async fn break_existing_active_grant(
5621        &self,
5622        identity: &AgentIdentity,
5623        grant: &LeaseGrant,
5624        detail: String,
5625    ) {
5626        let retained = {
5627            let mut entries = self.entries.write().await;
5628            entries.get_mut(identity).is_some_and(|entry| {
5629                if entry
5630                    .lease
5631                    .as_ref()
5632                    .is_some_and(|lease| lease.fencing_token == grant.fencing_token)
5633                {
5634                    entry.state = IdentityLifecycleState::Broken;
5635                    entry.lease = None;
5636                    entry.pending_lease_release = Some(grant.clone());
5637                    true
5638                } else {
5639                    false
5640                }
5641            })
5642        };
5643        if !retained {
5644            self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5645                .await;
5646        }
5647        self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Broken, Some(detail));
5648        self.emit_event(
5649            identity,
5650            IdentityEvent::StateChanged {
5651                identity: identity.clone(),
5652                new_state: IdentityLifecycleState::Broken,
5653            },
5654        )
5655        .await;
5656    }
5657
5658    async fn commit_staged_active_grant(
5659        &self,
5660        identity: &AgentIdentity,
5661        grant: &LeaseGrant,
5662        continuity: Option<ContinuityRecord>,
5663    ) -> Result<(), IdentityRuntimeError> {
5664        let committed = {
5665            let mut entries = self.entries.write().await;
5666            match entries.get_mut(identity) {
5667                Some(entry)
5668                    if entry
5669                        .pending_lease_release
5670                        .as_ref()
5671                        .is_some_and(|pending| pending.fencing_token == grant.fencing_token) =>
5672                {
5673                    if let Some(record) = continuity {
5674                        entry.checkpoint_version = record.checkpoint_version;
5675                        entry.continuity = Some(record);
5676                    }
5677                    entry.pending_lease_release = None;
5678                    entry.lease = Some(Self::lease_entry_from_grant(grant));
5679                    entry.state = IdentityLifecycleState::Active;
5680                    Ok(())
5681                }
5682                Some(entry) => Err(IdentityRuntimeError::Internal(format!(
5683                    "staged fencing token {} for {identity} was replaced before commit (state {:?})",
5684                    grant.fencing_token, entry.state
5685                ))),
5686                None => Err(IdentityRuntimeError::UnknownIdentity(identity.clone())),
5687            }
5688        };
5689        if let Err(error) = committed {
5690            self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5691                .await;
5692            return Err(error);
5693        }
5694        Ok(())
5695    }
5696
5697    async fn publish_active_grant(
5698        &self,
5699        identity: &AgentIdentity,
5700        expected_previous: Option<FencingToken>,
5701        grant: &LeaseGrant,
5702    ) -> Result<FencingToken, IdentityRuntimeError> {
5703        let mut continuity = self
5704            .stage_active_grant(identity, expected_previous, grant)
5705            .await?;
5706
5707        // `ensure_active_lease` suspends before invoking the provider so no
5708        // stale-token save can overlap the authority rotation. Keep this
5709        // idempotent suspension here as a defensive boundary for callers that
5710        // already hold a provider-returned active grant (restore compatibility).
5711        if let Err(error) = self.suspend_bridge_continuity(continuity.as_ref()).await {
5712            self.fail_staged_active_grant(identity, grant, error.to_string())
5713                .await;
5714            return Err(error);
5715        }
5716
5717        if let Some(record) = continuity.as_mut() {
5718            if let Err(error) = self
5719                .continuity_store
5720                .upsert_continuity_record(record, grant.fencing_token)
5721                .await
5722            {
5723                let runtime_error = IdentityRuntimeError::Store(error);
5724                self.fail_staged_active_grant(
5725                    identity,
5726                    grant,
5727                    format!("active grant continuity publication failed: {runtime_error}"),
5728                )
5729                .await;
5730                return Err(runtime_error);
5731            }
5732            if let Some(bridge) = self.bridge.as_ref() {
5733                match bridge
5734                    .register_session_runtime_state(
5735                        &record.session_id,
5736                        identity,
5737                        record.generation,
5738                        record.checkpoint_version,
5739                        grant.fencing_token,
5740                    )
5741                    .await
5742                {
5743                    Ok(version) => {
5744                        record.checkpoint_version = CheckpointVersion::new(
5745                            record.checkpoint_version.get().max(version.get()),
5746                        );
5747                    }
5748                    Err(error) => {
5749                        let runtime_error = IdentityRuntimeError::Internal(format!(
5750                            "bridge refresh session runtime state after active grant publication: {error}"
5751                        ));
5752                        let _ = self.suspend_bridge_continuity(Some(record)).await;
5753                        self.fail_staged_active_grant(identity, grant, runtime_error.to_string())
5754                            .await;
5755                        return Err(runtime_error);
5756                    }
5757                }
5758            }
5759        }
5760
5761        let session_to_reference = continuity.as_ref().map(|record| record.session_id.clone());
5762        if let Err(error) = self
5763            .commit_staged_active_grant(identity, grant, continuity)
5764            .await
5765        {
5766            if let (Some(bridge), Some(session_id)) =
5767                (self.bridge.as_ref(), session_to_reference.as_ref())
5768            {
5769                let _ = bridge.suspend_session_runtime_state(session_id).await;
5770            }
5771            return Err(error);
5772        }
5773        self.lease_renewal_notify.notify_one();
5774        self.emit_event(
5775            identity,
5776            IdentityEvent::LeaseUpdated {
5777                identity: identity.clone(),
5778                fencing_token: grant.fencing_token,
5779            },
5780        )
5781        .await;
5782        Ok(grant.fencing_token)
5783    }
5784
5785    async fn ensure_active_lease(
5786        &self,
5787        identity: &AgentIdentity,
5788    ) -> Result<FencingToken, IdentityRuntimeError> {
5789        let (grant, continuity) = {
5790            let entries = self.entries.read().await;
5791            let entry = entries
5792                .get(identity)
5793                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5794            let lease = match &entry.lease {
5795                Some(lease) if lease.is_healthy() => return Ok(lease.fencing_token),
5796                Some(lease) => lease,
5797                None => return Err(IdentityRuntimeError::NoActiveLease(identity.clone())),
5798            };
5799            (
5800                LeaseGrant {
5801                    identity: identity.clone(),
5802                    fencing_token: lease.fencing_token,
5803                    ttl: lease.ttl,
5804                },
5805                entry.continuity.clone(),
5806            )
5807        };
5808
5809        // Drain every already-admitted persistence mutation before asking the
5810        // provider to rotate N to N+1. Once this returns, all session-store
5811        // mutations fail closed until register_session_runtime_state publishes
5812        // the exact replacement token.
5813        if let Err(suspend_error) = self.suspend_bridge_continuity(continuity.as_ref()).await {
5814            if let Err(resume_error) = self
5815                .resume_bridge_continuity(identity, continuity.as_ref(), &grant)
5816                .await
5817            {
5818                self.break_existing_active_grant(
5819                    identity,
5820                    &grant,
5821                    format!("{suspend_error}; bridge rollback failed: {resume_error}"),
5822                )
5823                .await;
5824                return Err(IdentityRuntimeError::Internal(format!(
5825                    "{suspend_error}; bridge rollback failed: {resume_error}"
5826                )));
5827            }
5828            return Err(suspend_error);
5829        }
5830
5831        let renewed = match self
5832            .lease_provider
5833            .renew_leases(std::slice::from_ref(&grant))
5834            .await
5835        {
5836            Ok(renewed) => renewed,
5837            Err(error) => {
5838                let runtime_error = IdentityRuntimeError::Lease(error);
5839                if let Err(resume_error) = self
5840                    .resume_bridge_continuity(identity, continuity.as_ref(), &grant)
5841                    .await
5842                {
5843                    self.break_existing_active_grant(
5844                        identity,
5845                        &grant,
5846                        format!("{runtime_error}; bridge rollback failed: {resume_error}"),
5847                    )
5848                    .await;
5849                    return Err(IdentityRuntimeError::Internal(format!(
5850                        "{runtime_error}; bridge rollback failed: {resume_error}"
5851                    )));
5852                }
5853                return Err(runtime_error);
5854            }
5855        };
5856        let renewed_grant = match renewed.get(identity) {
5857            Some(super::types::LeaseRenewResult::Renewed(grant)) => grant.clone(),
5858            Some(super::types::LeaseRenewResult::Lost { .. }) | None => {
5859                self.mark_lease_lost(identity).await?;
5860                return Err(IdentityRuntimeError::LeaseLost(identity.clone()));
5861            }
5862        };
5863
5864        self.publish_active_grant(identity, Some(grant.fencing_token), &renewed_grant)
5865            .await
5866    }
5867
5868    async fn mark_lifecycle_in_progress(
5869        &self,
5870        identity: &AgentIdentity,
5871        state: IdentityLifecycleState,
5872    ) -> Result<IdentityEntry, IdentityRuntimeError> {
5873        let mut entries = self.entries.write().await;
5874        let entry = entries
5875            .get_mut(identity)
5876            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5877        let snapshot = entry.clone();
5878        entry.state = state;
5879        entry.lease = None;
5880        drop(entries);
5881        self.mark_bootstrap_materialization_started(identity, None);
5882        Ok(snapshot)
5883    }
5884
5885    async fn restore_entry(&self, identity: &AgentIdentity, entry: IdentityEntry) {
5886        let state = entry.state;
5887        self.entries.write().await.insert(identity.clone(), entry);
5888        self.mark_bootstrap_from_lifecycle(identity, state, None);
5889    }
5890
5891    async fn restore_entry_with_grant(
5892        &self,
5893        identity: &AgentIdentity,
5894        mut entry: IdentityEntry,
5895        grant: &LeaseGrant,
5896    ) {
5897        let restore_live_lease = entry.state == IdentityLifecycleState::Active;
5898        if let Some(record) = entry.continuity.as_ref() {
5899            if let Err(err) = self
5900                .continuity_store
5901                .upsert_continuity_record(record, grant.fencing_token)
5902                .await
5903            {
5904                tracing::warn!(
5905                    %identity,
5906                    error = %err,
5907                    "failed to advance restored continuity fencing token after lifecycle failure"
5908                );
5909                entry.state = IdentityLifecycleState::Broken;
5910            } else if let Some(bridge) = self.bridge.as_ref()
5911                && let Err(err) = bridge
5912                    .register_session_runtime_state(
5913                        &record.session_id,
5914                        identity,
5915                        record.generation,
5916                        record.checkpoint_version,
5917                        grant.fencing_token,
5918                    )
5919                    .await
5920            {
5921                tracing::warn!(
5922                    %identity,
5923                    error = %err,
5924                    "failed to refresh restored session runtime state after lifecycle failure"
5925                );
5926                entry.state = IdentityLifecycleState::Broken;
5927            }
5928        }
5929        // Preserve ownership for an originally Active entry even if bridge
5930        // repair marks it Broken; repair still needs the current fenced
5931        // grant. Originally non-Active entries must not retain a lease that
5932        // their restored local state does not expose.
5933        let retain_grant = restore_live_lease;
5934        entry.lease = retain_grant.then(|| Self::lease_entry_from_grant(grant));
5935        if !retain_grant {
5936            entry.pending_lease_release = Some(grant.clone());
5937            match self
5938                .lease_provider
5939                .release_leases(std::slice::from_ref(grant))
5940                .await
5941            {
5942                Ok(()) => entry.pending_lease_release = None,
5943                Err(err) => {
5944                    tracing::warn!(
5945                        %identity,
5946                        error = %err,
5947                        "failed to release lifecycle rollback lease for non-active identity; exact grant parked for retry"
5948                    );
5949                    entry.state = IdentityLifecycleState::Broken;
5950                }
5951            }
5952        }
5953        self.restore_entry(identity, entry).await;
5954        if retain_grant {
5955            self.lease_renewal_notify.notify_one();
5956        }
5957    }
5958
5959    /// Reuse an already-active embodiment during roster reconciliation.
5960    ///
5961    /// The lifecycle reservation held by the restore controller makes this
5962    /// snapshot stable. Crucially, this path preserves the exact live lease
5963    /// grant instead of reacquiring/rotating authority for a member that is
5964    /// already running.
5965    pub(crate) async fn reuse_active_restore_state(
5966        &self,
5967        spec: &DurableAgentSpec,
5968    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
5969        // The restore controller still owns this identity's lifecycle guard,
5970        // so validate time-sensitive external authority before snapshotting or
5971        // mutating the local projection. Healthy grants return unchanged;
5972        // due grants publish their exact renewal, and Lost authority marks the
5973        // identity Broken instead of reporting a successful active reconcile.
5974        self.ensure_active_lease(&spec.identity).await?;
5975
5976        let mut entries = self.entries.write().await;
5977        let entry = entries
5978            .get_mut(&spec.identity)
5979            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
5980        if entry.state != IdentityLifecycleState::Active {
5981            return Err(IdentityRuntimeError::InvalidState {
5982                identity: spec.identity.clone(),
5983                state: entry.state,
5984                operation: "reuse active restore state",
5985            });
5986        }
5987        if entry.lease.is_none() {
5988            return Err(IdentityRuntimeError::NoActiveLease(spec.identity.clone()));
5989        }
5990        let record = entry.continuity.clone().ok_or_else(|| {
5991            IdentityRuntimeError::Internal(format!(
5992                "active identity {} has no continuity record",
5993                spec.identity
5994            ))
5995        })?;
5996        entry.spec = spec.clone();
5997        Ok(record)
5998    }
5999
6000    /// Publish a fail-closed local state and relinquish the fresh external
6001    /// grant acquired for the failed lifecycle transaction.
6002    ///
6003    /// These ambiguous failure paths deliberately publish Broken with no local
6004    /// lease. Keeping the matching provider grant alive would make that
6005    /// projection a lie and could block another runtime from repairing the
6006    /// identity until the provider TTL expires (forever for the bundled
6007    /// single-process provider).
6008    async fn restore_broken_entry_and_release_grant(
6009        &self,
6010        identity: &AgentIdentity,
6011        mut entry: IdentityEntry,
6012        grant: &LeaseGrant,
6013    ) {
6014        entry.state = IdentityLifecycleState::Broken;
6015        entry.lease = None;
6016        // Park the exact grant before the provider call. If release fails (or
6017        // this future is abandoned after publication), reconcile/shutdown can
6018        // retry the same fencing token instead of leaving an invisible owner.
6019        entry.pending_lease_release = Some(grant.clone());
6020        self.restore_entry(identity, entry).await;
6021        match self
6022            .lease_provider
6023            .release_leases(std::slice::from_ref(grant))
6024            .await
6025        {
6026            Ok(()) => {
6027                let mut entries = self.entries.write().await;
6028                if let Some(entry) = entries.get_mut(identity)
6029                    && entry
6030                        .pending_lease_release
6031                        .as_ref()
6032                        .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
6033                {
6034                    entry.pending_lease_release = None;
6035                }
6036            }
6037            Err(err) => {
6038                self.mark_bootstrap_from_lifecycle(
6039                    identity,
6040                    IdentityLifecycleState::Broken,
6041                    Some(format!("pending lease release: {err}")),
6042                );
6043                tracing::warn!(
6044                    %identity,
6045                    error = %err,
6046                    "failed to release lease after lifecycle transaction became Broken; exact grant parked for retry"
6047                );
6048            }
6049        }
6050    }
6051
6052    async fn restore_broken_entry_with_fenced_store(
6053        &self,
6054        identity: &AgentIdentity,
6055        mut entry: IdentityEntry,
6056        grant: &LeaseGrant,
6057    ) {
6058        entry.state = IdentityLifecycleState::Broken;
6059        entry.lease = None;
6060        if let Some(record) = entry.continuity.as_ref()
6061            && let Err(err) = self
6062                .continuity_store
6063                .upsert_continuity_record(record, grant.fencing_token)
6064                .await
6065        {
6066            tracing::warn!(
6067                %identity,
6068                error = %err,
6069                "failed to preserve fenced continuity record for broken identity"
6070            );
6071        }
6072        self.restore_broken_entry_and_release_grant(identity, entry, grant)
6073            .await;
6074    }
6075
6076    /// A failed reset rollback means the caller's pre-reset entry is no
6077    /// longer authoritative. Re-resolve under the lifecycle transaction and
6078    /// publish only the store's actual record; if the store itself cannot be
6079    /// read, fail closed without continuity rather than claiming the old
6080    /// generation still owns the durable row.
6081    async fn restore_broken_entry_from_authoritative_continuity(
6082        &self,
6083        identity: &AgentIdentity,
6084        mut entry: IdentityEntry,
6085        grant: &LeaseGrant,
6086    ) {
6087        let authoritative = self
6088            .continuity_store
6089            .resolve_many(std::slice::from_ref(identity))
6090            .await;
6091        entry.continuity = match authoritative {
6092            Ok(resolved) => match resolved.get(identity) {
6093                Some(super::types::ContinuityResolveState::Ready { record }) => {
6094                    Some(record.clone())
6095                }
6096                Some(super::types::ContinuityResolveState::Broken { failure }) => {
6097                    failure.record.clone()
6098                }
6099                Some(super::types::ContinuityResolveState::Uninitialized) | None => None,
6100            },
6101            Err(error) => {
6102                tracing::warn!(
6103                    %identity,
6104                    error = %error,
6105                    "failed to resolve authoritative continuity after reset rollback failure"
6106                );
6107                None
6108            }
6109        };
6110        entry.checkpoint_version = entry
6111            .continuity
6112            .as_ref()
6113            .map(|record| record.checkpoint_version)
6114            .unwrap_or(CheckpointVersion::new(0));
6115        self.restore_broken_entry_and_release_grant(identity, entry, grant)
6116            .await;
6117    }
6118
6119    async fn mark_rebind_failure_broken(
6120        &self,
6121        identity: &AgentIdentity,
6122        mut entry: IdentityEntry,
6123        grant: &LeaseGrant,
6124        rebound_record: &ContinuityRecord,
6125    ) {
6126        entry.state = IdentityLifecycleState::Broken;
6127        entry.lease = None;
6128        entry.checkpoint_version = rebound_record.checkpoint_version;
6129        entry.continuity = Some(rebound_record.clone());
6130        if let Err(err) = self
6131            .continuity_store
6132            .upsert_continuity_record(rebound_record, grant.fencing_token)
6133            .await
6134        {
6135            tracing::warn!(
6136                %identity,
6137                session_id = %rebound_record.session_id,
6138                error = %err,
6139                "failed to preserve rebound continuity after live respawn rebind failure"
6140            );
6141        }
6142        self.restore_broken_entry_and_release_grant(identity, entry, grant)
6143            .await;
6144    }
6145
6146    async fn restore_entry_after_reset_bridge_failure(
6147        &self,
6148        identity: &AgentIdentity,
6149        expected_attempt: &ContinuityRecord,
6150        entry: IdentityEntry,
6151        grant: &LeaseGrant,
6152        force_broken: bool,
6153    ) -> Option<ContinuityStoreError> {
6154        let rollback_error = self
6155            .continuity_store
6156            .rollback_continuity_record(
6157                expected_attempt,
6158                entry.continuity.as_ref(),
6159                grant.fencing_token,
6160            )
6161            .await
6162            .err();
6163        if rollback_error.is_some() {
6164            self.restore_broken_entry_from_authoritative_continuity(identity, entry, grant)
6165                .await;
6166        } else if force_broken {
6167            self.restore_broken_entry_and_release_grant(identity, entry, grant)
6168                .await;
6169        } else {
6170            self.restore_entry_with_grant(identity, entry, grant).await;
6171        }
6172        rollback_error
6173    }
6174
6175    async fn restore_continuity_after_materialize_failure(
6176        &self,
6177        identity: &AgentIdentity,
6178        previous: Option<&ContinuityRecord>,
6179        grant: &LeaseGrant,
6180    ) -> Option<ContinuityStoreError> {
6181        match previous {
6182            Some(record) => self
6183                .continuity_store
6184                .upsert_continuity_record(record, grant.fencing_token)
6185                .await
6186                .err(),
6187            None => self
6188                .continuity_store
6189                .delete_continuity_record(identity, grant.fencing_token)
6190                .await
6191                .err(),
6192        }
6193    }
6194
6195    async fn unregister_bridge_session_runtime_states(
6196        bridge: &dyn SessionBridge,
6197        session_ids: &[SessionId],
6198    ) -> Option<String> {
6199        let mut errors = Vec::new();
6200        let mut seen = BTreeSet::new();
6201        for session_id in session_ids {
6202            if !seen.insert(session_id.to_string()) {
6203                continue;
6204            }
6205            if let Err(err) = bridge.unregister_session_runtime_state(session_id).await {
6206                errors.push(format!("{session_id}: {err}"));
6207            }
6208        }
6209        (!errors.is_empty()).then(|| errors.join("; "))
6210    }
6211
6212    async fn advance_existing_continuity_fence(
6213        &self,
6214        identity: &AgentIdentity,
6215        entry: &IdentityEntry,
6216        grant: &LeaseGrant,
6217    ) -> Result<(), IdentityRuntimeError> {
6218        if let Some(record) = entry.continuity.as_ref() {
6219            self.continuity_store
6220                .upsert_continuity_record(record, grant.fencing_token)
6221                .await
6222                .map_err(IdentityRuntimeError::Store)?;
6223        }
6224        let _ = identity;
6225        Ok(())
6226    }
6227
6228    async fn refresh_existing_session_runtime_state(
6229        &self,
6230        identity: &AgentIdentity,
6231        record: &ContinuityRecord,
6232        grant: &LeaseGrant,
6233    ) -> Result<CheckpointVersion, IdentityRuntimeError> {
6234        let Some(bridge) = self.bridge.as_ref() else {
6235            return Ok(record.checkpoint_version);
6236        };
6237        bridge
6238            .register_session_runtime_state(
6239                &record.session_id,
6240                identity,
6241                record.generation,
6242                record.checkpoint_version,
6243                grant.fencing_token,
6244            )
6245            .await
6246            .map_err(|err| {
6247                IdentityRuntimeError::Internal(format!(
6248                    "bridge refresh session runtime state: {err}"
6249                ))
6250            })
6251    }
6252
6253    // -----------------------------------------------------------------------
6254    // Delivery: send() — REQ-01, REQ-03
6255    // -----------------------------------------------------------------------
6256
6257    /// Send conversational content to an addressable identity.
6258    ///
6259    /// Enforces:
6260    /// - Identity must be registered and active
6261    /// - Identity must be Addressable (REQ-03)
6262    /// - Lease must be held (INV-01)
6263    /// - Lease must not be lost (INV-02)
6264    ///
6265    /// Returns the fencing token for the delivery (caller uses it for checkpoint).
6266    pub async fn send(
6267        &self,
6268        identity: &AgentIdentity,
6269        content: &meerkat_core::ContentInput,
6270    ) -> Result<FencingToken, IdentityRuntimeError> {
6271        self.send_with_mode(identity, content, HandlingMode::Queue)
6272            .await
6273    }
6274
6275    /// Cancellation-safe queue send for RPC/host request boundaries.
6276    pub async fn send_tracked(
6277        self: &Arc<Self>,
6278        identity: &AgentIdentity,
6279        content: &meerkat_core::ContentInput,
6280    ) -> Result<FencingToken, IdentityRuntimeError> {
6281        self.send_with_mode_tracked(identity, content, HandlingMode::Queue)
6282            .await
6283    }
6284
6285    /// Send conversational content using an explicit turn handling mode.
6286    ///
6287    /// This is the identity-first counterpart to the mob member send path used
6288    /// by the console. Ordinary API callers can keep using [`Self::send`],
6289    /// which preserves queue semantics.
6290    pub async fn send_with_mode(
6291        &self,
6292        identity: &AgentIdentity,
6293        content: &meerkat_core::ContentInput,
6294        handling_mode: HandlingMode,
6295    ) -> Result<FencingToken, IdentityRuntimeError> {
6296        self.send_with_mode_and_interaction(identity, content, handling_mode, None)
6297            .await
6298    }
6299
6300    /// Cancellation-safe send that also returns the completion baseline to
6301    /// wait past.
6302    ///
6303    /// This is the delivery entry point every surface should use when the
6304    /// caller intends to wait for the answer: it is the only one that hands
6305    /// back a [`CompletionCursor`] captured before delivery, which is what
6306    /// makes "wait for MY turn" expressible without comparing output text.
6307    /// `expected_alias` pins the delivery to a generated runtime alias the
6308    /// caller already resolved; pass `None` for the durable identity.
6309    pub async fn send_admission_tracked(
6310        self: &Arc<Self>,
6311        identity: &AgentIdentity,
6312        expected_alias: Option<&str>,
6313        content: &meerkat_core::ContentInput,
6314        handling_mode: HandlingMode,
6315        interaction_id: Option<&str>,
6316    ) -> Result<SendAdmission, IdentityRuntimeError> {
6317        let runtime = Arc::clone(self);
6318        let identity = identity.clone();
6319        let expected_alias = expected_alias.map(ToString::to_string);
6320        let content = content.clone();
6321        let interaction_id = interaction_id.map(ToString::to_string);
6322        self.run_tracked_foreground(async move {
6323            runtime
6324                .send_with_mode_and_interaction_with_expected_member_alias(
6325                    &identity,
6326                    expected_alias.as_deref(),
6327                    &content,
6328                    handling_mode,
6329                    interaction_id.as_deref(),
6330                )
6331                .await
6332                .map(|(fencing_token, completion_baseline)| SendAdmission {
6333                    fencing_token,
6334                    completion_baseline,
6335                })
6336        })
6337        .await
6338    }
6339
6340    /// Cancellation-safe explicit-mode send for RPC/host request boundaries.
6341    pub async fn send_with_mode_tracked(
6342        self: &Arc<Self>,
6343        identity: &AgentIdentity,
6344        content: &meerkat_core::ContentInput,
6345        handling_mode: HandlingMode,
6346    ) -> Result<FencingToken, IdentityRuntimeError> {
6347        self.send_with_mode_and_interaction_tracked(identity, content, handling_mode, None)
6348            .await
6349    }
6350
6351    /// Cancellation-safe interaction send for RPC/host request boundaries.
6352    pub async fn send_with_mode_and_interaction_tracked(
6353        self: &Arc<Self>,
6354        identity: &AgentIdentity,
6355        content: &meerkat_core::ContentInput,
6356        handling_mode: HandlingMode,
6357        interaction_id: Option<&str>,
6358    ) -> Result<FencingToken, IdentityRuntimeError> {
6359        self.send_admission_tracked(identity, None, content, handling_mode, interaction_id)
6360            .await
6361            .map(|admission| admission.fencing_token)
6362    }
6363
6364    /// Cancellation-safe interaction send pinned to the generated runtime
6365    /// alias that the caller resolved. Validation and delivery share the
6366    /// identity lifecycle lock, so a concurrent reset cannot retarget the
6367    /// request onto the replacement generation.
6368    pub async fn send_with_mode_and_interaction_member_alias_tracked(
6369        self: &Arc<Self>,
6370        identity: &AgentIdentity,
6371        expected_alias: &str,
6372        content: &meerkat_core::ContentInput,
6373        handling_mode: HandlingMode,
6374        interaction_id: Option<&str>,
6375    ) -> Result<FencingToken, IdentityRuntimeError> {
6376        self.send_admission_tracked(
6377            identity,
6378            Some(expected_alias),
6379            content,
6380            handling_mode,
6381            interaction_id,
6382        )
6383        .await
6384        .map(|admission| admission.fencing_token)
6385    }
6386
6387    /// [`Self::send_with_mode`] with a host-minted interaction id (meerkat
6388    /// 0.7.25 ask 15 addendum). The id rides `WorkSpec` into runtime
6389    /// admission, so the turn's live events and its committed transcript
6390    /// messages carry the same identity the console stamped on its frames —
6391    /// the exact live↔history join the console dedup needs. Only UUID-form
6392    /// ids thread; others are delivered without one.
6393    pub async fn send_with_mode_and_interaction(
6394        &self,
6395        identity: &AgentIdentity,
6396        content: &meerkat_core::ContentInput,
6397        handling_mode: HandlingMode,
6398        interaction_id: Option<&str>,
6399    ) -> Result<FencingToken, IdentityRuntimeError> {
6400        self.send_with_mode_and_interaction_with_expected_member_alias(
6401            identity,
6402            None,
6403            content,
6404            handling_mode,
6405            interaction_id,
6406        )
6407        .await
6408        .map(|(token, _)| token)
6409    }
6410
6411    async fn send_with_mode_and_interaction_with_expected_member_alias(
6412        &self,
6413        identity: &AgentIdentity,
6414        expected_alias: Option<&str>,
6415        content: &meerkat_core::ContentInput,
6416        handling_mode: HandlingMode,
6417        interaction_id: Option<&str>,
6418    ) -> Result<(FencingToken, CompletionCursor), IdentityRuntimeError> {
6419        let should_materialize = {
6420            let entries = self.entries.read().await;
6421            let entry = entries
6422                .get(identity)
6423                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6424
6425            // REQ-03: reject send to InternalOnly
6426            if entry.spec.addressability == AgentAddressability::InternalOnly {
6427                return Err(IdentityRuntimeError::NotAddressable(NotAddressable {
6428                    identity: identity.clone(),
6429                    addressability: entry.spec.addressability,
6430                }));
6431            }
6432            entry.state == IdentityLifecycleState::Dormant
6433                || entry.state == IdentityLifecycleState::Uninitialized
6434        };
6435        if should_materialize {
6436            self.materialize_with_expected_member_alias(identity, expected_alias)
6437                .await?;
6438        }
6439        // Live steers are latency-sensitive operator input for an already
6440        // active turn. Ordinary sends may hydrate the reachable topology first,
6441        // but a steer must reach the current session boundary before the tool
6442        // turn resumes; background/full-fleet materialization owns the peers.
6443        if handling_mode != HandlingMode::Steer {
6444            if let Some(expected_alias) = expected_alias {
6445                let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6446                let _lifecycle_guard = lifecycle_lock.lock().await;
6447                self.ensure_expected_member_alias_current(identity, expected_alias)
6448                    .await?;
6449            }
6450            self.materialize_reachable_peers(identity).await?;
6451        }
6452
6453        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6454        let _lifecycle_guard = lifecycle_lock.lock().await;
6455        if let Some(expected_alias) = expected_alias {
6456            self.ensure_expected_member_alias_current(identity, expected_alias)
6457                .await?;
6458        }
6459        {
6460            let entries = self.entries.read().await;
6461            let entry = entries
6462                .get(identity)
6463                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6464            if entry.state != IdentityLifecycleState::Active {
6465                return Err(IdentityRuntimeError::InvalidState {
6466                    identity: identity.clone(),
6467                    state: entry.state,
6468                    operation: "send",
6469                });
6470            }
6471        }
6472
6473        let mut token = self.ensure_active_lease(identity).await?;
6474        // Read the completion baseline BEFORE delivery is attempted. The turn
6475        // this send starts can only complete after this point, so a caller
6476        // waiting past the baseline cannot miss it. The converse ambiguity is
6477        // deliberate and documented: on an identity receiving concurrent
6478        // traffic, another delivery's completion can also satisfy the wait.
6479        // Waiting too little beats the failure this replaces (waiting forever).
6480        let completion_baseline = self.rebase_completion_cursor(identity, token);
6481        let (runtime_id, memory_session_key, memory_generation, bridge_interaction_id) = {
6482            let entries = self.entries.read().await;
6483            let entry = entries
6484                .get(identity)
6485                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6486            (
6487                entry
6488                    .continuity
6489                    .as_ref()
6490                    .map(|c| c.agent_runtime_id.clone()),
6491                // Scopes the injector's cross-turn dedup + cumulative budget.
6492                entry.continuity.as_ref().map(|c| c.session_id.to_string()),
6493                entry.continuity.as_ref().map(|c| c.generation.get()),
6494                interaction_id_for_delivery(&entry.spec, interaction_id),
6495            )
6496        };
6497        let (content_to_deliver, injected_context) = self
6498            .prepare_member_delivery(
6499                identity,
6500                content,
6501                memory_session_key.as_deref(),
6502                memory_generation,
6503                handling_mode == HandlingMode::Steer,
6504            )
6505            .await?;
6506
6507        // Deliver through the session bridge when available.
6508        if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id) {
6509            let delivered_session_id = bridge
6510                .deliver_with_mode_and_context(
6511                    rid,
6512                    &content_to_deliver,
6513                    &injected_context,
6514                    handling_mode,
6515                    bridge_interaction_id,
6516                )
6517                .await
6518                .map_err(|e| IdentityRuntimeError::Internal(format!("bridge deliver: {e}")))?;
6519            if let Some(rebound_token) = self
6520                .reconcile_delivered_session_locked(identity, delivered_session_id)
6521                .await?
6522            {
6523                token = rebound_token;
6524            }
6525        }
6526
6527        Ok((token, completion_baseline))
6528    }
6529
6530    // -----------------------------------------------------------------------
6531    // Delivery: dispatch() — REQ-02
6532    // -----------------------------------------------------------------------
6533
6534    /// Dispatch internal content to any identity (Addressable or InternalOnly).
6535    ///
6536    /// Enforces:
6537    /// - Identity must be registered and active
6538    /// - Lease must be held (INV-01)
6539    /// - Lease must not be lost (INV-02)
6540    ///
6541    /// Returns (fencing_token, is_durable) where is_durable indicates whether
6542    /// the dispatch is backed by a runtime_store (REQ-04).
6543    pub async fn dispatch(
6544        &self,
6545        identity: &AgentIdentity,
6546        input: &DispatchInput,
6547    ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6548        self.dispatch_with_expected_member_alias(identity, None, input)
6549            .await
6550            .map(|outcome| (outcome.admission.fencing_token, outcome.admission.durable))
6551    }
6552
6553    /// The ONE delivery preparation both member doors run (task #54): the
6554    /// send door always had it; the dispatch door previously delivered raw,
6555    /// so internal dispatches (schedules foremost) skipped defanging, taint
6556    /// session attribution, and ambient memory injection for the member's
6557    /// whole lifetime (HomeCore: zero surface=Turn injection-ledger rows).
6558    ///
6559    /// Steer is latency-sensitive live operator input: it bypasses both
6560    /// memory injection and inbound defanging by design. Every other
6561    /// delivery is defanged first (§9.1 anti-spoofing - even with injection
6562    /// off, forged memory envelopes are an inbound threat) and only then
6563    /// considered for ambient injection. Ask 1: the user content and the
6564    /// ambient recall travel as SEPARATE bodies - the (defanged) message and
6565    /// the recall as its own typed injected-context body, never fused.
6566    /// §10.1 taint hook: `note_current_session` keeps session attribution
6567    /// authoritative ahead of the async observe stream; the generation bind
6568    /// feeds the Distiller's EvidenceRefs (§8.4).
6569    async fn prepare_member_delivery(
6570        &self,
6571        identity: &AgentIdentity,
6572        content: &meerkat_core::ContentInput,
6573        memory_session_key: Option<&str>,
6574        memory_generation: Option<u64>,
6575        steer: bool,
6576    ) -> Result<(meerkat_core::ContentInput, Vec<meerkat_core::ContentInput>), IdentityRuntimeError>
6577    {
6578        if steer {
6579            return Ok((content.clone(), Vec::new()));
6580        }
6581        match self.agent_memory.read().await.clone() {
6582            Some(injector) => {
6583                if let Some(session_key) = memory_session_key {
6584                    injector.note_current_session(identity, session_key);
6585                    if let Some(generation) = memory_generation {
6586                        injector.note_session_generation(identity, session_key, generation);
6587                    }
6588                }
6589                let defanged = injector.defang_inbound(identity, content);
6590                let injected_context = injector
6591                    .inject_for_turn(identity, memory_session_key, &defanged)
6592                    .await
6593                    .map_err(|err| {
6594                        IdentityRuntimeError::Internal(format!("agent memory recall: {err}"))
6595                    })?;
6596                Ok((defanged, injected_context))
6597            }
6598            None => Ok((content.clone(), Vec::new())),
6599        }
6600    }
6601
6602    async fn dispatch_with_expected_member_alias(
6603        &self,
6604        identity: &AgentIdentity,
6605        expected_alias: Option<&str>,
6606        input: &DispatchInput,
6607    ) -> Result<DispatchOutcome, IdentityRuntimeError> {
6608        let should_materialize = {
6609            let entries = self.entries.read().await;
6610            let entry = entries
6611                .get(identity)
6612                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6613            entry.state == IdentityLifecycleState::Dormant
6614                || entry.state == IdentityLifecycleState::Uninitialized
6615        };
6616        if should_materialize {
6617            self.materialize_with_expected_member_alias(identity, expected_alias)
6618                .await?;
6619        }
6620        if let Some(expected_alias) = expected_alias {
6621            let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6622            let _lifecycle_guard = lifecycle_lock.lock().await;
6623            self.ensure_expected_member_alias_current(identity, expected_alias)
6624                .await?;
6625        }
6626        self.materialize_reachable_peers(identity).await?;
6627
6628        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6629        let _lifecycle_guard = lifecycle_lock.lock().await;
6630        if let Some(expected_alias) = expected_alias {
6631            self.ensure_expected_member_alias_current(identity, expected_alias)
6632                .await?;
6633        }
6634        {
6635            let entries = self.entries.read().await;
6636            let entry = entries
6637                .get(identity)
6638                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6639            if entry.state != IdentityLifecycleState::Active {
6640                return Err(IdentityRuntimeError::InvalidState {
6641                    identity: identity.clone(),
6642                    state: entry.state,
6643                    operation: "dispatch",
6644                });
6645            }
6646        }
6647
6648        let mut token = self.ensure_active_lease(identity).await?;
6649        // Same pre-delivery baseline contract as the send path — see
6650        // `send_with_mode_and_interaction_with_expected_member_alias`.
6651        let completion_baseline = self.rebase_completion_cursor(identity, token);
6652        let (is_durable, runtime_id, memory_session_key, memory_generation) = {
6653            let entries = self.entries.read().await;
6654            let entry = entries
6655                .get(identity)
6656                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6657            // REQ-04: durability depends on runtime_store
6658            let is_durable = entry.has_runtime_store;
6659
6660            let runtime_id = entry
6661                .continuity
6662                .as_ref()
6663                .map(|c| c.agent_runtime_id.clone());
6664
6665            (
6666                is_durable,
6667                runtime_id,
6668                entry.continuity.as_ref().map(|c| c.session_id.to_string()),
6669                entry.continuity.as_ref().map(|c| c.generation.get()),
6670            )
6671        };
6672
6673        // Task #54: the dispatch door runs the SAME delivery preparation as
6674        // the send door (defang, taint session attribution, ambient memory
6675        // injection). Dispatch has no Steer mode; `DispatchOrigin` rides the
6676        // input untouched.
6677        let (content_to_deliver, injected_context) = self
6678            .prepare_member_delivery(
6679                identity,
6680                &input.content,
6681                memory_session_key.as_deref(),
6682                memory_generation,
6683                false,
6684            )
6685            .await?;
6686        // Task #50 fail-closed matrix, validated BEFORE bridge admission:
6687        // (None, None) is an ordinary delivery; a full pair validates
6688        // upstream (canonical non-nil UUID correlation) and carries; EVERY
6689        // other combination - half-pair, invalid UUID - is a typed refusal
6690        // and NO delivery occurs. Never warn-and-degrade: a degraded
6691        // delivery under a broken identity is a silent dedup hole.
6692        let delivery_identity = match (&input.idempotency_key, &input.correlation_id) {
6693            (None, None) => None,
6694            (Some(idempotency_key), Some(correlation_id)) => {
6695                // App-supplied correlations canonicalize deterministically
6696                // instead of refusing on value shape (HomeCore admission
6697                // break: source-string correlations were tolerated through
6698                // the 0.8.15 pair and refused by 0.8.16 identity threading).
6699                // Half-pairs below stay typed refusals - structure is still
6700                // fail closed; only the VALUE domain is widened, and ONLY
6701                // for the app-reachable dispatch origins (Connector, plus
6702                // the RPC surface's System default). Code-owned lanes -
6703                // Scheduler, Policy, Flow - mint their own canonical UUIDs,
6704                // so a non-canonical value there is a defect to refuse
6705                // typed (the task #50 matrix), never to mask.
6706                let app_lane = matches!(
6707                    input.origin,
6708                    super::types::DispatchOrigin::Connector | super::types::DispatchOrigin::System
6709                );
6710                let canonical = if app_lane {
6711                    crate::member_comms_id::canonical_correlation_id(correlation_id.as_str())
6712                } else {
6713                    std::borrow::Cow::Borrowed(correlation_id.as_str())
6714                };
6715                if canonical.as_ref() != correlation_id.as_str() {
6716                    tracing::info!(
6717                        identity = %identity,
6718                        canonical_correlation = %canonical,
6719                        source_len = correlation_id.as_str().len(),
6720                        "app-supplied delivery correlation canonicalized to UUIDv5 \
6721                         for bridge admission"
6722                    );
6723                }
6724                Some(
6725                    meerkat_mob::MobDeliveryIdentity::new(
6726                        idempotency_key.as_str(),
6727                        canonical.as_ref(),
6728                    )
6729                    .map_err(|error| {
6730                        IdentityRuntimeError::InvalidDeliveryIdentity {
6731                            identity: identity.clone(),
6732                            detail: error.to_string(),
6733                        }
6734                    })?,
6735                )
6736            }
6737            (Some(_), None) => {
6738                return Err(IdentityRuntimeError::InvalidDeliveryIdentity {
6739                    identity: identity.clone(),
6740                    detail: "idempotency key without a correlation id (half-pair)".to_string(),
6741                });
6742            }
6743            (None, Some(_)) => {
6744                return Err(IdentityRuntimeError::InvalidDeliveryIdentity {
6745                    identity: identity.clone(),
6746                    detail: "correlation id without an idempotency key (half-pair)".to_string(),
6747                });
6748            }
6749        };
6750
6751        // Deliver through the session bridge when available. When the dedup
6752        // carrier is present its correlation id also rides as the
6753        // interaction id.
6754        let mut dispatched_session_id = None;
6755        if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id) {
6756            let mut delivery =
6757                super::bridge::BridgeDelivery::new(content_to_deliver.clone(), HandlingMode::Queue);
6758            delivery.injected_context = injected_context.clone();
6759            delivery.interaction_id = delivery_identity
6760                .as_ref()
6761                .map(|identity| identity.correlation_id.clone());
6762            delivery.delivery_identity = delivery_identity.clone();
6763            let delivered_session_id = bridge
6764                .deliver_admitted(rid, delivery)
6765                .await
6766                .map_err(|e| IdentityRuntimeError::Internal(format!("bridge dispatch: {e}")))?;
6767            dispatched_session_id = Some(delivered_session_id.clone());
6768            if let Some(rebound_token) = self
6769                .reconcile_delivered_session_locked(identity, delivered_session_id)
6770                .await?
6771            {
6772                token = rebound_token;
6773            }
6774        }
6775
6776        Ok(DispatchOutcome {
6777            admission: DispatchAdmission {
6778                fencing_token: token,
6779                durable: is_durable,
6780                completion_baseline,
6781            },
6782            session_id: dispatched_session_id,
6783        })
6784    }
6785
6786    /// Cancellation-safe dispatch for RPC/host request boundaries.
6787    pub async fn dispatch_tracked(
6788        self: &Arc<Self>,
6789        identity: &AgentIdentity,
6790        input: &DispatchInput,
6791    ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6792        let runtime = Arc::clone(self);
6793        let identity = identity.clone();
6794        let input = input.clone();
6795        self.run_tracked_foreground(async move { runtime.dispatch(&identity, &input).await })
6796            .await
6797    }
6798
6799    /// Cancellation-safe dispatch pinned to the generated runtime alias that
6800    /// the caller resolved.
6801    pub async fn dispatch_member_alias_tracked(
6802        self: &Arc<Self>,
6803        identity: &AgentIdentity,
6804        expected_alias: &str,
6805        input: &DispatchInput,
6806    ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6807        self.dispatch_admission_tracked(identity, Some(expected_alias), input)
6808            .await
6809            .map(|admission| (admission.fencing_token, admission.durable))
6810    }
6811
6812    /// Cancellation-safe dispatch that also returns the completion baseline to
6813    /// wait past. Dispatch counterpart of [`Self::send_admission_tracked`];
6814    /// pass `expected_alias = None` for the durable identity.
6815    pub async fn dispatch_admission_tracked(
6816        self: &Arc<Self>,
6817        identity: &AgentIdentity,
6818        expected_alias: Option<&str>,
6819        input: &DispatchInput,
6820    ) -> Result<DispatchAdmission, IdentityRuntimeError> {
6821        let runtime = Arc::clone(self);
6822        let identity = identity.clone();
6823        let expected_alias = expected_alias.map(ToString::to_string);
6824        let input = input.clone();
6825        self.run_tracked_foreground(async move {
6826            runtime
6827                .dispatch_with_expected_member_alias(&identity, expected_alias.as_deref(), &input)
6828                .await
6829                .map(|outcome| outcome.admission)
6830        })
6831        .await
6832    }
6833
6834    /// Scheduler delivery pinned to a generated alias, returning the exact
6835    /// bridge session that accepted the work while the lifecycle lock held.
6836    pub(crate) async fn dispatch_member_alias_with_session_tracked(
6837        self: &Arc<Self>,
6838        identity: &AgentIdentity,
6839        expected_alias: &str,
6840        input: &DispatchInput,
6841    ) -> Result<Option<SessionId>, IdentityRuntimeError> {
6842        let runtime = Arc::clone(self);
6843        let identity = identity.clone();
6844        let expected_alias = expected_alias.to_string();
6845        let input = input.clone();
6846        self.run_tracked_foreground(async move {
6847            runtime
6848                .dispatch_with_expected_member_alias(&identity, Some(&expected_alias), &input)
6849                .await
6850                .map(|outcome| outcome.session_id)
6851        })
6852        .await
6853    }
6854
6855    // -----------------------------------------------------------------------
6856    // Status: status() — REQ-07
6857    // -----------------------------------------------------------------------
6858
6859    /// Return the full identity status for the given identity.
6860    pub async fn status(
6861        &self,
6862        identity: &AgentIdentity,
6863    ) -> Result<IdentityStatus, IdentityRuntimeError> {
6864        let entries = self.entries.read().await;
6865        let entry = entries
6866            .get(identity)
6867            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6868
6869        let lease_info = entry.lease.as_ref().map(|l| LeaseInfo {
6870            fencing_token: l.fencing_token,
6871            ttl_remaining: l.ttl_remaining(),
6872            healthy: l.is_healthy(),
6873        });
6874
6875        let continuity_health = Some(ContinuityHealth {
6876            store_reachable: true, // tracked per-store in production
6877            durability_policy: self.durability_policy.clone(),
6878            last_checkpoint_version: if entry.checkpoint_version.get() > 0 {
6879                Some(entry.checkpoint_version)
6880            } else {
6881                None
6882            },
6883        });
6884
6885        Ok(IdentityStatus {
6886            identity: identity.clone(),
6887            state: entry.state,
6888            agent_runtime_id: entry
6889                .continuity
6890                .as_ref()
6891                .map(|c| c.agent_runtime_id.clone()),
6892            session_id: entry.continuity.as_ref().map(|c| c.session_id.clone()),
6893            profile: Some(entry.spec.profile.clone()),
6894            runtime_mode: entry.spec.runtime_mode_override,
6895            addressability: entry.spec.addressability,
6896            display_name: entry.spec.display_name.clone(),
6897            labels: entry.spec.labels.clone(),
6898            generation: entry.continuity.as_ref().map(|c| c.generation),
6899            checkpoint_version: if entry.checkpoint_version.get() > 0 {
6900                Some(entry.checkpoint_version)
6901            } else {
6902                None
6903            },
6904            lease: lease_info,
6905            continuity_health,
6906            continuity_unrecoverable: entry.continuity_unrecoverable.clone(),
6907        })
6908    }
6909
6910    /// Return statuses for every registered identity without materializing
6911    /// dormant members.
6912    pub async fn statuses(&self) -> Vec<IdentityStatus> {
6913        let identities = self
6914            .entries
6915            .read()
6916            .await
6917            .keys()
6918            .cloned()
6919            .collect::<Vec<_>>();
6920        let mut statuses = Vec::with_capacity(identities.len());
6921        for identity in identities {
6922            if let Ok(status) = self.status(&identity).await {
6923                statuses.push(status);
6924            }
6925        }
6926        statuses
6927    }
6928
6929    // -----------------------------------------------------------------------
6930    // Lifecycle: retire() — REQ-08
6931    // -----------------------------------------------------------------------
6932
6933    /// Retire an identity. Validates lease ownership and retires the mob member.
6934    pub async fn retire(
6935        &self,
6936        identity: &AgentIdentity,
6937    ) -> Result<FencingToken, IdentityRuntimeError> {
6938        self.retire_with_expected_member_alias(identity, None).await
6939    }
6940
6941    async fn retire_with_expected_member_alias(
6942        &self,
6943        identity: &AgentIdentity,
6944        expected_alias: Option<&str>,
6945    ) -> Result<FencingToken, IdentityRuntimeError> {
6946        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6947        let _lifecycle_guard = lifecycle_lock.lock().await;
6948        if let Some(expected_alias) = expected_alias {
6949            self.ensure_expected_member_alias_current(identity, expected_alias)
6950                .await?;
6951        }
6952        self.retire_locked(identity).await
6953    }
6954
6955    async fn retire_locked(
6956        &self,
6957        identity: &AgentIdentity,
6958    ) -> Result<FencingToken, IdentityRuntimeError> {
6959        self.ensure_active_lease(identity).await?;
6960        let registered_entry = self
6961            .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Retiring)
6962            .await?;
6963        let _previous_token = match Self::check_lease(&registered_entry) {
6964            Ok(token) => token,
6965            Err(err) => {
6966                self.restore_entry(identity, registered_entry).await;
6967                return Err(err);
6968            }
6969        };
6970        let runtime_id = registered_entry
6971            .continuity
6972            .as_ref()
6973            .map(|c| c.agent_runtime_id.clone());
6974        let session_id = registered_entry
6975            .continuity
6976            .as_ref()
6977            .map(|c| c.session_id.clone());
6978
6979        let acquire_result = match self
6980            .lease_provider
6981            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
6982            .await
6983        {
6984            Ok(result) => result,
6985            Err(err) => {
6986                self.restore_entry(identity, registered_entry).await;
6987                return Err(IdentityRuntimeError::Lease(err));
6988            }
6989        };
6990
6991        let grant = match acquire_result.get(identity) {
6992            Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
6993            _ => {
6994                self.restore_entry(identity, registered_entry).await;
6995                return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
6996            }
6997        };
6998        if let Err(err) = self
6999            .advance_existing_continuity_fence(identity, &registered_entry, &grant)
7000            .await
7001        {
7002            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7003                .await;
7004            return Err(err);
7005        }
7006        // The durable fence now belongs to the retire transaction. Refresh
7007        // the bridge adapter before retirement so Meerkat's terminal archive
7008        // projection carries that same token. Leaving the live session on the
7009        // prior token makes ArchiveSession fail closed with a stale fence and
7010        // strands the member in Retiring.
7011        if let Some(record) = registered_entry.continuity.as_ref()
7012            && let Err(error) = self
7013                .refresh_existing_session_runtime_state(identity, record, &grant)
7014                .await
7015        {
7016            self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
7017                .await;
7018            return Err(IdentityRuntimeError::Internal(format!(
7019                "bridge refresh session authority before retire: {error}"
7020            )));
7021        }
7022
7023        // §8.4 trigger (b): distill the outgoing session's tail BEFORE the
7024        // member retires. Best-effort and bounded — retirement proceeds at
7025        // the distiller's pre-rotation timeout.
7026        if let Some(injector) = self.agent_memory.read().await.clone() {
7027            if let Some(session_id) = session_id.as_ref() {
7028                injector
7029                    .distill_before_rotation(
7030                        identity,
7031                        &session_id.to_string(),
7032                        crate::memory::distiller::DistillCause::Retire,
7033                    )
7034                    .await;
7035            }
7036            // §8.5 exit interview: queue the retired identity's store for
7037            // the next dream's harvest sub-phase.
7038            injector
7039                .note_identity_retired(
7040                    identity,
7041                    session_id
7042                        .as_ref()
7043                        .map(std::string::ToString::to_string)
7044                        .as_deref(),
7045                    "retire",
7046                )
7047                .await;
7048        }
7049
7050        // Retire the mob member through the session bridge when available.
7051        if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id)
7052            && let Err(err) = bridge.retire_member(rid).await
7053        {
7054            self.restore_entry_with_grant(identity, registered_entry, &grant)
7055                .await;
7056            return Err(IdentityRuntimeError::Internal(format!(
7057                "bridge retire: {err}"
7058            )));
7059        }
7060        if let (Some(bridge), Some(session_id)) = (&self.bridge, &session_id)
7061            && let Err(err) = bridge.unregister_session_runtime_state(session_id).await
7062        {
7063            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7064                .await;
7065            return Err(IdentityRuntimeError::Internal(format!(
7066                "bridge unregister retired session: {err}"
7067            )));
7068        }
7069
7070        let release_result = self
7071            .lease_provider
7072            .release_leases(std::slice::from_ref(&grant))
7073            .await;
7074        let (state, bootstrap_error) = match &release_result {
7075            Ok(()) => (IdentityLifecycleState::Retiring, None),
7076            Err(error) => (
7077                IdentityLifecycleState::Broken,
7078                Some(format!("lease release after retire: {error}")),
7079            ),
7080        };
7081        {
7082            let mut entries = self.entries.write().await;
7083            let entry = entries
7084                .get_mut(identity)
7085                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7086            entry.state = state;
7087            entry.lease = None;
7088            entry.pending_lease_release = release_result.as_ref().err().map(|_| grant.clone());
7089        }
7090        self.mark_bootstrap_from_lifecycle(identity, state, bootstrap_error);
7091        release_result.map_err(IdentityRuntimeError::Lease)?;
7092        Ok(grant.fencing_token)
7093    }
7094
7095    // -----------------------------------------------------------------------
7096    // Lifecycle: respawn() — REQ-09
7097    // -----------------------------------------------------------------------
7098
7099    /// Respawn: non-destructive recovery.
7100    ///
7101    /// 1. Fence the current owner
7102    /// 2. Attempt final checkpoint
7103    /// 3. Reactivate from authoritative continuity with same record + runtime ID
7104    /// 4. ContinuityGeneration does NOT advance
7105    pub async fn respawn(
7106        &self,
7107        identity: &AgentIdentity,
7108    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7109        self.respawn_with_expected_member_alias(identity, None)
7110            .await
7111    }
7112
7113    async fn respawn_with_expected_member_alias(
7114        &self,
7115        identity: &AgentIdentity,
7116        expected_alias: Option<&str>,
7117    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7118        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7119        let _lifecycle_guard = lifecycle_lock.lock().await;
7120        if let Some(expected_alias) = expected_alias {
7121            self.ensure_expected_member_alias_current(identity, expected_alias)
7122                .await?;
7123        }
7124        self.respawn_locked(identity).await
7125    }
7126
7127    /// Perform the identity-side half of respawn while the caller holds the
7128    /// per-identity lifecycle lock.
7129    async fn respawn_locked(
7130        &self,
7131        identity: &AgentIdentity,
7132    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7133        let registered_entry = self
7134            .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7135            .await?;
7136
7137        // Fence the old owner by re-acquiring the lease
7138        let acquire_result = match self
7139            .lease_provider
7140            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7141            .await
7142        {
7143            Ok(result) => result,
7144            Err(err) => {
7145                self.restore_entry(identity, registered_entry).await;
7146                return Err(IdentityRuntimeError::Lease(err));
7147            }
7148        };
7149
7150        let grant = match acquire_result.get(identity) {
7151            Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7152            _ => {
7153                self.restore_entry(identity, registered_entry).await;
7154                return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7155            }
7156        };
7157
7158        if let Err(err) = self
7159            .advance_existing_continuity_fence(identity, &registered_entry, &grant)
7160            .await
7161        {
7162            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7163                .await;
7164            return Err(err);
7165        }
7166
7167        // Resolve current continuity state
7168        let resolved = match self
7169            .continuity_store
7170            .resolve_many(std::slice::from_ref(identity))
7171            .await
7172        {
7173            Ok(resolved) => resolved,
7174            Err(err) => {
7175                self.restore_entry_with_grant(identity, registered_entry.clone(), &grant)
7176                    .await;
7177                return Err(IdentityRuntimeError::Store(err));
7178            }
7179        };
7180
7181        let record = match resolved.get(identity) {
7182            Some(super::types::ContinuityResolveState::Ready { record }) => record.clone(),
7183            Some(super::types::ContinuityResolveState::Broken { failure }) => {
7184                self.restore_entry_with_grant(identity, registered_entry, &grant)
7185                    .await;
7186                return Err(IdentityRuntimeError::Internal(format!(
7187                    "broken continuity for {identity}: {}",
7188                    failure.detail
7189                )));
7190            }
7191            Some(super::types::ContinuityResolveState::Uninitialized) => {
7192                self.restore_entry_with_grant(identity, registered_entry, &grant)
7193                    .await;
7194                return Err(IdentityRuntimeError::Internal(format!(
7195                    "cannot respawn uninitialized identity {identity}"
7196                )));
7197            }
7198            None => {
7199                self.restore_entry_with_grant(identity, registered_entry, &grant)
7200                    .await;
7201                return Err(IdentityRuntimeError::Store(
7202                    ContinuityStoreError::NotFound {
7203                        identity: identity.clone(),
7204                    },
7205                ));
7206            }
7207        };
7208
7209        // §8.4 trigger (b): respawn is a recovery boundary — harvest the
7210        // session's window before the runtime refreshes (the SessionId does
7211        // not rotate here; the cursor stays valid). Bounded; respawn
7212        // proceeds at the pre-rotation timeout.
7213        if let Some(injector) = self.agent_memory.read().await.clone() {
7214            let session_key = record.session_id.to_string();
7215            injector.note_session_generation(identity, &session_key, record.generation.get());
7216            injector
7217                .distill_before_rotation(
7218                    identity,
7219                    &session_key,
7220                    crate::memory::distiller::DistillCause::Respawn,
7221                )
7222                .await;
7223        }
7224
7225        let effective_checkpoint_version = match self
7226            .refresh_existing_session_runtime_state(identity, &record, &grant)
7227            .await
7228        {
7229            Ok(version) => version,
7230            Err(err) => {
7231                self.restore_entry_with_grant(identity, registered_entry, &grant)
7232                    .await;
7233                return Err(err);
7234            }
7235        };
7236        let mut record = record;
7237        record.checkpoint_version = effective_checkpoint_version;
7238
7239        // Update runtime state: same record, new lease, back to Active
7240        let mut entries = self.entries.write().await;
7241        let Some(entry) = entries.get_mut(identity) else {
7242            drop(entries);
7243            if let Err(err) = self
7244                .release_or_park_untracked_leases(std::slice::from_ref(&grant))
7245                .await
7246            {
7247                tracing::warn!(
7248                    %identity,
7249                    error = %err,
7250                    "failed to release lease after respawn entry disappeared; exact grant parked for retry"
7251                );
7252            }
7253            return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7254        };
7255        entry.continuity = Some(record.clone());
7256        entry.lease = Some(LeaseEntry {
7257            fencing_token: grant.fencing_token,
7258            ttl: grant.ttl,
7259            acquired_at: Instant::now(),
7260        });
7261        entry.state = IdentityLifecycleState::Active;
7262        entry.checkpoint_version = record.checkpoint_version;
7263        drop(entries);
7264        self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
7265
7266        Ok(record)
7267    }
7268
7269    /// Rebind continuity to the concrete session created by a lower-level
7270    /// member respawn. This keeps identity-first status aligned when a control
7271    /// surface refreshes the mob member outside the identity runtime bridge.
7272    pub async fn rebind_session_after_live_respawn(
7273        &self,
7274        identity: &AgentIdentity,
7275        session_id: SessionId,
7276    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7277        self.rebind_session_after_live_respawn_with_expected_member_alias(
7278            identity, None, session_id,
7279        )
7280        .await
7281    }
7282
7283    async fn rebind_session_after_live_respawn_with_expected_member_alias(
7284        &self,
7285        identity: &AgentIdentity,
7286        expected_alias: Option<&str>,
7287        session_id: SessionId,
7288    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7289        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7290        let _lifecycle_guard = lifecycle_lock.lock().await;
7291        if let Some(expected_alias) = expected_alias {
7292            self.ensure_expected_member_alias_current(identity, expected_alias)
7293                .await?;
7294        }
7295        self.rebind_session_after_live_respawn_locked(identity, session_id)
7296            .await
7297    }
7298
7299    async fn reconcile_delivered_session_locked(
7300        &self,
7301        identity: &AgentIdentity,
7302        delivered_session_id: SessionId,
7303    ) -> Result<Option<FencingToken>, IdentityRuntimeError> {
7304        let current_session_id = {
7305            let entries = self.entries.read().await;
7306            let entry = entries
7307                .get(identity)
7308                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7309            entry
7310                .continuity
7311                .as_ref()
7312                .map(|record| record.session_id.clone())
7313        };
7314
7315        let Some(current_session_id) = current_session_id else {
7316            return Ok(None);
7317        };
7318        if current_session_id == delivered_session_id {
7319            return Ok(None);
7320        }
7321
7322        tracing::warn!(
7323            %identity,
7324            old_session_id = %current_session_id,
7325            new_session_id = %delivered_session_id,
7326            "identity bridge delivery returned a rotated session; rebinding continuity"
7327        );
7328        self.rebind_session_after_live_respawn_locked(identity, delivered_session_id)
7329            .await?;
7330
7331        let entries = self.entries.read().await;
7332        let entry = entries
7333            .get(identity)
7334            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7335        Ok(entry.lease.as_ref().map(|lease| lease.fencing_token))
7336    }
7337
7338    async fn rebind_session_after_live_respawn_locked(
7339        &self,
7340        identity: &AgentIdentity,
7341        session_id: SessionId,
7342    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7343        let registered_entry = self
7344            .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7345            .await?;
7346
7347        let acquire_result = match self
7348            .lease_provider
7349            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7350            .await
7351        {
7352            Ok(result) => result,
7353            Err(err) => {
7354                self.restore_entry(identity, registered_entry).await;
7355                return Err(IdentityRuntimeError::Lease(err));
7356            }
7357        };
7358
7359        let grant = match acquire_result.get(identity) {
7360            Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7361            _ => {
7362                self.restore_entry(identity, registered_entry).await;
7363                return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7364            }
7365        };
7366
7367        let mut record = match registered_entry.continuity.as_ref() {
7368            Some(record) => record.clone(),
7369            None => {
7370                self.restore_entry_with_grant(identity, registered_entry, &grant)
7371                    .await;
7372                return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7373            }
7374        };
7375        if let Err(err) = self
7376            .advance_existing_continuity_fence(identity, &registered_entry, &grant)
7377            .await
7378        {
7379            if let Some(bridge) = self.bridge.as_ref()
7380                && let Err(unregister_err) =
7381                    bridge.unregister_session_runtime_state(&session_id).await
7382            {
7383                tracing::warn!(
7384                    %identity,
7385                    session_id = %session_id,
7386                    error = %unregister_err,
7387                    "failed to unregister rebound session after continuity fence failure"
7388                );
7389            }
7390            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7391                .await;
7392            return Err(err);
7393        }
7394        let previous_session_id = record.session_id.clone();
7395        record.session_id = session_id;
7396
7397        if let Err(err) = self
7398            .continuity_store
7399            .upsert_continuity_record(&record, grant.fencing_token)
7400            .await
7401        {
7402            if let Some(bridge) = self.bridge.as_ref()
7403                && let Err(unregister_err) = bridge
7404                    .unregister_session_runtime_state(&record.session_id)
7405                    .await
7406            {
7407                tracing::warn!(
7408                    %identity,
7409                    session_id = %record.session_id,
7410                    error = %unregister_err,
7411                    "failed to unregister rebound session after continuity upsert failure"
7412                );
7413            }
7414            self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7415                .await;
7416            return Err(IdentityRuntimeError::Store(err));
7417        }
7418
7419        // A lower session-store save can advance the durable checkpoint head
7420        // while a respawn is rebinding to a different session id. The local
7421        // store preserves that same-generation maximum; read the effective
7422        // row back before seeding bridge runtime state so neither the in-memory
7423        // entry nor the new session counter rewinds to the stale captured
7424        // version.
7425        let effective = match self
7426            .continuity_store
7427            .resolve_many(std::slice::from_ref(identity))
7428            .await
7429        {
7430            Ok(effective) => effective,
7431            Err(error) => {
7432                self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7433                    .await;
7434                return Err(IdentityRuntimeError::Store(error));
7435            }
7436        };
7437        match effective.get(identity) {
7438            Some(super::types::ContinuityResolveState::Ready {
7439                record: effective_record,
7440            }) if effective_record.session_id == record.session_id
7441                && effective_record.generation == record.generation =>
7442            {
7443                record = effective_record.clone();
7444            }
7445            other => {
7446                self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7447                    .await;
7448                return Err(IdentityRuntimeError::Internal(format!(
7449                    "rebind continuity read-back did not return the committed session/generation: {other:?}"
7450                )));
7451            }
7452        }
7453
7454        if let Some(bridge) = self.bridge.as_ref() {
7455            match bridge
7456                .register_session_runtime_state(
7457                    &record.session_id,
7458                    identity,
7459                    record.generation,
7460                    record.checkpoint_version,
7461                    grant.fencing_token,
7462                )
7463                .await
7464            {
7465                Ok(version) => {
7466                    record.checkpoint_version =
7467                        CheckpointVersion::new(record.checkpoint_version.get().max(version.get()));
7468                }
7469                Err(err) => {
7470                    if let Err(unregister_err) = bridge
7471                        .unregister_session_runtime_state(&record.session_id)
7472                        .await
7473                    {
7474                        tracing::warn!(
7475                            %identity,
7476                            session_id = %record.session_id,
7477                            error = %unregister_err,
7478                            "failed to unregister rebound session after bridge register failure"
7479                        );
7480                    }
7481                    self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7482                        .await;
7483                    return Err(IdentityRuntimeError::Internal(format!(
7484                        "bridge rebind respawned session runtime state: {err}"
7485                    )));
7486                }
7487            }
7488            if previous_session_id != record.session_id
7489                && let Err(err) = bridge
7490                    .unregister_session_runtime_state(&previous_session_id)
7491                    .await
7492            {
7493                tracing::warn!(
7494                    %identity,
7495                    session_id = %previous_session_id,
7496                    error = %err,
7497                    "failed to unregister previous session after live respawn rebind"
7498                );
7499            }
7500        }
7501
7502        if let Err(err) = self
7503            .continuity_store
7504            .upsert_continuity_record(&record, grant.fencing_token)
7505            .await
7506        {
7507            if let Some(bridge) = self.bridge.as_ref()
7508                && let Err(unregister_err) = bridge
7509                    .unregister_session_runtime_state(&record.session_id)
7510                    .await
7511            {
7512                tracing::warn!(
7513                    %identity,
7514                    session_id = %record.session_id,
7515                    error = %unregister_err,
7516                    "failed to unregister rebound session after final continuity upsert failure"
7517                );
7518            }
7519            self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7520                .await;
7521            return Err(IdentityRuntimeError::Store(err));
7522        }
7523
7524        self.register(
7525            registered_entry.spec,
7526            IdentityLifecycleState::Active,
7527            Some(record.clone()),
7528            Some(grant),
7529        )
7530        .await;
7531        Ok(record)
7532    }
7533
7534    // -----------------------------------------------------------------------
7535    // Lifecycle: reset() — REQ-10
7536    // -----------------------------------------------------------------------
7537
7538    /// Reset: destructive continuity reset.
7539    ///
7540    /// 1. Fence old owner
7541    /// 2. Advance ContinuityGeneration
7542    /// 3. Create fresh continuity under the same AgentIdentity
7543    /// 4. Old-owner late writes rejected by stale fencing token
7544    pub async fn reset(
7545        &self,
7546        identity: &AgentIdentity,
7547    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7548        self.reset_with_expected_member_alias(identity, None).await
7549    }
7550
7551    async fn reset_with_expected_member_alias(
7552        &self,
7553        identity: &AgentIdentity,
7554        expected_alias: Option<&str>,
7555    ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7556        let mut foreground_shutdown = self.foreground_cancel.subscribe();
7557        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7558        let _lifecycle_guard = lifecycle_lock.lock().await;
7559        if let Some(expected_alias) = expected_alias {
7560            self.ensure_expected_member_alias_current(identity, expected_alias)
7561                .await?;
7562        }
7563        // Re-profile before snapshotting the lifecycle entry so the rebuilt
7564        // session uses the current roster spec, not the old checkpoint spec.
7565        self.adopt_current_roster_spec_for_reset(identity).await;
7566        let registered_entry = self
7567            .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7568            .await?;
7569
7570        // INV-05: fence the old owner first
7571        let acquire_result = match self
7572            .lease_provider
7573            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7574            .await
7575        {
7576            Ok(result) => result,
7577            Err(err) => {
7578                self.restore_entry(identity, registered_entry).await;
7579                return Err(IdentityRuntimeError::Lease(err));
7580            }
7581        };
7582
7583        let grant = match acquire_result.get(identity) {
7584            Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7585            _ => {
7586                self.restore_entry(identity, registered_entry).await;
7587                return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7588            }
7589        };
7590        if let Err(err) = self
7591            .advance_existing_continuity_fence(identity, &registered_entry, &grant)
7592            .await
7593        {
7594            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7595                .await;
7596            return Err(err);
7597        }
7598
7599        // Resolve to get current generation
7600        let resolved = match self
7601            .continuity_store
7602            .resolve_many(std::slice::from_ref(identity))
7603            .await
7604        {
7605            Ok(resolved) => resolved,
7606            Err(err) => {
7607                self.restore_entry_with_grant(identity, registered_entry.clone(), &grant)
7608                    .await;
7609                return Err(IdentityRuntimeError::Store(err));
7610            }
7611        };
7612
7613        let current_gen = match resolved.get(identity) {
7614            Some(super::types::ContinuityResolveState::Ready { record }) => record.generation,
7615            Some(super::types::ContinuityResolveState::Uninitialized) => {
7616                ContinuityGeneration::new(0)
7617            }
7618            _ => ContinuityGeneration::new(0),
7619        };
7620
7621        // Advance generation
7622        let new_gen = ContinuityGeneration::new(current_gen.get() + 1);
7623        let new_session_id = meerkat_core::types::SessionId::new();
7624        let new_runtime_id = AgentRuntimeId::parse(&format!("rt:{identity}:{}", new_gen.get()))
7625            .map_err(|e| {
7626                IdentityRuntimeError::Internal(format!("failed to mint runtime id: {e}"))
7627            })?;
7628
7629        let new_record = ContinuityRecord {
7630            identity: identity.clone(),
7631            agent_runtime_id: new_runtime_id,
7632            session_id: new_session_id,
7633            generation: new_gen,
7634            checkpoint_version: CheckpointVersion::new(0),
7635        };
7636        let spec = registered_entry.spec.clone();
7637        let mut draft = super::types::AgentBuildDraft {
7638            model: None,
7639            system_prompt: None,
7640            additional_instructions: spec.additional_instructions.clone(),
7641            labels: spec.labels.clone(),
7642            app_context: spec.context.clone(),
7643            external_tools: Vec::new(),
7644            local_external_tools: Default::default(),
7645            provider_params: None,
7646        };
7647        if self.bridge.is_some() {
7648            let active_peers = self.entries.read().await.keys().cloned().collect();
7649            let managed_edges = self.desired_peer_edges.read().await.clone();
7650            let build_context = AgentBuildContext {
7651                identity: identity.clone(),
7652                active_peers,
7653                managed_edges,
7654                runtime_services: self.runtime_services(),
7655            };
7656            if let Some(customizer) = self.customizer.read().await.clone() {
7657                let customize = customizer.customize_build(&build_context, &spec, &mut draft);
7658                tokio::pin!(customize);
7659                let customize_result = if *foreground_shutdown.borrow() {
7660                    None
7661                } else {
7662                    tokio::select! {
7663                        result = &mut customize => Some(result),
7664                        changed = foreground_shutdown.changed() => {
7665                            if changed.is_ok() && *foreground_shutdown.borrow() {
7666                                None
7667                            } else {
7668                                Some(customize.await)
7669                            }
7670                        }
7671                    }
7672                };
7673                let Some(customize_result) = customize_result else {
7674                    self.restore_entry_with_grant(identity, registered_entry, &grant)
7675                        .await;
7676                    return Err(IdentityRuntimeError::Internal(
7677                        "identity reset cancelled during shutdown before session installation"
7678                            .to_string(),
7679                    ));
7680                };
7681                if let Err(err) = customize_result {
7682                    self.restore_entry_with_grant(identity, registered_entry, &grant)
7683                        .await;
7684                    return Err(IdentityRuntimeError::Internal(format!(
7685                        "customizer after reset: {err}"
7686                    )));
7687                }
7688            }
7689        }
7690
7691        // Bridge: retire old mob member and create fresh session for the new identity.
7692        if let Some(bridge) = &self.bridge {
7693            if let Err(err) = self
7694                .continuity_store
7695                .upsert_continuity_record(&new_record, grant.fencing_token)
7696                .await
7697            {
7698                self.restore_entry_with_grant(identity, registered_entry, &grant)
7699                    .await;
7700                return Err(IdentityRuntimeError::Store(err));
7701            }
7702            let provisional_record = new_record.clone();
7703
7704            let old_runtime_id = registered_entry
7705                .continuity
7706                .as_ref()
7707                .map(|c| c.agent_runtime_id.clone());
7708            let old_session_id = registered_entry
7709                .continuity
7710                .as_ref()
7711                .map(|c| c.session_id.clone());
7712
7713            let session_id = bridge
7714                .create_session(
7715                    identity,
7716                    &new_record.agent_runtime_id,
7717                    &spec,
7718                    &draft,
7719                    &new_record.session_id,
7720                )
7721                .await
7722                .map_err(|e| {
7723                    IdentityRuntimeError::Internal(format!(
7724                        "bridge create_session after reset: {e}"
7725                    ))
7726                });
7727            let session_id = match session_id {
7728                Ok(session_id) => session_id,
7729                Err(err) => {
7730                    let cleanup_error = bridge
7731                        .retire_member(&new_record.agent_runtime_id)
7732                        .await
7733                        .err();
7734                    let delete_error = self
7735                        .restore_entry_after_reset_bridge_failure(
7736                            identity,
7737                            &provisional_record,
7738                            registered_entry.clone(),
7739                            &grant,
7740                            cleanup_error.is_some(),
7741                        )
7742                        .await;
7743                    if cleanup_error.is_some() || delete_error.is_some() {
7744                        return Err(IdentityRuntimeError::Internal(format!(
7745                            "{err}{}{}",
7746                            cleanup_error
7747                                .as_ref()
7748                                .map(|e| format!("; cleanup retire failed: {e}"))
7749                                .unwrap_or_default(),
7750                            delete_error
7751                                .as_ref()
7752                                .map(|e| format!("; tentative continuity cleanup failed: {e}"))
7753                                .unwrap_or_default()
7754                        )));
7755                    }
7756                    return Err(err);
7757                }
7758            };
7759            // Update the record with the actual session ID
7760            let mut new_record = new_record;
7761            new_record.session_id = session_id;
7762            tracing::debug!(
7763                identity = %identity,
7764                runtime_id = %new_record.agent_runtime_id,
7765                session_id = %new_record.session_id,
7766                "reset bridge create_session completed",
7767            );
7768
7769            if let Err(err) = self
7770                .continuity_store
7771                .upsert_continuity_record(&new_record, grant.fencing_token)
7772                .await
7773            {
7774                let unregister_error = Self::unregister_bridge_session_runtime_states(
7775                    bridge.as_ref(),
7776                    std::slice::from_ref(&new_record.session_id),
7777                )
7778                .await;
7779                let cleanup_error = bridge
7780                    .retire_member(&new_record.agent_runtime_id)
7781                    .await
7782                    .err();
7783                let delete_error = self
7784                    .restore_entry_after_reset_bridge_failure(
7785                        identity,
7786                        &provisional_record,
7787                        registered_entry.clone(),
7788                        &grant,
7789                        unregister_error.is_some() || cleanup_error.is_some(),
7790                    )
7791                    .await;
7792                if unregister_error.is_some() || cleanup_error.is_some() || delete_error.is_some() {
7793                    return Err(IdentityRuntimeError::Internal(format!(
7794                        "continuity upsert actual session after reset: {err}{}{}{}",
7795                        unregister_error
7796                            .as_ref()
7797                            .map(|e| format!("; unregister session failed: {e}"))
7798                            .unwrap_or_default(),
7799                        cleanup_error
7800                            .as_ref()
7801                            .map(|e| format!("; cleanup retire failed: {e}"))
7802                            .unwrap_or_default(),
7803                        delete_error
7804                            .as_ref()
7805                            .map(|e| format!("; tentative continuity cleanup failed: {e}"))
7806                            .unwrap_or_default(),
7807                    )));
7808                }
7809                return Err(IdentityRuntimeError::Store(err));
7810            }
7811
7812            let register_result = bridge
7813                .register_session_runtime_state(
7814                    &new_record.session_id,
7815                    identity,
7816                    new_record.generation,
7817                    new_record.checkpoint_version,
7818                    grant.fencing_token,
7819                )
7820                .await;
7821            let effective_checkpoint_version = match register_result {
7822                Ok(version) => version,
7823                Err(err) => {
7824                    let unregister_error = Self::unregister_bridge_session_runtime_states(
7825                        bridge.as_ref(),
7826                        std::slice::from_ref(&new_record.session_id),
7827                    )
7828                    .await;
7829                    let cleanup_error = bridge
7830                        .retire_member(&new_record.agent_runtime_id)
7831                        .await
7832                        .err();
7833                    let mut detail =
7834                        format!("bridge register actual session runtime state after reset: {err}");
7835                    if let Some(unregister_error) = unregister_error.as_ref() {
7836                        detail
7837                            .push_str(&format!("; unregister session failed: {unregister_error}"));
7838                    }
7839                    if let Some(cleanup_error) = cleanup_error.as_ref() {
7840                        detail.push_str(&format!("; cleanup retire failed: {cleanup_error}"));
7841                    }
7842                    let delete_error = self
7843                        .restore_entry_after_reset_bridge_failure(
7844                            identity,
7845                            &new_record,
7846                            registered_entry.clone(),
7847                            &grant,
7848                            unregister_error.is_some() || cleanup_error.is_some(),
7849                        )
7850                        .await;
7851                    if let Some(delete_error) = delete_error {
7852                        return Err(IdentityRuntimeError::Internal(format!(
7853                            "{detail}; tentative continuity cleanup failed: {delete_error}"
7854                        )));
7855                    }
7856                    return Err(IdentityRuntimeError::Internal(detail));
7857                }
7858            };
7859            new_record.checkpoint_version = effective_checkpoint_version;
7860            tracing::debug!(
7861                identity = %identity,
7862                runtime_id = %new_record.agent_runtime_id,
7863                session_id = %new_record.session_id,
7864                checkpoint_version = new_record.checkpoint_version.get(),
7865                "reset bridge session runtime state registered",
7866            );
7867
7868            if let Err(err) = self
7869                .continuity_store
7870                .upsert_continuity_record(&new_record, grant.fencing_token)
7871                .await
7872            {
7873                tracing::warn!(
7874                    identity = %identity,
7875                    runtime_id = %new_record.agent_runtime_id,
7876                    session_id = %new_record.session_id,
7877                    error = %err,
7878                    "reset final continuity upsert failed after bridge materialization; rolling back new generation",
7879                );
7880                let unregister_error = bridge
7881                    .unregister_session_runtime_state(&new_record.session_id)
7882                    .await
7883                    .err();
7884                let cleanup_error = bridge
7885                    .retire_member(&new_record.agent_runtime_id)
7886                    .await
7887                    .err();
7888                let rollback_error = self
7889                    .continuity_store
7890                    .rollback_continuity_record(
7891                        &new_record,
7892                        registered_entry.continuity.as_ref(),
7893                        grant.fencing_token,
7894                    )
7895                    .await
7896                    .err();
7897                if rollback_error.is_some() {
7898                    self.restore_broken_entry_from_authoritative_continuity(
7899                        identity,
7900                        registered_entry,
7901                        &grant,
7902                    )
7903                    .await;
7904                } else {
7905                    self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7906                        .await;
7907                }
7908                if unregister_error.is_some() || cleanup_error.is_some() || rollback_error.is_some()
7909                {
7910                    return Err(IdentityRuntimeError::Internal(format!(
7911                        "continuity upsert after reset: {err}{}{}{}",
7912                        unregister_error
7913                            .as_ref()
7914                            .map(|e| format!("; unregister session failed: {e}"))
7915                            .unwrap_or_default(),
7916                        cleanup_error
7917                            .as_ref()
7918                            .map(|e| format!("; cleanup retire failed: {e}"))
7919                            .unwrap_or_default(),
7920                        rollback_error
7921                            .as_ref()
7922                            .map(|e| format!("; continuity rollback failed: {e}"))
7923                            .unwrap_or_default()
7924                    )));
7925                }
7926                return Err(IdentityRuntimeError::Store(err));
7927            }
7928
7929            // Update runtime state
7930            let mut entries = self.entries.write().await;
7931            let Some(entry) = entries.get_mut(identity) else {
7932                drop(entries);
7933                tracing::warn!(
7934                    identity = %identity,
7935                    runtime_id = %new_record.agent_runtime_id,
7936                    session_id = %new_record.session_id,
7937                    "reset entry disappeared after bridge materialization; rolling back new generation",
7938                );
7939                let _ = bridge
7940                    .unregister_session_runtime_state(&new_record.session_id)
7941                    .await;
7942                let _ = bridge.retire_member(&new_record.agent_runtime_id).await;
7943                if let Err(err) = self
7944                    .continuity_store
7945                    .rollback_continuity_record(
7946                        &new_record,
7947                        registered_entry.continuity.as_ref(),
7948                        grant.fencing_token,
7949                    )
7950                    .await
7951                {
7952                    tracing::warn!(
7953                        %identity,
7954                        error = %err,
7955                        "failed to roll back continuity after reset entry disappeared"
7956                    );
7957                }
7958                if let Err(err) = self
7959                    .release_or_park_untracked_leases(std::slice::from_ref(&grant))
7960                    .await
7961                {
7962                    tracing::warn!(
7963                        %identity,
7964                        error = %err,
7965                        "failed to release lease after reset entry disappeared; exact grant parked for retry"
7966                    );
7967                }
7968                return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7969            };
7970            entry.continuity = Some(new_record.clone());
7971            entry.lease = Some(Self::lease_entry_from_grant(&grant));
7972            entry.state = IdentityLifecycleState::Active;
7973            entry.checkpoint_version = new_record.checkpoint_version;
7974            tracing::debug!(
7975                identity = %identity,
7976                runtime_id = %new_record.agent_runtime_id,
7977                session_id = %new_record.session_id,
7978                "reset completed",
7979            );
7980            drop(entries);
7981
7982            // The prior bridge projection remains rollback authority until
7983            // the final continuity row and in-memory entry both commit. In
7984            // particular, blocking continuity stores introduce scheduler
7985            // yield points here, so scheduling cleanup before the final
7986            // upsert can unregister the old session while rollback is still
7987            // possible.
7988            let cleanup_old_runtime_id = old_runtime_id
7989                .as_ref()
7990                .filter(|old_id| *old_id != &new_record.agent_runtime_id)
7991                .cloned();
7992            let cleanup_old_session_id = old_session_id
7993                .as_ref()
7994                .filter(|old_session_id| *old_session_id != &new_record.session_id)
7995                .cloned();
7996            let reset_memory_injector = self.agent_memory.read().await.clone();
7997            let reset_memory_capture = reset_memory_injector.as_ref().and_then(|injector| {
7998                registered_entry.continuity.as_ref().map(|old_continuity| {
7999                    PendingResetMemoryCapture {
8000                        injector: injector.clone(),
8001                        identity: identity.clone(),
8002                        session_key: old_continuity.session_id.to_string(),
8003                        generation: old_continuity.generation.get(),
8004                    }
8005                })
8006            });
8007            // Record exact cleanup debt immediately after the durable and
8008            // in-memory replacement commit. The owned debt carries its memory
8009            // capture prerequisite, so reset remains latency-neutral while
8010            // graceful shutdown still joins/retries the complete sequence.
8011            let reset_bridge_cleanup = self
8012                .record_old_bridge_cleanup_after_reset(
8013                    cleanup_old_runtime_id,
8014                    cleanup_old_session_id,
8015                    reset_memory_capture.clone(),
8016                )
8017                .await;
8018            // §10.1: reset is the deliberate clean-slate boundary — clear
8019            // session taint explicitly (rotation clears implicitly; this
8020            // also drops pending pre-attribution taint). Mark the outgoing
8021            // boundary synchronously; the runtime-owned cleanup task performs
8022            // bounded distillation before it may CAS-delete the superseded
8023            // session projection. This preserves detached reset latency
8024            // without racing the evidence source.
8025            if let Some(injector) = reset_memory_injector.as_ref() {
8026                injector.clear_taint_for_identity(identity);
8027                injector.note_session_generation(
8028                    identity,
8029                    &new_record.session_id.to_string(),
8030                    new_record.generation.get(),
8031                );
8032                if let Some(old_continuity) = registered_entry.continuity.as_ref() {
8033                    let old_session_key = old_continuity.session_id.to_string();
8034                    injector.note_reset_boundary(&old_session_key);
8035                    injector.note_session_generation(
8036                        identity,
8037                        &old_session_key,
8038                        old_continuity.generation.get(),
8039                    );
8040                }
8041            }
8042
8043            if let Some(cleanup) = reset_bridge_cleanup {
8044                self.spawn_old_bridge_cleanup_after_reset(bridge.clone(), cleanup)
8045                    .await;
8046            }
8047            tracing::debug!(
8048                identity = %identity,
8049                runtime_id = %new_record.agent_runtime_id,
8050                session_id = %new_record.session_id,
8051                "reset old bridge cleanup debt recorded after continuity commit",
8052            );
8053            self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
8054            return Ok(new_record);
8055        }
8056
8057        // Persist the new record (fencing token from new lease protects against old writes)
8058        if let Err(err) = self
8059            .continuity_store
8060            .upsert_continuity_record(&new_record, grant.fencing_token)
8061            .await
8062        {
8063            self.restore_entry_with_grant(identity, registered_entry, &grant)
8064                .await;
8065            return Err(IdentityRuntimeError::Store(err));
8066        }
8067
8068        // No bridge — update runtime state only (validation mode)
8069        let mut entries = self.entries.write().await;
8070        let Some(entry) = entries.get_mut(identity) else {
8071            drop(entries);
8072            if let Err(err) = self
8073                .continuity_store
8074                .rollback_continuity_record(
8075                    &new_record,
8076                    registered_entry.continuity.as_ref(),
8077                    grant.fencing_token,
8078                )
8079                .await
8080            {
8081                tracing::warn!(
8082                    %identity,
8083                    error = %err,
8084                    "failed to roll back continuity after reset validation entry disappeared"
8085                );
8086            }
8087            if let Err(err) = self
8088                .release_or_park_untracked_leases(std::slice::from_ref(&grant))
8089                .await
8090            {
8091                tracing::warn!(
8092                    %identity,
8093                    error = %err,
8094                    "failed to release lease after reset validation entry disappeared; exact grant parked for retry"
8095                );
8096            }
8097            return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
8098        };
8099        entry.continuity = Some(new_record.clone());
8100        entry.lease = Some(Self::lease_entry_from_grant(&grant));
8101        entry.state = IdentityLifecycleState::Active;
8102        entry.checkpoint_version = CheckpointVersion::new(0);
8103        drop(entries);
8104        if let Some(injector) = self.agent_memory.read().await.as_ref() {
8105            injector.clear_taint_for_identity(identity);
8106            // No-bridge (validation) reset: same §8.4 boundary semantics.
8107            if let Some(old_continuity) = registered_entry.continuity.as_ref() {
8108                let old_session_key = old_continuity.session_id.to_string();
8109                injector.note_reset_boundary(&old_session_key);
8110                injector.spawn_rotation_distillation(
8111                    identity,
8112                    &old_session_key,
8113                    crate::memory::distiller::DistillCause::Reset,
8114                );
8115            }
8116        }
8117
8118        self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
8119        Ok(new_record)
8120    }
8121
8122    // -----------------------------------------------------------------------
8123    // Lifecycle: delete_identity() — REQ-11
8124    // -----------------------------------------------------------------------
8125
8126    /// Delete an identity: removes continuity record.
8127    ///
8128    /// 1. Fence old owner
8129    /// 2. Remove ContinuityRecord
8130    /// 3. Future bootstrap treats identity as Uninitialized
8131    pub async fn delete_identity(
8132        &self,
8133        identity: &AgentIdentity,
8134    ) -> Result<(), IdentityRuntimeError> {
8135        self.delete_identity_with_expected_member_alias(identity, None)
8136            .await
8137    }
8138
8139    async fn delete_identity_with_expected_member_alias(
8140        &self,
8141        identity: &AgentIdentity,
8142        expected_alias: Option<&str>,
8143    ) -> Result<(), IdentityRuntimeError> {
8144        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
8145        let _lifecycle_guard = lifecycle_lock.lock().await;
8146        if let Some(expected_alias) = expected_alias {
8147            self.ensure_expected_member_alias_current(identity, expected_alias)
8148                .await?;
8149        }
8150        self.delete_identity_locked(identity).await
8151    }
8152
8153    async fn delete_identity_locked(
8154        &self,
8155        identity: &AgentIdentity,
8156    ) -> Result<(), IdentityRuntimeError> {
8157        let registered_entry = self
8158            .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Retiring)
8159            .await?;
8160        let runtime_id = registered_entry
8161            .continuity
8162            .as_ref()
8163            .map(|c| c.agent_runtime_id.clone());
8164        let session_id = registered_entry
8165            .continuity
8166            .as_ref()
8167            .map(|c| c.session_id.clone());
8168
8169        // INV-05: fence the old owner first
8170        let acquire_result = match self
8171            .lease_provider
8172            .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
8173            .await
8174        {
8175            Ok(result) => result,
8176            Err(err) => {
8177                self.restore_entry(identity, registered_entry).await;
8178                return Err(IdentityRuntimeError::Lease(err));
8179            }
8180        };
8181
8182        let grant = match acquire_result.get(identity) {
8183            Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
8184            _ => {
8185                self.restore_entry(identity, registered_entry).await;
8186                return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
8187            }
8188        };
8189        if let Err(err) = self
8190            .advance_existing_continuity_fence(identity, &registered_entry, &grant)
8191            .await
8192        {
8193            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8194                .await;
8195            return Err(err);
8196        }
8197        // The durable fence now belongs to the delete transaction. Refresh
8198        // the bridge adapter before retirement so Meerkat's terminal archive
8199        // projection carries that same token. Leaving the live session on the
8200        // prior token makes ArchiveSession fail closed, retains a Retiring Mob
8201        // anchor, and later prevents strict shutdown attestation.
8202        if let Some(record) = registered_entry.continuity.as_ref()
8203            && let Err(error) = self
8204                .refresh_existing_session_runtime_state(identity, record, &grant)
8205                .await
8206        {
8207            self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
8208                .await;
8209            return Err(IdentityRuntimeError::Internal(format!(
8210                "bridge refresh session authority before delete: {error}"
8211            )));
8212        }
8213
8214        // §8.4 trigger (b): delete is the identity's LAST boundary — harvest
8215        // the outgoing session before teardown (its exit-interview analog,
8216        // §8.5, is the steward's; the distillate is what it will read).
8217        // Bounded; deletion proceeds at the pre-rotation timeout.
8218        if let Some(injector) = self.agent_memory.read().await.clone() {
8219            if let Some(session_id) = session_id.as_ref() {
8220                let session_key = session_id.to_string();
8221                injector
8222                    .distill_before_rotation(
8223                        identity,
8224                        &session_key,
8225                        crate::memory::distiller::DistillCause::Delete,
8226                    )
8227                    .await;
8228                // Ask 2 GC: delete permanently abandons this session id, and
8229                // its knowledge has just been distilled into the identity-keyed
8230                // MobKit store (which the exit-interview harvest below reads —
8231                // a DIFFERENT store from the meerkat session-memory scope). So
8232                // reclaiming the orphaned meerkat scope here frees dead
8233                // re-embed weight without starving any downstream read.
8234                injector
8235                    .drop_orphaned_session_scope(
8236                        &session_key,
8237                        crate::memory::distiller::DistillCause::Delete,
8238                    )
8239                    .await;
8240            }
8241            // §8.5 exit interview (delete is the identity's LAST boundary).
8242            injector
8243                .note_identity_retired(
8244                    identity,
8245                    session_id
8246                        .as_ref()
8247                        .map(std::string::ToString::to_string)
8248                        .as_deref(),
8249                    "delete",
8250                )
8251                .await;
8252        }
8253
8254        // Retire the mob member through the session bridge before removing
8255        // the continuity record. This ensures the mob actor is cleaned up.
8256        if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id)
8257            && let Err(err) = bridge.retire_member(rid).await
8258        {
8259            self.restore_entry_with_grant(identity, registered_entry, &grant)
8260                .await;
8261            return Err(IdentityRuntimeError::Internal(format!(
8262                "bridge retire before delete: {err}"
8263            )));
8264        }
8265
8266        if let (Some(bridge), Some(session_id)) = (&self.bridge, &session_id)
8267            && let Some(err) = Self::unregister_bridge_session_runtime_states(
8268                bridge.as_ref(),
8269                std::slice::from_ref(session_id),
8270            )
8271            .await
8272        {
8273            self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
8274                .await;
8275            return Err(IdentityRuntimeError::Internal(format!(
8276                "bridge unregister session before delete: {err}"
8277            )));
8278        }
8279
8280        // Destroy-deprojection (2026-07-31 verdict): the durable session row
8281        // must not outlive the identity - a leftover external body is
8282        // exactly what the ephemeral-runtime-store activation mint would
8283        // faithfully resurrect on the next cold pod. Projection writes are
8284        // already quiesced (member retired, session unregistered) and this
8285        // delete transaction owns the ADVANCED identity fence, so the
8286        // revision-CAS delete cannot race a live writer. `delete_continuity_record` below removes any
8287        // remainder atomically for conforming stores; a store that cannot
8288        // support session-scoped deletion (`Ok(false)` default) or a row the
8289        // current decoder cannot token (an unimported released envelope)
8290        // keeps the record-scoped contract and is surfaced loudly instead of
8291        // silently retained.
8292        if let Some(session_id) = session_id.as_ref() {
8293            match self
8294                .continuity_store
8295                .load_session_snapshot(session_id)
8296                .await
8297            {
8298                Ok(Some(snapshot)) => {
8299                    let cas_token = serde_json::from_slice::<meerkat_core::Session>(&snapshot.data)
8300                        .ok()
8301                        .and_then(|current| {
8302                            meerkat_core::session_store::session_projection_cas_token(&current).ok()
8303                        });
8304                    match cas_token {
8305                        Some(token) => match self
8306                            .continuity_store
8307                            .delete_session_snapshot_if_current_revision(session_id, &token)
8308                            .await
8309                        {
8310                            Ok(true) => {}
8311                            Ok(false) => {
8312                                tracing::warn!(
8313                                    identity = %identity,
8314                                    session_id = %session_id,
8315                                    "session-scoped snapshot delete unsupported or superseded; \
8316                                     relying on delete_continuity_record's record-scoped \
8317                                     deletion - a non-conforming external store may retain a \
8318                                     resurrectable session body"
8319                                );
8320                            }
8321                            Err(err) => {
8322                                self.restore_broken_entry_and_release_grant(
8323                                    identity,
8324                                    registered_entry,
8325                                    &grant,
8326                                )
8327                                .await;
8328                                return Err(IdentityRuntimeError::Store(err));
8329                            }
8330                        },
8331                        None => {
8332                            tracing::warn!(
8333                                identity = %identity,
8334                                session_id = %session_id,
8335                                "durable session row cannot be revision-tokened for CAS \
8336                                 delete; relying on delete_continuity_record's record-scoped \
8337                                 deletion"
8338                            );
8339                        }
8340                    }
8341                }
8342                Ok(None) => {}
8343                Err(err) => {
8344                    self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8345                        .await;
8346                    return Err(IdentityRuntimeError::Store(err));
8347                }
8348            }
8349        }
8350
8351        // Remove authoritative continuity record from the store
8352        if let Err(err) = self
8353            .continuity_store
8354            .delete_continuity_record(identity, grant.fencing_token)
8355            .await
8356        {
8357            self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8358                .await;
8359            return Err(IdentityRuntimeError::Store(err));
8360        }
8361
8362        // Authoritative deletion is committed before ownership is released,
8363        // so another holder can never race the continuity delete. Surface a
8364        // provider release failure explicitly; the still-held external grant
8365        // then remains the fail-closed ownership fence.
8366        let release_result = self
8367            .lease_provider
8368            .release_leases(std::slice::from_ref(&grant))
8369            .await;
8370
8371        match release_result {
8372            Ok(()) => {
8373                self.event_channels.write().await.remove(identity);
8374                self.entries.write().await.remove(identity);
8375                // The identity remains part of the desired bootstrap roster
8376                // until a roster reconcile removes it. Successful deletion
8377                // therefore makes readiness Dormant.
8378                self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Dormant, None);
8379                Ok(())
8380            }
8381            Err(error) => {
8382                // The physical member and continuity row are already gone,
8383                // but the provider still owns this exact fencing grant. Keep
8384                // an explicit Broken tombstone so reconcile and shutdown can
8385                // retry it; removing the entry here would make non-expiring
8386                // provider authority permanently unreachable.
8387                let mut entries = self.entries.write().await;
8388                let entry = entries
8389                    .get_mut(identity)
8390                    .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8391                entry.state = IdentityLifecycleState::Broken;
8392                entry.continuity = None;
8393                entry.lease = None;
8394                entry.pending_lease_release = Some(grant);
8395                drop(entries);
8396                self.mark_bootstrap_from_lifecycle(
8397                    identity,
8398                    IdentityLifecycleState::Broken,
8399                    Some(format!("lease release after delete: {error}")),
8400                );
8401                Err(IdentityRuntimeError::Lease(error))
8402            }
8403        }
8404    }
8405
8406    // -----------------------------------------------------------------------
8407    // Checkpoint — REQ-14, REQ-15, REQ-16, REQ-17
8408    // -----------------------------------------------------------------------
8409
8410    /// Save a checkpoint snapshot. Enforces version ordering and fencing.
8411    pub async fn checkpoint(
8412        &self,
8413        identity: &AgentIdentity,
8414        snapshot: &SessionSnapshot,
8415    ) -> Result<CheckpointVersion, IdentityRuntimeError> {
8416        let lifecycle_lock = self.lifecycle_lock_for(identity).await;
8417        let _lifecycle_guard = lifecycle_lock.lock().await;
8418        {
8419            let entries = self.entries.read().await;
8420            let entry = entries
8421                .get(identity)
8422                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8423            if entry.state != IdentityLifecycleState::Active {
8424                return Err(IdentityRuntimeError::InvalidState {
8425                    identity: identity.clone(),
8426                    state: entry.state,
8427                    operation: "checkpoint",
8428                });
8429            }
8430        }
8431
8432        let token = self.ensure_active_lease(identity).await?;
8433        let (record, new_version) = {
8434            let entries = self.entries.read().await;
8435            let entry = entries
8436                .get(identity)
8437                .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8438            let record = entry
8439                .continuity
8440                .as_ref()
8441                .ok_or_else(|| {
8442                    IdentityRuntimeError::Internal(format!("no continuity record for {identity}"))
8443                })?
8444                .clone();
8445
8446            let new_version = CheckpointVersion::new(entry.checkpoint_version.get() + 1);
8447            (record, new_version)
8448        };
8449
8450        // REQ-15 + REQ-16: store enforces version ordering and fencing
8451        self.continuity_store
8452            .save_session_snapshot(
8453                identity,
8454                &record.session_id,
8455                record.generation,
8456                new_version,
8457                token,
8458                snapshot,
8459            )
8460            .await?;
8461
8462        // Update local checkpoint version
8463        {
8464            let mut entries = self.entries.write().await;
8465            if let Some(entry) = entries.get_mut(identity) {
8466                entry.checkpoint_version = new_version;
8467            }
8468        }
8469
8470        self.emit_event(
8471            identity,
8472            IdentityEvent::CheckpointCompleted {
8473                identity: identity.clone(),
8474                version: new_version,
8475            },
8476        )
8477        .await;
8478
8479        Ok(new_version)
8480    }
8481
8482    // -----------------------------------------------------------------------
8483    // Roster inspection — REQ-32
8484    // -----------------------------------------------------------------------
8485
8486    /// Return all active identities with their specs and status.
8487    pub async fn roster_inspect(
8488        &self,
8489    ) -> BTreeMap<AgentIdentity, (DurableAgentSpec, IdentityStatus)> {
8490        let entries = self.entries.read().await;
8491        let mut result = BTreeMap::new();
8492        for (identity, entry) in entries.iter() {
8493            let lease_info = entry.lease.as_ref().map(|l| LeaseInfo {
8494                fencing_token: l.fencing_token,
8495                ttl_remaining: l.ttl_remaining(),
8496                healthy: l.is_healthy(),
8497            });
8498            let continuity_health = Some(ContinuityHealth {
8499                store_reachable: true,
8500                durability_policy: self.durability_policy.clone(),
8501                last_checkpoint_version: if entry.checkpoint_version.get() > 0 {
8502                    Some(entry.checkpoint_version)
8503                } else {
8504                    None
8505                },
8506            });
8507            let status = IdentityStatus {
8508                identity: identity.clone(),
8509                state: entry.state,
8510                agent_runtime_id: entry
8511                    .continuity
8512                    .as_ref()
8513                    .map(|c| c.agent_runtime_id.clone()),
8514                session_id: entry.continuity.as_ref().map(|c| c.session_id.clone()),
8515                profile: Some(entry.spec.profile.clone()),
8516                runtime_mode: entry.spec.runtime_mode_override,
8517                addressability: entry.spec.addressability,
8518                display_name: entry.spec.display_name.clone(),
8519                labels: entry.spec.labels.clone(),
8520                generation: entry.continuity.as_ref().map(|c| c.generation),
8521                checkpoint_version: if entry.checkpoint_version.get() > 0 {
8522                    Some(entry.checkpoint_version)
8523                } else {
8524                    None
8525                },
8526                lease: lease_info,
8527                continuity_health,
8528                continuity_unrecoverable: entry.continuity_unrecoverable.clone(),
8529            };
8530            result.insert(identity.clone(), (entry.spec.clone(), status));
8531        }
8532        result
8533    }
8534
8535    // -----------------------------------------------------------------------
8536    // Roster uniqueness validation — INV-06
8537    // -----------------------------------------------------------------------
8538
8539    /// Validate that a roster contains no duplicate identities.
8540    pub fn validate_roster_uniqueness(
8541        specs: &[DurableAgentSpec],
8542    ) -> Result<(), IdentityRuntimeError> {
8543        let mut seen = std::collections::BTreeSet::new();
8544        for spec in specs {
8545            if !seen.insert(&spec.identity) {
8546                return Err(IdentityRuntimeError::DuplicateIdentity(
8547                    spec.identity.clone(),
8548                ));
8549            }
8550        }
8551        Ok(())
8552    }
8553
8554    // -----------------------------------------------------------------------
8555    // Accessors for internal state
8556    // -----------------------------------------------------------------------
8557
8558    /// Get the current entries (read-only snapshot).
8559    #[allow(dead_code)]
8560    pub(crate) async fn entries(&self) -> BTreeMap<AgentIdentity, IdentityEntry> {
8561        self.entries.read().await.clone()
8562    }
8563
8564    /// Check if an identity is registered.
8565    pub async fn contains(&self, identity: &AgentIdentity) -> bool {
8566        self.entries.read().await.contains_key(identity)
8567    }
8568
8569    /// Check if an identity is registered AND in Active state.
8570    pub async fn is_active(&self, identity: &AgentIdentity) -> bool {
8571        self.entries
8572            .read()
8573            .await
8574            .get(identity)
8575            .is_some_and(|e| e.state == IdentityLifecycleState::Active)
8576    }
8577
8578    fn identity_from_member_alias(alias: &str) -> Option<(AgentIdentity, bool)> {
8579        let alias = crate::member_comms_id::runtime_alias_str(alias);
8580        let alias = alias.as_ref();
8581        alias
8582            .strip_prefix("rt:")
8583            .and_then(|rest| rest.rsplit_once(':'))
8584            .filter(|(identity, generation)| {
8585                !identity.is_empty()
8586                    && !generation.is_empty()
8587                    && generation.chars().all(|ch| ch.is_ascii_digit())
8588            })
8589            .and_then(|(identity, _)| AgentIdentity::parse(identity).ok())
8590            .map(|identity| (identity, true))
8591            .or_else(|| {
8592                AgentIdentity::parse(alias)
8593                    .ok()
8594                    .map(|identity| (identity, false))
8595            })
8596    }
8597
8598    /// Parse the durable owner encoded in the reserved generated-alias
8599    /// namespace, whether or not that identity is still registered. Mutating
8600    /// member surfaces use this to fail closed after a concurrent delete
8601    /// instead of treating an orphaned `rt:*` alias as an ordinary mob member.
8602    pub(crate) fn identity_for_generated_member_alias(alias: &str) -> Option<AgentIdentity> {
8603        Self::identity_from_member_alias(alias).and_then(|(identity, generated)| {
8604            if generated { Some(identity) } else { None }
8605        })
8606    }
8607
8608    /// Resolve a mutating member request without allowing a generated alias
8609    /// to fall back to the raw mob plane after concurrent deletion. Plain
8610    /// durable identities retain their historical registered-only behavior.
8611    pub(crate) async fn identity_for_member_mutation(&self, alias: &str) -> Option<AgentIdentity> {
8612        match Self::identity_from_member_alias(alias) {
8613            Some((identity, true)) => Some(identity),
8614            Some((identity, false)) => self.contains(&identity).await.then_some(identity),
8615            None => None,
8616        }
8617    }
8618
8619    async fn ensure_expected_member_alias_current(
8620        &self,
8621        identity: &AgentIdentity,
8622        expected_alias: &str,
8623    ) -> Result<(), IdentityRuntimeError> {
8624        let canonical_alias = crate::member_comms_id::runtime_alias_str(expected_alias);
8625        let Some((alias_identity, generated_runtime_alias)) =
8626            Self::identity_from_member_alias(canonical_alias.as_ref())
8627        else {
8628            return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
8629        };
8630        if alias_identity != *identity {
8631            return Err(IdentityRuntimeError::UnknownIdentity(alias_identity));
8632        }
8633        let entries = self.entries.read().await;
8634        let entry = entries
8635            .get(identity)
8636            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8637        if generated_runtime_alias {
8638            let current = entry
8639                .continuity
8640                .as_ref()
8641                .map(|record| record.agent_runtime_id.clone());
8642            if current.as_ref().map(AgentRuntimeId::as_str) != Some(canonical_alias.as_ref()) {
8643                return Err(IdentityRuntimeError::StaleRuntimeAlias {
8644                    identity: identity.clone(),
8645                    requested: canonical_alias.into_owned(),
8646                    current,
8647                });
8648            }
8649        }
8650        Ok(())
8651    }
8652
8653    /// Resolve a member alias to the durable identity that OWNS it, if any.
8654    ///
8655    /// Accepts both a generated runtime alias (`rt:<identity>:<generation>`)
8656    /// and a plain identity string; returns the identity only when it is
8657    /// actually registered in this runtime. Member-scoped RPCs use this to
8658    /// route lifecycle mutations of identity-owned members through the
8659    /// identity authority — a classic `handle.retire()`/`respawn()` on such
8660    /// a member would mutate it behind the IdentityRuntime's back (stale
8661    /// continuity binding, generation drift), which is the doctrine's
8662    /// "mob plane must not mangle durables" rule.
8663    pub async fn owned_identity_for_member_alias(&self, alias: &str) -> Option<AgentIdentity> {
8664        let (identity, _) = Self::identity_from_member_alias(alias)?;
8665        self.contains(&identity).await.then_some(identity)
8666    }
8667
8668    /// Identities currently in the Broken lifecycle state.
8669    pub async fn broken_identities(&self) -> Vec<AgentIdentity> {
8670        self.entries
8671            .read()
8672            .await
8673            .iter()
8674            .filter(|(_, entry)| entry.state == IdentityLifecycleState::Broken)
8675            .map(|(identity, _)| identity.clone())
8676            .collect()
8677    }
8678
8679    /// Broken identities that do NOT carry a terminal heal verdict — the set
8680    /// the continuity repair supervisor is allowed to keep retrying.
8681    pub async fn repairable_broken_identities(&self) -> Vec<AgentIdentity> {
8682        self.entries
8683            .read()
8684            .await
8685            .iter()
8686            .filter(|(_, entry)| {
8687                entry.state == IdentityLifecycleState::Broken
8688                    && entry.continuity_unrecoverable.is_none()
8689                    && entry
8690                        .host_rejected_build_park
8691                        .as_ref()
8692                        .is_none_or(|park| park.spec_digest != durable_spec_digest(&entry.spec))
8693            })
8694            .map(|(identity, _)| identity.clone())
8695            .collect()
8696    }
8697
8698    /// The terminal heal verdict recorded for an identity, if any.
8699    pub async fn continuity_unrecoverable(
8700        &self,
8701        identity: &AgentIdentity,
8702    ) -> Option<ContinuityUnrecoverable> {
8703        self.entries
8704            .read()
8705            .await
8706            .get(identity)
8707            .and_then(|entry| entry.continuity_unrecoverable.clone())
8708    }
8709
8710    /// Record a terminal heal verdict against a Broken identity.
8711    ///
8712    /// Returns `false` (without writing) when the identity is unknown or no
8713    /// longer Broken — the verdict only ever parks an already-Broken entry;
8714    /// it never degrades a live one. While recorded, the repair supervisor
8715    /// skips the identity and reconcile keeps its Broken projection instead
8716    /// of cosmetically resetting it (2026-07-29 heal/re-Break incident).
8717    pub async fn mark_continuity_unrecoverable(
8718        &self,
8719        identity: &AgentIdentity,
8720        reason: String,
8721    ) -> bool {
8722        let marked = {
8723            let mut entries = self.entries.write().await;
8724            match entries.get_mut(identity) {
8725                Some(entry) if entry.state == IdentityLifecycleState::Broken => {
8726                    entry.continuity_unrecoverable = Some(ContinuityUnrecoverable {
8727                        reason: reason.clone(),
8728                    });
8729                    true
8730                }
8731                _ => false,
8732            }
8733        };
8734        if marked {
8735            // Keep the bootstrap status surface honest about WHY the
8736            // identity stays broken (operators read this, not the log).
8737            self.mark_bootstrap_from_lifecycle(
8738                identity,
8739                IdentityLifecycleState::Broken,
8740                Some(reason),
8741            );
8742        }
8743        marked
8744    }
8745
8746    /// Clear a previously recorded terminal heal verdict (operator retry).
8747    pub async fn clear_continuity_unrecoverable(&self, identity: &AgentIdentity) -> bool {
8748        let mut entries = self.entries.write().await;
8749        match entries.get_mut(identity) {
8750            Some(entry) => entry.continuity_unrecoverable.take().is_some(),
8751            None => false,
8752        }
8753    }
8754
8755    /// The ACTIVE host-rejected-build park for an identity: `Some` only
8756    /// while the identity's current spec still digests to the parked value.
8757    /// A mismatch (the roster/policy changed) clears the park in place and
8758    /// returns `None` — the retry is permitted.
8759    pub async fn host_rejected_build_park(
8760        &self,
8761        identity: &AgentIdentity,
8762    ) -> Option<HostRejectedBuildPark> {
8763        let mut entries = self.entries.write().await;
8764        let entry = entries.get_mut(identity)?;
8765        let park = entry.host_rejected_build_park.clone()?;
8766        if park.spec_digest == durable_spec_digest(&entry.spec) {
8767            Some(park)
8768        } else {
8769            entry.host_rejected_build_park = None;
8770            None
8771        }
8772    }
8773
8774    /// Park an identity whose build the host deterministically rejected,
8775    /// scoped to the identity's CURRENT spec. Operator-visible through the
8776    /// bootstrap status surface and the warn line; no automatic retry until
8777    /// the spec changes.
8778    pub(crate) async fn mark_host_rejected_build_park(
8779        &self,
8780        identity: &AgentIdentity,
8781        reason: String,
8782    ) -> bool {
8783        let marked_state = {
8784            let mut entries = self.entries.write().await;
8785            match entries.get_mut(identity) {
8786                Some(entry) => {
8787                    entry.host_rejected_build_park = Some(HostRejectedBuildPark {
8788                        reason: reason.clone(),
8789                        spec_digest: durable_spec_digest(&entry.spec),
8790                    });
8791                    Some(entry.state)
8792                }
8793                None => None,
8794            }
8795        };
8796        let Some(state) = marked_state else {
8797            return false;
8798        };
8799        tracing::warn!(
8800            identity = %identity,
8801            reason = %reason,
8802            "host deterministically rejected this identity's build; parking (no automatic \
8803             retry) until the roster spec changes"
8804        );
8805        // Keep the bootstrap status surface honest about WHY the identity is
8806        // stuck (operators read this, not the log). The detail renders on
8807        // Broken projections; for a Dormant park the typed materialize error
8808        // carries the reason to every send instead.
8809        self.mark_bootstrap_from_lifecycle(identity, state, Some(reason));
8810        true
8811    }
8812
8813    /// Clear a host-rejected-build park (operator retry with an unchanged
8814    /// spec — e.g. after fixing the app-side gate's policy out of band).
8815    pub async fn clear_host_rejected_build_park(&self, identity: &AgentIdentity) -> bool {
8816        let mut entries = self.entries.write().await;
8817        match entries.get_mut(identity) {
8818            Some(entry) => entry.host_rejected_build_park.take().is_some(),
8819            None => false,
8820        }
8821    }
8822
8823    /// The durable session currently bound to an identity, if any.
8824    pub(crate) async fn continuity_session_id(
8825        &self,
8826        identity: &AgentIdentity,
8827    ) -> Option<SessionId> {
8828        self.entries
8829            .read()
8830            .await
8831            .get(identity)
8832            .and_then(|entry| entry.continuity.as_ref().map(|c| c.session_id.clone()))
8833    }
8834
8835    /// Get the continuity store reference.
8836    pub fn continuity_store(&self) -> &Arc<dyn ContinuityStore> {
8837        &self.continuity_store
8838    }
8839
8840    /// Get the lease provider reference.
8841    pub fn lease_provider(&self) -> &Arc<dyn LeaseProvider> {
8842        &self.lease_provider
8843    }
8844
8845    /// Get the runtime instance ID.
8846    pub fn runtime_instance_id(&self) -> &str {
8847        &self.runtime_instance_id
8848    }
8849
8850    /// Get the durability policy.
8851    pub fn durability_policy(&self) -> &DurabilityPolicy {
8852        &self.durability_policy
8853    }
8854
8855    /// Get whether a runtime store is configured.
8856    pub fn has_runtime_store(&self) -> bool {
8857        self.has_runtime_store
8858    }
8859
8860    /// Get the session bridge reference, if configured.
8861    pub fn bridge(&self) -> Option<&Arc<dyn SessionBridge>> {
8862        self.bridge.as_ref()
8863    }
8864
8865    // -----------------------------------------------------------------------
8866    // Convenience methods
8867    // -----------------------------------------------------------------------
8868
8869    /// Send plain text to an addressable identity.
8870    pub async fn send_text(
8871        &self,
8872        identity: &AgentIdentity,
8873        text: impl Into<String>,
8874    ) -> Result<FencingToken, IdentityRuntimeError> {
8875        self.send(identity, &meerkat_core::ContentInput::Text(text.into()))
8876            .await
8877    }
8878
8879    /// Dispatch plain text with system origin.
8880    pub async fn dispatch_text(
8881        &self,
8882        identity: &AgentIdentity,
8883        text: impl Into<String>,
8884    ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
8885        self.dispatch(identity, &DispatchInput::system(text)).await
8886    }
8887
8888    /// Execute the restore flow for the given roster.
8889    pub async fn restore_flow(
8890        &self,
8891        roster: &[DurableAgentSpec],
8892        topology_provider: Option<&dyn super::contracts::TopologyProvider>,
8893        customizer: Option<&dyn super::contracts::AgentCustomizer>,
8894    ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
8895        super::orchestrator::restore_flow(self, roster, topology_provider, customizer).await
8896    }
8897
8898    /// Resolve the AgentRuntimeId for a registered identity.
8899    pub async fn runtime_id_for(
8900        &self,
8901        identity: &AgentIdentity,
8902    ) -> Result<AgentRuntimeId, IdentityRuntimeError> {
8903        let entries = self.entries.read().await;
8904        let entry = entries
8905            .get(identity)
8906            .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8907        entry
8908            .continuity
8909            .as_ref()
8910            .map(|c| c.agent_runtime_id.clone())
8911            .ok_or_else(|| {
8912                IdentityRuntimeError::Internal(format!("no continuity record for {identity}"))
8913            })
8914    }
8915
8916    /// Inspect the current execution state of an identity via the bridge.
8917    pub async fn inspect(
8918        &self,
8919        identity: &AgentIdentity,
8920    ) -> Result<super::bridge::MemberInspection, IdentityRuntimeError> {
8921        let runtime_id = self.runtime_id_for(identity).await?;
8922        let bridge = self
8923            .bridge
8924            .as_ref()
8925            .ok_or_else(|| IdentityRuntimeError::Internal("no bridge configured".to_string()))?;
8926        bridge
8927            .inspect_member(&runtime_id)
8928            .await
8929            .map_err(|e| IdentityRuntimeError::Internal(format!("inspect: {e}")))
8930    }
8931
8932    // -----------------------------------------------------------------------
8933    // Turn-completion cursor
8934    // -----------------------------------------------------------------------
8935
8936    /// Completion epoch of a REGISTERED identity: its live lease token, or
8937    /// token 0 when it holds no grant (dormant, or lease lost). `None` when
8938    /// the identity is not registered at all.
8939    async fn registered_completion_epoch(&self, identity: &AgentIdentity) -> Option<FencingToken> {
8940        let entries = self.entries.read().await;
8941        entries.get(identity).map(|entry| {
8942            entry
8943                .lease
8944                .as_ref()
8945                .map_or_else(|| FencingToken::new(0), |lease| lease.fencing_token)
8946        })
8947    }
8948
8949    /// Move the stored cursor onto `epoch` if the lease incarnation advanced,
8950    /// and return it. Never rewinds: a stale epoch leaves the cursor alone.
8951    fn rebase_completion_cursor(
8952        &self,
8953        identity: &AgentIdentity,
8954        epoch: FencingToken,
8955    ) -> CompletionCursor {
8956        let mut cursors = self
8957            .completion_cursors
8958            .lock()
8959            .unwrap_or_else(std::sync::PoisonError::into_inner);
8960        let cursor = cursors
8961            .entry(identity.clone())
8962            .or_insert_with(|| CompletionCursor::start(epoch));
8963        *cursor = cursor.rebased(epoch);
8964        *cursor
8965    }
8966
8967    /// Whatever was last published for `identity`, without minting a ledger
8968    /// entry. Reads for unregistered names go through here so probing
8969    /// arbitrary strings cannot grow the map.
8970    fn retained_completion_cursor(&self, identity: &AgentIdentity) -> CompletionCursor {
8971        self.completion_cursors
8972            .lock()
8973            .unwrap_or_else(std::sync::PoisonError::into_inner)
8974            .get(identity)
8975            .copied()
8976            .unwrap_or_default()
8977    }
8978
8979    /// Current [`CompletionCursor`] for `identity`.
8980    ///
8981    /// A registered identity's cursor is rebased onto its live lease
8982    /// incarnation first, so a poller observes an incarnation change
8983    /// immediately rather than comparing against a turn count that no longer
8984    /// means anything. An identity that is gone (retired, deleted, or never
8985    /// registered) reports its last published cursor — retained precisely so
8986    /// this read cannot rewind.
8987    pub async fn completion_cursor(&self, identity: &AgentIdentity) -> CompletionCursor {
8988        match self.registered_completion_epoch(identity).await {
8989            Some(epoch) => self.rebase_completion_cursor(identity, epoch),
8990            None => self.retained_completion_cursor(identity),
8991        }
8992    }
8993
8994    /// Record that a turn completed for `identity`, advancing its cursor by
8995    /// one within the current lease incarnation.
8996    ///
8997    /// Production drives this from `AgentEvent::RunCompleted` on the always-on
8998    /// identity agent-event monitor. It is deliberately event-driven rather
8999    /// than derived from a polled projection: a poll cannot distinguish "new
9000    /// turn, identical text" from "no new turn", which is the entire defect
9001    /// this cursor exists to close.
9002    ///
9003    /// Not idempotent by design — one observed completion advances the cursor
9004    /// once, which is exactly what a poller comparing against a pre-delivery
9005    /// baseline needs.
9006    pub async fn record_turn_completed(&self, identity: &AgentIdentity) -> CompletionCursor {
9007        // A completion racing deregistration still counts under the last
9008        // known epoch rather than being dropped.
9009        let epoch = self
9010            .registered_completion_epoch(identity)
9011            .await
9012            .unwrap_or_else(|| self.retained_completion_cursor(identity).epoch);
9013        let mut cursors = self
9014            .completion_cursors
9015            .lock()
9016            .unwrap_or_else(std::sync::PoisonError::into_inner);
9017        let cursor = cursors
9018            .entry(identity.clone())
9019            .or_insert_with(|| CompletionCursor::start(epoch));
9020        *cursor = cursor.rebased(epoch).advanced();
9021        *cursor
9022    }
9023
9024    /// Wait until a turn completes past `baseline`, or the timeout expires.
9025    ///
9026    /// This is the correct completion barrier: it compares cursors, never
9027    /// output text, so two consecutive turns emitting byte-identical text are
9028    /// still two distinct completions. `baseline` is the
9029    /// `completion_baseline` returned by [`Self::dispatch_admission_tracked`]
9030    /// or [`Self::send_admission_tracked`].
9031    ///
9032    /// A genuinely stalled turn still times out rather than hanging forever,
9033    /// and an incarnation change is reported as its own error rather than
9034    /// being read as either completion or continued waiting.
9035    pub async fn wait_for_completion(
9036        &self,
9037        identity: &AgentIdentity,
9038        baseline: CompletionCursor,
9039        timeout: Duration,
9040    ) -> Result<CompletionCursor, IdentityRuntimeError> {
9041        let deadline = Instant::now() + timeout;
9042        loop {
9043            let cursor = self.completion_cursor(identity).await;
9044            match cursor.progress_since(baseline) {
9045                CompletionProgress::Completed => return Ok(cursor),
9046                CompletionProgress::IncarnationChanged => {
9047                    return Err(IdentityRuntimeError::CompletionIncarnationChanged {
9048                        identity: identity.clone(),
9049                        baseline,
9050                        observed: cursor,
9051                    });
9052                }
9053                CompletionProgress::Pending => {}
9054            }
9055            let now = Instant::now();
9056            if now >= deadline {
9057                return Err(IdentityRuntimeError::Internal(format!(
9058                    "timed out after {}s waiting for a turn past {baseline} on {identity}",
9059                    timeout.as_secs_f64()
9060                )));
9061            }
9062            tokio::time::sleep(
9063                COMPLETION_POLL_INTERVAL.min(deadline.saturating_duration_since(now)),
9064            )
9065            .await;
9066        }
9067    }
9068
9069    /// The configured default timeout for wait operations.
9070    pub fn default_timeout(&self) -> Duration {
9071        self.default_timeout
9072    }
9073
9074    async fn execute_reset_bridge_cleanup(
9075        bridge: &Arc<dyn SessionBridge>,
9076        cleanup: &PendingResetBridgeCleanup,
9077    ) -> Result<(), BridgeError> {
9078        match (&cleanup.runtime_id, &cleanup.session_id) {
9079            (Some(runtime_id), Some(session_id)) => {
9080                bridge
9081                    .retire_reset_superseded_member(runtime_id, session_id)
9082                    .await
9083            }
9084            (Some(runtime_id), None) => bridge.retire_member(runtime_id).await,
9085            (None, Some(session_id)) => bridge.unregister_session_runtime_state(session_id).await,
9086            (None, None) => Ok(()),
9087        }
9088    }
9089
9090    async fn clear_reset_bridge_cleanup_if_current(
9091        pending: &RwLock<BTreeMap<String, PendingResetBridgeCleanup>>,
9092        key: &str,
9093        cleanup: &PendingResetBridgeCleanup,
9094    ) {
9095        let mut pending = pending.write().await;
9096        if pending.get(key) == Some(cleanup) {
9097            pending.remove(key);
9098        }
9099    }
9100
9101    async fn run_reset_memory_capture_if_pending(
9102        pending: &RwLock<BTreeMap<String, PendingResetBridgeCleanup>>,
9103        key: &str,
9104        cleanup: &mut PendingResetBridgeCleanup,
9105    ) {
9106        let Some(capture) = cleanup.memory_capture.take() else {
9107            return;
9108        };
9109        capture.run().await;
9110
9111        // Publish the state-machine advance only after the bounded capture
9112        // returns. Cancellation before this write leaves the capture on the
9113        // authoritative debt and a shutdown retry runs it again.
9114        let mut pending = pending.write().await;
9115        if let Some(current) = pending.get_mut(key)
9116            && current == cleanup
9117        {
9118            current.memory_capture = None;
9119        }
9120    }
9121
9122    async fn record_old_bridge_cleanup_after_reset(
9123        &self,
9124        old_runtime_id: Option<AgentRuntimeId>,
9125        old_session_id: Option<SessionId>,
9126        memory_capture: Option<PendingResetMemoryCapture>,
9127    ) -> Option<PendingResetBridgeCleanup> {
9128        if old_runtime_id.is_none() && old_session_id.is_none() {
9129            return None;
9130        }
9131        let cleanup = PendingResetBridgeCleanup {
9132            runtime_id: old_runtime_id,
9133            session_id: old_session_id,
9134            memory_capture,
9135        };
9136        let key = cleanup.key();
9137        self.pending_reset_bridge_cleanups
9138            .write()
9139            .await
9140            .insert(key, cleanup.clone());
9141        Some(cleanup)
9142    }
9143
9144    async fn spawn_old_bridge_cleanup_after_reset(
9145        &self,
9146        bridge: Arc<dyn SessionBridge>,
9147        mut cleanup: PendingResetBridgeCleanup,
9148    ) {
9149        let key = cleanup.key();
9150        let runtime_instance_id = self.runtime_instance_id.clone();
9151        let pending = self.pending_reset_bridge_cleanups.clone();
9152        let mut tasks = self.reset_bridge_cleanup_tasks.lock().await;
9153        while let Some(result) = tasks.try_join_next() {
9154            if let Err(error) = result {
9155                tracing::error!(
9156                    runtime_instance_id = %self.runtime_instance_id,
9157                    %error,
9158                    "reset bridge cleanup task panicked"
9159                );
9160            }
9161        }
9162        tasks.spawn(async move {
9163            Self::run_reset_memory_capture_if_pending(&pending, &key, &mut cleanup).await;
9164            match Self::execute_reset_bridge_cleanup(&bridge, &cleanup).await {
9165                Ok(()) => {
9166                    Self::clear_reset_bridge_cleanup_if_current(&pending, &key, &cleanup).await;
9167                }
9168                Err(error) => tracing::warn!(
9169                    runtime_instance_id = %runtime_instance_id,
9170                    cleanup_key = %key,
9171                    %error,
9172                    "reset-superseded bridge cleanup failed; retaining exact shutdown debt",
9173                ),
9174            }
9175        });
9176    }
9177
9178    /// Join every post-commit reset cleanup task after foreground lifecycle
9179    /// admission is closed. The exact debt remains in the ledger on failure
9180    /// and is retried synchronously before physical mob shutdown.
9181    pub(crate) async fn join_reset_bridge_cleanup_tasks(&self) {
9182        let mut tasks = self.reset_bridge_cleanup_tasks.lock().await;
9183        while let Some(result) = tasks.join_next().await {
9184            if let Err(error) = result {
9185                tracing::error!(
9186                    runtime_instance_id = %self.runtime_instance_id,
9187                    %error,
9188                    "reset bridge cleanup task panicked"
9189                );
9190            }
9191        }
9192    }
9193
9194    /// Retry all exact reset cleanup debt. Success is the only path that
9195    /// removes an obligation; callers must retain identity grants when this
9196    /// returns an error.
9197    pub(crate) async fn drain_pending_reset_bridge_cleanups(
9198        &self,
9199    ) -> Result<usize, IdentityRuntimeError> {
9200        let Some(bridge) = self.bridge.as_ref() else {
9201            return Ok(0);
9202        };
9203        let pending = self
9204            .pending_reset_bridge_cleanups
9205            .read()
9206            .await
9207            .values()
9208            .cloned()
9209            .collect::<Vec<_>>();
9210        let mut completed = 0_usize;
9211        let mut errors = Vec::new();
9212        for mut cleanup in pending {
9213            let key = cleanup.key();
9214            Self::run_reset_memory_capture_if_pending(
9215                &self.pending_reset_bridge_cleanups,
9216                &key,
9217                &mut cleanup,
9218            )
9219            .await;
9220            match Self::execute_reset_bridge_cleanup(bridge, &cleanup).await {
9221                Ok(()) => {
9222                    Self::clear_reset_bridge_cleanup_if_current(
9223                        &self.pending_reset_bridge_cleanups,
9224                        &key,
9225                        &cleanup,
9226                    )
9227                    .await;
9228                    completed += 1;
9229                }
9230                Err(error) => errors.push(format!("{key}: {error}")),
9231            }
9232        }
9233        if errors.is_empty() {
9234            Ok(completed)
9235        } else {
9236            Err(IdentityRuntimeError::Internal(format!(
9237                "reset bridge cleanup debt remains: {}",
9238                errors.join("; ")
9239            )))
9240        }
9241    }
9242
9243    /// Poll until the identity produces an output_preview, or timeout.
9244    ///
9245    /// **Unsound as a completion barrier.** `output_preview` is the last
9246    /// committed assistant text, so this returns immediately when a PREVIOUS
9247    /// turn already left a preview, and it cannot tell "new turn, identical
9248    /// text" from "no new turn" at all. Use
9249    /// [`Self::wait_for_completion`] with the `completion_baseline` from
9250    /// [`Self::send_admission_tracked`] / [`Self::dispatch_admission_tracked`]
9251    /// when you need to wait for a specific turn. Retained for callers that
9252    /// only need "has this identity ever spoken".
9253    pub async fn wait_for_output(
9254        &self,
9255        identity: &AgentIdentity,
9256        timeout: Duration,
9257    ) -> Result<String, IdentityRuntimeError> {
9258        let deadline = Instant::now() + timeout;
9259        loop {
9260            if let Ok(inspection) = self.inspect(identity).await
9261                && let Some(preview) = inspection.output_preview
9262            {
9263                return Ok(preview);
9264            }
9265            if Instant::now() >= deadline {
9266                return Err(IdentityRuntimeError::Internal(format!(
9267                    "timed out waiting for output from {identity}"
9268                )));
9269            }
9270            tokio::time::sleep(Duration::from_millis(500)).await;
9271        }
9272    }
9273
9274    /// Poll until output_preview contains the given substring, or timeout.
9275    pub async fn wait_for_output_containing(
9276        &self,
9277        identity: &AgentIdentity,
9278        needle: &str,
9279        timeout: Duration,
9280    ) -> Result<String, IdentityRuntimeError> {
9281        let deadline = Instant::now() + timeout;
9282        loop {
9283            if let Ok(inspection) = self.inspect(identity).await
9284                && let Some(ref preview) = inspection.output_preview
9285                && preview.contains(needle)
9286            {
9287                return Ok(preview.clone());
9288            }
9289            if Instant::now() >= deadline {
9290                return Err(IdentityRuntimeError::Internal(format!(
9291                    "timed out waiting for output containing '{needle}' from {identity}"
9292                )));
9293            }
9294            tokio::time::sleep(Duration::from_millis(500)).await;
9295        }
9296    }
9297}
9298
9299/// Wire two identities across mobs, resolving runtime IDs from both IdentityRuntimes.
9300///
9301/// This is a convenience function for same-process cross-mob scenarios where both
9302/// IdentityRuntimes are available. It resolves the AgentRuntimeId for each identity
9303/// and delegates to `UnifiedRuntime::wire_cross_mob()`.
9304pub async fn wire_cross_mob_by_identity(
9305    local_irt: &IdentityRuntime,
9306    local_identity: &AgentIdentity,
9307    remote_irt: &IdentityRuntime,
9308    remote_identity: &AgentIdentity,
9309    local_unified: &crate::UnifiedRuntime,
9310    remote_mob_id: &str,
9311) -> Result<(), IdentityRuntimeError> {
9312    let local_rt = local_irt.runtime_id_for(local_identity).await?;
9313    let remote_rt = remote_irt.runtime_id_for(remote_identity).await?;
9314    Box::pin(local_unified.wire_cross_mob(local_rt.as_str(), remote_rt.as_str(), remote_mob_id))
9315        .await
9316        .map_err(|e| IdentityRuntimeError::Internal(format!("wire_cross_mob: {e}")))
9317}
9318
9319#[cfg(test)]
9320mod reset_reprofile_tests {
9321    use super::*;
9322    use std::sync::Arc;
9323    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
9324    use tokio::sync::{Mutex as AsyncMutex, RwLock as AsyncRwLock};
9325
9326    use super::super::bridge::{
9327        BridgeDelivery, BridgeError, MemberInspection, ResumeSessionOutcome,
9328    };
9329    use super::super::contracts::RosterProvider;
9330    use super::super::local_lease::LocalLeaseProvider;
9331    use super::super::local_store::LocalContinuityStore;
9332    use super::super::types::{
9333        AgentBuildDraft, ContinuityResolveState, CustomizerError, LeaseAcquireResult, LeaseError,
9334        LeaseRenewResult, RosterError, SessionSnapshot,
9335    };
9336
9337    struct MutableRoster {
9338        specs: AsyncRwLock<Vec<DurableAgentSpec>>,
9339    }
9340
9341    struct LostRenewLeaseProvider {
9342        inner: LocalLeaseProvider,
9343        lose_renewals: AtomicBool,
9344    }
9345
9346    impl Default for LostRenewLeaseProvider {
9347        fn default() -> Self {
9348            Self {
9349                inner: LocalLeaseProvider::new(),
9350                lose_renewals: AtomicBool::new(false),
9351            }
9352        }
9353    }
9354
9355    #[async_trait::async_trait]
9356    impl LeaseProvider for LostRenewLeaseProvider {
9357        async fn acquire_leases(
9358            &self,
9359            identities: &[AgentIdentity],
9360            runtime_instance: &str,
9361        ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
9362            self.inner
9363                .acquire_leases(identities, runtime_instance)
9364                .await
9365        }
9366
9367        async fn renew_leases(
9368            &self,
9369            grants: &[LeaseGrant],
9370        ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
9371            if self.lose_renewals.load(Ordering::SeqCst) {
9372                return Ok(grants
9373                    .iter()
9374                    .map(|grant| {
9375                        (
9376                            grant.identity.clone(),
9377                            LeaseRenewResult::Lost {
9378                                identity: grant.identity.clone(),
9379                            },
9380                        )
9381                    })
9382                    .collect());
9383            }
9384            self.inner.renew_leases(grants).await
9385        }
9386
9387        async fn release_leases(&self, grants: &[LeaseGrant]) -> Result<(), LeaseError> {
9388            self.inner.release_leases(grants).await
9389        }
9390    }
9391
9392    struct RecordingReleaseLeaseProvider {
9393        inner: LocalLeaseProvider,
9394        fail_next_release: AtomicBool,
9395        release_attempts: AsyncMutex<Vec<LeaseGrant>>,
9396    }
9397
9398    impl Default for RecordingReleaseLeaseProvider {
9399        fn default() -> Self {
9400            Self {
9401                inner: LocalLeaseProvider::new(),
9402                fail_next_release: AtomicBool::new(false),
9403                release_attempts: AsyncMutex::new(Vec::new()),
9404            }
9405        }
9406    }
9407
9408    impl RecordingReleaseLeaseProvider {
9409        fn fail_next_release(&self) {
9410            self.fail_next_release.store(true, Ordering::SeqCst);
9411        }
9412
9413        async fn release_attempts(&self) -> Vec<LeaseGrant> {
9414            self.release_attempts.lock().await.clone()
9415        }
9416    }
9417
9418    #[async_trait::async_trait]
9419    impl LeaseProvider for RecordingReleaseLeaseProvider {
9420        async fn acquire_leases(
9421            &self,
9422            identities: &[AgentIdentity],
9423            runtime_instance: &str,
9424        ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
9425            self.inner
9426                .acquire_leases(identities, runtime_instance)
9427                .await
9428        }
9429
9430        async fn renew_leases(
9431            &self,
9432            grants: &[LeaseGrant],
9433        ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
9434            self.inner.renew_leases(grants).await
9435        }
9436
9437        async fn release_leases(&self, grants: &[LeaseGrant]) -> Result<(), LeaseError> {
9438            self.release_attempts.lock().await.extend_from_slice(grants);
9439            if self.fail_next_release.swap(false, Ordering::SeqCst) {
9440                return Err(LeaseError::ProviderUnavailable(
9441                    "synthetic retained Broken lease release failure".to_string(),
9442                ));
9443            }
9444            self.inner.release_leases(grants).await
9445        }
9446    }
9447
9448    struct GatedResetContinuityStore {
9449        inner: Arc<LocalContinuityStore>,
9450        upsert_calls: AtomicUsize,
9451        fail_on_call: AtomicUsize,
9452        failure_started: Notify,
9453        release_failure: Notify,
9454    }
9455
9456    #[derive(Default)]
9457    struct GatedResetCustomizer {
9458        entered: Notify,
9459    }
9460
9461    impl GatedResetCustomizer {
9462        async fn wait_for_entry(&self) {
9463            self.entered.notified().await;
9464        }
9465    }
9466
9467    #[async_trait::async_trait]
9468    impl AgentCustomizer for GatedResetCustomizer {
9469        async fn customize_build(
9470            &self,
9471            _context: &AgentBuildContext,
9472            _spec: &DurableAgentSpec,
9473            _draft: &mut AgentBuildDraft,
9474        ) -> Result<(), CustomizerError> {
9475            self.entered.notify_one();
9476            futures::future::pending::<()>().await;
9477            Ok(())
9478        }
9479    }
9480
9481    impl GatedResetContinuityStore {
9482        fn new() -> Result<Self, ContinuityStoreError> {
9483            Ok(Self {
9484                inner: Arc::new(LocalContinuityStore::in_memory()?),
9485                upsert_calls: AtomicUsize::new(0),
9486                fail_on_call: AtomicUsize::new(usize::MAX),
9487                failure_started: Notify::new(),
9488                release_failure: Notify::new(),
9489            })
9490        }
9491
9492        fn fail_after_successful_upserts(&self, successful_upserts: usize) {
9493            let current = self.upsert_calls.load(Ordering::SeqCst);
9494            self.fail_on_call
9495                .store(current + successful_upserts + 1, Ordering::SeqCst);
9496        }
9497
9498        async fn wait_for_failure(&self) {
9499            self.failure_started.notified().await;
9500        }
9501
9502        fn release_failure(&self) {
9503            self.release_failure.notify_one();
9504        }
9505    }
9506
9507    #[async_trait::async_trait]
9508    impl ContinuityStore for GatedResetContinuityStore {
9509        async fn resolve_many(
9510            &self,
9511            identities: &[AgentIdentity],
9512        ) -> Result<BTreeMap<AgentIdentity, ContinuityResolveState>, ContinuityStoreError> {
9513            self.inner.resolve_many(identities).await
9514        }
9515
9516        async fn load_session_snapshot(
9517            &self,
9518            session_id: &SessionId,
9519        ) -> Result<Option<SessionSnapshot>, ContinuityStoreError> {
9520            self.inner.load_session_snapshot(session_id).await
9521        }
9522
9523        async fn save_session_snapshot(
9524            &self,
9525            identity: &AgentIdentity,
9526            session_id: &SessionId,
9527            generation: ContinuityGeneration,
9528            version: CheckpointVersion,
9529            fencing_token: FencingToken,
9530            snapshot: &SessionSnapshot,
9531        ) -> Result<(), ContinuityStoreError> {
9532            self.inner
9533                .save_session_snapshot(
9534                    identity,
9535                    session_id,
9536                    generation,
9537                    version,
9538                    fencing_token,
9539                    snapshot,
9540                )
9541                .await
9542        }
9543
9544        async fn upsert_continuity_record(
9545            &self,
9546            record: &ContinuityRecord,
9547            fencing_token: FencingToken,
9548        ) -> Result<(), ContinuityStoreError> {
9549            let call = self.upsert_calls.fetch_add(1, Ordering::SeqCst) + 1;
9550            if self
9551                .fail_on_call
9552                .compare_exchange(call, usize::MAX, Ordering::SeqCst, Ordering::SeqCst)
9553                .is_ok()
9554            {
9555                self.failure_started.notify_one();
9556                self.release_failure.notified().await;
9557                return Err(ContinuityStoreError::Io(
9558                    "gated final reset upsert failure".to_string(),
9559                ));
9560            }
9561            self.inner
9562                .upsert_continuity_record(record, fencing_token)
9563                .await
9564        }
9565
9566        async fn rollback_continuity_record(
9567            &self,
9568            expected_attempt: &ContinuityRecord,
9569            previous: Option<&ContinuityRecord>,
9570            fencing_token: FencingToken,
9571        ) -> Result<(), ContinuityStoreError> {
9572            self.inner
9573                .rollback_continuity_record(expected_attempt, previous, fencing_token)
9574                .await
9575        }
9576
9577        async fn delete_continuity_record(
9578            &self,
9579            identity: &AgentIdentity,
9580            fencing_token: FencingToken,
9581        ) -> Result<(), ContinuityStoreError> {
9582            self.inner
9583                .delete_continuity_record(identity, fencing_token)
9584                .await
9585        }
9586    }
9587
9588    impl MutableRoster {
9589        fn new(specs: Vec<DurableAgentSpec>) -> Self {
9590            Self {
9591                specs: AsyncRwLock::new(specs),
9592            }
9593        }
9594
9595        async fn set(&self, specs: Vec<DurableAgentSpec>) {
9596            *self.specs.write().await = specs;
9597        }
9598    }
9599
9600    #[async_trait::async_trait]
9601    impl RosterProvider for MutableRoster {
9602        async fn roster(
9603            &self,
9604            _context: &RosterContext,
9605        ) -> Result<Vec<DurableAgentSpec>, RosterError> {
9606            Ok(self.specs.read().await.clone())
9607        }
9608    }
9609
9610    #[derive(Default)]
9611    struct RecordingBridge {
9612        create_profiles: AsyncMutex<Vec<String>>,
9613        create_delay: Duration,
9614        creates_in_flight: AtomicUsize,
9615        max_creates_in_flight: AtomicUsize,
9616        retired_runtime_ids: AsyncMutex<Vec<String>>,
9617        hanging_retire_runtime_ids: AsyncMutex<BTreeSet<String>>,
9618        failing_register_session_ids: AsyncMutex<BTreeSet<String>>,
9619        failing_unregister_session_ids: AsyncMutex<BTreeSet<String>>,
9620        registered_fencing_tokens: AsyncMutex<Vec<FencingToken>>,
9621        authority_transitions: AsyncMutex<Vec<String>>,
9622    }
9623
9624    impl RecordingBridge {
9625        async fn create_profiles(&self) -> Vec<String> {
9626            self.create_profiles.lock().await.clone()
9627        }
9628
9629        fn max_creates_in_flight(&self) -> usize {
9630            self.max_creates_in_flight.load(Ordering::SeqCst)
9631        }
9632
9633        async fn retired_runtime_ids(&self) -> Vec<String> {
9634            self.retired_runtime_ids.lock().await.clone()
9635        }
9636
9637        async fn hang_retire_for(&self, runtime_id: &AgentRuntimeId) {
9638            self.hanging_retire_runtime_ids
9639                .lock()
9640                .await
9641                .insert(runtime_id.to_string());
9642        }
9643
9644        async fn fail_unregister_for(&self, session_id: &SessionId) {
9645            self.failing_unregister_session_ids
9646                .lock()
9647                .await
9648                .insert(session_id.to_string());
9649        }
9650
9651        async fn allow_unregister_for(&self, session_id: &SessionId) {
9652            self.failing_unregister_session_ids
9653                .lock()
9654                .await
9655                .remove(&session_id.to_string());
9656        }
9657
9658        async fn fail_register_for(&self, session_id: &SessionId) {
9659            self.failing_register_session_ids
9660                .lock()
9661                .await
9662                .insert(session_id.to_string());
9663        }
9664
9665        async fn registered_fencing_tokens(&self) -> Vec<FencingToken> {
9666            self.registered_fencing_tokens.lock().await.clone()
9667        }
9668
9669        async fn authority_transitions(&self) -> Vec<String> {
9670            self.authority_transitions.lock().await.clone()
9671        }
9672    }
9673
9674    #[async_trait::async_trait]
9675    impl SessionBridge for RecordingBridge {
9676        async fn create_session(
9677            &self,
9678            _identity: &AgentIdentity,
9679            _runtime_id: &AgentRuntimeId,
9680            spec: &DurableAgentSpec,
9681            _draft: &AgentBuildDraft,
9682            session_id: &SessionId,
9683        ) -> Result<SessionId, BridgeError> {
9684            let in_flight = self.creates_in_flight.fetch_add(1, Ordering::SeqCst) + 1;
9685            self.max_creates_in_flight
9686                .fetch_max(in_flight, Ordering::SeqCst);
9687            tokio::time::sleep(self.create_delay).await;
9688            self.creates_in_flight.fetch_sub(1, Ordering::SeqCst);
9689            self.create_profiles
9690                .lock()
9691                .await
9692                .push(spec.profile.to_string());
9693            Ok(session_id.clone())
9694        }
9695
9696        async fn resume_session(
9697            &self,
9698            _identity: &AgentIdentity,
9699            _runtime_id: &AgentRuntimeId,
9700            _spec: &DurableAgentSpec,
9701            _draft: &AgentBuildDraft,
9702            _session_id: &SessionId,
9703            _snapshot: &SessionSnapshot,
9704        ) -> Result<ResumeSessionOutcome, BridgeError> {
9705            Err(BridgeError::Mob(
9706                "resume not used in reset test".to_string(),
9707            ))
9708        }
9709
9710        async fn deliver_admitted(
9711            &self,
9712            _runtime_id: &AgentRuntimeId,
9713            _delivery: BridgeDelivery,
9714        ) -> Result<SessionId, BridgeError> {
9715            Err(BridgeError::Mob(
9716                "deliver not used in reset test".to_string(),
9717            ))
9718        }
9719
9720        async fn checkpoint_session(
9721            &self,
9722            _runtime_id: &AgentRuntimeId,
9723            _session_id: &SessionId,
9724        ) -> Result<SessionSnapshot, BridgeError> {
9725            Err(BridgeError::Mob(
9726                "checkpoint not used in reset test".to_string(),
9727            ))
9728        }
9729
9730        async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9731            self.retired_runtime_ids
9732                .lock()
9733                .await
9734                .push(runtime_id.to_string());
9735            if self
9736                .hanging_retire_runtime_ids
9737                .lock()
9738                .await
9739                .contains(runtime_id.as_str())
9740            {
9741                futures::future::pending::<()>().await;
9742            }
9743            Ok(())
9744        }
9745
9746        async fn inspect_member(
9747            &self,
9748            _runtime_id: &AgentRuntimeId,
9749        ) -> Result<MemberInspection, BridgeError> {
9750            Err(BridgeError::Mob(
9751                "inspect not used in reset test".to_string(),
9752            ))
9753        }
9754
9755        async fn register_session_runtime_state(
9756            &self,
9757            session_id: &SessionId,
9758            _identity: &AgentIdentity,
9759            _generation: ContinuityGeneration,
9760            checkpoint_version: CheckpointVersion,
9761            fencing_token: FencingToken,
9762        ) -> Result<CheckpointVersion, BridgeError> {
9763            self.registered_fencing_tokens
9764                .lock()
9765                .await
9766                .push(fencing_token);
9767            self.authority_transitions
9768                .lock()
9769                .await
9770                .push(format!("register:{}", fencing_token.get()));
9771            if self
9772                .failing_register_session_ids
9773                .lock()
9774                .await
9775                .contains(&session_id.to_string())
9776            {
9777                return Err(BridgeError::Mob(
9778                    "synthetic live-session rebind failure".to_string(),
9779                ));
9780            }
9781            Ok(checkpoint_version)
9782        }
9783
9784        async fn suspend_session_runtime_state(
9785            &self,
9786            session_id: &SessionId,
9787        ) -> Result<(), BridgeError> {
9788            self.authority_transitions
9789                .lock()
9790                .await
9791                .push(format!("suspend:{session_id}"));
9792            Ok(())
9793        }
9794
9795        async fn unregister_session_runtime_state(
9796            &self,
9797            session_id: &SessionId,
9798        ) -> Result<(), BridgeError> {
9799            if self
9800                .failing_unregister_session_ids
9801                .lock()
9802                .await
9803                .contains(&session_id.to_string())
9804            {
9805                return Err(BridgeError::Mob("old session still draining".to_string()));
9806            }
9807            Ok(())
9808        }
9809    }
9810
9811    /// Bridge used to prove REQ-33 metadata hot reloads never cross the
9812    /// retire/resume boundary. Initial creation and ordinary dispatch work;
9813    /// any attempted resume fails loudly, reproducing the real gateway's
9814    /// still-draining-member collision without an LLM or network dependency.
9815    #[derive(Default)]
9816    struct HotReloadBridge {
9817        sessions: AsyncMutex<BTreeMap<String, SessionId>>,
9818        retire_calls: AtomicUsize,
9819        resume_calls: AtomicUsize,
9820    }
9821
9822    #[async_trait::async_trait]
9823    impl SessionBridge for HotReloadBridge {
9824        async fn create_session(
9825            &self,
9826            _identity: &AgentIdentity,
9827            runtime_id: &AgentRuntimeId,
9828            _spec: &DurableAgentSpec,
9829            _draft: &AgentBuildDraft,
9830            session_id: &SessionId,
9831        ) -> Result<SessionId, BridgeError> {
9832            self.sessions
9833                .lock()
9834                .await
9835                .insert(runtime_id.to_string(), session_id.clone());
9836            Ok(session_id.clone())
9837        }
9838
9839        async fn resume_session(
9840            &self,
9841            _identity: &AgentIdentity,
9842            _runtime_id: &AgentRuntimeId,
9843            _spec: &DurableAgentSpec,
9844            _draft: &AgentBuildDraft,
9845            _session_id: &SessionId,
9846            _snapshot: &SessionSnapshot,
9847        ) -> Result<ResumeSessionOutcome, BridgeError> {
9848            self.resume_calls.fetch_add(1, Ordering::SeqCst);
9849            Err(BridgeError::Mob(
9850                "same-profile metadata hot reload attempted session resume".to_string(),
9851            ))
9852        }
9853
9854        async fn deliver_admitted(
9855            &self,
9856            runtime_id: &AgentRuntimeId,
9857            _delivery: BridgeDelivery,
9858        ) -> Result<SessionId, BridgeError> {
9859            self.sessions
9860                .lock()
9861                .await
9862                .get(runtime_id.as_str())
9863                .cloned()
9864                .ok_or_else(|| BridgeError::Mob(format!("missing session for {runtime_id}")))
9865        }
9866
9867        async fn checkpoint_session(
9868            &self,
9869            _runtime_id: &AgentRuntimeId,
9870            _session_id: &SessionId,
9871        ) -> Result<SessionSnapshot, BridgeError> {
9872            Err(BridgeError::Mob(
9873                "checkpoint not used in hot-reload test".to_string(),
9874            ))
9875        }
9876
9877        async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9878            self.retire_calls.fetch_add(1, Ordering::SeqCst);
9879            self.sessions.lock().await.remove(runtime_id.as_str());
9880            Ok(())
9881        }
9882
9883        async fn inspect_member(
9884            &self,
9885            _runtime_id: &AgentRuntimeId,
9886        ) -> Result<MemberInspection, BridgeError> {
9887            Err(BridgeError::Mob(
9888                "inspect not used in hot-reload test".to_string(),
9889            ))
9890        }
9891    }
9892
9893    /// Lower-plane model for lease-loss reconciliation. Resume rejects a
9894    /// duplicate concrete alias, so the tests prove Broken cleanup happens
9895    /// before profile replacement rather than merely observing call counts.
9896    #[derive(Default)]
9897    struct LostCleanupBridge {
9898        members: AsyncMutex<BTreeSet<String>>,
9899        session_runtime_states: AsyncMutex<BTreeSet<String>>,
9900        retire_calls: AtomicUsize,
9901        unregister_calls: AtomicUsize,
9902        resume_collisions: AtomicUsize,
9903    }
9904
9905    impl LostCleanupBridge {
9906        async fn member_count(&self) -> usize {
9907            self.members.lock().await.len()
9908        }
9909
9910        async fn session_runtime_state_count(&self) -> usize {
9911            self.session_runtime_states.lock().await.len()
9912        }
9913    }
9914
9915    async fn force_renewal_lost(
9916        runtime: &IdentityRuntime,
9917        lease_provider: &LostRenewLeaseProvider,
9918        identity: &AgentIdentity,
9919    ) -> Result<(), Box<dyn std::error::Error>> {
9920        lease_provider.lose_renewals.store(true, Ordering::SeqCst);
9921        {
9922            let mut entries = runtime.entries.write().await;
9923            let entry = entries
9924                .get_mut(identity)
9925                .ok_or("identity disappeared before forced renewal")?;
9926            let lease = entry
9927                .lease
9928                .as_mut()
9929                .ok_or("active identity has no lease before forced renewal")?;
9930            lease.ttl = Duration::ZERO;
9931        }
9932        assert!(matches!(
9933            runtime.renew_due_leases_once().await,
9934            Err(IdentityRuntimeError::LeaseLost(lost)) if lost == *identity
9935        ));
9936        Ok(())
9937    }
9938
9939    #[async_trait::async_trait]
9940    impl SessionBridge for LostCleanupBridge {
9941        async fn create_session(
9942            &self,
9943            _identity: &AgentIdentity,
9944            runtime_id: &AgentRuntimeId,
9945            _spec: &DurableAgentSpec,
9946            _draft: &AgentBuildDraft,
9947            session_id: &SessionId,
9948        ) -> Result<SessionId, BridgeError> {
9949            if !self.members.lock().await.insert(runtime_id.to_string()) {
9950                return Err(BridgeError::Mob(format!(
9951                    "member collision for {runtime_id}"
9952                )));
9953            }
9954            Ok(session_id.clone())
9955        }
9956
9957        async fn resume_session(
9958            &self,
9959            _identity: &AgentIdentity,
9960            runtime_id: &AgentRuntimeId,
9961            _spec: &DurableAgentSpec,
9962            _draft: &AgentBuildDraft,
9963            session_id: &SessionId,
9964            _snapshot: &SessionSnapshot,
9965        ) -> Result<ResumeSessionOutcome, BridgeError> {
9966            if !self.members.lock().await.insert(runtime_id.to_string()) {
9967                self.resume_collisions.fetch_add(1, Ordering::SeqCst);
9968                return Err(BridgeError::Mob(format!(
9969                    "member collision for {runtime_id}"
9970                )));
9971            }
9972            Ok(ResumeSessionOutcome::Resumed {
9973                session_id: session_id.clone(),
9974            })
9975        }
9976
9977        async fn deliver_admitted(
9978            &self,
9979            _runtime_id: &AgentRuntimeId,
9980            _delivery: BridgeDelivery,
9981        ) -> Result<SessionId, BridgeError> {
9982            Err(BridgeError::Mob(
9983                "deliver not used in lost cleanup test".to_string(),
9984            ))
9985        }
9986
9987        async fn checkpoint_session(
9988            &self,
9989            _runtime_id: &AgentRuntimeId,
9990            _session_id: &SessionId,
9991        ) -> Result<SessionSnapshot, BridgeError> {
9992            Err(BridgeError::Mob(
9993                "checkpoint not used in lost cleanup test".to_string(),
9994            ))
9995        }
9996
9997        async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9998            self.retire_calls.fetch_add(1, Ordering::SeqCst);
9999            self.members.lock().await.remove(runtime_id.as_str());
10000            Ok(())
10001        }
10002
10003        async fn inspect_member(
10004            &self,
10005            _runtime_id: &AgentRuntimeId,
10006        ) -> Result<MemberInspection, BridgeError> {
10007            Err(BridgeError::Mob(
10008                "inspect not used in lost cleanup test".to_string(),
10009            ))
10010        }
10011
10012        async fn register_session_runtime_state(
10013            &self,
10014            session_id: &SessionId,
10015            _identity: &AgentIdentity,
10016            _generation: ContinuityGeneration,
10017            checkpoint_version: CheckpointVersion,
10018            _fencing_token: FencingToken,
10019        ) -> Result<CheckpointVersion, BridgeError> {
10020            self.session_runtime_states
10021                .lock()
10022                .await
10023                .insert(session_id.to_string());
10024            Ok(checkpoint_version)
10025        }
10026
10027        async fn unregister_session_runtime_state(
10028            &self,
10029            session_id: &SessionId,
10030        ) -> Result<(), BridgeError> {
10031            self.unregister_calls.fetch_add(1, Ordering::SeqCst);
10032            self.session_runtime_states
10033                .lock()
10034                .await
10035                .remove(&session_id.to_string());
10036            Ok(())
10037        }
10038    }
10039
10040    fn durable_spec(identity: AgentIdentity, profile: &str) -> DurableAgentSpec {
10041        DurableAgentSpec {
10042            identity,
10043            profile: meerkat_mob::ProfileName::from(profile),
10044            addressability: AgentAddressability::Addressable,
10045            display_name: None,
10046            labels: BTreeMap::new(),
10047            context: None,
10048            additional_instructions: Vec::new(),
10049            initial_message: None,
10050            runtime_mode_override: None,
10051            backend: None,
10052            binding: None,
10053        }
10054    }
10055
10056    /// Always answers create with the host's deterministic rejection — the
10057    /// exact string shape rpc_gateway mints when the app-side
10058    /// `callback/build_agent` round trip COMPLETES with an error.
10059    #[derive(Default)]
10060    struct HostRejectingBridge {
10061        create_attempts: std::sync::atomic::AtomicUsize,
10062    }
10063
10064    impl HostRejectingBridge {
10065        fn attempts(&self) -> usize {
10066            self.create_attempts
10067                .load(std::sync::atomic::Ordering::SeqCst)
10068        }
10069    }
10070
10071    #[async_trait::async_trait]
10072    impl SessionBridge for HostRejectingBridge {
10073        async fn create_session(
10074            &self,
10075            _identity: &AgentIdentity,
10076            _runtime_id: &AgentRuntimeId,
10077            _spec: &DurableAgentSpec,
10078            _draft: &AgentBuildDraft,
10079            _session_id: &SessionId,
10080        ) -> Result<SessionId, BridgeError> {
10081            self.create_attempts
10082                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
10083            Err(BridgeError::Mob(
10084                "spawn_member: callback/build_agent failed: callback error: candidate-mode \
10085                 effect gate refused this build"
10086                    .to_string(),
10087            ))
10088        }
10089
10090        async fn resume_session(
10091            &self,
10092            _identity: &AgentIdentity,
10093            _runtime_id: &AgentRuntimeId,
10094            _spec: &DurableAgentSpec,
10095            _draft: &AgentBuildDraft,
10096            _session_id: &SessionId,
10097            _snapshot: &SessionSnapshot,
10098        ) -> Result<ResumeSessionOutcome, BridgeError> {
10099            Err(BridgeError::Mob(
10100                "resume not used in host-reject test".to_string(),
10101            ))
10102        }
10103
10104        async fn deliver_admitted(
10105            &self,
10106            _runtime_id: &AgentRuntimeId,
10107            _delivery: BridgeDelivery,
10108        ) -> Result<SessionId, BridgeError> {
10109            Err(BridgeError::Mob(
10110                "deliver not used in host-reject test".to_string(),
10111            ))
10112        }
10113
10114        async fn checkpoint_session(
10115            &self,
10116            _runtime_id: &AgentRuntimeId,
10117            _session_id: &SessionId,
10118        ) -> Result<SessionSnapshot, BridgeError> {
10119            Err(BridgeError::Mob(
10120                "checkpoint not used in host-reject test".to_string(),
10121            ))
10122        }
10123
10124        async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
10125            Ok(())
10126        }
10127    }
10128
10129    /// Herd-investigation park: a build the HOST deterministically rejects
10130    /// (the candidate-mode effect gate) parks the identity typed on the
10131    /// FIRST attempt — no repair-loop churn — and a roster spec change
10132    /// re-admits exactly one new attempt.
10133    #[tokio::test]
10134    async fn host_rejected_build_parks_identity_until_spec_changes()
10135    -> Result<(), Box<dyn std::error::Error>> {
10136        let identity = AgentIdentity::parse("domain:gated")?;
10137        let bridge = Arc::new(HostRejectingBridge::default());
10138        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10139            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10140            lease_provider: Arc::new(LocalLeaseProvider::new()),
10141            runtime_instance_id: "host-reject-park-test".to_string(),
10142            has_runtime_store: true,
10143            durability_policy: DurabilityPolicy::SyncWriteThrough,
10144            bridge: Some(bridge.clone()),
10145            default_timeout: None,
10146        }));
10147        runtime
10148            .register(
10149                durable_spec(identity.clone(), "domain"),
10150                IdentityLifecycleState::Dormant,
10151                None,
10152                None,
10153            )
10154            .await;
10155
10156        // The first attempt reaches the host exactly once; the deterministic
10157        // rejection parks the identity typed.
10158        assert!(
10159            runtime.materialize(&identity).await.is_err(),
10160            "gated build must fail"
10161        );
10162        assert_eq!(bridge.attempts(), 1);
10163        let park = runtime
10164            .host_rejected_build_park(&identity)
10165            .await
10166            .ok_or("the first host rejection must park the identity")?;
10167        assert!(park.reason.contains("callback error"));
10168
10169        // Parked: the next attempt fails fast typed WITHOUT reaching the
10170        // host, and the repair supervisor's Broken selection excludes the
10171        // identity even when a reconcile re-registers it Broken with the
10172        // same spec.
10173        match runtime.materialize(&identity).await {
10174            Err(IdentityRuntimeError::HostRejectedBuild { reason, .. }) => {
10175                assert!(reason.contains("callback error"));
10176            }
10177            other => return Err(format!("expected the typed park, got {other:?}").into()),
10178        }
10179        assert_eq!(
10180            bridge.attempts(),
10181            1,
10182            "a parked identity must not re-ask the host"
10183        );
10184        runtime
10185            .register(
10186                durable_spec(identity.clone(), "domain"),
10187                IdentityLifecycleState::Broken,
10188                None,
10189                None,
10190            )
10191            .await;
10192        assert!(
10193            runtime.repairable_broken_identities().await.is_empty(),
10194            "the repair supervisor must skip a parked identity"
10195        );
10196
10197        // A spec change clears the park: the retry is permitted and reaches
10198        // the host exactly once more.
10199        let mut changed = durable_spec(identity.clone(), "domain");
10200        changed
10201            .labels
10202            .insert("policy_epoch".to_string(), "2".to_string());
10203        runtime
10204            .register(changed, IdentityLifecycleState::Dormant, None, None)
10205            .await;
10206        assert!(
10207            runtime.host_rejected_build_park(&identity).await.is_none(),
10208            "a spec change must clear the park"
10209        );
10210        assert!(runtime.materialize(&identity).await.is_err());
10211        assert_eq!(
10212            bridge.attempts(),
10213            2,
10214            "a changed spec re-admits exactly one new build attempt"
10215        );
10216        Ok(())
10217    }
10218
10219    #[test]
10220    fn external_delivery_omits_unrepresentable_interaction_id_carrier()
10221    -> Result<(), Box<dyn std::error::Error>> {
10222        let identity = AgentIdentity::parse("target-smoke")?;
10223        let mut external = durable_spec(identity.clone(), "target");
10224        external.backend = Some(meerkat_mob::MobBackendKind::External);
10225        assert_eq!(
10226            interaction_id_for_delivery(&external, Some("interaction-1")),
10227            None
10228        );
10229
10230        let local = durable_spec(identity, "target");
10231        assert_eq!(
10232            interaction_id_for_delivery(&local, Some("interaction-1")),
10233            Some("interaction-1")
10234        );
10235        Ok(())
10236    }
10237
10238    async fn lazy_context_with_broken_retained_lease(
10239        identity: AgentIdentity,
10240        spec: DurableAgentSpec,
10241        roster: Arc<MutableRoster>,
10242        lease_provider: Arc<RecordingReleaseLeaseProvider>,
10243        runtime_instance_id: &str,
10244    ) -> Result<
10245        (
10246            Arc<IdentityRuntime>,
10247            IdentityFirstRuntimeContext,
10248            LeaseGrant,
10249        ),
10250        Box<dyn std::error::Error>,
10251    > {
10252        let continuity_store = Arc::new(LocalContinuityStore::in_memory()?);
10253        let acquired = lease_provider
10254            .acquire_leases(std::slice::from_ref(&identity), runtime_instance_id)
10255            .await?;
10256        let grant = match acquired.get(&identity) {
10257            Some(LeaseAcquireResult::Acquired(grant)) => grant.clone(),
10258            other => return Err(format!("expected retained lease grant, got {other:?}").into()),
10259        };
10260        let record = ContinuityRecord {
10261            identity: identity.clone(),
10262            agent_runtime_id: AgentRuntimeId::parse(&format!("rt:{identity}:0"))?,
10263            session_id: SessionId::new(),
10264            generation: ContinuityGeneration::new(0),
10265            checkpoint_version: CheckpointVersion::new(0),
10266        };
10267        continuity_store
10268            .upsert_continuity_record(&record, grant.fencing_token)
10269            .await?;
10270        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10271            continuity_store,
10272            lease_provider,
10273            runtime_instance_id: runtime_instance_id.to_string(),
10274            has_runtime_store: true,
10275            durability_policy: DurabilityPolicy::SyncWriteThrough,
10276            bridge: None,
10277            default_timeout: None,
10278        }));
10279        runtime
10280            .register(
10281                spec,
10282                IdentityLifecycleState::Active,
10283                Some(record),
10284                Some(grant.clone()),
10285            )
10286            .await;
10287        // Model the public rollback state exercised by
10288        // `identity_first_runtime_reset_register_failure_cleans_new_member_and_preserves_old_continuity`:
10289        // the identity is fail-closed Broken while its exact current grant is
10290        // retained in `entry.lease` for repair.
10291        runtime
10292            .entries
10293            .write()
10294            .await
10295            .get_mut(&identity)
10296            .ok_or("seeded identity disappeared")?
10297            .state = IdentityLifecycleState::Broken;
10298        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
10299            runtime.clone(),
10300            roster,
10301            None,
10302            None,
10303            None,
10304            IdentityBootstrapMode::LazyMaterialize,
10305        );
10306        Ok((runtime, context, grant))
10307    }
10308
10309    async fn active_alias_runtime(
10310        runtime_instance_id: &str,
10311        identity: &str,
10312    ) -> Result<(Arc<IdentityRuntime>, String), Box<dyn std::error::Error>> {
10313        let identity = AgentIdentity::parse(identity)?;
10314        let alias = format!("rt:{identity}:0");
10315        let record = ContinuityRecord {
10316            identity: identity.clone(),
10317            agent_runtime_id: AgentRuntimeId::parse(&alias)?,
10318            session_id: SessionId::new(),
10319            generation: ContinuityGeneration::new(0),
10320            checkpoint_version: CheckpointVersion::new(0),
10321        };
10322        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10323            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10324            lease_provider: Arc::new(LocalLeaseProvider::new()),
10325            runtime_instance_id: runtime_instance_id.to_string(),
10326            has_runtime_store: true,
10327            durability_policy: DurabilityPolicy::SyncWriteThrough,
10328            bridge: None,
10329            default_timeout: None,
10330        }));
10331        runtime
10332            .register(
10333                durable_spec(identity, "domain"),
10334                IdentityLifecycleState::Active,
10335                Some(record),
10336                None,
10337            )
10338            .await;
10339        Ok((runtime, alias))
10340    }
10341
10342    async fn alias_target(
10343        runtime: &Arc<IdentityRuntime>,
10344        alias: &str,
10345    ) -> Result<MemberAliasLifecycleTarget, Box<dyn std::error::Error>> {
10346        runtime
10347            .member_alias_lifecycle_target(alias)
10348            .await?
10349            .ok_or_else(|| {
10350                format!("generated alias did not resolve to lifecycle target: {alias}").into()
10351            })
10352    }
10353
10354    #[tokio::test]
10355    async fn permanent_stream_loss_breaks_only_the_current_active_embodiment()
10356    -> Result<(), Box<dyn std::error::Error>> {
10357        let identity = AgentIdentity::parse("domain:stream-loss")?;
10358        let (runtime, alias) =
10359            active_alias_runtime("stream-loss-runtime", identity.as_str()).await?;
10360        runtime
10361            .update_lease(
10362                &identity,
10363                LeaseGrant {
10364                    identity: identity.clone(),
10365                    fencing_token: FencingToken::new(7),
10366                    ttl: Duration::from_mins(1),
10367                },
10368            )
10369            .await?;
10370
10371        assert!(
10372            runtime
10373                .mark_active_runtime_broken(&identity, &alias, 7, "event stream closed")
10374                .await?
10375        );
10376        assert_eq!(
10377            runtime.status(&identity).await?.state,
10378            IdentityLifecycleState::Broken
10379        );
10380        assert!(
10381            runtime
10382                .entries
10383                .read()
10384                .await
10385                .get(&identity)
10386                .and_then(|entry| entry.lease.as_ref())
10387                .is_some(),
10388            "repair must retain the exact live lease for fenced cleanup"
10389        );
10390        Ok(())
10391    }
10392
10393    #[tokio::test]
10394    async fn stale_stream_loss_does_not_break_replacement_embodiment()
10395    -> Result<(), Box<dyn std::error::Error>> {
10396        let identity = AgentIdentity::parse("domain:replacement")?;
10397        let (runtime, _) = active_alias_runtime("replacement-runtime", identity.as_str()).await?;
10398
10399        assert!(
10400            !runtime
10401                .mark_active_runtime_broken(
10402                    &identity,
10403                    "rt:domain:replacement:stale",
10404                    0,
10405                    "stale event stream closed",
10406                )
10407                .await?
10408        );
10409        assert_eq!(
10410            runtime.status(&identity).await?.state,
10411            IdentityLifecycleState::Active
10412        );
10413        Ok(())
10414    }
10415
10416    #[tokio::test]
10417    async fn stale_fence_stream_loss_does_not_break_same_alias_replacement()
10418    -> Result<(), Box<dyn std::error::Error>> {
10419        let identity = AgentIdentity::parse("domain:replacement-fence")?;
10420        let (runtime, alias) =
10421            active_alias_runtime("replacement-fence-runtime", identity.as_str()).await?;
10422        runtime
10423            .update_lease(
10424                &identity,
10425                LeaseGrant {
10426                    identity: identity.clone(),
10427                    fencing_token: FencingToken::new(8),
10428                    ttl: Duration::from_mins(1),
10429                },
10430            )
10431            .await?;
10432
10433        assert!(
10434            !runtime
10435                .mark_active_runtime_broken(&identity, &alias, 7, "old-fence event stream closed",)
10436                .await?
10437        );
10438        assert_eq!(
10439            runtime.status(&identity).await?.state,
10440            IdentityLifecycleState::Active
10441        );
10442        Ok(())
10443    }
10444
10445    #[tokio::test]
10446    async fn compound_alias_targets_sort_opposite_orders_without_deadlock()
10447    -> Result<(), Box<dyn std::error::Error>> {
10448        let (runtime_a, alias_a) = active_alias_runtime("00-alias-runtime", "domain:alpha").await?;
10449        let (runtime_b, alias_b) = active_alias_runtime("01-alias-runtime", "domain:beta").await?;
10450
10451        // Hold the globally first lock while both transactions are admitted.
10452        // Correct ordering makes both wait on A without touching B. An input-
10453        // ordered implementation lets the reverse request take B first and
10454        // then deadlocks once the forward request receives A.
10455        let held_a = alias_target(&runtime_a, &alias_a)
10456            .await?
10457            .lock
10458            .clone()
10459            .lock_owned()
10460            .await;
10461        let completed = Arc::new(AtomicUsize::new(0));
10462
10463        let forward = tokio::spawn({
10464            let targets = vec![
10465                alias_target(&runtime_a, &alias_a).await?,
10466                alias_target(&runtime_b, &alias_b).await?,
10467            ];
10468            let completed = Arc::clone(&completed);
10469            async move {
10470                IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10471                    let completed = Arc::clone(&completed);
10472                    async move {
10473                        completed.fetch_add(1, Ordering::SeqCst);
10474                        Ok(())
10475                    }
10476                })
10477                .await
10478            }
10479        });
10480        tokio::time::timeout(Duration::from_secs(1), async {
10481            loop {
10482                if !runtime_a.foreground_operations.lock().await.is_empty() {
10483                    break;
10484                }
10485                tokio::task::yield_now().await;
10486            }
10487        })
10488        .await?;
10489
10490        let reverse = tokio::spawn({
10491            let targets = vec![
10492                alias_target(&runtime_b, &alias_b).await?,
10493                alias_target(&runtime_a, &alias_a).await?,
10494            ];
10495            let completed = Arc::clone(&completed);
10496            async move {
10497                IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10498                    let completed = Arc::clone(&completed);
10499                    async move {
10500                        completed.fetch_add(1, Ordering::SeqCst);
10501                        Ok(())
10502                    }
10503                })
10504                .await
10505            }
10506        });
10507        tokio::time::timeout(Duration::from_secs(1), async {
10508            loop {
10509                if runtime_a.foreground_operations.lock().await.len() >= 2 {
10510                    break;
10511                }
10512                tokio::task::yield_now().await;
10513            }
10514        })
10515        .await?;
10516        tokio::task::yield_now().await;
10517
10518        let b_probe = alias_target(&runtime_b, &alias_b)
10519            .await?
10520            .lock
10521            .try_lock_owned()
10522            .map_err(|_| "reverse-order request acquired B before globally-first A")?;
10523        drop(b_probe);
10524        drop(held_a);
10525
10526        tokio::time::timeout(Duration::from_secs(2), forward).await???;
10527        tokio::time::timeout(Duration::from_secs(2), reverse).await???;
10528        assert_eq!(completed.load(Ordering::SeqCst), 2);
10529
10530        runtime_a.close_foreground_operations();
10531        runtime_b.close_foreground_operations();
10532        tokio::time::timeout(Duration::from_secs(2), async {
10533            tokio::join!(
10534                runtime_a.join_foreground_operations(),
10535                runtime_b.join_foreground_operations()
10536            );
10537        })
10538        .await?;
10539        Ok(())
10540    }
10541
10542    #[tokio::test]
10543    async fn dropped_compound_alias_caller_still_completes_operation_boundary()
10544    -> Result<(), Box<dyn std::error::Error>> {
10545        let (runtime_a, alias_a) = active_alias_runtime("00-drop-runtime", "domain:alpha").await?;
10546        let (runtime_b, alias_b) = active_alias_runtime("01-drop-runtime", "domain:beta").await?;
10547        let targets = vec![
10548            alias_target(&runtime_a, &alias_a).await?,
10549            alias_target(&runtime_b, &alias_b).await?,
10550        ];
10551        let entered = Arc::new(tokio::sync::Semaphore::new(0));
10552        let release = Arc::new(tokio::sync::Semaphore::new(0));
10553        let completed = Arc::new(AtomicUsize::new(0));
10554        let caller = tokio::spawn({
10555            let entered = Arc::clone(&entered);
10556            let release = Arc::clone(&release);
10557            let completed = Arc::clone(&completed);
10558            async move {
10559                IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10560                    let entered = Arc::clone(&entered);
10561                    let release = Arc::clone(&release);
10562                    let completed = Arc::clone(&completed);
10563                    async move {
10564                        entered.add_permits(1);
10565                        release
10566                            .acquire()
10567                            .await
10568                            .map_err(|error| error.to_string())?
10569                            .forget();
10570                        completed.fetch_add(1, Ordering::SeqCst);
10571                        Ok(())
10572                    }
10573                })
10574                .await
10575            }
10576        });
10577        tokio::time::timeout(Duration::from_secs(2), entered.acquire())
10578            .await??
10579            .forget();
10580
10581        caller.abort();
10582        match caller.await {
10583            Err(error) => assert!(error.is_cancelled()),
10584            Ok(result) => {
10585                return Err(format!("aborted caller unexpectedly returned: {result:?}").into());
10586            }
10587        }
10588        release.add_permits(1);
10589
10590        runtime_a.close_foreground_operations();
10591        runtime_b.close_foreground_operations();
10592        tokio::time::timeout(Duration::from_secs(2), async {
10593            tokio::join!(
10594                runtime_a.join_foreground_operations(),
10595                runtime_b.join_foreground_operations()
10596            );
10597        })
10598        .await?;
10599        assert_eq!(
10600            completed.load(Ordering::SeqCst),
10601            1,
10602            "runtime-owned compound transaction must reach its boundary"
10603        );
10604        Ok(())
10605    }
10606
10607    #[tokio::test]
10608    async fn either_compound_alias_runtime_shutdown_waits_for_operation_boundary()
10609    -> Result<(), Box<dyn std::error::Error>> {
10610        for shutdown_first_runtime in [true, false] {
10611            let suffix = if shutdown_first_runtime {
10612                "first"
10613            } else {
10614                "second"
10615            };
10616            let (runtime_a, alias_a) =
10617                active_alias_runtime(&format!("00-shutdown-{suffix}"), "domain:alpha").await?;
10618            let (runtime_b, alias_b) =
10619                active_alias_runtime(&format!("01-shutdown-{suffix}"), "domain:beta").await?;
10620            let targets = vec![
10621                alias_target(&runtime_a, &alias_a).await?,
10622                alias_target(&runtime_b, &alias_b).await?,
10623            ];
10624            let entered = Arc::new(tokio::sync::Semaphore::new(0));
10625            let release = Arc::new(tokio::sync::Semaphore::new(0));
10626            let operation = tokio::spawn({
10627                let entered = Arc::clone(&entered);
10628                let release = Arc::clone(&release);
10629                async move {
10630                    IdentityRuntime::run_member_alias_targets_operation_tracked(
10631                        targets,
10632                        move || {
10633                            let entered = Arc::clone(&entered);
10634                            let release = Arc::clone(&release);
10635                            async move {
10636                                entered.add_permits(1);
10637                                release
10638                                    .acquire()
10639                                    .await
10640                                    .map_err(|error| error.to_string())?
10641                                    .forget();
10642                                Ok(())
10643                            }
10644                        },
10645                    )
10646                    .await
10647                }
10648            });
10649            tokio::time::timeout(Duration::from_secs(2), entered.acquire())
10650                .await??
10651                .forget();
10652
10653            let shutting_down = if shutdown_first_runtime {
10654                Arc::clone(&runtime_a)
10655            } else {
10656                Arc::clone(&runtime_b)
10657            };
10658            shutting_down.close_foreground_operations();
10659            let mut shutdown = tokio::spawn(async move {
10660                shutting_down.join_foreground_operations().await;
10661            });
10662            assert!(
10663                tokio::time::timeout(Duration::from_millis(50), &mut shutdown)
10664                    .await
10665                    .is_err(),
10666                "shutdown of {suffix} participating runtime returned before compound boundary"
10667            );
10668
10669            release.add_permits(1);
10670            tokio::time::timeout(Duration::from_secs(2), &mut shutdown).await??;
10671            tokio::time::timeout(Duration::from_secs(2), operation).await???;
10672
10673            runtime_a.close_foreground_operations();
10674            runtime_b.close_foreground_operations();
10675            tokio::time::timeout(Duration::from_secs(2), async {
10676                tokio::join!(
10677                    runtime_a.join_foreground_operations(),
10678                    runtime_b.join_foreground_operations()
10679                );
10680            })
10681            .await?;
10682        }
10683        Ok(())
10684    }
10685
10686    #[tokio::test]
10687    async fn restore_flow_bounds_parallel_member_creation() -> Result<(), Box<dyn std::error::Error>>
10688    {
10689        let specs = (0..8)
10690            .map(|index| {
10691                Ok(durable_spec(
10692                    AgentIdentity::parse(&format!("domain:restore-{index}"))?,
10693                    "domain",
10694                ))
10695            })
10696            .collect::<Result<Vec<_>, Box<dyn std::error::Error>>>()?;
10697        let bridge = Arc::new(RecordingBridge {
10698            create_delay: Duration::from_millis(25),
10699            ..RecordingBridge::default()
10700        });
10701        let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
10702            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10703            lease_provider: Arc::new(LocalLeaseProvider::new()),
10704            runtime_instance_id: "parallel-restore-test".to_string(),
10705            has_runtime_store: true,
10706            durability_policy: DurabilityPolicy::SyncWriteThrough,
10707            bridge: Some(bridge.clone()),
10708            default_timeout: None,
10709        });
10710
10711        let result = super::super::orchestrator::restore_flow(&runtime, &specs, None, None).await?;
10712
10713        assert_eq!(result.outcomes.len(), specs.len());
10714        assert!(
10715            bridge.max_creates_in_flight() > 1,
10716            "member creation should no longer be serial"
10717        );
10718        assert!(
10719            bridge.max_creates_in_flight()
10720                <= super::super::orchestrator::IDENTITY_RESTORE_CONCURRENCY,
10721            "restore concurrency must remain bounded"
10722        );
10723        Ok(())
10724    }
10725
10726    #[tokio::test]
10727    async fn reset_reprofiles_session_from_runtime_configured_roster_provider()
10728    -> Result<(), Box<dyn std::error::Error>> {
10729        let identity = AgentIdentity::parse("domain:security")?;
10730        let roster = Arc::new(MutableRoster::new(vec![durable_spec(
10731            identity.clone(),
10732            "domain",
10733        )]));
10734        let bridge = Arc::new(RecordingBridge::default());
10735        let runtime = Arc::new(
10736            IdentityRuntime::new(IdentityRuntimeConfig {
10737                continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10738                lease_provider: Arc::new(LocalLeaseProvider::new()),
10739                runtime_instance_id: "reset-reprofile-test".to_string(),
10740                has_runtime_store: true,
10741                durability_policy: DurabilityPolicy::SyncWriteThrough,
10742                bridge: Some(bridge.clone()),
10743                default_timeout: None,
10744            })
10745            .with_reset_roster_provider(roster.clone()),
10746        );
10747
10748        super::super::orchestrator::restore_flow(
10749            &runtime,
10750            &roster
10751                .roster(&RosterContext {
10752                    mob_definition: None,
10753                    previous_identities: Vec::new(),
10754                })
10755                .await?,
10756            None,
10757            None,
10758        )
10759        .await?;
10760        roster
10761            .set(vec![durable_spec(identity.clone(), "security")])
10762            .await;
10763
10764        let record = runtime.reset(&identity).await?;
10765
10766        assert_eq!(record.generation.get(), 1);
10767        assert_eq!(
10768            bridge.create_profiles().await,
10769            vec!["domain".to_string(), "security".to_string()]
10770        );
10771        let status = runtime.status(&identity).await?;
10772        assert_eq!(
10773            status.profile.map(|profile| profile.to_string()).as_deref(),
10774            Some("security")
10775        );
10776        Ok(())
10777    }
10778
10779    #[tokio::test]
10780    async fn shutdown_joins_reset_after_outer_abort_at_final_upsert()
10781    -> Result<(), Box<dyn std::error::Error>> {
10782        let identity = AgentIdentity::parse("domain:security")?;
10783        let roster = Arc::new(MutableRoster::new(vec![durable_spec(
10784            identity.clone(),
10785            "domain",
10786        )]));
10787        let bridge = Arc::new(RecordingBridge::default());
10788        let store = Arc::new(GatedResetContinuityStore::new()?);
10789        let runtime = Arc::new(
10790            IdentityRuntime::new(IdentityRuntimeConfig {
10791                continuity_store: store.clone(),
10792                lease_provider: Arc::new(LocalLeaseProvider::new()),
10793                runtime_instance_id: "tracked-reset-shutdown-test".to_string(),
10794                has_runtime_store: true,
10795                durability_policy: DurabilityPolicy::SyncWriteThrough,
10796                bridge: Some(bridge.clone()),
10797                default_timeout: None,
10798            })
10799            .with_reset_roster_provider(roster.clone()),
10800        );
10801
10802        super::super::orchestrator::restore_flow(
10803            &runtime,
10804            &roster
10805                .roster(&RosterContext {
10806                    mob_definition: None,
10807                    previous_identities: Vec::new(),
10808                })
10809                .await?,
10810            None,
10811            None,
10812        )
10813        .await?;
10814        let old_record = match store
10815            .resolve_many(std::slice::from_ref(&identity))
10816            .await?
10817            .remove(&identity)
10818        {
10819            Some(ContinuityResolveState::Ready { record }) => record,
10820            other => return Err(format!("expected initial continuity, got {other:?}").into()),
10821        };
10822        store.fail_after_successful_upserts(3);
10823
10824        let outer = tokio::spawn({
10825            let runtime = Arc::clone(&runtime);
10826            let identity = identity.clone();
10827            async move { runtime.reset_tracked(&identity).await }
10828        });
10829        store.wait_for_failure().await;
10830        outer.abort();
10831        match outer.await {
10832            Err(error) => assert!(error.is_cancelled()),
10833            Ok(result) => {
10834                return Err(
10835                    format!("outer reset waiter unexpectedly completed: {result:?}").into(),
10836                );
10837            }
10838        }
10839
10840        runtime.close_foreground_operations();
10841        let mut join = tokio::spawn({
10842            let runtime = Arc::clone(&runtime);
10843            async move { runtime.join_foreground_operations().await }
10844        });
10845        assert!(
10846            tokio::time::timeout(Duration::from_millis(50), &mut join)
10847                .await
10848                .is_err(),
10849            "shutdown must wait for the runtime-owned reset transaction"
10850        );
10851        assert!(
10852            bridge.retired_runtime_ids().await.is_empty(),
10853            "rollback cleanup cannot run before the final upsert resolves"
10854        );
10855
10856        store.release_failure();
10857        tokio::time::timeout(Duration::from_secs(2), &mut join)
10858            .await
10859            .map_err(|_| "shutdown did not join reset rollback")??;
10860
10861        let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
10862        assert_eq!(
10863            resolved.get(&identity),
10864            Some(&ContinuityResolveState::Ready {
10865                record: old_record.clone(),
10866            }),
10867            "abandoned reset must roll durable continuity back before shutdown completes"
10868        );
10869        let status = runtime.status(&identity).await?;
10870        assert_eq!(
10871            status.agent_runtime_id.as_ref(),
10872            Some(&old_record.agent_runtime_id)
10873        );
10874        assert_eq!(status.state, IdentityLifecycleState::Broken);
10875        assert_eq!(
10876            bridge.retired_runtime_ids().await,
10877            vec!["rt:domain:security:1".to_string()],
10878            "rollback must retire only the tentative reset generation"
10879        );
10880        Ok(())
10881    }
10882
10883    #[tokio::test]
10884    async fn shutdown_cancels_reset_customizer_and_restores_old_generation()
10885    -> Result<(), Box<dyn std::error::Error>> {
10886        let identity = AgentIdentity::parse("domain:security")?;
10887        let spec = durable_spec(identity.clone(), "domain");
10888        let bridge = Arc::new(RecordingBridge::default());
10889        let store = Arc::new(LocalContinuityStore::in_memory()?);
10890        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10891            continuity_store: store.clone(),
10892            lease_provider: Arc::new(LocalLeaseProvider::new()),
10893            runtime_instance_id: "reset-customizer-shutdown-test".to_string(),
10894            has_runtime_store: true,
10895            durability_policy: DurabilityPolicy::SyncWriteThrough,
10896            bridge: Some(bridge.clone()),
10897            default_timeout: None,
10898        }));
10899
10900        super::super::orchestrator::restore_flow(&runtime, std::slice::from_ref(&spec), None, None)
10901            .await?;
10902        let old_record = match store
10903            .resolve_many(std::slice::from_ref(&identity))
10904            .await?
10905            .remove(&identity)
10906        {
10907            Some(ContinuityResolveState::Ready { record }) => record,
10908            other => return Err(format!("expected initial continuity, got {other:?}").into()),
10909        };
10910
10911        let customizer = Arc::new(GatedResetCustomizer::default());
10912        runtime.set_agent_customizer(Some(customizer.clone())).await;
10913        let outer = tokio::spawn({
10914            let runtime = Arc::clone(&runtime);
10915            let identity = identity.clone();
10916            async move { runtime.reset_tracked(&identity).await }
10917        });
10918        tokio::time::timeout(Duration::from_secs(2), customizer.wait_for_entry())
10919            .await
10920            .map_err(|_| "reset did not enter the gated customizer")?;
10921        outer.abort();
10922        match outer.await {
10923            Err(error) => assert!(error.is_cancelled()),
10924            Ok(result) => {
10925                return Err(
10926                    format!("outer reset waiter unexpectedly completed: {result:?}").into(),
10927                );
10928            }
10929        }
10930
10931        runtime.close_foreground_operations();
10932        tokio::time::timeout(Duration::from_secs(2), runtime.join_foreground_operations())
10933            .await
10934            .map_err(|_| "shutdown hung on the reset customizer")?;
10935
10936        let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
10937        assert_eq!(
10938            resolved.get(&identity),
10939            Some(&ContinuityResolveState::Ready {
10940                record: old_record.clone(),
10941            }),
10942            "shutdown cancellation must roll durable continuity back to the old generation"
10943        );
10944        let status = runtime.status(&identity).await?;
10945        assert_eq!(status.state, IdentityLifecycleState::Active);
10946        assert_eq!(
10947            status.agent_runtime_id.as_ref(),
10948            Some(&old_record.agent_runtime_id)
10949        );
10950        assert_eq!(status.session_id.as_ref(), Some(&old_record.session_id));
10951        assert_eq!(bridge.create_profiles().await, vec!["domain".to_string()]);
10952        assert!(
10953            bridge.retired_runtime_ids().await.is_empty(),
10954            "cancellation before session installation must not touch either bridge generation"
10955        );
10956        Ok(())
10957    }
10958
10959    #[tokio::test]
10960    async fn shutdown_cancelled_dormant_reset_releases_temporary_lease()
10961    -> Result<(), Box<dyn std::error::Error>> {
10962        let identity = AgentIdentity::parse("domain:dormant")?;
10963        let bridge = Arc::new(RecordingBridge::default());
10964        let store = Arc::new(LocalContinuityStore::in_memory()?);
10965        let lease_provider = Arc::new(LocalLeaseProvider::new());
10966        let runtime_instance_id = "dormant-reset-customizer-shutdown-test";
10967        let acquired = lease_provider
10968            .acquire_leases(std::slice::from_ref(&identity), runtime_instance_id)
10969            .await?;
10970        let initial_grant = match acquired.get(&identity) {
10971            Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
10972            other => return Err(format!("expected initial lease, got {other:?}").into()),
10973        };
10974        let old_record = ContinuityRecord {
10975            identity: identity.clone(),
10976            agent_runtime_id: AgentRuntimeId::parse("rt:domain:dormant:0")?,
10977            session_id: SessionId::new(),
10978            generation: ContinuityGeneration::new(0),
10979            checkpoint_version: CheckpointVersion::new(0),
10980        };
10981        store
10982            .upsert_continuity_record(&old_record, initial_grant.fencing_token)
10983            .await?;
10984        lease_provider
10985            .release_leases(std::slice::from_ref(&initial_grant))
10986            .await?;
10987        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10988            continuity_store: store.clone(),
10989            lease_provider: lease_provider.clone(),
10990            runtime_instance_id: runtime_instance_id.to_string(),
10991            has_runtime_store: true,
10992            durability_policy: DurabilityPolicy::SyncWriteThrough,
10993            bridge: Some(bridge.clone()),
10994            default_timeout: None,
10995        }));
10996        runtime
10997            .register(
10998                durable_spec(identity.clone(), "domain"),
10999                IdentityLifecycleState::Dormant,
11000                Some(old_record.clone()),
11001                None,
11002            )
11003            .await;
11004
11005        let customizer = Arc::new(GatedResetCustomizer::default());
11006        runtime.set_agent_customizer(Some(customizer.clone())).await;
11007        let outer = tokio::spawn({
11008            let runtime = Arc::clone(&runtime);
11009            let identity = identity.clone();
11010            async move { runtime.reset_tracked(&identity).await }
11011        });
11012        tokio::time::timeout(Duration::from_secs(2), customizer.wait_for_entry())
11013            .await
11014            .map_err(|_| "dormant reset did not enter the gated customizer")?;
11015        outer.abort();
11016        match outer.await {
11017            Err(error) => assert!(error.is_cancelled()),
11018            Ok(result) => {
11019                return Err(
11020                    format!("outer dormant reset unexpectedly completed: {result:?}").into(),
11021                );
11022            }
11023        }
11024        runtime.close_foreground_operations();
11025        tokio::time::timeout(Duration::from_secs(2), runtime.join_foreground_operations())
11026            .await
11027            .map_err(|_| "shutdown hung on the dormant reset customizer")?;
11028
11029        let status = runtime.status(&identity).await?;
11030        assert_eq!(status.state, IdentityLifecycleState::Dormant);
11031        assert!(status.lease.is_none());
11032        let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
11033        assert_eq!(
11034            resolved.get(&identity),
11035            Some(&ContinuityResolveState::Ready {
11036                record: old_record.clone(),
11037            })
11038        );
11039        let failover = lease_provider
11040            .acquire_leases(std::slice::from_ref(&identity), "dormant-reset-failover")
11041            .await?;
11042        assert!(
11043            matches!(
11044                failover.get(&identity),
11045                Some(super::super::types::LeaseAcquireResult::Acquired(_))
11046            ),
11047            "shutdown rollback must release the dormant reset lease: {failover:?}"
11048        );
11049        assert!(bridge.create_profiles().await.is_empty());
11050        assert!(bridge.retired_runtime_ids().await.is_empty());
11051        Ok(())
11052    }
11053
11054    #[tokio::test]
11055    async fn tracked_identity_respawn_preserves_authoritative_continuity()
11056    -> Result<(), Box<dyn std::error::Error>> {
11057        let identity = AgentIdentity::parse("domain:durable-respawn")?;
11058        let spec = durable_spec(identity.clone(), "domain");
11059        let bridge = Arc::new(RecordingBridge::default());
11060        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11061            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11062            lease_provider: Arc::new(LocalLeaseProvider::new()),
11063            runtime_instance_id: "durable-respawn".to_string(),
11064            has_runtime_store: true,
11065            durability_policy: DurabilityPolicy::SyncWriteThrough,
11066            bridge: Some(bridge.clone()),
11067            default_timeout: None,
11068        }));
11069        super::super::orchestrator::restore_flow(&runtime, std::slice::from_ref(&spec), None, None)
11070            .await?;
11071
11072        let before = runtime.status(&identity).await?;
11073        let runtime_alias = before
11074            .agent_runtime_id
11075            .clone()
11076            .ok_or("missing durable runtime id")?;
11077        let respawned = runtime
11078            .respawn_identity_in_place_tracked(&identity, Some(runtime_alias.as_str()))
11079            .await?;
11080
11081        assert_eq!(Some(respawned.session_id), before.session_id);
11082        assert_eq!(Some(respawned.agent_runtime_id), before.agent_runtime_id);
11083        assert_eq!(Some(respawned.generation), before.generation);
11084        assert_eq!(
11085            runtime.status(&identity).await?.state,
11086            IdentityLifecycleState::Active
11087        );
11088        assert!(
11089            bridge.retired_runtime_ids().await.is_empty(),
11090            "identity respawn must not retire and recreate the authoritative session"
11091        );
11092        Ok(())
11093    }
11094
11095    #[tokio::test]
11096    async fn stale_alias_materialization_preserves_active_and_dormant_bootstrap_status()
11097    -> Result<(), Box<dyn std::error::Error>> {
11098        let identity = AgentIdentity::parse("domain:bootstrap-alias")?;
11099        let spec = durable_spec(identity.clone(), "domain");
11100        let current_alias = AgentRuntimeId::parse("rt:domain:bootstrap-alias:1")?;
11101        let stale_alias = "rt:domain:bootstrap-alias:0";
11102        let store = Arc::new(LocalContinuityStore::in_memory()?);
11103        let lease_provider = Arc::new(LocalLeaseProvider::new());
11104        let grants = lease_provider
11105            .acquire_leases(std::slice::from_ref(&identity), "bootstrap-alias-status")
11106            .await?;
11107        let grant = match grants.get(&identity) {
11108            Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
11109            other => return Err(format!("expected acquired lease, got {other:?}").into()),
11110        };
11111        let record = ContinuityRecord {
11112            identity: identity.clone(),
11113            agent_runtime_id: current_alias,
11114            session_id: SessionId::new(),
11115            generation: ContinuityGeneration::new(1),
11116            checkpoint_version: CheckpointVersion::new(0),
11117        };
11118        store
11119            .upsert_continuity_record(&record, grant.fencing_token)
11120            .await?;
11121        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11122            continuity_store: store,
11123            lease_provider,
11124            runtime_instance_id: "bootstrap-alias-status".to_string(),
11125            has_runtime_store: true,
11126            durability_policy: DurabilityPolicy::SyncWriteThrough,
11127            bridge: None,
11128            default_timeout: None,
11129        }));
11130        runtime
11131            .register(
11132                spec.clone(),
11133                IdentityLifecycleState::Active,
11134                Some(record),
11135                Some(grant),
11136            )
11137            .await;
11138
11139        let active_generation =
11140            runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11141        runtime
11142            .begin_identity_bootstrap(
11143                active_generation,
11144                IdentityBootstrapMode::LazyMaterialize,
11145                std::slice::from_ref(&spec),
11146            )
11147            .await;
11148        runtime.modify_bootstrap_status(Some(active_generation), |status| {
11149            status.complete = true;
11150            status.refresh_aggregates();
11151        });
11152        let active_before = runtime.identity_bootstrap_status();
11153        assert_eq!(
11154            active_before
11155                .identities
11156                .get(&identity)
11157                .map(|entry| entry.state),
11158            Some(IdentityBootstrapState::Active)
11159        );
11160        assert!(matches!(
11161            runtime
11162                .materialize_with_expected_member_alias(&identity, Some(stale_alias))
11163                .await,
11164            Err(IdentityRuntimeError::StaleRuntimeAlias { .. })
11165        ));
11166        assert_eq!(runtime.identity_bootstrap_status(), active_before);
11167
11168        runtime.retire(&identity).await?;
11169        let dormant_generation =
11170            runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11171        runtime
11172            .begin_identity_bootstrap(
11173                dormant_generation,
11174                IdentityBootstrapMode::LazyMaterialize,
11175                std::slice::from_ref(&spec),
11176            )
11177            .await;
11178        runtime.modify_bootstrap_status(Some(dormant_generation), |status| {
11179            status.complete = true;
11180            status.refresh_aggregates();
11181        });
11182        let dormant_before = runtime.identity_bootstrap_status();
11183        assert_eq!(
11184            dormant_before
11185                .identities
11186                .get(&identity)
11187                .map(|entry| entry.state),
11188            Some(IdentityBootstrapState::Dormant)
11189        );
11190        assert!(matches!(
11191            runtime
11192                .materialize_with_expected_member_alias(&identity, Some(stale_alias))
11193                .await,
11194            Err(IdentityRuntimeError::StaleRuntimeAlias { .. })
11195        ));
11196        assert_eq!(runtime.identity_bootstrap_status(), dormant_before);
11197        Ok(())
11198    }
11199
11200    #[tokio::test]
11201    async fn eager_reconcile_hot_reloads_active_metadata_without_retire_or_resume()
11202    -> Result<(), Box<dyn std::error::Error>> {
11203        let identity = AgentIdentity::parse("identity:luka")?;
11204        let mut initial_spec = durable_spec(identity.clone(), "personal");
11205        initial_spec
11206            .labels
11207            .insert("revision".to_string(), "v1".to_string());
11208        let roster = Arc::new(MutableRoster::new(vec![initial_spec.clone()]));
11209        let bridge = Arc::new(HotReloadBridge::default());
11210        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11211            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11212            lease_provider: Arc::new(LocalLeaseProvider::new()),
11213            runtime_instance_id: "eager-hot-reload-contract".to_string(),
11214            has_runtime_store: true,
11215            durability_policy: DurabilityPolicy::SyncWriteThrough,
11216            bridge: Some(bridge.clone()),
11217            default_timeout: None,
11218        }));
11219        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11220            runtime.clone(),
11221            roster.clone(),
11222            None,
11223            None,
11224            None,
11225            IdentityBootstrapMode::EagerMaterialize,
11226        );
11227        context
11228            .bootstrap_roster(std::slice::from_ref(&initial_spec))
11229            .await?;
11230
11231        let before = runtime.status(&identity).await?;
11232        assert_eq!(before.state, IdentityLifecycleState::Active);
11233        let before_session = before.session_id.clone();
11234        let before_runtime_id = before.agent_runtime_id.clone();
11235        let before_generation = before.generation;
11236        let before_token = before
11237            .lease
11238            .as_ref()
11239            .ok_or("initial active identity must have a lease")?
11240            .fencing_token;
11241
11242        let mut updated_spec = initial_spec;
11243        updated_spec.addressability = AgentAddressability::InternalOnly;
11244        updated_spec
11245            .labels
11246            .insert("timezone".to_string(), "Europe/Stockholm".to_string());
11247        updated_spec
11248            .labels
11249            .insert("revision".to_string(), "v2".to_string());
11250        roster.set(vec![updated_spec]).await;
11251
11252        let result = context.refresh_desired_topology().await?;
11253        assert!(matches!(
11254            result.outcomes.get(&identity),
11255            Some(super::super::orchestrator::RestoreOutcome::Resumed { .. })
11256        ));
11257        let after = runtime.status(&identity).await?;
11258        assert_eq!(after.state, IdentityLifecycleState::Active);
11259        assert_eq!(after.addressability, AgentAddressability::InternalOnly);
11260        assert_eq!(
11261            after.labels.get("timezone").map(String::as_str),
11262            Some("Europe/Stockholm")
11263        );
11264        assert_eq!(after.session_id, before_session);
11265        assert_eq!(after.agent_runtime_id, before_runtime_id);
11266        assert_eq!(after.generation, before_generation);
11267        assert_eq!(
11268            after
11269                .lease
11270                .as_ref()
11271                .ok_or("hot-reloaded identity must retain its lease")?
11272                .fencing_token,
11273            before_token,
11274            "same-profile reconcile must preserve exact live authority"
11275        );
11276        assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 0);
11277        assert_eq!(bridge.resume_calls.load(Ordering::SeqCst), 0);
11278
11279        assert!(matches!(
11280            runtime
11281                .send(
11282                    &identity,
11283                    &meerkat_core::ContentInput::Text("external".to_string()),
11284                )
11285                .await,
11286            Err(IdentityRuntimeError::NotAddressable(_))
11287        ));
11288        let (dispatch_token, durable) = runtime
11289            .dispatch(&identity, &DispatchInput::system("system notice"))
11290            .await?;
11291        assert_eq!(dispatch_token, before_token);
11292        assert!(durable);
11293        Ok(())
11294    }
11295
11296    #[tokio::test]
11297    async fn lazy_unchanged_reconcile_retained_broken_lease_failure_stays_shutdown_visible()
11298    -> Result<(), Box<dyn std::error::Error>> {
11299        let identity = AgentIdentity::parse("identity:broken-retained-unchanged")?;
11300        let spec = durable_spec(identity.clone(), "personal");
11301        let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11302        let lease_provider = Arc::new(RecordingReleaseLeaseProvider::default());
11303        let (runtime, context, retained_grant) = lazy_context_with_broken_retained_lease(
11304            identity.clone(),
11305            spec,
11306            roster,
11307            lease_provider.clone(),
11308            "broken-retained-unchanged",
11309        )
11310        .await?;
11311
11312        lease_provider.fail_next_release();
11313        let error = match context.refresh_desired_topology().await {
11314            Err(error) => error,
11315            Ok(_) => return Err("failed exact release allowed lazy registration".into()),
11316        };
11317        assert!(
11318            error
11319                .to_string()
11320                .contains("synthetic retained Broken lease release failure")
11321        );
11322        let status = runtime.status(&identity).await?;
11323        assert_eq!(status.state, IdentityLifecycleState::Broken);
11324        assert!(
11325            status.lease.is_none(),
11326            "failed release authority must be pending, never advertised as active"
11327        );
11328        assert_eq!(
11329            runtime
11330                .entries
11331                .read()
11332                .await
11333                .get(&identity)
11334                .and_then(|entry| entry.pending_lease_release.as_ref())
11335                .map(|grant| grant.fencing_token),
11336            Some(retained_grant.fencing_token),
11337            "the exact retained token must survive provider failure"
11338        );
11339        assert_eq!(
11340            lease_provider
11341                .release_attempts()
11342                .await
11343                .iter()
11344                .map(|grant| grant.fencing_token)
11345                .collect::<Vec<_>>(),
11346            vec![retained_grant.fencing_token]
11347        );
11348
11349        assert_eq!(
11350            runtime.release_all_leases_for_shutdown().await?,
11351            1,
11352            "shutdown must retain visibility of the staged exact grant"
11353        );
11354        assert_eq!(
11355            lease_provider
11356                .release_attempts()
11357                .await
11358                .iter()
11359                .map(|grant| grant.fencing_token)
11360                .collect::<Vec<_>>(),
11361            vec![retained_grant.fencing_token, retained_grant.fencing_token],
11362            "shutdown must retry the same exact fencing token"
11363        );
11364        assert!(
11365            runtime
11366                .entries
11367                .read()
11368                .await
11369                .get(&identity)
11370                .is_some_and(|entry| {
11371                    entry.lease.is_none() && entry.pending_lease_release.is_none()
11372                })
11373        );
11374
11375        context.refresh_desired_topology().await?;
11376        let recovered = runtime.status(&identity).await?;
11377        assert_eq!(recovered.state, IdentityLifecycleState::Dormant);
11378        assert!(recovered.lease.is_none());
11379        let failover = lease_provider
11380            .acquire_leases(std::slice::from_ref(&identity), "other-runtime")
11381            .await?;
11382        assert!(matches!(
11383            failover.get(&identity),
11384            Some(LeaseAcquireResult::Acquired(_))
11385        ));
11386        Ok(())
11387    }
11388
11389    #[tokio::test]
11390    async fn lazy_profile_replace_releases_retained_broken_lease_before_overwrite()
11391    -> Result<(), Box<dyn std::error::Error>> {
11392        let identity = AgentIdentity::parse("identity:broken-retained-replace")?;
11393        let original = durable_spec(identity.clone(), "personal-v1");
11394        let roster = Arc::new(MutableRoster::new(vec![original.clone()]));
11395        let lease_provider = Arc::new(RecordingReleaseLeaseProvider::default());
11396        let (runtime, context, retained_grant) = lazy_context_with_broken_retained_lease(
11397            identity.clone(),
11398            original,
11399            roster.clone(),
11400            lease_provider.clone(),
11401            "broken-retained-replace",
11402        )
11403        .await?;
11404
11405        roster
11406            .set(vec![durable_spec(identity.clone(), "personal-v2")])
11407            .await;
11408        context.refresh_desired_topology().await?;
11409
11410        let status = runtime.status(&identity).await?;
11411        assert_eq!(status.state, IdentityLifecycleState::Dormant);
11412        assert_eq!(
11413            status.profile.as_ref().map(ToString::to_string).as_deref(),
11414            Some("personal-v2")
11415        );
11416        assert!(status.lease.is_none());
11417        assert_eq!(
11418            lease_provider
11419                .release_attempts()
11420                .await
11421                .iter()
11422                .map(|grant| grant.fencing_token)
11423                .collect::<Vec<_>>(),
11424            vec![retained_grant.fencing_token],
11425            "profile replacement must release the retained exact token before lazy registration"
11426        );
11427        assert!(
11428            runtime
11429                .entries
11430                .read()
11431                .await
11432                .get(&identity)
11433                .is_some_and(|entry| entry.pending_lease_release.is_none())
11434        );
11435        let failover = lease_provider
11436            .acquire_leases(std::slice::from_ref(&identity), "replacement-failover")
11437            .await?;
11438        assert!(matches!(
11439            failover.get(&identity),
11440            Some(LeaseAcquireResult::Acquired(_))
11441        ));
11442        Ok(())
11443    }
11444
11445    #[tokio::test]
11446    async fn lease_lost_then_roster_remove_cleans_lower_plane_before_dropping_entry()
11447    -> Result<(), Box<dyn std::error::Error>> {
11448        let identity = AgentIdentity::parse("identity:lost-remove")?;
11449        let spec = durable_spec(identity.clone(), "personal");
11450        let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11451        let bridge = Arc::new(LostCleanupBridge::default());
11452        let lease_provider = Arc::new(LostRenewLeaseProvider::default());
11453        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11454            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11455            lease_provider: lease_provider.clone(),
11456            runtime_instance_id: "lost-remove-cleanup".to_string(),
11457            has_runtime_store: true,
11458            durability_policy: DurabilityPolicy::SyncWriteThrough,
11459            bridge: Some(bridge.clone()),
11460            default_timeout: None,
11461        }));
11462        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11463            runtime.clone(),
11464            roster.clone(),
11465            None,
11466            None,
11467            None,
11468            IdentityBootstrapMode::EagerMaterialize,
11469        );
11470        context
11471            .bootstrap_roster(std::slice::from_ref(&spec))
11472            .await?;
11473        assert_eq!(bridge.member_count().await, 1);
11474        assert_eq!(bridge.session_runtime_state_count().await, 1);
11475
11476        force_renewal_lost(&runtime, &lease_provider, &identity).await?;
11477        assert_eq!(
11478            runtime.status(&identity).await?.state,
11479            IdentityLifecycleState::Broken
11480        );
11481        assert_eq!(bridge.member_count().await, 1);
11482        let unregisters_after_lost = bridge.unregister_calls.load(Ordering::SeqCst);
11483
11484        roster.set(Vec::new()).await;
11485        context.refresh_desired_topology().await?;
11486        assert!(!runtime.contains(&identity).await);
11487        assert_eq!(
11488            bridge.member_count().await,
11489            0,
11490            "stale member survived removal"
11491        );
11492        assert_eq!(
11493            bridge.session_runtime_state_count().await,
11494            0,
11495            "stale session authority survived removal"
11496        );
11497        assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 1);
11498        assert_eq!(
11499            bridge.unregister_calls.load(Ordering::SeqCst),
11500            unregisters_after_lost + 1,
11501            "roster removal must idempotently unregister after Lost"
11502        );
11503        Ok(())
11504    }
11505
11506    #[tokio::test]
11507    async fn lease_lost_then_profile_replace_cleans_old_member_before_resume()
11508    -> Result<(), Box<dyn std::error::Error>> {
11509        let identity = AgentIdentity::parse("identity:lost-replace")?;
11510        let original = durable_spec(identity.clone(), "personal-v1");
11511        let roster = Arc::new(MutableRoster::new(vec![original.clone()]));
11512        let bridge = Arc::new(LostCleanupBridge::default());
11513        let lease_provider = Arc::new(LostRenewLeaseProvider::default());
11514        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11515            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11516            lease_provider: lease_provider.clone(),
11517            runtime_instance_id: "lost-replace-cleanup".to_string(),
11518            has_runtime_store: true,
11519            durability_policy: DurabilityPolicy::SyncWriteThrough,
11520            bridge: Some(bridge.clone()),
11521            default_timeout: None,
11522        }));
11523        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11524            runtime.clone(),
11525            roster.clone(),
11526            None,
11527            None,
11528            None,
11529            IdentityBootstrapMode::EagerMaterialize,
11530        );
11531        context
11532            .bootstrap_roster(std::slice::from_ref(&original))
11533            .await?;
11534        force_renewal_lost(&runtime, &lease_provider, &identity).await?;
11535        let unregisters_after_lost = bridge.unregister_calls.load(Ordering::SeqCst);
11536
11537        let replacement = durable_spec(identity.clone(), "personal-v2");
11538        roster.set(vec![replacement]).await;
11539        context.refresh_desired_topology().await?;
11540
11541        let status = runtime.status(&identity).await?;
11542        assert_eq!(status.state, IdentityLifecycleState::Active);
11543        assert_eq!(
11544            status.profile.as_ref().map(ToString::to_string).as_deref(),
11545            Some("personal-v2")
11546        );
11547        assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 1);
11548        assert_eq!(
11549            bridge.unregister_calls.load(Ordering::SeqCst),
11550            unregisters_after_lost + 1,
11551            "profile replacement must idempotently unregister before resume"
11552        );
11553        assert_eq!(
11554            bridge.resume_collisions.load(Ordering::SeqCst),
11555            0,
11556            "replacement encountered the stale lower-plane alias"
11557        );
11558        assert_eq!(bridge.member_count().await, 1);
11559        assert_eq!(bridge.session_runtime_state_count().await, 1);
11560        Ok(())
11561    }
11562
11563    #[tokio::test]
11564    async fn foreground_materialization_completion_cannot_overwrite_newer_lazy_reconcile_status()
11565    -> Result<(), Box<dyn std::error::Error>> {
11566        let identity = AgentIdentity::parse("domain:foreground-generation")?;
11567        let mut v1 = durable_spec(identity.clone(), "domain");
11568        v1.labels
11569            .insert("roster_revision".to_string(), "v1".to_string());
11570        let mut v2 = v1.clone();
11571        v2.profile = meerkat_mob::ProfileName::from("replacement");
11572        v2.labels
11573            .insert("roster_revision".to_string(), "v2".to_string());
11574
11575        let roster = Arc::new(MutableRoster::new(vec![v1.clone()]));
11576        let bridge = Arc::new(RecordingBridge::default());
11577        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11578            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11579            lease_provider: Arc::new(LocalLeaseProvider::new()),
11580            runtime_instance_id: "foreground-generation-fence".to_string(),
11581            has_runtime_store: true,
11582            durability_policy: DurabilityPolicy::SyncWriteThrough,
11583            bridge: Some(bridge.clone()),
11584            default_timeout: None,
11585        }));
11586        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11587            runtime.clone(),
11588            roster.clone(),
11589            None,
11590            None,
11591            None,
11592            IdentityBootstrapMode::LazyMaterialize,
11593        );
11594        context.bootstrap_roster(std::slice::from_ref(&v1)).await?;
11595        let (v1_generation, initial) = runtime.identity_bootstrap_status_with_generation();
11596        assert!(initial.complete);
11597        assert!(!initial.ready);
11598        assert_eq!(
11599            initial.identities.get(&identity).map(|entry| entry.state),
11600            Some(IdentityBootstrapState::Dormant)
11601        );
11602
11603        // Pause the v1 foreground call after its lifecycle transaction has
11604        // committed Active but before its outer readiness bookkeeping runs.
11605        // The lifecycle lock is free at this seam, so a v2 roster pass can
11606        // retire the v1 member and install a new lazy Dormant snapshot.
11607        let completion_entered = Arc::new(Notify::new());
11608        let release_completion = Arc::new(Notify::new());
11609        let materialize = tokio::spawn({
11610            let runtime = runtime.clone();
11611            let identity = identity.clone();
11612            let completion_entered = completion_entered.clone();
11613            let release_completion = release_completion.clone();
11614            async move {
11615                runtime
11616                    .materialize_with_expected_member_alias_after_inner(
11617                        &identity,
11618                        None,
11619                        async move {
11620                            completion_entered.notify_one();
11621                            release_completion.notified().await;
11622                        },
11623                    )
11624                    .await
11625            }
11626        });
11627        completion_entered.notified().await;
11628        assert_eq!(
11629            runtime.status(&identity).await?.state,
11630            IdentityLifecycleState::Active
11631        );
11632
11633        roster.set(vec![v2]).await;
11634        context.refresh_desired_topology().await?;
11635        let after_v2 = runtime.identity_bootstrap_status();
11636        assert!(after_v2.complete);
11637        assert!(!after_v2.ready);
11638        assert_eq!(
11639            after_v2.identities.get(&identity).map(|entry| entry.state),
11640            Some(IdentityBootstrapState::Dormant)
11641        );
11642        let lifecycle = runtime.status(&identity).await?;
11643        assert_eq!(lifecycle.state, IdentityLifecycleState::Dormant);
11644        assert_eq!(
11645            lifecycle.labels.get("roster_revision").map(String::as_str),
11646            Some("v2")
11647        );
11648        assert_eq!(bridge.retired_runtime_ids().await.len(), 1);
11649
11650        // Let the older v1 outer future run its terminal status update. Its
11651        // captured generation must make the update a no-op against v2.
11652        release_completion.notify_one();
11653        materialize.await??;
11654        assert_eq!(runtime.identity_bootstrap_status(), after_v2);
11655        assert_eq!(
11656            runtime.status(&identity).await?.state,
11657            IdentityLifecycleState::Dormant
11658        );
11659        assert_ne!(
11660            runtime.identity_bootstrap_status_with_generation().0,
11661            v1_generation
11662        );
11663        Ok(())
11664    }
11665
11666    #[tokio::test]
11667    async fn foreground_materialization_uses_generation_of_reconcile_that_wins_lifecycle_lock()
11668    -> Result<(), Box<dyn std::error::Error>> {
11669        let identity = AgentIdentity::parse("domain:foreground-reconcile-first")?;
11670        let spec = durable_spec(identity.clone(), "domain");
11671        let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11672        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11673            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11674            lease_provider: Arc::new(LocalLeaseProvider::new()),
11675            runtime_instance_id: "foreground-reconcile-first".to_string(),
11676            has_runtime_store: true,
11677            durability_policy: DurabilityPolicy::SyncWriteThrough,
11678            bridge: None,
11679            default_timeout: None,
11680        }));
11681        let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11682            runtime.clone(),
11683            roster,
11684            None,
11685            None,
11686            None,
11687            IdentityBootstrapMode::LazyMaterialize,
11688        );
11689        context
11690            .bootstrap_roster(std::slice::from_ref(&spec))
11691            .await?;
11692        let first_generation = runtime.identity_bootstrap_status_with_generation().0;
11693
11694        // Publish G+1 with an unchanged desired spec, then pause its reconcile
11695        // while it owns the lifecycle lock. This is the ordering that a
11696        // pre-lock global generation sample gets wrong: materialization starts
11697        // during G+1 and must operate on the G+1-stamped entry, even though the
11698        // entry itself was originally registered by G.
11699        let next_generation =
11700            runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11701        assert_ne!(next_generation, first_generation);
11702        runtime
11703            .begin_identity_bootstrap(
11704                next_generation,
11705                IdentityBootstrapMode::LazyMaterialize,
11706                std::slice::from_ref(&spec),
11707            )
11708            .await;
11709
11710        let reconcile_holds_lock = Arc::new(Notify::new());
11711        let release_reconcile = Arc::new(Notify::new());
11712        let paused = Arc::new(AtomicBool::new(false));
11713        let reconcile = tokio::spawn({
11714            let runtime = runtime.clone();
11715            let spec = spec.clone();
11716            let reconcile_holds_lock = reconcile_holds_lock.clone();
11717            let release_reconcile = release_reconcile.clone();
11718            let paused = paused.clone();
11719            async move {
11720                runtime
11721                    .reconcile_roster_members_after_lifecycle_lock(
11722                        std::slice::from_ref(&spec),
11723                        next_generation,
11724                        move |_| {
11725                            let should_pause = !paused.swap(true, Ordering::SeqCst);
11726                            let reconcile_holds_lock = reconcile_holds_lock.clone();
11727                            let release_reconcile = release_reconcile.clone();
11728                            async move {
11729                                if should_pause {
11730                                    reconcile_holds_lock.notify_one();
11731                                    release_reconcile.notified().await;
11732                                }
11733                            }
11734                        },
11735                    )
11736                    .await
11737            }
11738        });
11739        reconcile_holds_lock.notified().await;
11740
11741        let materialize = tokio::spawn({
11742            let runtime = runtime.clone();
11743            let identity = identity.clone();
11744            async move { runtime.materialize(&identity).await }
11745        });
11746        // Poll the spawned future until it queues behind the lifecycle owner.
11747        tokio::task::yield_now().await;
11748        release_reconcile.notify_one();
11749        reconcile.await??;
11750        materialize.await??;
11751
11752        let (published_generation, status) = runtime.identity_bootstrap_status_with_generation();
11753        assert_eq!(published_generation, next_generation);
11754        assert_eq!(
11755            status.identities.get(&identity).map(|entry| entry.state),
11756            Some(IdentityBootstrapState::Active),
11757            "G+1 materialization completion was misbound to the older pass"
11758        );
11759        let entries = runtime.entries.read().await;
11760        assert_eq!(
11761            entries
11762                .get(&identity)
11763                .map(|entry| entry.bootstrap_generation),
11764            Some(next_generation),
11765            "unchanged roster acceptance must still stamp the new generation"
11766        );
11767        Ok(())
11768    }
11769
11770    #[tokio::test]
11771    async fn stale_runtime_alias_preflight_cannot_mutate_new_generation()
11772    -> Result<(), Box<dyn std::error::Error>> {
11773        let identity = AgentIdentity::parse("domain:alias-race")?;
11774        let old_alias = "rt:domain:alias-race:0";
11775        let store = Arc::new(LocalContinuityStore::in_memory()?);
11776        let lease_provider = Arc::new(LocalLeaseProvider::new());
11777        let acquired = lease_provider
11778            .acquire_leases(std::slice::from_ref(&identity), "alias-race-test")
11779            .await?;
11780        let grant = match acquired.get(&identity) {
11781            Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
11782            other => return Err(format!("expected initial lease, got {other:?}").into()),
11783        };
11784        let old_record = ContinuityRecord {
11785            identity: identity.clone(),
11786            agent_runtime_id: AgentRuntimeId::parse(old_alias)?,
11787            session_id: SessionId::new(),
11788            generation: ContinuityGeneration::new(0),
11789            checkpoint_version: CheckpointVersion::new(0),
11790        };
11791        store
11792            .upsert_continuity_record(&old_record, grant.fencing_token)
11793            .await?;
11794        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11795            continuity_store: store,
11796            lease_provider,
11797            runtime_instance_id: "alias-race-test".to_string(),
11798            has_runtime_store: true,
11799            durability_policy: DurabilityPolicy::SyncWriteThrough,
11800            bridge: None,
11801            default_timeout: None,
11802        }));
11803        runtime
11804            .register(
11805                durable_spec(identity.clone(), "domain"),
11806                IdentityLifecycleState::Active,
11807                Some(old_record),
11808                Some(grant),
11809            )
11810            .await;
11811
11812        let preflight_identity = runtime
11813            .owned_identity_for_member_alias(old_alias)
11814            .await
11815            .ok_or("old alias did not pass ownership preflight")?;
11816        let new_record = runtime.reset_tracked(&identity).await?;
11817        assert_eq!(new_record.generation, ContinuityGeneration::new(1));
11818
11819        let respawn_error = match runtime
11820            .respawn_member_alias_tracked(&preflight_identity, old_alias)
11821            .await
11822        {
11823            Ok(_) => return Err("old alias respawned the replacement generation".into()),
11824            Err(error) => error,
11825        };
11826        assert!(matches!(
11827            respawn_error,
11828            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11829                if requested == old_alias
11830        ));
11831        let content = meerkat_core::ContentInput::Text("stale alias delivery".to_string());
11832        let send_error = match runtime
11833            .send_with_mode_and_interaction_member_alias_tracked(
11834                &preflight_identity,
11835                old_alias,
11836                &content,
11837                HandlingMode::Queue,
11838                None,
11839            )
11840            .await
11841        {
11842            Ok(_) => return Err("old alias delivered to the replacement generation".into()),
11843            Err(error) => error,
11844        };
11845        assert!(matches!(
11846            send_error,
11847            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11848                if requested == old_alias
11849        ));
11850        let dispatch = DispatchInput {
11851            content,
11852            origin: super::super::types::DispatchOrigin::System,
11853            correlation_id: None,
11854            idempotency_key: None,
11855        };
11856        let dispatch_error = match runtime
11857            .dispatch_member_alias_tracked(&preflight_identity, old_alias, &dispatch)
11858            .await
11859        {
11860            Ok(_) => return Err("old alias dispatched to the replacement generation".into()),
11861            Err(error) => error,
11862        };
11863        assert!(matches!(
11864            dispatch_error,
11865            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11866                if requested == old_alias
11867        ));
11868        let rebind_error = match runtime
11869            .rebind_session_after_live_respawn_member_alias_tracked(
11870                &preflight_identity,
11871                old_alias,
11872                SessionId::new(),
11873            )
11874            .await
11875        {
11876            Ok(_) => return Err("old alias rebound the replacement generation".into()),
11877            Err(error) => error,
11878        };
11879        assert!(matches!(
11880            rebind_error,
11881            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11882                if requested == old_alias
11883        ));
11884        let retire_error = match runtime
11885            .retire_member_alias_tracked(&preflight_identity, old_alias)
11886            .await
11887        {
11888            Ok(_) => return Err("old alias retired the replacement generation".into()),
11889            Err(error) => error,
11890        };
11891        assert!(matches!(
11892            retire_error,
11893            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11894                if requested == old_alias
11895        ));
11896        let reset_error = match runtime
11897            .reset_member_alias_tracked(&preflight_identity, old_alias)
11898            .await
11899        {
11900            Ok(_) => return Err("old alias reset the replacement generation".into()),
11901            Err(error) => error,
11902        };
11903        assert!(matches!(
11904            reset_error,
11905            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11906                if requested == old_alias
11907        ));
11908        let delete_error = match runtime
11909            .delete_identity_member_alias_tracked(&preflight_identity, old_alias)
11910            .await
11911        {
11912            Ok(()) => return Err("old alias deleted the replacement generation".into()),
11913            Err(error) => error,
11914        };
11915        assert!(matches!(
11916            delete_error,
11917            IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11918                if requested == old_alias
11919        ));
11920        let status = runtime.status(&identity).await?;
11921        assert_eq!(status.state, IdentityLifecycleState::Active);
11922        assert_eq!(status.generation, Some(ContinuityGeneration::new(1)));
11923        assert_eq!(
11924            status.agent_runtime_id.as_ref(),
11925            Some(&new_record.agent_runtime_id)
11926        );
11927        Ok(())
11928    }
11929
11930    #[tokio::test]
11931    async fn reset_reprofiles_session_from_identity_first_context_roster_provider()
11932    -> Result<(), Box<dyn std::error::Error>> {
11933        let identity = AgentIdentity::parse("domain:security")?;
11934        let roster = Arc::new(MutableRoster::new(vec![durable_spec(
11935            identity.clone(),
11936            "domain",
11937        )]));
11938        let bridge = Arc::new(RecordingBridge::default());
11939        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11940            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11941            lease_provider: Arc::new(LocalLeaseProvider::new()),
11942            runtime_instance_id: "reset-reprofile-context-test".to_string(),
11943            has_runtime_store: true,
11944            durability_policy: DurabilityPolicy::SyncWriteThrough,
11945            bridge: Some(bridge.clone()),
11946            default_timeout: None,
11947        }));
11948        let context =
11949            IdentityFirstRuntimeContext::new(runtime.clone(), roster.clone(), None, None, None);
11950
11951        context.refresh_desired_topology().await?;
11952        roster
11953            .set(vec![durable_spec(identity.clone(), "security")])
11954            .await;
11955
11956        let record = runtime.reset(&identity).await?;
11957
11958        assert_eq!(record.generation.get(), 1);
11959        assert_eq!(
11960            bridge.create_profiles().await,
11961            vec!["domain".to_string(), "security".to_string()]
11962        );
11963        let status = runtime.status(&identity).await?;
11964        assert_eq!(
11965            status.profile.map(|profile| profile.to_string()).as_deref(),
11966            Some("security")
11967        );
11968        Ok(())
11969    }
11970
11971    #[tokio::test]
11972    async fn reset_records_exact_cleanup_debt_without_waiting_for_hung_old_retire()
11973    -> Result<(), Box<dyn std::error::Error>> {
11974        let identity = AgentIdentity::parse("domain:security")?;
11975        let roster = Arc::new(MutableRoster::new(vec![durable_spec(
11976            identity.clone(),
11977            "domain",
11978        )]));
11979        let bridge = Arc::new(RecordingBridge::default());
11980        let runtime = Arc::new(
11981            IdentityRuntime::new(IdentityRuntimeConfig {
11982                continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11983                lease_provider: Arc::new(LocalLeaseProvider::new()),
11984                runtime_instance_id: "reset-skips-old-retire-test".to_string(),
11985                has_runtime_store: true,
11986                durability_policy: DurabilityPolicy::SyncWriteThrough,
11987                bridge: Some(bridge.clone()),
11988                default_timeout: Some(Duration::from_millis(50)),
11989            })
11990            .with_reset_roster_provider(roster.clone()),
11991        );
11992
11993        super::super::orchestrator::restore_flow(
11994            &runtime,
11995            &roster
11996                .roster(&RosterContext {
11997                    mob_definition: None,
11998                    previous_identities: Vec::new(),
11999                })
12000                .await?,
12001            None,
12002            None,
12003        )
12004        .await?;
12005
12006        let old_runtime_id = AgentRuntimeId::parse("rt:domain:security:0")?;
12007        bridge.hang_retire_for(&old_runtime_id).await;
12008        roster
12009            .set(vec![durable_spec(identity.clone(), "security")])
12010            .await;
12011
12012        let record = tokio::time::timeout(Duration::from_secs(1), runtime.reset(&identity))
12013            .await
12014            .map_err(|_| "reset timed out waiting for old generation retirement")??;
12015
12016        assert_eq!(record.generation.get(), 1);
12017        assert_eq!(
12018            bridge.create_profiles().await,
12019            vec!["domain".to_string(), "security".to_string()]
12020        );
12021        tokio::time::timeout(Duration::from_secs(1), async {
12022            loop {
12023                if bridge
12024                    .retired_runtime_ids()
12025                    .await
12026                    .contains(&old_runtime_id.to_string())
12027                {
12028                    break;
12029                }
12030                tokio::task::yield_now().await;
12031            }
12032        })
12033        .await
12034        .map_err(|_| "reset cleanup task never attempted the old generation")?;
12035        assert!(
12036            runtime
12037                .pending_reset_bridge_cleanups
12038                .read()
12039                .await
12040                .values()
12041                .any(|cleanup| cleanup.runtime_id.as_ref() == Some(&old_runtime_id)),
12042            "hung cleanup must retain the exact old runtime/session debt for shutdown"
12043        );
12044        let status = runtime.status(&identity).await?;
12045        assert_eq!(
12046            status.profile.map(|profile| profile.to_string()).as_deref(),
12047            Some("security")
12048        );
12049        Ok(())
12050    }
12051
12052    #[tokio::test]
12053    async fn reset_cleanup_failure_stays_retryable_after_new_generation()
12054    -> Result<(), Box<dyn std::error::Error>> {
12055        let identity = AgentIdentity::parse("domain:security")?;
12056        let roster = Arc::new(MutableRoster::new(vec![durable_spec(
12057            identity.clone(),
12058            "domain",
12059        )]));
12060        let bridge = Arc::new(RecordingBridge::default());
12061        let runtime = Arc::new(
12062            IdentityRuntime::new(IdentityRuntimeConfig {
12063                continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12064                lease_provider: Arc::new(LocalLeaseProvider::new()),
12065                runtime_instance_id: "reset-unregister-failure-test".to_string(),
12066                has_runtime_store: true,
12067                durability_policy: DurabilityPolicy::SyncWriteThrough,
12068                bridge: Some(bridge.clone()),
12069                default_timeout: Some(Duration::from_millis(50)),
12070            })
12071            .with_reset_roster_provider(roster.clone()),
12072        );
12073
12074        super::super::orchestrator::restore_flow(
12075            &runtime,
12076            &roster
12077                .roster(&RosterContext {
12078                    mob_definition: None,
12079                    previous_identities: Vec::new(),
12080                })
12081                .await?,
12082            None,
12083            None,
12084        )
12085        .await?;
12086
12087        let old_status = runtime.status(&identity).await?;
12088        let Some(old_session_id) = old_status.session_id else {
12089            return Err("initial session id missing".into());
12090        };
12091        bridge.fail_unregister_for(&old_session_id).await;
12092        roster
12093            .set(vec![durable_spec(identity.clone(), "security")])
12094            .await;
12095
12096        let record = tokio::time::timeout(Duration::from_secs(1), runtime.reset(&identity))
12097            .await
12098            .map_err(|_| "reset timed out waiting for old session unregister cleanup")??;
12099
12100        assert_eq!(record.generation.get(), 1);
12101        assert_eq!(
12102            bridge.create_profiles().await,
12103            vec!["domain".to_string(), "security".to_string()]
12104        );
12105        runtime.join_reset_bridge_cleanup_tasks().await;
12106        assert_eq!(
12107            runtime.pending_reset_bridge_cleanups.read().await.len(),
12108            1,
12109            "failed unregister must retain exact cleanup debt"
12110        );
12111        bridge.allow_unregister_for(&old_session_id).await;
12112        assert_eq!(runtime.drain_pending_reset_bridge_cleanups().await?, 1);
12113        assert!(
12114            runtime
12115                .pending_reset_bridge_cleanups
12116                .read()
12117                .await
12118                .is_empty()
12119        );
12120        let status = runtime.status(&identity).await?;
12121        assert_eq!(
12122            status.profile.map(|profile| profile.to_string()).as_deref(),
12123            Some("security")
12124        );
12125        Ok(())
12126    }
12127
12128    #[tokio::test]
12129    async fn active_renewal_suspends_bridge_before_publishing_replacement_authority()
12130    -> Result<(), Box<dyn std::error::Error>> {
12131        struct RotatingLeaseProvider;
12132
12133        #[async_trait::async_trait]
12134        impl LeaseProvider for RotatingLeaseProvider {
12135            async fn acquire_leases(
12136                &self,
12137                _identities: &[AgentIdentity],
12138                _runtime_instance: &str,
12139            ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
12140                Ok(BTreeMap::new())
12141            }
12142
12143            async fn renew_leases(
12144                &self,
12145                grants: &[LeaseGrant],
12146            ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
12147                Ok(grants
12148                    .iter()
12149                    .map(|grant| {
12150                        let renewed = LeaseGrant {
12151                            identity: grant.identity.clone(),
12152                            fencing_token: FencingToken::new(grant.fencing_token.get() + 1),
12153                            ttl: Duration::from_mins(5),
12154                        };
12155                        (grant.identity.clone(), LeaseRenewResult::Renewed(renewed))
12156                    })
12157                    .collect())
12158            }
12159
12160            async fn release_leases(&self, _grants: &[LeaseGrant]) -> Result<(), LeaseError> {
12161                Ok(())
12162            }
12163        }
12164
12165        let identity = AgentIdentity::parse("domain:renewal-barrier")?;
12166        let runtime_instance = "active-renewal-barrier-test";
12167        let store = Arc::new(LocalContinuityStore::in_memory()?);
12168        let lease = Arc::new(RotatingLeaseProvider);
12169        let bridge = Arc::new(RecordingBridge::default());
12170        let first = LeaseGrant {
12171            identity: identity.clone(),
12172            fencing_token: FencingToken::new(1),
12173            ttl: Duration::ZERO,
12174        };
12175        let record = ContinuityRecord {
12176            identity: identity.clone(),
12177            agent_runtime_id: AgentRuntimeId::parse("rt:domain:renewal-barrier:0")?,
12178            session_id: SessionId::new(),
12179            generation: ContinuityGeneration::new(0),
12180            checkpoint_version: CheckpointVersion::new(0),
12181        };
12182        store
12183            .upsert_continuity_record(&record, first.fencing_token)
12184            .await?;
12185        bridge
12186            .register_session_runtime_state(
12187                &record.session_id,
12188                &identity,
12189                record.generation,
12190                record.checkpoint_version,
12191                first.fencing_token,
12192            )
12193            .await?;
12194        let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12195            continuity_store: store,
12196            lease_provider: lease,
12197            runtime_instance_id: runtime_instance.to_string(),
12198            has_runtime_store: true,
12199            durability_policy: DurabilityPolicy::SyncWriteThrough,
12200            bridge: Some(bridge.clone()),
12201            default_timeout: None,
12202        });
12203        runtime
12204            .register(
12205                durable_spec(identity.clone(), "domain"),
12206                IdentityLifecycleState::Active,
12207                Some(record.clone()),
12208                Some(first.clone()),
12209            )
12210            .await;
12211
12212        let renewed_token = runtime.ensure_active_lease(&identity).await?;
12213        assert!(renewed_token > first.fencing_token);
12214        assert_eq!(
12215            bridge.authority_transitions().await,
12216            vec![
12217                format!("register:{}", first.fencing_token.get()),
12218                format!("suspend:{}", record.session_id),
12219                format!("suspend:{}", record.session_id),
12220                format!("register:{}", renewed_token.get()),
12221            ],
12222            "the old bridge authority must be quiesced before replacement publication"
12223        );
12224        assert_eq!(
12225            runtime
12226                .status(&identity)
12227                .await?
12228                .lease
12229                .map(|lease| lease.fencing_token),
12230            Some(renewed_token)
12231        );
12232        Ok(())
12233    }
12234
12235    #[tokio::test]
12236    async fn active_grant_bridge_failure_parks_rotated_token_for_exact_shutdown_release()
12237    -> Result<(), Box<dyn std::error::Error>> {
12238        let identity = AgentIdentity::parse("domain:bridge-fence")?;
12239        let runtime_instance = "active-grant-bridge-failure-test";
12240        let store = Arc::new(LocalContinuityStore::in_memory()?);
12241        let lease = Arc::new(LocalLeaseProvider::new());
12242        let bridge = Arc::new(RecordingBridge::default());
12243        let first = match lease
12244            .acquire_leases(std::slice::from_ref(&identity), runtime_instance)
12245            .await?
12246            .remove(&identity)
12247        {
12248            Some(LeaseAcquireResult::Acquired(grant)) => grant,
12249            other => return Err(format!("initial lease was not acquired: {other:?}").into()),
12250        };
12251        let record = ContinuityRecord {
12252            identity: identity.clone(),
12253            agent_runtime_id: AgentRuntimeId::parse("rt:domain:bridge-fence:0")?,
12254            session_id: SessionId::new(),
12255            generation: ContinuityGeneration::new(0),
12256            checkpoint_version: CheckpointVersion::new(0),
12257        };
12258        store
12259            .upsert_continuity_record(&record, first.fencing_token)
12260            .await?;
12261        let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12262            continuity_store: store,
12263            lease_provider: lease.clone(),
12264            runtime_instance_id: runtime_instance.to_string(),
12265            has_runtime_store: true,
12266            durability_policy: DurabilityPolicy::SyncWriteThrough,
12267            bridge: Some(bridge.clone()),
12268            default_timeout: None,
12269        });
12270        runtime
12271            .register(
12272                durable_spec(identity.clone(), "domain"),
12273                IdentityLifecycleState::Active,
12274                Some(record.clone()),
12275                Some(first.clone()),
12276            )
12277            .await;
12278
12279        let rotated = match lease
12280            .renew_leases(std::slice::from_ref(&first))
12281            .await?
12282            .remove(&identity)
12283        {
12284            Some(LeaseRenewResult::Renewed(grant)) => grant,
12285            other => return Err(format!("lease did not rotate: {other:?}").into()),
12286        };
12287        runtime
12288            .publish_active_grant(&identity, None, &rotated)
12289            .await?;
12290        {
12291            let entries = runtime.entries.read().await;
12292            let entry = entries
12293                .get(&identity)
12294                .ok_or("identity entry disappeared after active grant refresh")?;
12295            assert_eq!(entry.state, IdentityLifecycleState::Active);
12296            assert_eq!(
12297                entry.lease.as_ref().map(|lease| lease.fencing_token),
12298                Some(rotated.fencing_token)
12299            );
12300        }
12301
12302        let failed_grant = match lease
12303            .renew_leases(std::slice::from_ref(&rotated))
12304            .await?
12305            .remove(&identity)
12306        {
12307            Some(LeaseRenewResult::Renewed(grant)) => grant,
12308            other => return Err(format!("second lease did not rotate: {other:?}").into()),
12309        };
12310        bridge.fail_register_for(&record.session_id).await;
12311        let error = match runtime
12312            .publish_active_grant(&identity, None, &failed_grant)
12313            .await
12314        {
12315            Ok(_) => return Err("bridge publication failure unexpectedly succeeded".into()),
12316            Err(error) => error,
12317        };
12318        assert!(
12319            error
12320                .to_string()
12321                .contains("synthetic live-session rebind failure")
12322        );
12323        assert_eq!(
12324            bridge.registered_fencing_tokens().await,
12325            vec![rotated.fencing_token, failed_grant.fencing_token],
12326            "every active refresh must project the provider-committed token into the bridge"
12327        );
12328        {
12329            let entries = runtime.entries.read().await;
12330            let entry = entries
12331                .get(&identity)
12332                .ok_or("identity entry disappeared after bridge failure")?;
12333            assert_eq!(entry.state, IdentityLifecycleState::Broken);
12334            assert!(entry.lease.is_none());
12335            assert_eq!(
12336                entry
12337                    .pending_lease_release
12338                    .as_ref()
12339                    .map(|grant| grant.fencing_token),
12340                Some(failed_grant.fencing_token)
12341            );
12342        }
12343
12344        assert_eq!(runtime.release_all_leases_for_shutdown().await?, 1);
12345        let failover = lease
12346            .acquire_leases(std::slice::from_ref(&identity), "bridge-failure-failover")
12347            .await?;
12348        assert!(matches!(
12349            failover.get(&identity),
12350            Some(LeaseAcquireResult::Acquired(_))
12351        ));
12352        Ok(())
12353    }
12354}
12355
12356#[cfg(test)]
12357mod lease_renewal_backoff_tests {
12358    use super::*;
12359
12360    /// Regression: a lease provider that errors persistently must not spin the
12361    /// renewal task at the TTL-derived floor (down to 10ms). The failure
12362    /// backoff grows from a 1s base and caps at the max poll interval.
12363    #[test]
12364    fn lease_renewal_failure_backoff_grows_and_caps() {
12365        let max = Duration::from_mins(1);
12366        assert_eq!(
12367            lease_renewal_failure_backoff(0, max),
12368            LEASE_RENEWAL_FAILURE_BACKOFF_BASE
12369        );
12370        assert_eq!(
12371            lease_renewal_failure_backoff(1, max),
12372            LEASE_RENEWAL_FAILURE_BACKOFF_BASE * 2
12373        );
12374        assert_eq!(lease_renewal_failure_backoff(6, max), max);
12375        // Saturates at the cap for arbitrarily many failures (no shift overflow).
12376        assert_eq!(lease_renewal_failure_backoff(99, max), max);
12377        assert!(lease_renewal_failure_backoff(2, max) > lease_renewal_failure_backoff(1, max));
12378    }
12379}
12380
12381#[cfg(test)]
12382mod continuity_repair_supervisor_tests {
12383    use super::*;
12384    use crate::identity_first::bridge::{
12385        BridgeDelivery, BridgeError, ResumeSessionOutcome, SessionBridge,
12386    };
12387    use crate::identity_first::types::{AgentBuildDraft, SessionSnapshot};
12388    use crate::identity_first::{LocalContinuityStore, LocalLeaseProvider, MutableRosterProvider};
12389
12390    /// A resume path that is deterministically wedged: every attempt fails
12391    /// with the SAME error bytes, and every attempt is counted. This is the
12392    /// shape whose blind re-execution the bounded-identical-retry park
12393    /// exists to stop (each real repair attempt re-runs destructive dispose
12394    /// steps against the same blocking precondition).
12395    struct IdenticallyWedgedResumeBridge {
12396        resume_attempts: std::sync::atomic::AtomicUsize,
12397    }
12398
12399    impl IdenticallyWedgedResumeBridge {
12400        fn attempts(&self) -> usize {
12401            self.resume_attempts
12402                .load(std::sync::atomic::Ordering::SeqCst)
12403        }
12404    }
12405
12406    #[async_trait::async_trait]
12407    impl SessionBridge for IdenticallyWedgedResumeBridge {
12408        async fn create_session(
12409            &self,
12410            _identity: &AgentIdentity,
12411            _runtime_id: &AgentRuntimeId,
12412            _spec: &DurableAgentSpec,
12413            _draft: &AgentBuildDraft,
12414            _session_id: &SessionId,
12415        ) -> Result<SessionId, BridgeError> {
12416            Err(BridgeError::Mob(
12417                "create not expected: the identity resolves Ready".to_string(),
12418            ))
12419        }
12420
12421        async fn resume_session(
12422            &self,
12423            _identity: &AgentIdentity,
12424            _runtime_id: &AgentRuntimeId,
12425            _spec: &DurableAgentSpec,
12426            _draft: &AgentBuildDraft,
12427            _session_id: &SessionId,
12428            _snapshot: &SessionSnapshot,
12429        ) -> Result<ResumeSessionOutcome, BridgeError> {
12430            self.resume_attempts
12431                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
12432            Err(BridgeError::Mob(
12433                "collision retire blocked: disposal precondition holds (wedged for park test)"
12434                    .to_string(),
12435            ))
12436        }
12437
12438        async fn deliver_admitted(
12439            &self,
12440            _runtime_id: &AgentRuntimeId,
12441            _delivery: BridgeDelivery,
12442        ) -> Result<SessionId, BridgeError> {
12443            Err(BridgeError::Mob("deliver not used".to_string()))
12444        }
12445
12446        async fn checkpoint_session(
12447            &self,
12448            _runtime_id: &AgentRuntimeId,
12449            _session_id: &SessionId,
12450        ) -> Result<SessionSnapshot, BridgeError> {
12451            Err(BridgeError::Mob("checkpoint not used".to_string()))
12452        }
12453
12454        async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
12455            Ok(())
12456        }
12457    }
12458
12459    fn park_test_spec(identity: AgentIdentity) -> DurableAgentSpec {
12460        DurableAgentSpec {
12461            identity,
12462            profile: meerkat_mob::ProfileName::from("domain"),
12463            addressability: crate::identity_first::AgentAddressability::Addressable,
12464            display_name: None,
12465            labels: BTreeMap::new(),
12466            context: None,
12467            additional_instructions: Vec::new(),
12468            initial_message: None,
12469            runtime_mode_override: None,
12470            backend: None,
12471            binding: None,
12472        }
12473    }
12474
12475    /// Task #48 (c) — bounded non-identical retries: a Broken identity whose
12476    /// repair fails byte-identically on three consecutive passes is parked
12477    /// TYPED (`continuity_unrecoverable`, reason naming the blocking
12478    /// failure), and the supervisor never re-executes the repair afterwards.
12479    #[tokio::test]
12480    async fn repair_loop_parks_typed_after_three_byte_identical_failures()
12481    -> Result<(), Box<dyn std::error::Error>> {
12482        let identity = AgentIdentity::parse("domain:wedged")?;
12483        let spec = park_test_spec(identity.clone());
12484        let session_id = SessionId::new();
12485        let record = ContinuityRecord {
12486            identity: identity.clone(),
12487            agent_runtime_id: AgentRuntimeId::parse(&format!("rt:{identity}:0"))?,
12488            session_id: session_id.clone(),
12489            generation: ContinuityGeneration::new(1),
12490            checkpoint_version: CheckpointVersion::new(1),
12491        };
12492        let continuity_store = Arc::new(LocalContinuityStore::in_memory()?);
12493        continuity_store
12494            .upsert_continuity_record(&record, FencingToken::new(1))
12495            .await?;
12496        let bridge = Arc::new(IdenticallyWedgedResumeBridge {
12497            resume_attempts: std::sync::atomic::AtomicUsize::new(0),
12498        });
12499        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
12500            continuity_store,
12501            lease_provider: Arc::new(LocalLeaseProvider::new()),
12502            runtime_instance_id: "identical-failure-park-test".to_string(),
12503            has_runtime_store: true,
12504            durability_policy: DurabilityPolicy::SyncWriteThrough,
12505            bridge: Some(bridge.clone()),
12506            default_timeout: None,
12507        }));
12508        runtime
12509            .register(
12510                spec.clone(),
12511                IdentityLifecycleState::Broken,
12512                Some(record),
12513                None,
12514            )
12515            .await;
12516        let context = Arc::new(IdentityFirstRuntimeContext::new(
12517            Arc::clone(&runtime),
12518            Arc::new(MutableRosterProvider::new(vec![spec])),
12519            None,
12520            None,
12521            None,
12522        ));
12523        let task = context.spawn_tracked_broken_identity_repair_task(ContinuityRepairPolicy {
12524            initial_backoff: Duration::from_millis(10),
12525            max_backoff: Duration::from_millis(10),
12526        });
12527
12528        // The park must arrive after EXACTLY the bounded attempt count.
12529        let deadline = Instant::now() + Duration::from_secs(30);
12530        let park = loop {
12531            if let Some(park) = runtime.continuity_unrecoverable(&identity).await {
12532                break park;
12533            }
12534            if Instant::now() > deadline {
12535                task.cancel_and_join().await;
12536                return Err("repair loop never parked the identically-failing identity".into());
12537            }
12538            tokio::time::sleep(Duration::from_millis(20)).await;
12539        };
12540        assert!(
12541            park.reason.contains("byte-identical"),
12542            "park reason must name the bounded-identical-retry cause: {}",
12543            park.reason
12544        );
12545        assert!(
12546            park.reason
12547                .contains("disposal precondition holds (wedged for park test)"),
12548            "park reason must carry the blocking failure verbatim: {}",
12549            park.reason
12550        );
12551        let attempts_at_park = bridge.attempts();
12552        assert_eq!(
12553            attempts_at_park, 3,
12554            "the destructive repair must run exactly the bounded attempt count"
12555        );
12556
12557        // Parked = no further destructive re-execution on the timer.
12558        tokio::time::sleep(Duration::from_millis(200)).await;
12559        assert_eq!(
12560            bridge.attempts(),
12561            attempts_at_park,
12562            "a parked identity must not be re-repaired on the timer"
12563        );
12564        assert!(
12565            runtime
12566                .repairable_broken_identities()
12567                .await
12568                .iter()
12569                .all(|id| id != &identity),
12570            "a parked identity must leave the repairable set"
12571        );
12572        task.cancel_and_join().await;
12573        Ok(())
12574    }
12575
12576    #[tokio::test]
12577    async fn repair_loop_exits_when_supervisor_sender_is_dropped()
12578    -> Result<(), Box<dyn std::error::Error>> {
12579        let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
12580            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12581            lease_provider: Arc::new(LocalLeaseProvider::new()),
12582            runtime_instance_id: "dropped-repair-supervisor-test".to_string(),
12583            has_runtime_store: true,
12584            durability_policy: DurabilityPolicy::SyncWriteThrough,
12585            bridge: None,
12586            default_timeout: None,
12587        }));
12588        let context = Arc::new(IdentityFirstRuntimeContext::new(
12589            runtime,
12590            Arc::new(MutableRosterProvider::new(Vec::new())),
12591            None,
12592            None,
12593            None,
12594        ));
12595        let TrackedContinuityRepairTask { cancel, join } = context
12596            .spawn_tracked_broken_identity_repair_task(ContinuityRepairPolicy {
12597                initial_backoff: Duration::from_mins(1),
12598                max_backoff: Duration::from_mins(1),
12599            });
12600
12601        // This is what happens if the owning runtime is dropped without an
12602        // explicit shutdown: JoinHandle detaches, while the sender disappears.
12603        drop(cancel);
12604        tokio::time::timeout(Duration::from_millis(100), join)
12605            .await
12606            .map_err(|_| "repair loop spun after its cancellation channel closed")??;
12607        Ok(())
12608    }
12609}
12610
12611#[cfg(test)]
12612mod foreground_shutdown_tests {
12613    use super::*;
12614    use crate::identity_first::{LocalContinuityStore, LocalLeaseProvider};
12615
12616    #[test]
12617    fn foreground_shutdown_value_is_retained_for_late_subscribers()
12618    -> Result<(), ContinuityStoreError> {
12619        let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12620            continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12621            lease_provider: Arc::new(LocalLeaseProvider::new()),
12622            runtime_instance_id: "late-foreground-cancel-test".to_string(),
12623            has_runtime_store: true,
12624            durability_policy: DurabilityPolicy::SyncWriteThrough,
12625            bridge: None,
12626            default_timeout: None,
12627        });
12628
12629        assert_eq!(runtime.foreground_cancel.receiver_count(), 0);
12630        runtime.close_foreground_operations();
12631        let receiver = runtime.foreground_cancel.subscribe();
12632        assert!(
12633            *receiver.borrow(),
12634            "a task subscribed after close must still observe shutdown"
12635        );
12636        Ok(())
12637    }
12638}