1use std::collections::{BTreeMap, BTreeSet, HashMap};
10use std::future::Future;
11use std::sync::atomic::{AtomicBool, Ordering};
12use std::sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock, Weak};
13use std::time::{Duration, Instant};
14
15use futures::stream::{self, StreamExt};
16use meerkat_core::types::{HandlingMode, SessionId};
17use tokio::sync::{Mutex, Notify, RwLock, broadcast, oneshot, watch};
18use tokio::task::{JoinHandle, JoinSet};
19
20use super::agent_memory::{
21 AgentMemoryError, AgentMemoryForgetResult, AgentMemoryRecallRequest, AgentMemoryRecord,
22 AgentMemoryRuntimeInjector, NewAgentMemory,
23};
24use super::bridge::{
25 BridgeError, CommittedBoundaryRepair, ResumeRejectionKind, SessionBridge,
26 archived_not_revivable_park_reason,
27};
28use super::contracts::{
29 AgentCustomizer, ContinuityStore, LeaseProvider, RosterProvider, TopologyProvider,
30};
31use super::types::{
32 AgentAddressability, AgentBuildContext, AgentIdentity, AgentRuntimeId, AgentRuntimeServices,
33 CheckpointVersion, CompletionCursor, CompletionProgress, ContinuityGeneration,
34 ContinuityHealth, ContinuityRecord, ContinuityStoreError, ContinuityUnrecoverable,
35 DispatchAdmission, DispatchInput, DurabilityPolicy, DurableAgentSpec, FencingToken,
36 HostRejectedBuildPark, IdentityBootstrapEntry, IdentityBootstrapMode, IdentityBootstrapState,
37 IdentityBootstrapStatus, IdentityLifecycleState, IdentityStatus, LeaseGrant, LeaseInfo,
38 ManagedPeerEdge, NotAddressable, RosterContext, SendAdmission, SessionSnapshot,
39 TopologyContext,
40};
41use crate::memory::records::{
42 ManifestTier, MemoryId, MemoryKind, MemoryScope, NewMemoryRecord, RecordMeta, UsageEvent,
43};
44
45const MANAGED_PEER_RECONCILE_CONCURRENCY: usize = 64;
46const COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(100);
49const MATERIALIZATION_FAILURE_BACKOFF: Duration = Duration::from_secs(30);
50const RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN: usize = 256;
51const BACKGROUND_WARM_CANCELLED: &str =
52 "identity background warm cancelled before session installation";
53fn durable_spec_uses_external_binding(spec: &DurableAgentSpec) -> bool {
54 matches!(spec.backend, Some(meerkat_mob::MobBackendKind::External))
55 || matches!(
56 spec.binding.as_ref(),
57 Some(meerkat_contracts::WireRuntimeBinding::External { .. })
58 )
59}
60
61fn interaction_id_for_delivery<'a>(
62 spec: &DurableAgentSpec,
63 interaction_id: Option<&'a str>,
64) -> Option<&'a str> {
65 (!durable_spec_uses_external_binding(spec))
70 .then_some(interaction_id)
71 .flatten()
72}
73
74#[derive(Debug)]
80pub enum IdentityRuntimeError {
81 UnknownIdentity(AgentIdentity),
83 NotAddressable(NotAddressable),
85 NoActiveLease(AgentIdentity),
87 AlreadyEmbodied {
93 identity: AgentIdentity,
94 holder: String,
95 },
96 LeaseLost(AgentIdentity),
98 InvalidState {
100 identity: AgentIdentity,
101 state: IdentityLifecycleState,
102 operation: &'static str,
103 },
104 InvalidDeliveryIdentity {
111 identity: AgentIdentity,
112 detail: String,
113 },
114 Store(ContinuityStoreError),
116 Lease(super::types::LeaseError),
118 DuplicateIdentity(AgentIdentity),
120 StaleFencingToken {
122 identity: AgentIdentity,
123 presented: FencingToken,
124 current: FencingToken,
125 },
126 StaleCheckpointVersion {
128 identity: AgentIdentity,
129 presented: CheckpointVersion,
130 current: CheckpointVersion,
131 },
132 StaleContinuityGeneration {
133 identity: AgentIdentity,
134 presented: ContinuityGeneration,
135 current: ContinuityGeneration,
136 },
137 StaleRuntimeAlias {
140 identity: AgentIdentity,
141 requested: String,
142 current: Option<AgentRuntimeId>,
143 },
144 CompletionIncarnationChanged {
149 identity: AgentIdentity,
150 baseline: CompletionCursor,
151 observed: CompletionCursor,
152 },
153 HostRejectedBuild {
159 identity: AgentIdentity,
160 reason: String,
161 },
162 Internal(String),
164}
165
166impl std::fmt::Display for IdentityRuntimeError {
167 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168 match self {
169 Self::UnknownIdentity(id) => write!(f, "unknown identity: {id}"),
170 Self::NotAddressable(err) => write!(f, "{err}"),
171 Self::NoActiveLease(id) => write!(f, "no active lease for {id}"),
172 Self::AlreadyEmbodied { identity, holder } => write!(
173 f,
174 "identity {identity} is already embodied by runtime instance '{holder}' \
175 (single-embodiment guard: refusing a second live bind)"
176 ),
177 Self::LeaseLost(id) => write!(f, "lease lost for {id}"),
178 Self::InvalidState {
179 identity,
180 state,
181 operation,
182 } => write!(
183 f,
184 "cannot {operation} identity {identity} in state {state:?}"
185 ),
186 Self::InvalidDeliveryIdentity { identity, detail } => write!(
187 f,
188 "dispatch to {identity} refused before bridge admission: invalid delivery \
189 identity ({detail}); dedup or typed refusal, never a degraded delivery"
190 ),
191 Self::Store(err) => write!(f, "continuity store: {err}"),
192 Self::Lease(err) => write!(f, "lease provider: {err}"),
193 Self::DuplicateIdentity(id) => write!(f, "duplicate identity in roster: {id}"),
194 Self::StaleFencingToken {
195 identity,
196 presented,
197 current,
198 } => write!(
199 f,
200 "stale fencing token for {identity}: presented {presented}, current {current}"
201 ),
202 Self::StaleCheckpointVersion {
203 identity,
204 presented,
205 current,
206 } => write!(
207 f,
208 "stale checkpoint version for {identity}: presented {presented}, current {current}"
209 ),
210 Self::StaleContinuityGeneration {
211 identity,
212 presented,
213 current,
214 } => write!(
215 f,
216 "stale continuity generation for {identity}: presented {presented}, current {current}"
217 ),
218 Self::StaleRuntimeAlias {
219 identity,
220 requested,
221 current,
222 } => write!(
223 f,
224 "stale runtime alias for {identity}: requested {requested}, current {}",
225 current
226 .as_ref()
227 .map(AgentRuntimeId::as_str)
228 .unwrap_or("<none>")
229 ),
230 Self::CompletionIncarnationChanged {
231 identity,
232 baseline,
233 observed,
234 } => write!(
235 f,
236 "completion baseline {baseline} for {identity} belongs to a superseded runtime \
237 incarnation (now {observed}); capture a fresh baseline"
238 ),
239 Self::HostRejectedBuild { identity, reason } => write!(
240 f,
241 "identity {identity} is parked: the host deterministically rejected its build \
242 ({reason}); a roster/policy (spec) change clears the park"
243 ),
244 Self::Internal(msg) => write!(f, "internal: {msg}"),
245 }
246 }
247}
248
249pub(crate) fn durable_spec_digest(spec: &DurableAgentSpec) -> u64 {
253 use std::hash::{Hash, Hasher};
254 let mut hasher = std::collections::hash_map::DefaultHasher::new();
255 match serde_json::to_string(spec) {
256 Ok(json) => json.hash(&mut hasher),
257 Err(_) => spec.identity.as_str().hash(&mut hasher),
260 }
261 hasher.finish()
262}
263
264pub(crate) fn is_host_rejected_build_error(detail: &str) -> bool {
273 detail.contains("callback/build_agent failed: callback error:")
274}
275
276impl std::error::Error for IdentityRuntimeError {}
277
278impl From<ContinuityStoreError> for IdentityRuntimeError {
279 fn from(err: ContinuityStoreError) -> Self {
280 match err {
281 ContinuityStoreError::StaleFencingToken {
282 identity,
283 presented,
284 current,
285 } => Self::StaleFencingToken {
286 identity,
287 presented,
288 current,
289 },
290 ContinuityStoreError::StaleCheckpointVersion {
291 identity,
292 presented,
293 current,
294 } => Self::StaleCheckpointVersion {
295 identity,
296 presented,
297 current,
298 },
299 ContinuityStoreError::StaleContinuityGeneration {
300 identity,
301 presented,
302 current,
303 } => Self::StaleContinuityGeneration {
304 identity,
305 presented,
306 current,
307 },
308 other => Self::Store(other),
309 }
310 }
311}
312
313#[derive(Debug, Clone)]
319pub(crate) struct IdentityEntry {
320 pub spec: DurableAgentSpec,
321 pub bootstrap_generation: u64,
327 pub state: IdentityLifecycleState,
328 pub continuity: Option<ContinuityRecord>,
329 pub lease: Option<LeaseEntry>,
330 pub pending_lease_release: Option<LeaseGrant>,
336 pub checkpoint_version: CheckpointVersion,
337 pub has_runtime_store: bool,
339 pub continuity_unrecoverable: Option<ContinuityUnrecoverable>,
345 pub host_rejected_build_park: Option<HostRejectedBuildPark>,
351}
352
353#[derive(Debug, Clone)]
355pub(crate) struct LeaseEntry {
356 pub fencing_token: FencingToken,
357 pub ttl: Duration,
358 pub acquired_at: Instant,
359}
360
361impl LeaseEntry {
362 pub fn is_expired(&self) -> bool {
363 self.acquired_at.elapsed() > self.ttl
364 }
365
366 pub fn ttl_remaining(&self) -> Duration {
367 self.ttl.saturating_sub(self.acquired_at.elapsed())
368 }
369
370 pub fn is_healthy(&self) -> bool {
371 let remaining = self.ttl_remaining();
373 remaining > self.ttl / 5
374 }
375}
376
377#[derive(Debug, Clone)]
383pub enum IdentityEvent {
384 StateChanged {
386 identity: AgentIdentity,
387 new_state: IdentityLifecycleState,
388 },
389 LeaseUpdated {
391 identity: AgentIdentity,
392 fencing_token: FencingToken,
393 },
394 LeaseLost { identity: AgentIdentity },
396 CheckpointCompleted {
398 identity: AgentIdentity,
399 version: CheckpointVersion,
400 },
401 ResumeFallback {
404 identity: AgentIdentity,
405 reason: super::bridge::ResumeFallbackReason,
406 },
407}
408
409const IDENTITY_EVENT_CHANNEL_CAPACITY: usize = 64;
411const DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL: Duration = Duration::from_mins(1);
412const DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL: Duration = Duration::from_millis(10);
413const LEASE_RENEWAL_FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(1);
419
420fn lease_renewal_failure_backoff(
421 consecutive_failures: u32,
422 max_poll_interval: Duration,
423) -> Duration {
424 LEASE_RENEWAL_FAILURE_BACKOFF_BASE
425 .saturating_mul(1u32 << consecutive_failures.min(6))
426 .min(max_poll_interval)
427}
428
429pub struct IdentityRuntimeConfig {
435 pub continuity_store: Arc<dyn ContinuityStore>,
436 pub lease_provider: Arc<dyn LeaseProvider>,
437 pub runtime_instance_id: String,
438 pub has_runtime_store: bool,
439 pub durability_policy: DurabilityPolicy,
440 pub bridge: Option<Arc<dyn SessionBridge>>,
444 pub default_timeout: Option<Duration>,
447}
448
449#[derive(Clone)]
450pub struct IdentityFirstRuntimeContext {
451 pub runtime: Arc<IdentityRuntime>,
452 pub roster_provider: Arc<dyn RosterProvider>,
453 pub topology_provider: Option<Arc<dyn TopologyProvider>>,
454 pub customizer: Option<Arc<dyn AgentCustomizer>>,
455 mob_definition: Option<meerkat_mob::MobDefinition>,
456 bootstrap_mode: IdentityBootstrapMode,
457}
458
459impl IdentityFirstRuntimeContext {
460 pub(crate) async fn topology_snapshot_inputs(
461 &self,
462 ) -> Result<(Vec<DurableAgentSpec>, Vec<ManagedPeerEdge>), IdentityRuntimeError> {
463 let previous_identities = self.runtime.registered_identities().await;
464 let roster = self
465 .roster_provider
466 .roster(&RosterContext {
467 mob_definition: self.mob_definition.clone(),
468 previous_identities,
469 })
470 .await
471 .map_err(|error| IdentityRuntimeError::Internal(format!("roster provider: {error}")))?;
472 let identities = roster
473 .iter()
474 .map(|spec| spec.identity.clone())
475 .collect::<Vec<_>>();
476 let declared = match self.topology_provider.as_deref() {
477 Some(provider) => provider
478 .compute_edges(
479 &identities,
480 &TopologyContext {
481 roster: roster.clone(),
482 },
483 )
484 .await
485 .map_err(|error| {
486 IdentityRuntimeError::Internal(format!("topology provider: {error}"))
487 })?,
488 None => self.runtime.desired_peer_edges.read().await.clone(),
489 };
490 Ok((roster, declared))
491 }
492
493 pub fn new(
494 runtime: Arc<IdentityRuntime>,
495 roster_provider: Arc<dyn RosterProvider>,
496 topology_provider: Option<Arc<dyn TopologyProvider>>,
497 customizer: Option<Arc<dyn AgentCustomizer>>,
498 mob_definition: Option<meerkat_mob::MobDefinition>,
499 ) -> Self {
500 Self::new_with_lazy_materialization(
501 runtime,
502 roster_provider,
503 topology_provider,
504 customizer,
505 mob_definition,
506 false,
507 )
508 }
509
510 pub fn new_with_lazy_materialization(
511 runtime: Arc<IdentityRuntime>,
512 roster_provider: Arc<dyn RosterProvider>,
513 topology_provider: Option<Arc<dyn TopologyProvider>>,
514 customizer: Option<Arc<dyn AgentCustomizer>>,
515 mob_definition: Option<meerkat_mob::MobDefinition>,
516 lazy_materialization: bool,
517 ) -> Self {
518 Self::new_with_bootstrap_mode(
519 runtime,
520 roster_provider,
521 topology_provider,
522 customizer,
523 mob_definition,
524 if lazy_materialization {
525 IdentityBootstrapMode::LazyMaterialize
526 } else {
527 IdentityBootstrapMode::EagerMaterialize
528 },
529 )
530 }
531
532 pub fn new_with_bootstrap_mode(
535 runtime: Arc<IdentityRuntime>,
536 roster_provider: Arc<dyn RosterProvider>,
537 topology_provider: Option<Arc<dyn TopologyProvider>>,
538 customizer: Option<Arc<dyn AgentCustomizer>>,
539 mob_definition: Option<meerkat_mob::MobDefinition>,
540 bootstrap_mode: IdentityBootstrapMode,
541 ) -> Self {
542 runtime.set_reset_roster_provider_context(
543 Some(roster_provider.clone()),
544 mob_definition.clone(),
545 );
546 Self {
547 runtime,
548 roster_provider,
549 topology_provider,
550 customizer,
551 mob_definition,
552 bootstrap_mode,
553 }
554 }
555
556 pub fn bootstrap_mode(&self) -> &IdentityBootstrapMode {
557 &self.bootstrap_mode
558 }
559
560 pub async fn bootstrap_roster(
564 &self,
565 roster: &[DurableAgentSpec],
566 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
567 let _controller = self.runtime.bootstrap_controller.lock().await;
568 let generation = self
569 .runtime
570 .begin_identity_bootstrap_pending(self.bootstrap_mode.clone());
571 if let Err(error) = self.prepare_controlled_bootstrap().await {
572 self.runtime.fail_identity_bootstrap(generation, &error);
573 return Err(error);
574 }
575 self.apply_roster_controlled(generation, roster, true).await
576 }
577
578 async fn prepare_controlled_bootstrap(&self) -> Result<(), IdentityRuntimeError> {
585 if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
586 return Err(IdentityRuntimeError::Internal(
587 "identity bootstrap is shutting down".to_string(),
588 ));
589 }
590 self.runtime.request_identity_bootstrap_stop();
593 self.runtime.join_identity_bootstrap_task().await;
594 if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
595 return Err(IdentityRuntimeError::Internal(
596 "identity bootstrap is shutting down".to_string(),
597 ));
598 }
599 Ok(())
600 }
601
602 async fn apply_roster_controlled(
603 &self,
604 generation: u64,
605 roster: &[DurableAgentSpec],
606 optimize_startup_snapshot_load: bool,
607 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
608 if let Err(message) = self.bootstrap_mode.validate() {
609 let error = IdentityRuntimeError::Internal(message);
610 self.runtime
611 .begin_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster)
612 .await;
613 self.runtime.fail_identity_bootstrap(generation, &error);
614 return Err(error);
615 }
616 self.runtime
620 .begin_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster)
621 .await;
622 if let Err(error) = self
629 .runtime
630 .reconcile_roster_members(roster, generation)
631 .await
632 {
633 self.runtime.fail_identity_bootstrap(generation, &error);
634 return Err(error);
635 }
636 let result = match (&self.bootstrap_mode, optimize_startup_snapshot_load) {
637 (IdentityBootstrapMode::EagerMaterialize, true) => {
638 super::orchestrator::restore_flow_for_bootstrap(
639 &self.runtime,
640 roster,
641 self.topology_provider.as_deref(),
642 self.customizer.as_deref(),
643 )
644 .await
645 }
646 (IdentityBootstrapMode::EagerMaterialize, false) => {
647 super::orchestrator::restore_flow(
648 &self.runtime,
649 roster,
650 self.topology_provider.as_deref(),
651 self.customizer.as_deref(),
652 )
653 .await
654 }
655 (
656 IdentityBootstrapMode::LazyMaterialize
657 | IdentityBootstrapMode::LazyWithBackgroundWarm { .. },
658 _,
659 ) => {
660 super::orchestrator::lazy_register_flow(
661 &self.runtime,
662 roster,
663 self.topology_provider.as_deref(),
664 )
665 .await
666 }
667 };
668 let result = match result {
669 Ok(result) => result,
670 Err(error) => {
671 self.runtime.fail_identity_bootstrap(generation, &error);
672 return Err(error);
673 }
674 };
675 if self.runtime.bootstrap_shutdown.load(Ordering::Acquire) {
676 let error =
677 IdentityRuntimeError::Internal("identity bootstrap is shutting down".to_string());
678 self.runtime.fail_identity_bootstrap(generation, &error);
679 return Err(error);
680 }
681 self.runtime
682 .install_identity_bootstrap(generation, self.bootstrap_mode.clone(), roster, &result)
683 .await;
684 Ok(result)
685 }
686
687 pub async fn refresh_desired_topology(
688 &self,
689 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
690 let _controller = self.runtime.bootstrap_controller.lock().await;
695 let generation = self
696 .runtime
697 .begin_identity_bootstrap_pending(self.bootstrap_mode.clone());
698 if let Err(error) = self.prepare_controlled_bootstrap().await {
699 self.runtime.fail_identity_bootstrap(generation, &error);
700 return Err(error);
701 }
702 let roster = match self
703 .roster_provider
704 .roster(&RosterContext {
705 mob_definition: self.mob_definition.clone(),
706 previous_identities: Vec::new(),
707 })
708 .await
709 {
710 Ok(roster) => roster,
711 Err(err) => {
712 let error = IdentityRuntimeError::Internal(format!("roster provider: {err}"));
713 self.runtime.fail_identity_bootstrap(generation, &error);
714 return Err(error);
715 }
716 };
717
718 self.apply_roster_controlled(generation, &roster, false)
719 .await
720 }
721
722 pub async fn refresh_desired_topology_tracked(
727 self: &Arc<Self>,
728 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
729 let context = Arc::clone(self);
730 let runtime = Arc::clone(&self.runtime);
731 runtime
732 .run_tracked_foreground(async move { context.refresh_desired_topology().await })
733 .await
734 }
735
736 pub fn spawn_broken_identity_repair_task(
754 self: Arc<Self>,
755 policy: ContinuityRepairPolicy,
756 ) -> JoinHandle<()> {
757 tokio::spawn(self.run_broken_identity_repair_loop(policy, None))
758 }
759
760 pub(crate) fn spawn_tracked_broken_identity_repair_task(
761 self: Arc<Self>,
762 policy: ContinuityRepairPolicy,
763 ) -> TrackedContinuityRepairTask {
764 let (cancel, receiver) = watch::channel(false);
765 let join = tokio::spawn(self.run_broken_identity_repair_loop(policy, Some(receiver)));
766 TrackedContinuityRepairTask { cancel, join }
767 }
768
769 async fn run_broken_identity_repair_loop(
770 self: Arc<Self>,
771 policy: ContinuityRepairPolicy,
772 mut cancellation: Option<watch::Receiver<bool>>,
773 ) {
774 let mut backoff = policy.initial_backoff;
775 let mut identical_failure_streaks: HashMap<AgentIdentity, (String, u32)> = HashMap::new();
782 loop {
783 if let Some(cancellation) = cancellation.as_mut() {
784 if *cancellation.borrow() {
785 return;
786 }
787 tokio::select! {
788 () = tokio::time::sleep(backoff) => {}
789 changed = cancellation.changed() => {
790 match changed {
791 Ok(()) if *cancellation.borrow() => return,
792 Ok(()) => continue,
793 Err(_) => return,
798 }
799 }
800 }
801 } else {
802 tokio::time::sleep(backoff).await;
803 }
804 let broken = self.runtime.broken_identities().await;
805 if broken.is_empty() {
806 backoff = policy.initial_backoff;
807 continue;
808 }
809 let mut repairable = Vec::new();
817 let mut recovery_failures = 0usize;
818 for identity in &broken {
819 if self
820 .runtime
821 .continuity_unrecoverable(identity)
822 .await
823 .is_some()
824 {
825 continue;
829 }
830 if self
831 .runtime
832 .host_rejected_build_park(identity)
833 .await
834 .is_some()
835 {
836 continue;
843 }
844 match self.attempt_committed_boundary_recovery(identity).await {
845 BrokenRepairDisposition::Repairable => repairable.push(identity.clone()),
846 BrokenRepairDisposition::Unprovable => {}
847 BrokenRepairDisposition::RetryLater => recovery_failures += 1,
848 }
849 }
850 if repairable.is_empty() {
851 backoff = if recovery_failures > 0 {
856 (backoff * 2).min(policy.max_backoff)
857 } else {
858 policy.initial_backoff
859 };
860 if cancellation
861 .as_ref()
862 .is_some_and(|cancellation| *cancellation.borrow())
863 {
864 return;
865 }
866 continue;
867 }
868 tracing::info!(
869 broken = repairable.len(),
870 "continuity repair: retrying restore for Broken identities"
871 );
872 let pass = match self.refresh_desired_topology().await {
873 Ok(pass) => pass,
874 Err(err) => {
875 tracing::warn!(
876 error = %err,
877 "continuity repair reconcile failed; backing off"
878 );
879 backoff = (backoff * 2).min(policy.max_backoff);
880 if cancellation
881 .as_ref()
882 .is_some_and(|cancellation| *cancellation.borrow())
883 {
884 return;
885 }
886 continue;
887 }
888 };
889 let still_broken = self.runtime.repairable_broken_identities().await;
890 let healed = repairable
891 .iter()
892 .filter(|id| !still_broken.contains(id))
893 .count();
894 if healed > 0 {
895 tracing::info!(
896 healed,
897 still_broken = still_broken.len(),
898 "continuity repair healed identities"
899 );
900 }
901 for identity in &repairable {
902 if !still_broken.contains(identity) {
903 identical_failure_streaks.remove(identity);
904 continue;
905 }
906 let Some(super::orchestrator::RestoreOutcome::Broken(failure)) =
907 pass.outcomes.get(identity)
908 else {
909 identical_failure_streaks.remove(identity);
913 continue;
914 };
915 let signature = format!("{:?}: {}", failure.kind, failure.detail);
916 let streak = identical_failure_streaks
917 .entry(identity.clone())
918 .or_insert_with(|| (signature.clone(), 0));
919 if streak.0 == signature {
920 streak.1 += 1;
921 } else {
922 *streak = (signature.clone(), 1);
923 }
924 if streak.1 >= REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS {
925 identical_failure_streaks.remove(identity);
926 tracing::error!(
927 %identity,
928 attempts = REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS,
929 blocking_failure = %signature,
930 "continuity repair failed byte-identically on every attempt; \
931 parking the identity typed instead of re-executing destructive \
932 repair steps on a timer. Operator path back after fixing the \
933 blocking failure: restart the gateway (the park is \
934 process-local; boot re-attempts repair once) or reset the \
935 identity via `mobkit/reset` (deliberate fresh start)"
936 );
937 if !self
938 .runtime
939 .mark_continuity_unrecoverable(
940 identity,
941 format!(
942 "continuity repair parked after \
943 {REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS} consecutive \
944 byte-identical repair failures; blocking failure: \
945 {signature}. After fixing it, restart the gateway \
946 (process-local park; boot re-attempts repair) or reset \
947 the identity via `mobkit/reset`"
948 ),
949 )
950 .await
951 {
952 tracing::debug!(
953 %identity,
954 "identity left Broken before the repair park could be recorded"
955 );
956 }
957 }
958 }
959 backoff = if still_broken.is_empty() && recovery_failures == 0 {
960 policy.initial_backoff
961 } else {
962 (backoff * 2).min(policy.max_backoff)
963 };
964 if cancellation
965 .as_ref()
966 .is_some_and(|cancellation| *cancellation.borrow())
967 {
968 return;
969 }
970 }
971 }
972
973 async fn attempt_committed_boundary_recovery(
977 &self,
978 identity: &AgentIdentity,
979 ) -> BrokenRepairDisposition {
980 let Some(bridge) = self.runtime.bridge() else {
981 return BrokenRepairDisposition::Repairable;
983 };
984 let Some(session_id) = self.runtime.continuity_session_id(identity).await else {
985 return BrokenRepairDisposition::Repairable;
988 };
989 match bridge.recover_committed_boundary(&session_id).await {
990 Ok(
991 CommittedBoundaryRepair::AlreadyCommitted | CommittedBoundaryRepair::Unsupported,
992 ) => BrokenRepairDisposition::Repairable,
993 Ok(CommittedBoundaryRepair::Recovered) => {
994 tracing::info!(
995 %identity,
996 %session_id,
997 "continuity heal: recovery persisted a committed durable head; \
998 proceeding to reconcile"
999 );
1000 BrokenRepairDisposition::Repairable
1001 }
1002 Ok(CommittedBoundaryRepair::Unprovable { reason }) => {
1003 tracing::error!(
1004 %identity,
1005 %session_id,
1006 reason = %reason,
1007 "continuity heal verdict: durable head unprovable; parking the \
1008 identity as Broken until an operator intervenes"
1009 );
1010 if !self
1011 .runtime
1012 .mark_continuity_unrecoverable(identity, reason)
1013 .await
1014 {
1015 tracing::debug!(
1018 %identity,
1019 "unprovable verdict arrived after the identity left Broken"
1020 );
1021 }
1022 BrokenRepairDisposition::Unprovable
1023 }
1024 Err(error) => {
1025 tracing::warn!(
1028 %identity,
1029 %session_id,
1030 error = %error,
1031 "committed-boundary recovery failed; retrying next repair pass"
1032 );
1033 BrokenRepairDisposition::RetryLater
1034 }
1035 }
1036 }
1037}
1038
1039enum BrokenRepairDisposition {
1042 Repairable,
1045 Unprovable,
1047 RetryLater,
1049}
1050
1051pub(crate) struct TrackedContinuityRepairTask {
1057 cancel: watch::Sender<bool>,
1058 join: JoinHandle<()>,
1059}
1060
1061impl TrackedContinuityRepairTask {
1062 pub(crate) fn cancel(&self) {
1063 let _ = self.cancel.send(true);
1064 }
1065
1066 pub(crate) async fn cancel_and_join(self) {
1067 self.cancel();
1068 let _ = self.join.await;
1069 }
1070}
1071
1072pub(crate) struct TrackedLeaseRenewalTask {
1078 cancel: watch::Sender<bool>,
1079 join: JoinHandle<()>,
1080}
1081
1082impl TrackedLeaseRenewalTask {
1083 pub(crate) fn cancel(&self) {
1084 let _ = self.cancel.send(true);
1085 }
1086
1087 pub(crate) async fn cancel_and_join(self) {
1088 self.cancel();
1089 let _ = self.join.await;
1090 }
1091}
1092
1093const REPAIR_IDENTICAL_FAILURE_PARK_ATTEMPTS: u32 = 3;
1098
1099#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1101pub struct ContinuityRepairPolicy {
1102 pub initial_backoff: Duration,
1104 pub max_backoff: Duration,
1106}
1107
1108impl Default for ContinuityRepairPolicy {
1109 fn default() -> Self {
1110 Self {
1111 initial_backoff: Duration::from_secs(30),
1112 max_backoff: Duration::from_mins(10),
1113 }
1114 }
1115}
1116
1117struct RawMemberAliasLockTable {
1122 entries: BTreeMap<String, Weak<Mutex<()>>>,
1123 next_sweep_len: usize,
1124 #[cfg(test)]
1125 sweep_count: usize,
1126}
1127
1128impl Default for RawMemberAliasLockTable {
1129 fn default() -> Self {
1130 Self {
1131 entries: BTreeMap::new(),
1132 next_sweep_len: RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN,
1133 #[cfg(test)]
1134 sweep_count: 0,
1135 }
1136 }
1137}
1138
1139impl RawMemberAliasLockTable {
1140 fn sweep_if_needed(&mut self) {
1141 if self.entries.len() < self.next_sweep_len {
1142 return;
1143 }
1144 self.entries.retain(|_, lock| lock.upgrade().is_some());
1145 self.next_sweep_len = self
1146 .entries
1147 .len()
1148 .saturating_mul(2)
1149 .max(RAW_MEMBER_ALIAS_LOCK_SWEEP_MIN);
1150 #[cfg(test)]
1151 {
1152 self.sweep_count += 1;
1153 }
1154 }
1155}
1156
1157pub struct IdentityRuntime {
1160 entries: RwLock<BTreeMap<AgentIdentity, IdentityEntry>>,
1161 event_channels: RwLock<BTreeMap<AgentIdentity, broadcast::Sender<IdentityEvent>>>,
1162 continuity_store: Arc<dyn ContinuityStore>,
1163 lease_provider: Arc<dyn LeaseProvider>,
1164 runtime_instance_id: String,
1165 has_runtime_store: bool,
1166 durability_policy: DurabilityPolicy,
1167 bridge: Option<Arc<dyn SessionBridge>>,
1168 reset_roster_source: StdRwLock<Option<ResetRosterSource>>,
1169 runtime_services: AgentRuntimeServices,
1170 managed_peer_edges: RwLock<BTreeSet<(AgentIdentity, AgentIdentity)>>,
1171 managed_peer_reconcile_lock: Mutex<()>,
1172 desired_peer_edges: RwLock<Vec<ManagedPeerEdge>>,
1173 topology_controller: StdRwLock<Option<crate::topology_control::TopologyController>>,
1174 materialization_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1175 best_effort_materialization_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1176 lifecycle_locks: RwLock<BTreeMap<AgentIdentity, Arc<Mutex<()>>>>,
1177 raw_member_alias_locks: RwLock<RawMemberAliasLockTable>,
1178 customizer: RwLock<Option<Arc<dyn AgentCustomizer>>>,
1179 agent_memory: RwLock<Option<AgentMemoryRuntimeInjector>>,
1180 lease_renewal_notify: Notify,
1181 pending_unactivated_lease_releases: RwLock<Vec<LeaseGrant>>,
1185 pending_unactivated_lease_release_gate: Mutex<()>,
1186 default_timeout: Duration,
1187 materialization_failure_backoff: RwLock<BTreeMap<AgentIdentity, MaterializationFailureBackoff>>,
1188 error_hook: StdRwLock<Option<crate::unified_runtime::ErrorHook>>,
1189 bootstrap_status: watch::Sender<IdentityBootstrapStatus>,
1190 bootstrap_generation: StdMutex<u64>,
1191 bootstrap_controller: Mutex<()>,
1192 bootstrap_task: Mutex<Option<JoinHandle<()>>>,
1193 bootstrap_cancel: StdRwLock<Option<watch::Sender<bool>>>,
1194 bootstrap_shutdown: AtomicBool,
1195 foreground_operations: Mutex<JoinSet<()>>,
1196 foreground_cancel: watch::Sender<bool>,
1197 foreground_shutdown: AtomicBool,
1198 pending_reset_bridge_cleanups: Arc<RwLock<BTreeMap<String, PendingResetBridgeCleanup>>>,
1203 reset_bridge_cleanup_tasks: Mutex<JoinSet<()>>,
1204 completion_cursors: StdMutex<BTreeMap<AgentIdentity, CompletionCursor>>,
1212}
1213
1214pub(crate) struct MemberAliasLifecycleTarget {
1219 runtime: Arc<IdentityRuntime>,
1220 identity: AgentIdentity,
1221 alias: String,
1222 lock: Arc<Mutex<()>>,
1223}
1224
1225struct MultiRuntimeForegroundCompletion(watch::Sender<bool>);
1226
1227impl Drop for MultiRuntimeForegroundCompletion {
1228 fn drop(&mut self) {
1229 self.0.send_replace(true);
1230 }
1231}
1232
1233struct DispatchOutcome {
1237 admission: DispatchAdmission,
1238 session_id: Option<SessionId>,
1239}
1240
1241#[derive(Clone)]
1242struct ResetRosterSource {
1243 provider: Arc<dyn RosterProvider>,
1244 mob_definition: Option<meerkat_mob::MobDefinition>,
1245}
1246
1247#[derive(Debug, Clone)]
1248struct MaterializationFailureBackoff {
1249 suppress_until: Instant,
1250 error: String,
1251}
1252
1253#[derive(Clone)]
1254struct PendingResetMemoryCapture {
1255 injector: AgentMemoryRuntimeInjector,
1256 identity: AgentIdentity,
1257 session_key: String,
1258 generation: u64,
1259}
1260
1261impl PendingResetMemoryCapture {
1262 async fn run(&self) {
1263 self.injector.note_reset_boundary(&self.session_key);
1267 self.injector
1268 .note_session_generation(&self.identity, &self.session_key, self.generation);
1269 self.injector
1270 .distill_before_rotation(
1271 &self.identity,
1272 &self.session_key,
1273 crate::memory::distiller::DistillCause::Reset,
1274 )
1275 .await;
1276 }
1277}
1278
1279#[derive(Clone)]
1280struct PendingResetBridgeCleanup {
1281 runtime_id: Option<AgentRuntimeId>,
1282 session_id: Option<SessionId>,
1283 memory_capture: Option<PendingResetMemoryCapture>,
1284}
1285
1286impl PartialEq for PendingResetBridgeCleanup {
1287 fn eq(&self, other: &Self) -> bool {
1288 self.runtime_id == other.runtime_id && self.session_id == other.session_id
1289 }
1290}
1291
1292impl Eq for PendingResetBridgeCleanup {}
1293
1294impl PendingResetBridgeCleanup {
1295 fn key(&self) -> String {
1296 format!(
1297 "{}|{}",
1298 self.runtime_id
1299 .as_ref()
1300 .map(AgentRuntimeId::as_str)
1301 .unwrap_or("-"),
1302 self.session_id
1303 .as_ref()
1304 .map(std::string::ToString::to_string)
1305 .as_deref()
1306 .unwrap_or("-")
1307 )
1308 }
1309}
1310
1311impl IdentityRuntime {
1312 pub fn new(config: IdentityRuntimeConfig) -> Self {
1314 let (bootstrap_status, _) = watch::channel(IdentityBootstrapStatus::empty(
1315 IdentityBootstrapMode::EagerMaterialize,
1316 ));
1317 let (foreground_cancel, _) = watch::channel(false);
1318 Self {
1319 entries: RwLock::new(BTreeMap::new()),
1320 event_channels: RwLock::new(BTreeMap::new()),
1321 continuity_store: config.continuity_store,
1322 lease_provider: config.lease_provider,
1323 runtime_instance_id: config.runtime_instance_id,
1324 has_runtime_store: config.has_runtime_store,
1325 durability_policy: config.durability_policy,
1326 bridge: config.bridge,
1327 reset_roster_source: StdRwLock::new(None),
1328 runtime_services: AgentRuntimeServices::empty(),
1329 managed_peer_edges: RwLock::new(BTreeSet::new()),
1330 managed_peer_reconcile_lock: Mutex::new(()),
1331 desired_peer_edges: RwLock::new(Vec::new()),
1332 topology_controller: StdRwLock::new(None),
1333 materialization_locks: RwLock::new(BTreeMap::new()),
1334 best_effort_materialization_locks: RwLock::new(BTreeMap::new()),
1335 lifecycle_locks: RwLock::new(BTreeMap::new()),
1336 raw_member_alias_locks: RwLock::new(RawMemberAliasLockTable::default()),
1337 customizer: RwLock::new(None),
1338 agent_memory: RwLock::new(None),
1339 lease_renewal_notify: Notify::new(),
1340 pending_unactivated_lease_releases: RwLock::new(Vec::new()),
1341 pending_unactivated_lease_release_gate: Mutex::new(()),
1342 default_timeout: config.default_timeout.unwrap_or(Duration::from_secs(90)),
1343 materialization_failure_backoff: RwLock::new(BTreeMap::new()),
1344 error_hook: StdRwLock::new(None),
1345 bootstrap_status,
1346 bootstrap_generation: StdMutex::new(0),
1347 bootstrap_controller: Mutex::new(()),
1348 bootstrap_task: Mutex::new(None),
1349 bootstrap_cancel: StdRwLock::new(None),
1350 bootstrap_shutdown: AtomicBool::new(false),
1351 foreground_operations: Mutex::new(JoinSet::new()),
1352 foreground_cancel,
1353 foreground_shutdown: AtomicBool::new(false),
1354 pending_reset_bridge_cleanups: Arc::new(RwLock::new(BTreeMap::new())),
1355 reset_bridge_cleanup_tasks: Mutex::new(JoinSet::new()),
1356 completion_cursors: StdMutex::new(BTreeMap::new()),
1357 }
1358 }
1359
1360 pub fn with_runtime_services(mut self, runtime_services: AgentRuntimeServices) -> Self {
1361 self.runtime_services = runtime_services;
1362 self
1363 }
1364
1365 pub fn with_reset_roster_provider(self, provider: Arc<dyn RosterProvider>) -> Self {
1366 self.set_reset_roster_provider(Some(provider));
1367 self
1368 }
1369
1370 pub fn with_reset_roster_provider_context(
1371 self,
1372 provider: Arc<dyn RosterProvider>,
1373 mob_definition: Option<meerkat_mob::MobDefinition>,
1374 ) -> Self {
1375 self.set_reset_roster_provider_context(Some(provider), mob_definition);
1376 self
1377 }
1378
1379 pub(crate) fn runtime_services(&self) -> AgentRuntimeServices {
1380 self.runtime_services.clone()
1381 }
1382
1383 pub fn identity_bootstrap_status(&self) -> IdentityBootstrapStatus {
1386 self.identity_bootstrap_status_with_generation().1
1387 }
1388
1389 pub(crate) fn identity_bootstrap_status_with_generation(
1390 &self,
1391 ) -> (u64, IdentityBootstrapStatus) {
1392 let generation = self
1393 .bootstrap_generation
1394 .lock()
1395 .unwrap_or_else(std::sync::PoisonError::into_inner);
1396 (*generation, self.bootstrap_status.borrow().clone())
1397 }
1398
1399 pub(crate) fn subscribe_identity_bootstrap_status(
1400 &self,
1401 ) -> watch::Receiver<IdentityBootstrapStatus> {
1402 self.bootstrap_status.subscribe()
1403 }
1404
1405 pub async fn wait_identity_bootstrap_terminal(
1409 &self,
1410 timeout: Duration,
1411 ) -> (IdentityBootstrapStatus, bool) {
1412 let (status, timed_out, _) = self
1413 .wait_identity_bootstrap_terminal_with_generation(timeout)
1414 .await;
1415 (status, timed_out)
1416 }
1417
1418 pub(crate) async fn wait_identity_bootstrap_terminal_with_generation(
1419 &self,
1420 timeout: Duration,
1421 ) -> (IdentityBootstrapStatus, bool, u64) {
1422 let mut receiver = self.subscribe_identity_bootstrap_status();
1423 let wait = async {
1424 loop {
1425 let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1426 if snapshot.complete && snapshot.materialization_terminal() {
1427 return (snapshot, generation);
1428 }
1429 if receiver.changed().await.is_err() {
1430 let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1431 return (snapshot, generation);
1432 }
1433 }
1434 };
1435 match tokio::time::timeout(timeout, wait).await {
1436 Ok((snapshot, generation)) => (snapshot, false, generation),
1437 Err(_) => {
1438 let (generation, snapshot) = self.identity_bootstrap_status_with_generation();
1439 (snapshot, true, generation)
1440 }
1441 }
1442 }
1443
1444 fn request_identity_bootstrap_stop(&self) {
1445 if let Some(cancel) = self
1446 .bootstrap_cancel
1447 .read()
1448 .unwrap_or_else(std::sync::PoisonError::into_inner)
1449 .clone()
1450 {
1451 let _ = cancel.send(true);
1452 }
1453 }
1454
1455 async fn join_identity_bootstrap_task(&self) {
1456 if let Some(task) = self.bootstrap_task.lock().await.take() {
1457 let _ = task.await;
1458 }
1459 *self
1460 .bootstrap_cancel
1461 .write()
1462 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
1463 }
1464
1465 fn modify_bootstrap_status<F>(&self, expected_generation: Option<u64>, modify: F) -> bool
1469 where
1470 F: FnOnce(&mut IdentityBootstrapStatus),
1471 {
1472 let current_generation = self
1473 .bootstrap_generation
1474 .lock()
1475 .unwrap_or_else(std::sync::PoisonError::into_inner);
1476 if expected_generation.is_some_and(|expected| expected != *current_generation) {
1477 return false;
1478 }
1479 self.bootstrap_status.send_modify(modify);
1480 true
1481 }
1482
1483 fn replace_bootstrap_status(
1484 &self,
1485 expected_generation: u64,
1486 status: IdentityBootstrapStatus,
1487 ) -> bool {
1488 let current_generation = self
1489 .bootstrap_generation
1490 .lock()
1491 .unwrap_or_else(std::sync::PoisonError::into_inner);
1492 if expected_generation != *current_generation {
1493 return false;
1494 }
1495 self.bootstrap_status.send_replace(status);
1496 true
1497 }
1498
1499 pub(crate) async fn cancel_identity_bootstrap(&self) {
1505 self.bootstrap_shutdown.store(true, Ordering::Release);
1506 self.request_identity_bootstrap_stop();
1507 let _controller = self.bootstrap_controller.lock().await;
1508 self.request_identity_bootstrap_stop();
1509 self.join_identity_bootstrap_task().await;
1510 self.modify_bootstrap_status(None, |snapshot| {
1511 for entry in snapshot.identities.values_mut() {
1512 if entry.state == IdentityBootstrapState::Warming {
1513 entry.state = IdentityBootstrapState::Dormant;
1514 }
1515 }
1516 snapshot.complete = true;
1517 snapshot.refresh_aggregates();
1518 });
1519 }
1520
1521 async fn begin_identity_bootstrap(
1522 &self,
1523 generation: u64,
1524 mode: IdentityBootstrapMode,
1525 roster: &[DurableAgentSpec],
1526 ) {
1527 let entries = self.entries.read().await;
1528 let identities = roster
1529 .iter()
1530 .map(|spec| {
1531 let lifecycle = entries.get(&spec.identity).map(|entry| entry.state);
1532 let state = match lifecycle {
1533 Some(IdentityLifecycleState::Active) => IdentityBootstrapState::Active,
1534 _ if matches!(&mode, IdentityBootstrapMode::EagerMaterialize) => {
1535 IdentityBootstrapState::Warming
1536 }
1537 _ => IdentityBootstrapState::Dormant,
1538 };
1539 (
1540 spec.identity.clone(),
1541 IdentityBootstrapEntry { state, error: None },
1542 )
1543 })
1544 .collect();
1545 drop(entries);
1546 let mut status = IdentityBootstrapStatus {
1547 mode,
1548 complete: false,
1549 ready: false,
1550 error: None,
1551 counts: Default::default(),
1552 identities,
1553 };
1554 status.refresh_aggregates();
1555 status.complete = false;
1559 status.ready = false;
1560 self.replace_bootstrap_status(generation, status);
1561 }
1562
1563 fn begin_identity_bootstrap_pending(&self, mode: IdentityBootstrapMode) -> u64 {
1564 let mut generation = self
1565 .bootstrap_generation
1566 .lock()
1567 .unwrap_or_else(std::sync::PoisonError::into_inner);
1568 *generation = generation.wrapping_add(1);
1569 if *generation == 0 {
1570 *generation = 1;
1571 }
1572 let current_generation = *generation;
1573 self.bootstrap_status.send_modify(|snapshot| {
1574 snapshot.mode = mode;
1575 snapshot.complete = false;
1576 snapshot.ready = false;
1577 snapshot.error = None;
1578 });
1579 current_generation
1580 }
1581
1582 fn fail_identity_bootstrap(&self, generation: u64, error: &IdentityRuntimeError) {
1583 self.modify_bootstrap_status(Some(generation), |snapshot| {
1584 let detail = error.to_string();
1585 snapshot.complete = true;
1586 snapshot.error = Some(detail.clone());
1587 for entry in snapshot.identities.values_mut() {
1588 if entry.state != IdentityBootstrapState::Active {
1589 entry.state = IdentityBootstrapState::Broken;
1590 entry.error = Some(detail.clone());
1591 }
1592 }
1593 snapshot.refresh_aggregates();
1594 });
1595 }
1596
1597 #[cfg(test)]
1598 pub(crate) fn test_supersede_identity_bootstrap_ready(&self) {
1599 let generation =
1600 self.begin_identity_bootstrap_pending(IdentityBootstrapMode::EagerMaterialize);
1601 self.modify_bootstrap_status(Some(generation), |snapshot| {
1602 snapshot.identities.clear();
1603 snapshot.complete = true;
1604 snapshot.error = None;
1605 snapshot.refresh_aggregates();
1606 });
1607 }
1608
1609 #[cfg(test)]
1610 pub(crate) fn test_fail_identity_bootstrap(&self, detail: &str) {
1611 let generation =
1612 self.begin_identity_bootstrap_pending(IdentityBootstrapMode::EagerMaterialize);
1613 self.modify_bootstrap_status(Some(generation), |snapshot| {
1614 snapshot.identities.clear();
1615 if let Ok(identity) = AgentIdentity::parse("agent:test-bootstrap-failure") {
1616 snapshot.identities.insert(
1617 identity,
1618 IdentityBootstrapEntry {
1619 state: IdentityBootstrapState::Dormant,
1620 error: None,
1621 },
1622 );
1623 }
1624 snapshot.refresh_aggregates();
1625 });
1626 self.fail_identity_bootstrap(
1627 generation,
1628 &IdentityRuntimeError::Internal(detail.to_string()),
1629 );
1630 }
1631
1632 async fn install_identity_bootstrap(
1633 self: &Arc<Self>,
1634 generation: u64,
1635 mode: IdentityBootstrapMode,
1636 roster: &[DurableAgentSpec],
1637 result: &super::orchestrator::RestoreFlowResult,
1638 ) {
1639 let background_concurrency = match mode {
1640 IdentityBootstrapMode::LazyWithBackgroundWarm { concurrency } => Some(concurrency),
1641 _ => None,
1642 };
1643 let mut status = IdentityBootstrapStatus {
1644 mode: mode.clone(),
1645 complete: background_concurrency.is_none(),
1646 ready: false,
1647 error: None,
1648 counts: Default::default(),
1649 identities: BTreeMap::new(),
1650 };
1651 for spec in roster {
1652 let broken_outcome = result.outcomes.get(&spec.identity).and_then(|outcome| {
1653 if let super::orchestrator::RestoreOutcome::Broken(failure) = outcome {
1654 Some(failure.detail.clone())
1655 } else {
1656 None
1657 }
1658 });
1659 let lifecycle = self
1660 .status(&spec.identity)
1661 .await
1662 .ok()
1663 .map(|item| item.state);
1664 let (state, error) = match (broken_outcome, lifecycle) {
1665 (Some(detail), _) => (IdentityBootstrapState::Broken, Some(detail)),
1670 (None, Some(IdentityLifecycleState::Active)) => {
1671 (IdentityBootstrapState::Active, None)
1672 }
1673 (None, Some(IdentityLifecycleState::Broken)) => {
1674 (IdentityBootstrapState::Broken, None)
1675 }
1676 (None, _) => (IdentityBootstrapState::Dormant, None),
1677 };
1678 status.identities.insert(
1679 spec.identity.clone(),
1680 IdentityBootstrapEntry { state, error },
1681 );
1682 }
1683 status.refresh_aggregates();
1684 if !self.replace_bootstrap_status(generation, status.clone()) {
1685 return;
1686 }
1687
1688 let Some(concurrency) = background_concurrency else {
1689 return;
1690 };
1691 let identities = status
1692 .identities
1693 .iter()
1694 .filter(|(_, entry)| entry.state == IdentityBootstrapState::Dormant)
1695 .map(|(identity, _)| identity.clone())
1696 .collect::<Vec<_>>();
1697 if identities.is_empty() {
1698 self.modify_bootstrap_status(Some(generation), |snapshot| {
1699 snapshot.complete = true;
1700 snapshot.refresh_aggregates();
1701 });
1702 return;
1703 }
1704
1705 let runtime = Arc::clone(self);
1706 let (cancel, task_cancel) = watch::channel(false);
1707 let task = tokio::spawn(async move {
1708 stream::iter(identities.into_iter().map(|identity| {
1709 let runtime = Arc::clone(&runtime);
1710 let mut cancel = task_cancel.clone();
1711 async move {
1712 if *cancel.borrow() {
1713 return;
1714 }
1715 let Some(result) = runtime
1716 .materialize_for_background(&identity, &mut cancel, generation)
1717 .await
1718 else {
1719 return;
1720 };
1721 if let Err(error) = result {
1722 tracing::warn!(
1723 %identity,
1724 error = %error,
1725 "identity background warm failed"
1726 );
1727 runtime
1728 .record_best_effort_materialization_failure(
1729 &identity,
1730 None,
1731 "background_warm",
1732 &error,
1733 )
1734 .await;
1735 }
1736 }
1737 }))
1738 .buffer_unordered(concurrency)
1739 .collect::<Vec<_>>()
1740 .await;
1741 if !*task_cancel.borrow() {
1742 runtime.modify_bootstrap_status(Some(generation), |snapshot| {
1743 snapshot.complete = true;
1744 snapshot.refresh_aggregates();
1745 });
1746 }
1747 });
1748 *self
1749 .bootstrap_cancel
1750 .write()
1751 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cancel);
1752 *self.bootstrap_task.lock().await = Some(task);
1753 }
1754
1755 fn mark_bootstrap_materialization_started(
1756 &self,
1757 identity: &AgentIdentity,
1758 generation: Option<u64>,
1759 ) {
1760 self.modify_bootstrap_status(generation, |snapshot| {
1761 let Some(entry) = snapshot.identities.get_mut(identity) else {
1762 return;
1763 };
1764 entry.state = IdentityBootstrapState::Warming;
1765 entry.error = None;
1766 snapshot.refresh_aggregates();
1767 });
1768 }
1769
1770 fn mark_bootstrap_from_lifecycle(
1771 &self,
1772 identity: &AgentIdentity,
1773 lifecycle: IdentityLifecycleState,
1774 error: Option<String>,
1775 ) {
1776 self.modify_bootstrap_status(None, |snapshot| {
1777 let Some(entry) = snapshot.identities.get_mut(identity) else {
1778 return;
1779 };
1780 entry.state = match lifecycle {
1781 IdentityLifecycleState::Active => IdentityBootstrapState::Active,
1782 IdentityLifecycleState::Broken => IdentityBootstrapState::Broken,
1783 IdentityLifecycleState::Dormant
1784 | IdentityLifecycleState::Retiring
1785 | IdentityLifecycleState::Suspended
1786 | IdentityLifecycleState::Uninitialized => IdentityBootstrapState::Dormant,
1787 };
1788 entry.error = if entry.state == IdentityBootstrapState::Broken {
1789 error
1790 } else {
1791 None
1792 };
1793 snapshot.refresh_aggregates();
1794 });
1795 }
1796
1797 fn mark_bootstrap_materialization_finished(
1798 &self,
1799 identity: &AgentIdentity,
1800 result: &Result<ContinuityRecord, IdentityRuntimeError>,
1801 generation: Option<u64>,
1802 ) {
1803 self.modify_bootstrap_status(generation, |snapshot| {
1804 let Some(entry) = snapshot.identities.get_mut(identity) else {
1805 return;
1806 };
1807 match result {
1808 Ok(_) => {
1809 entry.state = IdentityBootstrapState::Active;
1810 entry.error = None;
1811 }
1812 Err(error) => {
1813 entry.state = IdentityBootstrapState::Broken;
1814 entry.error = Some(error.to_string());
1815 }
1816 }
1817 snapshot.refresh_aggregates();
1818 });
1819 }
1820
1821 fn mark_bootstrap_materialization_cancelled(
1822 &self,
1823 identity: &AgentIdentity,
1824 generation: Option<u64>,
1825 ) {
1826 self.modify_bootstrap_status(generation, |snapshot| {
1827 if let Some(entry) = snapshot.identities.get_mut(identity) {
1828 entry.state = IdentityBootstrapState::Dormant;
1829 entry.error = None;
1830 snapshot.refresh_aggregates();
1831 }
1832 });
1833 }
1834
1835 pub async fn identity_bootstrap_member_ids(&self) -> Vec<meerkat_mob::ids::AgentIdentity> {
1839 let tracked = self.identity_bootstrap_status();
1840 self.identity_bootstrap_member_ids_for_status(&tracked)
1841 .await
1842 }
1843
1844 pub(crate) async fn identity_bootstrap_member_ids_for_status(
1845 &self,
1846 tracked: &IdentityBootstrapStatus,
1847 ) -> Vec<meerkat_mob::ids::AgentIdentity> {
1848 let entries = self.entries.read().await;
1849 tracked
1850 .identities
1851 .keys()
1852 .filter_map(|identity| {
1853 let entry = entries.get(identity)?;
1854 let runtime_id = entry.continuity.as_ref()?.agent_runtime_id.as_str();
1855 let alias = if durable_spec_uses_external_binding(&entry.spec) {
1856 identity.as_str()
1857 } else {
1858 runtime_id
1859 };
1860 Some(crate::member_comms_id::mob_member_id(alias))
1861 })
1862 .collect()
1863 }
1864
1865 pub async fn set_agent_customizer(&self, customizer: Option<Arc<dyn AgentCustomizer>>) {
1866 *self.customizer.write().await = customizer;
1867 }
1868
1869 pub async fn set_agent_memory(&self, injector: Option<AgentMemoryRuntimeInjector>) {
1870 *self.agent_memory.write().await = injector;
1871 }
1872
1873 pub async fn agent_memory_supports_recall(&self) -> bool {
1874 self.agent_memory.read().await.is_some()
1875 }
1876
1877 pub async fn agent_memory_supports_remember(&self) -> bool {
1878 self.agent_memory
1879 .read()
1880 .await
1881 .as_ref()
1882 .is_some_and(|injector| injector.provider().supports_remember())
1883 }
1884
1885 pub async fn agent_memory_supports_forget(&self) -> bool {
1886 self.agent_memory
1887 .read()
1888 .await
1889 .as_ref()
1890 .is_some_and(|injector| injector.provider().supports_forget())
1891 }
1892
1893 pub async fn agent_memory_supports_update(&self) -> bool {
1894 self.agent_memory
1895 .read()
1896 .await
1897 .as_ref()
1898 .is_some_and(|injector| injector.provider().supports_supersede())
1899 }
1900
1901 pub async fn agent_memory_supports_manifest(&self) -> bool {
1902 self.agent_memory
1903 .read()
1904 .await
1905 .as_ref()
1906 .is_some_and(|injector| injector.provider().supports_manifest())
1907 }
1908
1909 pub async fn remember_agent_memory(
1910 &self,
1911 realm: &str,
1912 identity: &AgentIdentity,
1913 memory: NewAgentMemory,
1914 ) -> Result<AgentMemoryRecord, AgentMemoryError> {
1915 self.status(identity)
1916 .await
1917 .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1918 let provider = self
1919 .agent_memory
1920 .read()
1921 .await
1922 .as_ref()
1923 .map(AgentMemoryRuntimeInjector::provider)
1924 .ok_or_else(|| {
1925 AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1926 })?;
1927 provider.remember(realm, identity, memory).await
1928 }
1929
1930 pub async fn forget_agent_memory(
1931 &self,
1932 realm: &str,
1933 identity: &AgentIdentity,
1934 memory_id: &str,
1935 ) -> Result<AgentMemoryForgetResult, AgentMemoryError> {
1936 self.status(identity)
1937 .await
1938 .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1939 let provider = self
1940 .agent_memory
1941 .read()
1942 .await
1943 .as_ref()
1944 .map(AgentMemoryRuntimeInjector::provider)
1945 .ok_or_else(|| {
1946 AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1947 })?;
1948 provider.forget(realm, identity, memory_id).await
1949 }
1950
1951 pub async fn update_agent_memory(
1955 &self,
1956 realm: &str,
1957 identity: &AgentIdentity,
1958 memory_id: &str,
1959 memory: NewAgentMemory,
1960 ) -> Result<MemoryId, AgentMemoryError> {
1961 self.status(identity)
1962 .await
1963 .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1964 let provider = self
1965 .agent_memory
1966 .read()
1967 .await
1968 .as_ref()
1969 .map(AgentMemoryRuntimeInjector::provider)
1970 .ok_or_else(|| {
1971 AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
1972 })?;
1973 let scope = MemoryScope::Identity {
1974 realm: realm.to_string(),
1975 identity: identity.as_str().to_string(),
1976 };
1977 let record = NewMemoryRecord {
1978 kind: MemoryKind::Fact,
1979 title: memory.title,
1980 description: String::new(),
1981 body: memory.body,
1982 tags: memory.tags,
1983 evidence: Vec::new(),
1984 verification: None,
1985 };
1986 provider.supersede(&scope, memory_id, record).await
1987 }
1988
1989 pub async fn manifest_agent_memory(
1991 &self,
1992 realm: &str,
1993 identity: &AgentIdentity,
1994 tier: ManifestTier,
1995 ) -> Result<Vec<RecordMeta>, AgentMemoryError> {
1996 self.status(identity)
1997 .await
1998 .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
1999 let provider = self
2000 .agent_memory
2001 .read()
2002 .await
2003 .as_ref()
2004 .map(AgentMemoryRuntimeInjector::provider)
2005 .ok_or_else(|| {
2006 AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
2007 })?;
2008 let scope = MemoryScope::Identity {
2009 realm: realm.to_string(),
2010 identity: identity.as_str().to_string(),
2011 };
2012 provider.manifest(&[scope], tier).await
2013 }
2014
2015 pub async fn recall_agent_memory(
2016 &self,
2017 request: AgentMemoryRecallRequest,
2018 ) -> Result<Vec<AgentMemoryRecord>, AgentMemoryError> {
2019 self.status(&request.identity)
2020 .await
2021 .map_err(|err| AgentMemoryError::InvalidConfig(err.to_string()))?;
2022 let provider = self
2023 .agent_memory
2024 .read()
2025 .await
2026 .as_ref()
2027 .map(AgentMemoryRuntimeInjector::provider)
2028 .ok_or_else(|| {
2029 AgentMemoryError::InvalidConfig("agent memory is not configured".to_string())
2030 })?;
2031 let records = provider.recall(request).await?;
2032 if !records.is_empty() {
2036 let ids: Vec<MemoryId> = records
2037 .iter()
2038 .map(|record| record.memory_id.clone())
2039 .collect();
2040 if let Err(err) = provider.mark_usage(&ids, UsageEvent::ExplicitRecall).await {
2041 tracing::debug!(error = %err, "agent memory explicit-recall usage marking skipped");
2042 }
2043 }
2044 Ok(records)
2045 }
2046
2047 pub fn set_reset_roster_provider(&self, provider: Option<Arc<dyn RosterProvider>>) {
2049 self.set_reset_roster_provider_context(provider, None);
2050 }
2051
2052 pub fn set_reset_roster_provider_context(
2054 &self,
2055 provider: Option<Arc<dyn RosterProvider>>,
2056 mob_definition: Option<meerkat_mob::MobDefinition>,
2057 ) {
2058 let source = provider.map(|provider| ResetRosterSource {
2059 provider,
2060 mob_definition,
2061 });
2062 match self.reset_roster_source.write() {
2063 Ok(mut stored_source) => *stored_source = source,
2064 Err(err) => {
2065 tracing::warn!(
2066 error = %err,
2067 "identity runtime reset roster source lock poisoned; dropping provider update"
2068 );
2069 }
2070 }
2071 }
2072
2073 async fn adopt_current_roster_spec_for_reset(&self, identity: &AgentIdentity) {
2074 let source = match self.reset_roster_source.read() {
2075 Ok(stored_source) => stored_source.clone(),
2076 Err(err) => {
2077 tracing::warn!(
2078 error = %err,
2079 "reset: roster source lock poisoned; rebuilding on stored spec"
2080 );
2081 None
2082 }
2083 };
2084 if let Some(source) = source {
2085 self.adopt_roster_spec_with_context(
2086 &source.provider,
2087 identity,
2088 source.mob_definition.clone(),
2089 )
2090 .await;
2091 }
2092 }
2093
2094 pub fn set_error_hook(&self, hook: Option<crate::unified_runtime::ErrorHook>) {
2096 match self.error_hook.write() {
2097 Ok(mut stored_hook) => *stored_hook = hook,
2098 Err(err) => {
2099 tracing::warn!(
2100 error = %err,
2101 "identity runtime error hook lock poisoned; dropping hook update"
2102 );
2103 }
2104 }
2105 }
2106
2107 pub fn spawn_lease_renewal_task(self: Arc<Self>) -> JoinHandle<()> {
2110 self.spawn_lease_renewal_task_with_poll_interval(DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL)
2111 }
2112
2113 pub(crate) fn spawn_tracked_lease_renewal_task(self: Arc<Self>) -> TrackedLeaseRenewalTask {
2118 self.spawn_tracked_lease_renewal_task_with_poll_interval(
2119 DEFAULT_LEASE_RENEWAL_MAX_POLL_INTERVAL,
2120 )
2121 }
2122
2123 pub(crate) fn spawn_tracked_lease_renewal_task_with_poll_interval(
2124 self: Arc<Self>,
2125 max_poll_interval: Duration,
2126 ) -> TrackedLeaseRenewalTask {
2127 let (cancel, receiver) = watch::channel(false);
2128 let join = tokio::spawn(self.run_lease_renewal_loop(max_poll_interval, Some(receiver)));
2129 TrackedLeaseRenewalTask { cancel, join }
2130 }
2131
2132 pub fn spawn_lease_renewal_task_with_poll_interval(
2136 self: Arc<Self>,
2137 max_poll_interval: Duration,
2138 ) -> JoinHandle<()> {
2139 tokio::spawn(self.run_lease_renewal_loop(max_poll_interval, None))
2140 }
2141
2142 async fn run_lease_renewal_loop(
2143 self: Arc<Self>,
2144 max_poll_interval: Duration,
2145 mut cancellation: Option<watch::Receiver<bool>>,
2146 ) {
2147 let max_poll_interval = max_poll_interval.max(DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL);
2148 let mut consecutive_failures: u32 = 0;
2149 loop {
2150 if cancellation
2151 .as_ref()
2152 .is_some_and(|cancellation| *cancellation.borrow())
2153 {
2154 return;
2155 }
2156 let base = self.lease_renewal_sleep_interval(max_poll_interval).await;
2157 let sleep = if consecutive_failures > 0 {
2161 base.max(lease_renewal_failure_backoff(
2162 consecutive_failures,
2163 max_poll_interval,
2164 ))
2165 } else {
2166 base
2167 };
2168 if let Some(cancellation) = cancellation.as_mut() {
2169 tokio::select! {
2170 () = tokio::time::sleep(sleep) => {}
2171 () = self.lease_renewal_notify.notified() => {}
2172 changed = cancellation.changed() => {
2173 match changed {
2174 Ok(()) if *cancellation.borrow() => return,
2175 Ok(()) => continue,
2176 Err(_) => return,
2177 }
2178 }
2179 }
2180 } else {
2181 tokio::select! {
2182 () = tokio::time::sleep(sleep) => {}
2183 () = self.lease_renewal_notify.notified() => {}
2184 }
2185 }
2186
2187 match self.renew_due_leases_once().await {
2192 Ok(_) => consecutive_failures = 0,
2193 Err(err) => {
2194 if consecutive_failures == 0 {
2197 tracing::warn!(
2198 error = %err,
2199 "identity-first proactive lease renewal tick failed; backing off"
2200 );
2201 } else {
2202 tracing::debug!(
2203 error = %err,
2204 consecutive_failures,
2205 "identity-first lease renewal still failing; backing off"
2206 );
2207 }
2208 consecutive_failures = consecutive_failures.saturating_add(1);
2209 }
2210 }
2211 if cancellation
2212 .as_ref()
2213 .is_some_and(|cancellation| *cancellation.borrow())
2214 {
2215 return;
2216 }
2217 }
2218 }
2219
2220 async fn lease_renewal_sleep_interval(&self, max_poll_interval: Duration) -> Duration {
2221 let entries = self.entries.read().await;
2222 entries
2223 .values()
2224 .filter(|entry| entry.state == IdentityLifecycleState::Active)
2225 .filter_map(|entry| entry.lease.as_ref())
2226 .map(|lease| (lease.ttl / 10).max(DEFAULT_LEASE_RENEWAL_MIN_POLL_INTERVAL))
2227 .min()
2228 .unwrap_or(max_poll_interval)
2229 .min(max_poll_interval)
2230 }
2231
2232 pub async fn renew_due_leases_once(&self) -> Result<usize, IdentityRuntimeError> {
2234 let due = {
2235 let entries = self.entries.read().await;
2236 entries
2237 .iter()
2238 .filter(|(_, entry)| entry.state == IdentityLifecycleState::Active)
2239 .filter_map(|(identity, entry)| {
2240 entry
2241 .lease
2242 .as_ref()
2243 .filter(|lease| !lease.is_healthy())
2244 .map(|_| identity.clone())
2245 })
2246 .collect::<Vec<_>>()
2247 };
2248
2249 let mut renewed = 0;
2250 let mut first_error = None;
2251 for identity in due {
2252 let lifecycle_lock = self.lifecycle_lock_for(&identity).await;
2253 let _lifecycle_guard = lifecycle_lock.lock().await;
2254 match self.ensure_active_lease(&identity).await {
2255 Ok(_) => renewed += 1,
2256 Err(err) => {
2257 if first_error.is_none() {
2258 first_error = Some(err);
2259 }
2260 }
2261 }
2262 }
2263 if let Some(err) = first_error {
2264 Err(err)
2265 } else {
2266 Ok(renewed)
2267 }
2268 }
2269
2270 async fn release_uninstalled_materialize_lease(&self, grant: &LeaseGrant) -> Option<String> {
2271 self.release_or_park_untracked_leases(std::slice::from_ref(grant))
2272 .await
2273 .err()
2274 .map(|err| err.to_string())
2275 }
2276
2277 async fn cancel_uninstalled_background_materialization(
2278 &self,
2279 grant: &LeaseGrant,
2280 ) -> IdentityRuntimeError {
2281 let cleanup_error = self.release_uninstalled_materialize_lease(grant).await;
2282 IdentityRuntimeError::Internal(
2283 cleanup_error
2284 .map(|error| format!("{BACKGROUND_WARM_CANCELLED}; lease cleanup failed: {error}"))
2285 .unwrap_or_else(|| BACKGROUND_WARM_CANCELLED.to_string()),
2286 )
2287 }
2288
2289 pub async fn set_desired_peer_edges(&self, edges: Vec<ManagedPeerEdge>) {
2290 *self.desired_peer_edges.write().await = edges;
2291 }
2292
2293 pub(crate) fn set_topology_controller(
2294 &self,
2295 controller: crate::topology_control::TopologyController,
2296 ) {
2297 *self
2298 .topology_controller
2299 .write()
2300 .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(controller);
2301 }
2302
2303 fn topology_controller(&self) -> Option<crate::topology_control::TopologyController> {
2304 self.topology_controller
2305 .read()
2306 .unwrap_or_else(std::sync::PoisonError::into_inner)
2307 .clone()
2308 }
2309
2310 pub async fn desired_peer_edges(&self) -> Vec<ManagedPeerEdge> {
2311 let declared = self.desired_peer_edges.read().await.clone();
2312 match self.topology_controller() {
2313 Some(controller) => {
2314 let _admission = controller.mutation_guard().await;
2318 match controller.compose_managed_peer_edges(&declared).await {
2319 Ok(edges) => edges,
2320 Err(error) => {
2321 tracing::error!(error = %error, "failed to compose identity topology overlay");
2322 Vec::new()
2323 }
2324 }
2325 }
2326 None => declared,
2327 }
2328 }
2329
2330 async fn registered_identities(&self) -> Vec<AgentIdentity> {
2331 self.entries.read().await.keys().cloned().collect()
2332 }
2333
2334 async fn reachable_peer_identities(&self, identity: &AgentIdentity) -> Vec<AgentIdentity> {
2335 self.desired_peer_edges()
2336 .await
2337 .iter()
2338 .filter_map(|edge| {
2339 if edge.a() == identity {
2340 Some(edge.b().clone())
2341 } else if edge.b() == identity {
2342 Some(edge.a().clone())
2343 } else {
2344 None
2345 }
2346 })
2347 .collect::<BTreeSet<_>>()
2348 .into_iter()
2349 .collect()
2350 }
2351
2352 #[must_use]
2353 pub fn has_session_bridge(&self) -> bool {
2354 self.bridge.is_some()
2355 }
2356
2357 pub(crate) async fn logical_peer_edges(
2358 &self,
2359 ) -> Result<Vec<ManagedPeerEdge>, IdentityRuntimeError> {
2360 let Some(bridge) = self.bridge.as_ref() else {
2361 return Ok(Vec::new());
2362 };
2363 let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = self
2364 .entries
2365 .read()
2366 .await
2367 .iter()
2368 .filter_map(|(identity, entry)| {
2369 entry
2370 .continuity
2371 .as_ref()
2372 .map(|record| (record.agent_runtime_id.clone(), identity.clone()))
2373 })
2374 .collect();
2375 let runtime_edges = bridge.current_member_wires().await.map_err(|error| {
2376 IdentityRuntimeError::Internal(format!("bridge current_member_wires: {error}"))
2377 })?;
2378 Ok(runtime_edges
2379 .into_iter()
2380 .filter_map(|(runtime_a, runtime_b)| {
2381 let a = runtime_identities.get(&runtime_a)?.clone();
2382 let b = runtime_identities.get(&runtime_b)?.clone();
2383 ManagedPeerEdge::new(a, b).ok()
2384 })
2385 .collect())
2386 }
2387
2388 pub(crate) async fn logical_peer_edges_any_half(
2389 &self,
2390 ) -> Result<Vec<ManagedPeerEdge>, IdentityRuntimeError> {
2391 let Some(bridge) = self.bridge.as_ref() else {
2392 return Ok(Vec::new());
2393 };
2394 let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = self
2395 .entries
2396 .read()
2397 .await
2398 .iter()
2399 .filter_map(|(identity, entry)| {
2400 entry
2401 .continuity
2402 .as_ref()
2403 .map(|record| (record.agent_runtime_id.clone(), identity.clone()))
2404 })
2405 .collect();
2406 let runtime_edges = bridge
2407 .current_member_wires_any_half()
2408 .await
2409 .map_err(|error| {
2410 IdentityRuntimeError::Internal(format!(
2411 "bridge current_member_wires_any_half: {error}"
2412 ))
2413 })?;
2414 Ok(runtime_edges
2415 .into_iter()
2416 .filter_map(|(runtime_a, runtime_b)| {
2417 let a = runtime_identities.get(&runtime_a)?.clone();
2418 let b = runtime_identities.get(&runtime_b)?.clone();
2419 ManagedPeerEdge::new(a, b).ok()
2420 })
2421 .collect())
2422 }
2423
2424 pub(crate) async fn managed_peer_edges_snapshot(
2425 &self,
2426 ) -> BTreeSet<(AgentIdentity, AgentIdentity)> {
2427 self.managed_peer_edges.read().await.clone()
2428 }
2429
2430 pub(crate) async fn retain_managed_peer_edges(
2431 &self,
2432 edges: &BTreeSet<(AgentIdentity, AgentIdentity)>,
2433 ) {
2434 self.managed_peer_edges
2435 .write()
2436 .await
2437 .extend(edges.iter().cloned());
2438 }
2439
2440 pub(crate) async fn mutate_managed_peer_edge_admitted(
2443 &self,
2444 action: crate::topology_control::TopologyAction,
2445 edge: &ManagedPeerEdge,
2446 ) -> Result<(), IdentityRuntimeError> {
2447 let _guard = self.managed_peer_reconcile_lock.lock().await;
2448 let _lifecycle_guards = self
2449 .lifecycle_guards_for([edge.a().clone(), edge.b().clone()])
2450 .await;
2451 let Some(bridge) = self.bridge.clone() else {
2452 return Err(IdentityRuntimeError::Internal(
2453 "topology mutation requires a session bridge".to_string(),
2454 ));
2455 };
2456 let (runtime_a, runtime_b) = {
2457 let entries = self.entries.read().await;
2458 let resolve = |identity: &AgentIdentity| {
2459 entries
2460 .get(identity)
2461 .filter(|entry| entry.state == IdentityLifecycleState::Active)
2462 .and_then(|entry| entry.continuity.as_ref())
2463 .map(|record| record.agent_runtime_id.clone())
2464 .ok_or_else(|| {
2465 IdentityRuntimeError::Internal(format!(
2466 "topology endpoint is not active: {identity}"
2467 ))
2468 })
2469 };
2470 (resolve(edge.a())?, resolve(edge.b())?)
2471 };
2472 let key = (edge.a().clone(), edge.b().clone());
2473 let current = bridge.current_member_wires().await.map_err(|error| {
2474 IdentityRuntimeError::Internal(format!("bridge current_member_wires: {error}"))
2475 })?;
2476 let actual = current.iter().any(|(a, b)| {
2477 (a == &runtime_a && b == &runtime_b) || (a == &runtime_b && b == &runtime_a)
2478 });
2479 let any_half = bridge
2480 .current_member_wires_any_half()
2481 .await
2482 .map_err(|error| {
2483 IdentityRuntimeError::Internal(format!(
2484 "bridge current_member_wires_any_half: {error}"
2485 ))
2486 })?
2487 .iter()
2488 .any(|(a, b)| {
2489 (a == &runtime_a && b == &runtime_b) || (a == &runtime_b && b == &runtime_a)
2490 });
2491 match action {
2492 crate::topology_control::TopologyAction::Connect if !actual => bridge
2493 .wire_peers_batch(&[(runtime_a, runtime_b)])
2494 .await
2495 .map_err(|error| {
2496 IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {error}"))
2497 })?,
2498 crate::topology_control::TopologyAction::Reconnect => {
2499 if any_half {
2500 bridge
2501 .unwire_peer(&runtime_a, &runtime_b)
2502 .await
2503 .map_err(|error| {
2504 IdentityRuntimeError::Internal(format!("bridge unwire_peer: {error}"))
2505 })?;
2506 }
2507 bridge
2508 .wire_peers_batch(&[(runtime_a, runtime_b)])
2509 .await
2510 .map_err(|error| {
2511 IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {error}"))
2512 })?;
2513 }
2514 crate::topology_control::TopologyAction::Disconnect if any_half => bridge
2515 .unwire_peer(&runtime_a, &runtime_b)
2516 .await
2517 .map_err(|error| {
2518 IdentityRuntimeError::Internal(format!("bridge unwire_peer: {error}"))
2519 })?,
2520 _ => {}
2521 }
2522 let mut managed = self.managed_peer_edges.write().await;
2523 if matches!(action, crate::topology_control::TopologyAction::Disconnect) {
2524 managed.remove(&key);
2525 } else {
2526 managed.insert(key);
2527 }
2528 Ok(())
2529 }
2530
2531 pub async fn reconcile_managed_peer_edges(
2537 &self,
2538 desired_edges: &[ManagedPeerEdge],
2539 ) -> Result<(), IdentityRuntimeError> {
2540 let topology_controller = self.topology_controller();
2541 let _topology_guard = match topology_controller.as_ref() {
2542 Some(controller) => Some(controller.mutation_guard().await),
2543 None => None,
2544 };
2545 if let Some(controller) = topology_controller.as_ref() {
2546 controller
2547 .prepare_pending_recovery()
2548 .await
2549 .map_err(|error| {
2550 IdentityRuntimeError::Internal(format!("topology recovery journal: {error}"))
2551 })?;
2552 }
2553 let pending_recovery_edges = if let Some(controller) = topology_controller.as_ref() {
2554 controller
2555 .pending_local_recovery_edges()
2556 .await
2557 .map_err(|error| {
2558 IdentityRuntimeError::Internal(format!("topology recovery ownership: {error}"))
2559 })?
2560 } else {
2561 BTreeSet::new()
2562 };
2563 if !pending_recovery_edges.is_empty() {
2564 self.retain_managed_peer_edges(&pending_recovery_edges)
2565 .await;
2566 }
2567 let composed_edges;
2568 let desired_edges = if let Some(controller) = topology_controller.as_ref() {
2569 composed_edges = controller
2570 .compose_managed_peer_edges(desired_edges)
2571 .await
2572 .map_err(|error| {
2573 IdentityRuntimeError::Internal(format!("topology overlay: {error}"))
2574 })?;
2575 composed_edges.as_slice()
2576 } else {
2577 desired_edges
2578 };
2579 let result = self
2580 .reconcile_managed_peer_edges_admitted(desired_edges)
2581 .await;
2582 let mut recovery_inspection_error = None;
2583 if let Some(controller) = topology_controller.as_ref() {
2584 let recovery_complete = if result.is_ok() && !pending_recovery_edges.is_empty() {
2585 match self
2586 .pending_recovery_is_physically_complete(desired_edges, &pending_recovery_edges)
2587 .await
2588 {
2589 Ok(complete) => complete,
2590 Err(error) => {
2591 recovery_inspection_error = Some(error);
2592 false
2593 }
2594 }
2595 } else {
2596 false
2597 };
2598 controller
2599 .finalize_recovered_pending(result.is_ok() && recovery_complete)
2600 .await
2601 .map_err(|error| {
2602 IdentityRuntimeError::Internal(format!("topology recovery receipt: {error}"))
2603 })?;
2604 }
2605 result?;
2606 if let Some(error) = recovery_inspection_error {
2607 return Err(error);
2608 }
2609 Ok(())
2610 }
2611
2612 pub(crate) async fn pending_recovery_is_physically_complete(
2613 &self,
2614 desired_edges: &[ManagedPeerEdge],
2615 pending_recovery_edges: &BTreeSet<(AgentIdentity, AgentIdentity)>,
2616 ) -> Result<bool, IdentityRuntimeError> {
2617 if self.bridge.is_none() {
2618 return Ok(false);
2619 }
2620 let active_identities = self
2621 .entries
2622 .read()
2623 .await
2624 .iter()
2625 .filter_map(|(identity, entry)| {
2626 (entry.state == IdentityLifecycleState::Active && entry.continuity.is_some())
2627 .then_some(identity.clone())
2628 })
2629 .collect::<BTreeSet<_>>();
2630 if pending_recovery_edges
2631 .iter()
2632 .any(|(a, b)| !active_identities.contains(a) || !active_identities.contains(b))
2633 {
2634 return Ok(false);
2635 }
2636 let desired = desired_edges
2637 .iter()
2638 .map(|edge| (edge.a().clone(), edge.b().clone()))
2639 .collect::<BTreeSet<_>>();
2640 let actual = self
2641 .logical_peer_edges()
2642 .await?
2643 .into_iter()
2644 .map(|edge| (edge.a().clone(), edge.b().clone()))
2645 .collect::<BTreeSet<_>>();
2646 let actual_any_half = self
2647 .logical_peer_edges_any_half()
2648 .await?
2649 .into_iter()
2650 .map(|edge| (edge.a().clone(), edge.b().clone()))
2651 .collect::<BTreeSet<_>>();
2652
2653 Ok(desired.is_subset(&actual)
2654 && pending_recovery_edges
2655 .difference(&desired)
2656 .all(|edge| !actual_any_half.contains(edge)))
2657 }
2658
2659 pub(crate) async fn reconcile_managed_peer_edges_admitted(
2662 &self,
2663 desired_edges: &[ManagedPeerEdge],
2664 ) -> Result<(), IdentityRuntimeError> {
2665 let _guard = self.managed_peer_reconcile_lock.lock().await;
2666 let Some(bridge) = self.bridge.clone() else {
2667 return Ok(());
2668 };
2669
2670 let managed_snapshot = self.managed_peer_edges.read().await.clone();
2671 let topology_identities = desired_edges
2672 .iter()
2673 .flat_map(|edge| [edge.a().clone(), edge.b().clone()])
2674 .chain(
2675 managed_snapshot
2676 .iter()
2677 .flat_map(|(a, b)| [a.clone(), b.clone()]),
2678 );
2679 let _lifecycle_guards = self.lifecycle_guards_for(topology_identities).await;
2680
2681 let (known_runtimes, active_runtimes): (
2682 BTreeMap<AgentIdentity, AgentRuntimeId>,
2683 BTreeMap<AgentIdentity, AgentRuntimeId>,
2684 ) = {
2685 let entries = self.entries.read().await;
2686 let known = entries
2687 .iter()
2688 .filter_map(|(identity, entry)| {
2689 entry
2690 .continuity
2691 .as_ref()
2692 .map(|record| (identity.clone(), record.agent_runtime_id.clone()))
2693 })
2694 .collect::<BTreeMap<_, _>>();
2695 let active = entries
2696 .iter()
2697 .filter_map(|(identity, entry)| {
2698 if entry.state != IdentityLifecycleState::Active {
2699 return None;
2700 }
2701 entry
2702 .continuity
2703 .as_ref()
2704 .map(|record| (identity.clone(), record.agent_runtime_id.clone()))
2705 })
2706 .collect();
2707 (known, active)
2708 };
2709 let runtime_identities: BTreeMap<AgentRuntimeId, AgentIdentity> = known_runtimes
2710 .iter()
2711 .map(|(identity, runtime_id)| (runtime_id.clone(), identity.clone()))
2712 .collect();
2713 let current_logical_edges: Option<BTreeSet<(AgentIdentity, AgentIdentity)>> =
2714 match bridge.current_member_wires().await {
2715 Ok(current_runtime_edges) => Some(
2716 current_runtime_edges
2717 .iter()
2718 .filter_map(|(runtime_a, runtime_b)| {
2719 let a = runtime_identities.get(runtime_a)?;
2720 let b = runtime_identities.get(runtime_b)?;
2721 if a <= b {
2722 Some((a.clone(), b.clone()))
2723 } else {
2724 Some((b.clone(), a.clone()))
2725 }
2726 })
2727 .collect(),
2728 ),
2729 Err(err) => {
2730 tracing::debug!(
2731 error = %err,
2732 "identity-first topology reconcile could not inspect current member wires"
2733 );
2734 None
2735 }
2736 };
2737 let current_any_half_edges: Option<BTreeSet<(AgentIdentity, AgentIdentity)>> =
2738 match bridge.current_member_wires_any_half().await {
2739 Ok(current_runtime_edges) => Some(
2740 current_runtime_edges
2741 .iter()
2742 .filter_map(|(runtime_a, runtime_b)| {
2743 let a = runtime_identities.get(runtime_a)?;
2744 let b = runtime_identities.get(runtime_b)?;
2745 if a <= b {
2746 Some((a.clone(), b.clone()))
2747 } else {
2748 Some((b.clone(), a.clone()))
2749 }
2750 })
2751 .collect(),
2752 ),
2753 Err(error) => {
2754 tracing::debug!(
2755 %error,
2756 "identity-first topology reconcile could not inspect orphan wire halves"
2757 );
2758 None
2759 }
2760 };
2761
2762 let desired: BTreeSet<(AgentIdentity, AgentIdentity)> = desired_edges
2763 .iter()
2764 .map(|edge| (edge.a().clone(), edge.b().clone()))
2765 .collect();
2766
2767 let edge_is_managed_and_live = |edge: &(AgentIdentity, AgentIdentity)| {
2768 managed_snapshot.contains(edge)
2770 && current_logical_edges
2771 .as_ref()
2772 .is_none_or(|edges| edges.contains(edge))
2773 };
2774 let retained_logical_edges: Vec<(AgentIdentity, AgentIdentity)> = desired
2775 .iter()
2776 .filter(|edge| !edge_is_managed_and_live(edge))
2777 .filter(|edge| {
2778 current_logical_edges
2779 .as_ref()
2780 .is_some_and(|edges| edges.contains(*edge))
2781 })
2782 .filter(|(a, b)| active_runtimes.contains_key(a) && active_runtimes.contains_key(b))
2783 .cloned()
2784 .collect();
2785 let to_wire: Vec<(AgentIdentity, AgentIdentity, AgentRuntimeId, AgentRuntimeId)> = desired
2786 .iter()
2787 .filter(|edge| !edge_is_managed_and_live(edge))
2788 .filter(|edge| {
2789 current_logical_edges
2790 .as_ref()
2791 .is_none_or(|edges| !edges.contains(*edge))
2792 })
2793 .filter_map(|(a, b)| {
2794 let runtime_a = active_runtimes.get(a)?;
2795 let runtime_b = active_runtimes.get(b)?;
2796 Some((a.clone(), b.clone(), runtime_a.clone(), runtime_b.clone()))
2797 })
2798 .collect();
2799
2800 let stale: Vec<(AgentIdentity, AgentIdentity)> = managed_snapshot
2801 .iter()
2802 .filter(|edge| !desired.contains(*edge))
2803 .cloned()
2804 .collect();
2805 let to_unwire: Vec<(AgentIdentity, AgentIdentity, AgentRuntimeId, AgentRuntimeId)> = stale
2806 .iter()
2807 .filter_map(|(a, b)| {
2808 let runtime_a = active_runtimes.get(a)?;
2809 let runtime_b = active_runtimes.get(b)?;
2810 if current_any_half_edges
2811 .as_ref()
2812 .is_some_and(|edges| !edges.contains(&(a.clone(), b.clone())))
2813 {
2814 return None;
2815 }
2816 Some((a.clone(), b.clone(), runtime_a.clone(), runtime_b.clone()))
2817 })
2818 .collect();
2819
2820 let wire_logical_edges = to_wire
2821 .iter()
2822 .map(|(a, b, _, _)| (a.clone(), b.clone()))
2823 .collect::<Vec<_>>();
2824 let wire_runtime_edges = to_wire
2825 .iter()
2826 .map(|(_, _, runtime_a, runtime_b)| (runtime_a.clone(), runtime_b.clone()))
2827 .collect::<Vec<_>>();
2828 if !wire_runtime_edges.is_empty() {
2829 bridge
2830 .wire_peers_batch(&wire_runtime_edges)
2831 .await
2832 .map_err(|e| {
2833 IdentityRuntimeError::Internal(format!("bridge wire_peers_batch: {e}"))
2834 })?;
2835 }
2836
2837 let unwire_results =
2838 stream::iter(to_unwire.into_iter().map(|(a, b, runtime_a, runtime_b)| {
2839 let bridge = bridge.clone();
2840 async move {
2841 let result = bridge
2842 .unwire_peer(&runtime_a, &runtime_b)
2843 .await
2844 .map_err(|e| format!("{e}"));
2845 (a, b, result)
2846 }
2847 }))
2848 .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
2849 .collect::<Vec<_>>()
2850 .await;
2851
2852 let mut managed = self.managed_peer_edges.write().await;
2853 for (a, b) in retained_logical_edges {
2854 managed.insert((a, b));
2855 }
2856 for (a, b) in wire_logical_edges {
2857 managed.insert((a, b));
2858 }
2859
2860 for (a, b) in stale {
2861 let key = (a.clone(), b.clone());
2862 if current_any_half_edges
2863 .as_ref()
2864 .is_some_and(|edges| !edges.contains(&key))
2865 {
2866 managed.remove(&key);
2867 }
2868 }
2869 for (a, b, result) in unwire_results {
2870 result
2871 .map_err(|e| IdentityRuntimeError::Internal(format!("bridge unwire_peer: {e}")))?;
2872 managed.remove(&(a, b));
2873 }
2874
2875 Ok(())
2876 }
2877
2878 async fn emit_event(&self, identity: &AgentIdentity, event: IdentityEvent) {
2880 let channels = self.event_channels.read().await;
2881 if let Some(tx) = channels.get(identity) {
2882 let _ = tx.send(event);
2883 }
2884 }
2885
2886 fn emit_error(&self, event: crate::unified_runtime::types::ErrorEvent) {
2887 let hook = match self.error_hook.read() {
2888 Ok(stored_hook) => stored_hook.clone(),
2889 Err(err) => {
2890 tracing::warn!(
2891 error = %err,
2892 "identity runtime error hook lock poisoned; dropping error event"
2893 );
2894 None
2895 }
2896 };
2897 if let Some(hook) = hook {
2898 tokio::spawn(async move {
2899 let () = hook(event).await;
2900 });
2901 }
2902 }
2903
2904 async fn materialization_backoff_error(&self, identity: &AgentIdentity) -> Option<String> {
2905 let backoffs = self.materialization_failure_backoff.read().await;
2906 let backoff = backoffs.get(identity)?;
2907 if Instant::now() < backoff.suppress_until {
2908 Some(backoff.error.clone())
2909 } else {
2910 None
2911 }
2912 }
2913
2914 async fn clear_materialization_backoff(&self, identity: &AgentIdentity) {
2915 self.materialization_failure_backoff
2916 .write()
2917 .await
2918 .remove(identity);
2919 }
2920
2921 async fn record_best_effort_materialization_failure(
2922 &self,
2923 identity: &AgentIdentity,
2924 initiator: Option<&AgentIdentity>,
2925 operation: &'static str,
2926 err: &IdentityRuntimeError,
2927 ) {
2928 let error = err.to_string();
2929 let suppress_until = Instant::now() + MATERIALIZATION_FAILURE_BACKOFF;
2930 self.materialization_failure_backoff.write().await.insert(
2931 identity.clone(),
2932 MaterializationFailureBackoff {
2933 suppress_until,
2934 error: error.clone(),
2935 },
2936 );
2937 self.emit_error(
2938 crate::unified_runtime::types::ErrorEvent::IdentityMaterializationFailure {
2939 identity: identity.to_string(),
2940 initiator: initiator.map(ToString::to_string),
2941 operation: operation.to_string(),
2942 error,
2943 },
2944 );
2945 }
2946
2947 pub async fn register(
2953 &self,
2954 spec: DurableAgentSpec,
2955 state: IdentityLifecycleState,
2956 continuity: Option<ContinuityRecord>,
2957 lease: Option<LeaseGrant>,
2958 ) {
2959 let identity = spec.identity.clone();
2960 let bootstrap_generation = self.identity_bootstrap_status_with_generation().0;
2961 let cpv = continuity
2962 .as_ref()
2963 .map(|r| r.checkpoint_version)
2964 .unwrap_or(CheckpointVersion::new(0));
2965 let lease_entry = lease.map(|g| LeaseEntry {
2966 fencing_token: g.fencing_token,
2967 ttl: g.ttl,
2968 acquired_at: Instant::now(),
2969 });
2970 let has_active_lease = state == IdentityLifecycleState::Active && lease_entry.is_some();
2971 {
2972 let mut entries = self.entries.write().await;
2973 let continuity_unrecoverable = if state == IdentityLifecycleState::Broken {
2979 entries
2980 .get(&identity)
2981 .and_then(|existing| existing.continuity_unrecoverable.clone())
2982 } else {
2983 None
2984 };
2985 let host_rejected_build_park = entries.get(&identity).and_then(|existing| {
2991 existing
2992 .host_rejected_build_park
2993 .clone()
2994 .filter(|park| park.spec_digest == durable_spec_digest(&spec))
2995 });
2996 let entry = IdentityEntry {
2997 spec,
2998 bootstrap_generation,
2999 state,
3000 continuity,
3001 lease: lease_entry,
3002 pending_lease_release: None,
3003 checkpoint_version: cpv,
3004 has_runtime_store: self.has_runtime_store,
3005 continuity_unrecoverable,
3006 host_rejected_build_park,
3007 };
3008 entries.insert(identity.clone(), entry);
3009 }
3010
3011 let (tx, _) = broadcast::channel(IDENTITY_EVENT_CHANNEL_CAPACITY);
3013 self.event_channels
3014 .write()
3015 .await
3016 .insert(identity.clone(), tx);
3017 if has_active_lease {
3018 self.lease_renewal_notify.notify_one();
3019 }
3020 self.mark_bootstrap_from_lifecycle(&identity, state, None);
3021 }
3022
3023 async fn materialization_lock_for(&self, identity: &AgentIdentity) -> Arc<Mutex<()>> {
3024 if let Some(lock) = self.materialization_locks.read().await.get(identity) {
3025 return lock.clone();
3026 }
3027 let mut locks = self.materialization_locks.write().await;
3028 locks
3029 .entry(identity.clone())
3030 .or_insert_with(|| Arc::new(Mutex::new(())))
3031 .clone()
3032 }
3033
3034 async fn best_effort_materialization_lock_for(
3035 &self,
3036 identity: &AgentIdentity,
3037 ) -> Arc<Mutex<()>> {
3038 if let Some(lock) = self
3039 .best_effort_materialization_locks
3040 .read()
3041 .await
3042 .get(identity)
3043 {
3044 return lock.clone();
3045 }
3046 let mut locks = self.best_effort_materialization_locks.write().await;
3047 locks
3048 .entry(identity.clone())
3049 .or_insert_with(|| Arc::new(Mutex::new(())))
3050 .clone()
3051 }
3052
3053 pub(crate) async fn lifecycle_lock_for(&self, identity: &AgentIdentity) -> Arc<Mutex<()>> {
3054 if let Some(lock) = self.lifecycle_locks.read().await.get(identity) {
3055 return lock.clone();
3056 }
3057 let mut locks = self.lifecycle_locks.write().await;
3058 locks
3059 .entry(identity.clone())
3060 .or_insert_with(|| Arc::new(Mutex::new(())))
3061 .clone()
3062 }
3063
3064 pub(crate) async fn raw_member_alias_lock(&self, alias: &str) -> Arc<Mutex<()>> {
3065 let alias = crate::member_comms_id::runtime_alias_str(alias.trim()).into_owned();
3066 if let Some(lock) = self
3067 .raw_member_alias_locks
3068 .read()
3069 .await
3070 .entries
3071 .get(&alias)
3072 .and_then(Weak::upgrade)
3073 {
3074 return lock;
3075 }
3076 let mut table = self.raw_member_alias_locks.write().await;
3077 table.sweep_if_needed();
3078 if let Some(lock) = table.entries.get(&alias).and_then(Weak::upgrade) {
3082 return lock;
3083 }
3084 let lock = Arc::new(Mutex::new(()));
3085 table.entries.insert(alias, Arc::downgrade(&lock));
3086 lock
3087 }
3088
3089 #[cfg(test)]
3090 pub(crate) async fn raw_member_alias_lock_metrics(&self) -> (usize, usize, usize) {
3091 let table = self.raw_member_alias_locks.read().await;
3092 (table.entries.len(), table.next_sweep_len, table.sweep_count)
3093 }
3094
3095 pub(crate) async fn ensure_raw_member_alias_available(
3096 &self,
3097 identity: &AgentIdentity,
3098 ) -> Result<(), IdentityRuntimeError> {
3099 if let Some(bridge) = self.bridge.as_ref()
3100 && bridge
3101 .raw_member_alias_exists(identity.as_str())
3102 .await
3103 .map_err(|error| {
3104 IdentityRuntimeError::Internal(format!(
3105 "raw member namespace inspection for {identity}: {error}"
3106 ))
3107 })?
3108 {
3109 return Err(IdentityRuntimeError::Internal(format!(
3110 "durable identity '{identity}' collides with an existing raw member alias"
3111 )));
3112 }
3113 Ok(())
3114 }
3115
3116 pub(crate) async fn member_alias_lifecycle_target(
3120 self: &Arc<Self>,
3121 alias: &str,
3122 ) -> Result<Option<MemberAliasLifecycleTarget>, IdentityRuntimeError> {
3123 let alias = crate::member_comms_id::runtime_alias_str(alias).into_owned();
3124 match self.identity_for_member_mutation(&alias).await {
3125 Some(identity) => {
3126 self.ensure_expected_member_alias_current(&identity, &alias)
3132 .await?;
3133 Ok(Some(MemberAliasLifecycleTarget {
3134 runtime: Arc::clone(self),
3135 lock: self.lifecycle_lock_for(&identity).await,
3136 identity,
3137 alias,
3138 }))
3139 }
3140 None if crate::member_comms_id::is_reserved_generated_alias(&alias) => {
3141 Err(IdentityRuntimeError::Internal(format!(
3142 "generated member alias requires identity authority: {alias}"
3143 )))
3144 }
3145 None => Ok(None),
3146 }
3147 }
3148
3149 pub(crate) async fn acquire_member_alias_lifecycle_targets(
3154 mut targets: Vec<MemberAliasLifecycleTarget>,
3155 ) -> Result<Vec<tokio::sync::OwnedMutexGuard<()>>, IdentityRuntimeError> {
3156 targets.sort_by(|a, b| {
3157 a.runtime
3158 .runtime_instance_id
3159 .cmp(&b.runtime.runtime_instance_id)
3160 .then_with(|| {
3161 (Arc::as_ptr(&a.runtime) as usize).cmp(&(Arc::as_ptr(&b.runtime) as usize))
3162 })
3163 .then_with(|| a.identity.cmp(&b.identity))
3164 .then_with(|| a.alias.cmp(&b.alias))
3165 });
3166
3167 let mut guards = Vec::with_capacity(targets.len());
3168 let mut held: Option<(usize, AgentIdentity)> = None;
3169 for target in targets {
3170 let key = (
3171 Arc::as_ptr(&target.runtime) as usize,
3172 target.identity.clone(),
3173 );
3174 if held.as_ref() != Some(&key) {
3175 guards.push(target.lock.lock_owned().await);
3176 held = Some(key);
3177 }
3178 target
3179 .runtime
3180 .ensure_expected_member_alias_current(&target.identity, &target.alias)
3181 .await?;
3182 let state = target
3183 .runtime
3184 .entries
3185 .read()
3186 .await
3187 .get(&target.identity)
3188 .map(|entry| entry.state)
3189 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(target.identity.clone()))?;
3190 if state != IdentityLifecycleState::Active {
3191 return Err(IdentityRuntimeError::InvalidState {
3192 identity: target.identity,
3193 state,
3194 operation: "mutate member alias",
3195 });
3196 }
3197 }
3198 Ok(guards)
3199 }
3200
3201 pub(crate) async fn run_member_alias_targets_operation_tracked<T, F, Fut>(
3205 targets: Vec<MemberAliasLifecycleTarget>,
3206 operation: F,
3207 ) -> Result<T, IdentityRuntimeError>
3208 where
3209 T: Send + 'static,
3210 F: FnOnce() -> Fut + Send + 'static,
3211 Fut: Future<Output = Result<T, String>> + Send + 'static,
3212 {
3213 let mut runtimes = targets
3214 .iter()
3215 .map(|target| Arc::clone(&target.runtime))
3216 .collect::<Vec<_>>();
3217 runtimes.sort_by(|a, b| {
3218 a.runtime_instance_id
3219 .cmp(&b.runtime_instance_id)
3220 .then_with(|| (Arc::as_ptr(a) as usize).cmp(&(Arc::as_ptr(b) as usize)))
3221 });
3222 runtimes.dedup_by(|a, b| Arc::ptr_eq(a, b));
3223 let transaction = async move {
3224 let _guards = Self::acquire_member_alias_lifecycle_targets(targets).await?;
3225 operation().await.map_err(IdentityRuntimeError::Internal)
3226 };
3227 if !runtimes.is_empty() {
3228 return Self::run_tracked_foreground_multi(runtimes, transaction).await;
3229 }
3230 transaction.await
3231 }
3232
3233 async fn run_tracked_foreground_multi<T, F>(
3237 runtimes: Vec<Arc<Self>>,
3238 operation: F,
3239 ) -> Result<T, IdentityRuntimeError>
3240 where
3241 T: Send + 'static,
3242 F: Future<Output = Result<T, IdentityRuntimeError>> + Send + 'static,
3243 {
3244 let Some((supervisor, participants)) = runtimes.split_first() else {
3245 return operation.await;
3246 };
3247 if runtimes
3248 .iter()
3249 .any(|runtime| runtime.foreground_shutdown.load(Ordering::Acquire))
3250 {
3251 return Err(IdentityRuntimeError::Internal(
3252 "identity runtime is shutting down".to_string(),
3253 ));
3254 }
3255
3256 let (completion, _) = watch::channel(false);
3257 for runtime in participants {
3258 let mut operations = runtime.foreground_operations.lock().await;
3259 if runtime.foreground_shutdown.load(Ordering::Acquire) {
3260 completion.send_replace(true);
3261 return Err(IdentityRuntimeError::Internal(
3262 "identity runtime is shutting down".to_string(),
3263 ));
3264 }
3265 while let Some(result) = operations.try_join_next() {
3266 if let Err(error) = result {
3267 tracing::error!(
3268 error = %error,
3269 "tracked foreground identity operation panicked"
3270 );
3271 }
3272 }
3273 let mut completed = completion.subscribe();
3274 operations.spawn(async move {
3275 while !*completed.borrow() && completed.changed().await.is_ok() {}
3276 });
3277 }
3278
3279 let (sender, receiver) = oneshot::channel();
3280 {
3281 let mut operations = supervisor.foreground_operations.lock().await;
3282 if supervisor.foreground_shutdown.load(Ordering::Acquire) {
3283 completion.send_replace(true);
3284 return Err(IdentityRuntimeError::Internal(
3285 "identity runtime is shutting down".to_string(),
3286 ));
3287 }
3288 while let Some(result) = operations.try_join_next() {
3289 if let Err(error) = result {
3290 tracing::error!(
3291 error = %error,
3292 "tracked foreground identity operation panicked"
3293 );
3294 }
3295 }
3296 operations.spawn(async move {
3297 let _completion = MultiRuntimeForegroundCompletion(completion);
3298 let _ = sender.send(operation.await);
3299 });
3300 }
3301 receiver.await.map_err(|_| {
3302 IdentityRuntimeError::Internal(
3303 "tracked foreground identity operation terminated without a result".to_string(),
3304 )
3305 })?
3306 }
3307
3308 async fn lifecycle_guards_for(
3312 &self,
3313 identities: impl IntoIterator<Item = AgentIdentity>,
3314 ) -> Vec<tokio::sync::OwnedMutexGuard<()>> {
3315 let identities = identities.into_iter().collect::<BTreeSet<_>>();
3316 let mut guards = Vec::with_capacity(identities.len());
3317 for identity in identities {
3318 guards.push(self.lifecycle_lock_for(&identity).await.lock_owned().await);
3319 }
3320 guards
3321 }
3322
3323 async fn run_tracked_foreground<T, F>(
3330 self: &Arc<Self>,
3331 operation: F,
3332 ) -> Result<T, IdentityRuntimeError>
3333 where
3334 T: Send + 'static,
3335 F: Future<Output = Result<T, IdentityRuntimeError>> + Send + 'static,
3336 {
3337 if self.foreground_shutdown.load(Ordering::Acquire) {
3338 return Err(IdentityRuntimeError::Internal(
3339 "identity runtime is shutting down".to_string(),
3340 ));
3341 }
3342 let (sender, receiver) = oneshot::channel();
3343 {
3344 let mut operations = self.foreground_operations.lock().await;
3345 if self.foreground_shutdown.load(Ordering::Acquire) {
3346 return Err(IdentityRuntimeError::Internal(
3347 "identity runtime is shutting down".to_string(),
3348 ));
3349 }
3350 while let Some(result) = operations.try_join_next() {
3351 if let Err(error) = result {
3352 tracing::error!(
3353 error = %error,
3354 "tracked foreground identity operation panicked"
3355 );
3356 }
3357 }
3358 operations.spawn(async move {
3359 let _ = sender.send(operation.await);
3360 });
3361 }
3362 receiver.await.map_err(|_| {
3363 IdentityRuntimeError::Internal(
3364 "tracked foreground identity operation terminated without a result".to_string(),
3365 )
3366 })?
3367 }
3368
3369 pub(crate) fn close_foreground_operations(&self) {
3370 self.foreground_shutdown.store(true, Ordering::Release);
3371 self.foreground_cancel.send_replace(true);
3375 }
3376
3377 pub(crate) async fn join_foreground_operations(&self) {
3378 let mut operations = self.foreground_operations.lock().await;
3379 while let Some(result) = operations.join_next().await {
3380 if let Err(error) = result {
3381 tracing::error!(
3382 error = %error,
3383 "tracked foreground identity operation panicked during shutdown"
3384 );
3385 }
3386 }
3387 }
3388
3389 pub(crate) async fn park_unactivated_lease_releases(&self, grants: &[LeaseGrant]) {
3394 let mut pending = self.pending_unactivated_lease_releases.write().await;
3395 for grant in grants {
3396 if !pending.iter().any(|parked| {
3397 parked.identity == grant.identity && parked.fencing_token == grant.fencing_token
3398 }) {
3399 pending.push(grant.clone());
3400 }
3401 }
3402 }
3403
3404 pub(crate) async fn release_or_park_untracked_leases(
3408 &self,
3409 grants: &[LeaseGrant],
3410 ) -> Result<(), super::types::LeaseError> {
3411 let _release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3412 match self.lease_provider.release_leases(grants).await {
3413 Ok(()) => Ok(()),
3414 Err(error) => {
3415 self.park_unactivated_lease_releases(grants).await;
3416 Err(error)
3417 }
3418 }
3419 }
3420
3421 pub(crate) async fn release_parked_unactivated_leases(
3425 &self,
3426 ) -> Result<usize, IdentityRuntimeError> {
3427 self.release_parked_unactivated_leases_matching(None).await
3428 }
3429
3430 async fn release_parked_unactivated_leases_for_identity(
3435 &self,
3436 identity: &AgentIdentity,
3437 ) -> Result<usize, IdentityRuntimeError> {
3438 self.release_parked_unactivated_leases_matching(Some(identity))
3439 .await
3440 }
3441
3442 async fn release_parked_unactivated_leases_matching(
3443 &self,
3444 identity: Option<&AgentIdentity>,
3445 ) -> Result<usize, IdentityRuntimeError> {
3446 let _release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3447 let grants = self
3448 .pending_unactivated_lease_releases
3449 .read()
3450 .await
3451 .iter()
3452 .filter(|grant| identity.is_none_or(|identity| grant.identity == *identity))
3453 .cloned()
3454 .collect::<Vec<_>>();
3455 if grants.is_empty() {
3456 return Ok(0);
3457 }
3458 self.lease_provider
3459 .release_leases(&grants)
3460 .await
3461 .map_err(IdentityRuntimeError::Lease)?;
3462 let mut pending = self.pending_unactivated_lease_releases.write().await;
3463 pending.retain(|parked| {
3464 !grants.iter().any(|released| {
3465 released.identity == parked.identity
3466 && released.fencing_token == parked.fencing_token
3467 })
3468 });
3469 Ok(grants.len())
3470 }
3471
3472 pub(crate) async fn release_all_leases_for_shutdown(
3477 &self,
3478 ) -> Result<usize, IdentityRuntimeError> {
3479 let _parked_release_guard = self.pending_unactivated_lease_release_gate.lock().await;
3480 let mut grants = self
3481 .entries
3482 .read()
3483 .await
3484 .iter()
3485 .filter_map(|(identity, entry)| {
3486 entry.pending_lease_release.clone().or_else(|| {
3487 entry.lease.as_ref().map(|lease| LeaseGrant {
3488 identity: identity.clone(),
3489 fencing_token: lease.fencing_token,
3490 ttl: lease.ttl,
3491 })
3492 })
3493 })
3494 .collect::<Vec<_>>();
3495 let parked_grants = self.pending_unactivated_lease_releases.read().await.clone();
3496 for grant in parked_grants {
3497 if !grants.iter().any(|existing| {
3498 existing.identity == grant.identity && existing.fencing_token == grant.fencing_token
3499 }) {
3500 grants.push(grant);
3501 }
3502 }
3503 if grants.is_empty() {
3504 return Ok(0);
3505 }
3506
3507 self.lease_provider
3508 .release_leases(&grants)
3509 .await
3510 .map_err(IdentityRuntimeError::Lease)?;
3511
3512 let mut entries = self.entries.write().await;
3513 let mut lifecycle_updates = Vec::new();
3514 for grant in &grants {
3515 if let Some(entry) = entries.get_mut(&grant.identity) {
3516 let released_active_lease = entry
3517 .lease
3518 .as_ref()
3519 .is_some_and(|lease| lease.fencing_token == grant.fencing_token);
3520 if released_active_lease {
3521 entry.lease = None;
3522 }
3523 if entry
3524 .pending_lease_release
3525 .as_ref()
3526 .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
3527 {
3528 entry.pending_lease_release = None;
3529 }
3530 if released_active_lease
3531 && matches!(
3532 entry.state,
3533 IdentityLifecycleState::Active | IdentityLifecycleState::Suspended
3534 )
3535 {
3536 entry.state = IdentityLifecycleState::Dormant;
3537 lifecycle_updates.push(grant.identity.clone());
3538 }
3539 }
3540 }
3541 drop(entries);
3542 let mut pending = self.pending_unactivated_lease_releases.write().await;
3543 pending.retain(|parked| {
3544 !grants.iter().any(|released| {
3545 released.identity == parked.identity
3546 && released.fencing_token == parked.fencing_token
3547 })
3548 });
3549 drop(pending);
3550 for identity in lifecycle_updates {
3551 self.mark_bootstrap_from_lifecycle(&identity, IdentityLifecycleState::Dormant, None);
3552 }
3553 Ok(grants.len())
3554 }
3555
3556 pub async fn materialize(
3563 &self,
3564 identity: &AgentIdentity,
3565 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3566 self.materialize_with_expected_member_alias(identity, None)
3567 .await
3568 }
3569
3570 async fn materialize_with_expected_member_alias(
3571 &self,
3572 identity: &AgentIdentity,
3573 expected_alias: Option<&str>,
3574 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3575 self.materialize_with_expected_member_alias_after_inner(
3576 identity,
3577 expected_alias,
3578 std::future::ready(()),
3579 )
3580 .await
3581 }
3582
3583 async fn materialize_with_expected_member_alias_after_inner<F>(
3588 &self,
3589 identity: &AgentIdentity,
3590 expected_alias: Option<&str>,
3591 after_inner: F,
3592 ) -> Result<ContinuityRecord, IdentityRuntimeError>
3593 where
3594 F: Future<Output = ()>,
3595 {
3596 let mut bootstrap_generation = None;
3602 let mut shutdown = self.foreground_cancel.subscribe();
3603 let result = self
3604 .materialize_inner(
3605 identity,
3606 expected_alias,
3607 Some(&mut shutdown),
3608 None,
3609 &mut bootstrap_generation,
3610 )
3611 .await;
3612 after_inner.await;
3613 if matches!(
3614 &result,
3615 Err(IdentityRuntimeError::Internal(message)) if message == BACKGROUND_WARM_CANCELLED
3616 ) {
3617 if let Some(generation) = bootstrap_generation {
3618 self.mark_bootstrap_materialization_cancelled(identity, Some(generation));
3619 }
3620 return result;
3621 }
3622 if matches!(&result, Err(IdentityRuntimeError::StaleRuntimeAlias { .. })) {
3627 return result;
3628 }
3629 if result.is_ok() {
3630 let desired_edges = self.desired_peer_edges.read().await.clone();
3631 let has_pending_topology_recovery = match self.topology_controller() {
3632 Some(controller) => controller.has_pending().await,
3633 None => false,
3634 };
3635 if (!desired_edges.is_empty() || has_pending_topology_recovery)
3636 && let Err(error) = self.reconcile_managed_peer_edges(&desired_edges).await
3637 {
3638 tracing::warn!(
3639 identity = %identity,
3640 %error,
3641 "identity materialized with topology reconcile warning"
3642 );
3643 }
3644 }
3645 if let Some(generation) = bootstrap_generation {
3646 self.mark_bootstrap_materialization_finished(identity, &result, Some(generation));
3647 }
3648 result
3649 }
3650
3651 pub async fn materialize_tracked(
3653 self: &Arc<Self>,
3654 identity: &AgentIdentity,
3655 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3656 let runtime = Arc::clone(self);
3657 let identity = identity.clone();
3658 self.run_tracked_foreground(async move { runtime.materialize(&identity).await })
3659 .await
3660 }
3661
3662 pub(crate) async fn restore_flow_tracked(
3666 self: &Arc<Self>,
3667 roster: Vec<DurableAgentSpec>,
3668 topology_provider: Option<Arc<dyn TopologyProvider>>,
3669 customizer: Option<Arc<dyn AgentCustomizer>>,
3670 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
3671 let runtime = Arc::clone(self);
3672 self.run_tracked_foreground(async move {
3673 super::orchestrator::restore_flow(
3674 runtime.as_ref(),
3675 &roster,
3676 topology_provider.as_deref(),
3677 customizer.as_deref(),
3678 )
3679 .await
3680 })
3681 .await
3682 }
3683
3684 pub async fn retire_tracked(
3686 self: &Arc<Self>,
3687 identity: &AgentIdentity,
3688 ) -> Result<FencingToken, IdentityRuntimeError> {
3689 let runtime = Arc::clone(self);
3690 let identity = identity.clone();
3691 self.run_tracked_foreground(async move { runtime.retire(&identity).await })
3692 .await
3693 }
3694
3695 pub async fn retire_member_alias_tracked(
3698 self: &Arc<Self>,
3699 identity: &AgentIdentity,
3700 expected_alias: &str,
3701 ) -> Result<FencingToken, IdentityRuntimeError> {
3702 let runtime = Arc::clone(self);
3703 let identity = identity.clone();
3704 let expected_alias = expected_alias.to_string();
3705 self.run_tracked_foreground(async move {
3706 runtime
3707 .retire_with_expected_member_alias(&identity, Some(&expected_alias))
3708 .await
3709 })
3710 .await
3711 }
3712
3713 pub(crate) async fn retire_and_cleanup_live_members_tracked<T, F, Fut>(
3718 self: &Arc<Self>,
3719 identity: &AgentIdentity,
3720 expected_alias: Option<&str>,
3721 cleanup: F,
3722 ) -> Result<(FencingToken, T), IdentityRuntimeError>
3723 where
3724 T: Send + 'static,
3725 F: FnOnce(Option<AgentRuntimeId>) -> Fut + Send + 'static,
3726 Fut: Future<Output = T> + Send + 'static,
3727 {
3728 let runtime = Arc::clone(self);
3729 let identity = identity.clone();
3730 let expected_alias = expected_alias.map(str::to_owned);
3731 self.run_tracked_foreground(async move {
3732 let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3733 let _lifecycle_guard = lifecycle_lock.lock().await;
3734 if let Some(expected_alias) = expected_alias.as_deref() {
3735 runtime
3736 .ensure_expected_member_alias_current(&identity, expected_alias)
3737 .await?;
3738 }
3739 let retired_alias = runtime
3740 .entries
3741 .read()
3742 .await
3743 .get(&identity)
3744 .and_then(|entry| entry.continuity.as_ref())
3745 .map(|record| record.agent_runtime_id.clone());
3746 let token = runtime.retire_locked(&identity).await?;
3747 let metadata = cleanup(retired_alias).await;
3748 Ok((token, metadata))
3749 })
3750 .await
3751 }
3752
3753 pub async fn respawn_tracked(
3755 self: &Arc<Self>,
3756 identity: &AgentIdentity,
3757 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3758 let runtime = Arc::clone(self);
3759 let identity = identity.clone();
3760 self.run_tracked_foreground(async move { runtime.respawn(&identity).await })
3761 .await
3762 }
3763
3764 pub async fn respawn_member_alias_tracked(
3767 self: &Arc<Self>,
3768 identity: &AgentIdentity,
3769 expected_alias: &str,
3770 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3771 let runtime = Arc::clone(self);
3772 let identity = identity.clone();
3773 let expected_alias = expected_alias.to_string();
3774 self.run_tracked_foreground(async move {
3775 runtime
3776 .respawn_with_expected_member_alias(&identity, Some(&expected_alias))
3777 .await
3778 })
3779 .await
3780 }
3781
3782 pub(crate) async fn run_member_alias_operation_tracked<T, F, Fut>(
3787 self: &Arc<Self>,
3788 identity: &AgentIdentity,
3789 expected_alias: &str,
3790 operation: F,
3791 ) -> Result<T, IdentityRuntimeError>
3792 where
3793 T: Send + 'static,
3794 F: FnOnce() -> Fut + Send + 'static,
3795 Fut: Future<Output = Result<T, String>> + Send + 'static,
3796 {
3797 let runtime = Arc::clone(self);
3798 let identity = identity.clone();
3799 let expected_alias = expected_alias.to_string();
3800 self.run_tracked_foreground(async move {
3801 let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3802 let _lifecycle_guard = lifecycle_lock.lock().await;
3803 runtime
3804 .ensure_expected_member_alias_current(&identity, &expected_alias)
3805 .await?;
3806 operation().await.map_err(IdentityRuntimeError::Internal)
3807 })
3808 .await
3809 }
3810
3811 pub(crate) async fn respawn_identity_in_place_tracked(
3820 self: &Arc<Self>,
3821 identity: &AgentIdentity,
3822 expected_alias: Option<&str>,
3823 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3824 match expected_alias {
3825 Some(expected_alias) => {
3826 self.respawn_member_alias_tracked(identity, expected_alias)
3827 .await
3828 }
3829 None => self.respawn_tracked(identity).await,
3830 }
3831 }
3832
3833 pub async fn rebind_session_after_live_respawn_tracked(
3835 self: &Arc<Self>,
3836 identity: &AgentIdentity,
3837 session_id: SessionId,
3838 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3839 let runtime = Arc::clone(self);
3840 let identity = identity.clone();
3841 self.run_tracked_foreground(async move {
3842 runtime
3843 .rebind_session_after_live_respawn(&identity, session_id)
3844 .await
3845 })
3846 .await
3847 }
3848
3849 pub async fn rebind_session_after_live_respawn_member_alias_tracked(
3852 self: &Arc<Self>,
3853 identity: &AgentIdentity,
3854 expected_alias: &str,
3855 session_id: SessionId,
3856 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3857 let runtime = Arc::clone(self);
3858 let identity = identity.clone();
3859 let expected_alias = expected_alias.to_string();
3860 self.run_tracked_foreground(async move {
3861 runtime
3862 .rebind_session_after_live_respawn_with_expected_member_alias(
3863 &identity,
3864 Some(&expected_alias),
3865 session_id,
3866 )
3867 .await
3868 })
3869 .await
3870 }
3871
3872 pub async fn reset_tracked(
3874 self: &Arc<Self>,
3875 identity: &AgentIdentity,
3876 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3877 let runtime = Arc::clone(self);
3878 let identity = identity.clone();
3879 self.run_tracked_foreground(async move { runtime.reset(&identity).await })
3880 .await
3881 }
3882
3883 pub async fn reset_member_alias_tracked(
3886 self: &Arc<Self>,
3887 identity: &AgentIdentity,
3888 expected_alias: &str,
3889 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
3890 let runtime = Arc::clone(self);
3891 let identity = identity.clone();
3892 let expected_alias = expected_alias.to_string();
3893 self.run_tracked_foreground(async move {
3894 runtime
3895 .reset_with_expected_member_alias(&identity, Some(&expected_alias))
3896 .await
3897 })
3898 .await
3899 }
3900
3901 pub async fn delete_identity_tracked(
3903 self: &Arc<Self>,
3904 identity: &AgentIdentity,
3905 ) -> Result<(), IdentityRuntimeError> {
3906 let runtime = Arc::clone(self);
3907 let identity = identity.clone();
3908 self.run_tracked_foreground(async move { runtime.delete_identity(&identity).await })
3909 .await
3910 }
3911
3912 pub async fn delete_identity_member_alias_tracked(
3915 self: &Arc<Self>,
3916 identity: &AgentIdentity,
3917 expected_alias: &str,
3918 ) -> Result<(), IdentityRuntimeError> {
3919 let runtime = Arc::clone(self);
3920 let identity = identity.clone();
3921 let expected_alias = expected_alias.to_string();
3922 self.run_tracked_foreground(async move {
3923 runtime
3924 .delete_identity_with_expected_member_alias(&identity, Some(&expected_alias))
3925 .await
3926 })
3927 .await
3928 }
3929
3930 pub(crate) async fn delete_identity_and_cleanup_live_members_tracked<T, F, Fut>(
3933 self: &Arc<Self>,
3934 identity: &AgentIdentity,
3935 expected_alias: Option<&str>,
3936 cleanup: F,
3937 ) -> Result<T, IdentityRuntimeError>
3938 where
3939 T: Send + 'static,
3940 F: FnOnce(Option<AgentRuntimeId>) -> Fut + Send + 'static,
3941 Fut: Future<Output = T> + Send + 'static,
3942 {
3943 let runtime = Arc::clone(self);
3944 let identity = identity.clone();
3945 let expected_alias = expected_alias.map(str::to_owned);
3946 self.run_tracked_foreground(async move {
3947 let lifecycle_lock = runtime.lifecycle_lock_for(&identity).await;
3948 let _lifecycle_guard = lifecycle_lock.lock().await;
3949 if let Some(expected_alias) = expected_alias.as_deref() {
3950 runtime
3951 .ensure_expected_member_alias_current(&identity, expected_alias)
3952 .await?;
3953 }
3954 let deleted_alias = runtime
3955 .entries
3956 .read()
3957 .await
3958 .get(&identity)
3959 .and_then(|entry| entry.continuity.as_ref())
3960 .map(|record| record.agent_runtime_id.clone());
3961 runtime.delete_identity_locked(&identity).await?;
3962 Ok(cleanup(deleted_alias).await)
3963 })
3964 .await
3965 }
3966
3967 async fn materialize_for_background(
3972 &self,
3973 identity: &AgentIdentity,
3974 cancellation: &mut watch::Receiver<bool>,
3975 generation: u64,
3976 ) -> Option<Result<ContinuityRecord, IdentityRuntimeError>> {
3977 let mut bound_generation = None;
3978 let result = self
3979 .materialize_inner(
3980 identity,
3981 None,
3982 Some(cancellation),
3983 Some(generation),
3984 &mut bound_generation,
3985 )
3986 .await;
3987 if matches!(
3988 &result,
3989 Err(IdentityRuntimeError::Internal(message)) if message == BACKGROUND_WARM_CANCELLED
3990 ) {
3991 self.mark_bootstrap_materialization_cancelled(identity, Some(generation));
3992 return None;
3993 }
3994 if result.is_ok() {
3995 let desired_edges = self.desired_peer_edges.read().await.clone();
3996 if !desired_edges.is_empty()
3997 && let Err(error) = self.reconcile_managed_peer_edges(&desired_edges).await
3998 {
3999 tracing::warn!(
4000 identity = %identity,
4001 %error,
4002 "background identity materialized with topology reconcile warning"
4003 );
4004 }
4005 }
4006 self.mark_bootstrap_materialization_finished(identity, &result, Some(generation));
4007 Some(result)
4008 }
4009
4010 async fn materialize_inner(
4011 &self,
4012 identity: &AgentIdentity,
4013 expected_alias: Option<&str>,
4014 mut cancellation: Option<&mut watch::Receiver<bool>>,
4015 expected_bootstrap_generation: Option<u64>,
4016 bound_bootstrap_generation: &mut Option<u64>,
4017 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
4018 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
4019 let _lifecycle_guard = lifecycle_lock.lock().await;
4020 let bootstrap_generation = {
4021 let entries = self.entries.read().await;
4022 entries
4023 .get(identity)
4024 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?
4025 .bootstrap_generation
4026 };
4027 *bound_bootstrap_generation = Some(bootstrap_generation);
4028 if expected_bootstrap_generation.is_some_and(|expected| expected != bootstrap_generation) {
4033 return Err(IdentityRuntimeError::Internal(
4034 BACKGROUND_WARM_CANCELLED.to_string(),
4035 ));
4036 }
4037 let raw_alias_lock = self.raw_member_alias_lock(identity.as_str()).await;
4038 let _raw_alias_guard = raw_alias_lock.lock().await;
4039 self.ensure_raw_member_alias_available(identity).await?;
4040 if let Some(expected_alias) = expected_alias {
4041 self.ensure_expected_member_alias_current(identity, expected_alias)
4042 .await?;
4043 }
4044 let lock = self.materialization_lock_for(identity).await;
4045 let _guard = lock.lock().await;
4046
4047 let (spec, continuity, state) = {
4048 let entries = self.entries.read().await;
4049 let entry = entries
4050 .get(identity)
4051 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4052 if entry.state == IdentityLifecycleState::Active {
4053 let continuity = entry.continuity.clone().ok_or_else(|| {
4054 IdentityRuntimeError::Internal(format!(
4055 "active identity {identity} has no continuity record"
4056 ))
4057 })?;
4058 drop(entries);
4059 self.clear_materialization_backoff(identity).await;
4060 return Ok(continuity);
4061 }
4062 (entry.spec.clone(), entry.continuity.clone(), entry.state)
4063 };
4064 if let Some(park) = self.host_rejected_build_park(identity).await {
4070 return Err(IdentityRuntimeError::HostRejectedBuild {
4071 identity: identity.clone(),
4072 reason: park.reason,
4073 });
4074 }
4075 let original_continuity = continuity.clone();
4076 let continuity = if durable_spec_uses_external_binding(&spec) {
4077 None
4078 } else {
4079 continuity
4080 };
4081
4082 match state {
4083 IdentityLifecycleState::Dormant | IdentityLifecycleState::Uninitialized => {}
4084 IdentityLifecycleState::Broken
4085 | IdentityLifecycleState::Retiring
4086 | IdentityLifecycleState::Suspended => {
4087 return Err(IdentityRuntimeError::InvalidState {
4088 identity: identity.clone(),
4089 state,
4090 operation: "materialize",
4091 });
4092 }
4093 IdentityLifecycleState::Active => unreachable!("active handled above"),
4094 }
4095 self.mark_bootstrap_materialization_started(identity, Some(bootstrap_generation));
4100
4101 self.release_parked_unactivated_leases_for_identity(identity)
4107 .await?;
4108
4109 let lease_results = self
4114 .lease_provider
4115 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
4116 .await
4117 .map_err(IdentityRuntimeError::Lease)?;
4118 let grant = match lease_results.get(identity) {
4119 Some(super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
4120 Some(super::types::LeaseAcquireResult::AlreadyHeld { holder, .. }) => {
4121 tracing::error!(
4122 %identity,
4123 holder = %holder,
4124 "single-embodiment guard: refusing to materialize an identity whose \
4125 durable lease is held by another live runtime instance"
4126 );
4127 return Err(IdentityRuntimeError::AlreadyEmbodied {
4128 identity: identity.clone(),
4129 holder: holder.clone(),
4130 });
4131 }
4132 None => return Err(IdentityRuntimeError::NoActiveLease(identity.clone())),
4133 };
4134 if let Some(record) = continuity.as_ref()
4135 && let Err(err) = self
4136 .continuity_store
4137 .upsert_continuity_record(record, grant.fencing_token)
4138 .await
4139 {
4140 let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4141 return Err(IdentityRuntimeError::Internal(format!(
4142 "continuity upsert before materialize: {err}{}",
4143 cleanup_error
4144 .as_ref()
4145 .map(|e| format!("; lease cleanup failed: {e}"))
4146 .unwrap_or_default(),
4147 )));
4148 }
4149
4150 if cancellation
4151 .as_ref()
4152 .is_some_and(|cancellation| *cancellation.borrow())
4153 {
4154 return Err(self
4155 .cancel_uninstalled_background_materialization(&grant)
4156 .await);
4157 }
4158
4159 let active_peers = self.entries.read().await.keys().cloned().collect();
4160 let managed_edges = self.desired_peer_edges.read().await.clone();
4161 let build_context = AgentBuildContext {
4162 identity: identity.clone(),
4163 active_peers,
4164 managed_edges,
4165 runtime_services: self.runtime_services(),
4166 };
4167 let mut draft = super::types::AgentBuildDraft {
4168 model: None,
4169 system_prompt: None,
4170 additional_instructions: spec.additional_instructions.clone(),
4171 labels: spec.labels.clone(),
4172 app_context: spec.context.clone(),
4173 external_tools: Vec::new(),
4174 local_external_tools: Default::default(),
4175 provider_params: None,
4176 };
4177 if let Some(customizer) = self.customizer.read().await.clone() {
4178 let customize = customizer.customize_build(&build_context, &spec, &mut draft);
4179 tokio::pin!(customize);
4180 let customize_result = if let Some(cancellation) = cancellation.as_mut() {
4181 let cancellation = &mut **cancellation;
4182 tokio::select! {
4183 result = &mut customize => result,
4184 changed = cancellation.changed() => {
4185 if changed.is_ok() && *cancellation.borrow() {
4186 return Err(
4187 self.cancel_uninstalled_background_materialization(&grant)
4188 .await,
4189 );
4190 }
4191 customize.await
4192 }
4193 }
4194 } else {
4195 customize.await
4196 };
4197 if let Err(err) = customize_result {
4198 let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4199 return Err(IdentityRuntimeError::Internal(format!(
4200 "customizer: {err}{}",
4201 cleanup_error
4202 .as_ref()
4203 .map(|e| format!("; lease cleanup failed: {e}"))
4204 .unwrap_or_default(),
4205 )));
4206 }
4207 }
4208
4209 let mut abandoned_session_registrations: Vec<SessionId> = Vec::new();
4210 let mut record = if let Some(mut record) = continuity {
4211 let snapshot = if self
4212 .bridge
4213 .as_ref()
4214 .is_none_or(|bridge| bridge.requires_resume_snapshot())
4215 {
4216 match self
4217 .continuity_store
4218 .load_session_snapshot(&record.session_id)
4219 .await
4220 {
4221 Ok(snapshot) => snapshot,
4222 Err(err) => {
4223 let cleanup_error =
4224 self.release_uninstalled_materialize_lease(&grant).await;
4225 return Err(IdentityRuntimeError::Internal(format!(
4226 "load session snapshot before materialize: {err}{}",
4227 cleanup_error
4228 .as_ref()
4229 .map(|e| format!("; lease cleanup failed: {e}"))
4230 .unwrap_or_default(),
4231 )));
4232 }
4233 }
4234 } else {
4235 None
4236 };
4237
4238 if let Some(bridge) = self.bridge.as_ref() {
4239 if let Err(err) = bridge
4240 .register_session_runtime_state(
4241 &record.session_id,
4242 identity,
4243 record.generation,
4244 record.checkpoint_version,
4245 grant.fencing_token,
4246 )
4247 .await
4248 {
4249 let unregister_error = Self::unregister_bridge_session_runtime_states(
4250 bridge.as_ref(),
4251 std::slice::from_ref(&record.session_id),
4252 )
4253 .await;
4254 let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4255 return Err(IdentityRuntimeError::Internal(format!(
4256 "bridge register_session_runtime_state: {err}{}{}",
4257 unregister_error
4258 .as_ref()
4259 .map(|e| format!("; unregister session failed: {e}"))
4260 .unwrap_or_default(),
4261 cleanup_error
4262 .as_ref()
4263 .map(|e| format!("; lease cleanup failed: {e}"))
4264 .unwrap_or_default(),
4265 )));
4266 }
4267 let registered_session_id = record.session_id.clone();
4268 let snapshot = snapshot.unwrap_or(SessionSnapshot { data: Vec::new() });
4269 let outcome = bridge
4270 .resume_session(
4271 identity,
4272 &record.agent_runtime_id,
4273 &spec,
4274 &draft,
4275 &record.session_id,
4276 &snapshot,
4277 )
4278 .await;
4279 let outcome = match outcome {
4280 Ok(outcome) => outcome,
4281 Err(err) => {
4282 tracing::error!(
4291 %identity,
4292 session_id = %registered_session_id,
4293 error = %err,
4294 "materialize resume rejected; marking identity Broken and preserving \
4295 the durable session for reconcile retry"
4296 );
4297 let unregister_error = Self::unregister_bridge_session_runtime_states(
4298 bridge.as_ref(),
4299 std::slice::from_ref(®istered_session_id),
4300 )
4301 .await;
4302 let lease_cleanup_error =
4303 self.release_uninstalled_materialize_lease(&grant).await;
4304 {
4305 let mut entries = self.entries.write().await;
4306 if let Some(entry) = entries.get_mut(identity) {
4307 entry.state = IdentityLifecycleState::Broken;
4308 entry.lease = None;
4309 }
4310 }
4311 self.emit_event(
4312 identity,
4313 IdentityEvent::StateChanged {
4314 identity: identity.clone(),
4315 new_state: IdentityLifecycleState::Broken,
4316 },
4317 )
4318 .await;
4319 {
4320 let rejection = err.to_string();
4321 if is_host_rejected_build_error(&rejection) {
4322 self.mark_host_rejected_build_park(identity, rejection)
4323 .await;
4324 }
4325 }
4326 if let BridgeError::ResumeRejected {
4333 kind: ResumeRejectionKind::ArchivedNotRevivable,
4334 detail,
4335 } = &err
4336 && !self
4337 .mark_continuity_unrecoverable(
4338 identity,
4339 archived_not_revivable_park_reason(
4340 ®istered_session_id,
4341 detail,
4342 ),
4343 )
4344 .await
4345 {
4346 tracing::debug!(
4347 %identity,
4348 "identity left Broken before the archived-not-revivable \
4349 park could be recorded"
4350 );
4351 }
4352 let detail = format!(
4353 "bridge resume_session rejected (identity degraded, durable session \
4354 preserved for reconcile retry): {err}{}{}",
4355 unregister_error
4356 .as_ref()
4357 .map(|e| format!("; unregister session failed: {e}"))
4358 .unwrap_or_default(),
4359 lease_cleanup_error
4360 .as_ref()
4361 .map(|e| format!("; lease cleanup failed: {e}"))
4362 .unwrap_or_default(),
4363 );
4364 return Err(IdentityRuntimeError::Internal(detail));
4365 }
4366 };
4367 if let Some(reason) = outcome.fallback_reason().cloned() {
4368 tracing::warn!(
4369 %identity,
4370 reason = ?reason,
4371 "lazy identity materialization fresh-spawned after typed resume fallback"
4372 );
4373 self.emit_event(
4374 identity,
4375 IdentityEvent::ResumeFallback {
4376 identity: identity.clone(),
4377 reason,
4378 },
4379 )
4380 .await;
4381 }
4382 let effective_session_id = outcome.session_id().clone();
4383 if effective_session_id != registered_session_id {
4384 if let Some(injector) = self.agent_memory.read().await.as_ref() {
4388 let abandoned_key = registered_session_id.to_string();
4389 injector.note_session_generation(
4390 identity,
4391 &abandoned_key,
4392 record.generation.get(),
4393 );
4394 injector.spawn_rotation_distillation(
4395 identity,
4396 &abandoned_key,
4397 crate::memory::distiller::DistillCause::ResumeFallback,
4398 );
4399 }
4400 abandoned_session_registrations.push(registered_session_id);
4401 }
4402 record.session_id = effective_session_id;
4403 }
4404 record
4405 } else {
4406 let new_runtime_id =
4407 AgentRuntimeId::parse(&format!("rt:{identity}:0")).map_err(|err| {
4408 IdentityRuntimeError::Internal(format!("failed to mint runtime id: {err}"))
4409 })?;
4410 let mut record = ContinuityRecord {
4411 identity: identity.clone(),
4412 agent_runtime_id: new_runtime_id,
4413 session_id: meerkat_core::types::SessionId::new(),
4414 generation: ContinuityGeneration::new(0),
4415 checkpoint_version: CheckpointVersion::new(0),
4416 };
4417 if let Err(err) = self
4418 .continuity_store
4419 .upsert_continuity_record(&record, grant.fencing_token)
4420 .await
4421 {
4422 let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4423 return Err(IdentityRuntimeError::Internal(format!(
4424 "continuity upsert before materialize create: {err}{}",
4425 cleanup_error
4426 .as_ref()
4427 .map(|e| format!("; lease cleanup failed: {e}"))
4428 .unwrap_or_default(),
4429 )));
4430 }
4431 if let Some(bridge) = self.bridge.as_ref() {
4432 let provisional_session_id = record.session_id.clone();
4433 if let Err(err) = bridge
4434 .register_session_runtime_state(
4435 &record.session_id,
4436 identity,
4437 record.generation,
4438 record.checkpoint_version,
4439 grant.fencing_token,
4440 )
4441 .await
4442 .map_err(|err| {
4443 IdentityRuntimeError::Internal(format!(
4444 "bridge register_session_runtime_state: {err}"
4445 ))
4446 })
4447 {
4448 let unregister_error = Self::unregister_bridge_session_runtime_states(
4449 bridge.as_ref(),
4450 std::slice::from_ref(&provisional_session_id),
4451 )
4452 .await;
4453 let delete_error = self
4454 .continuity_store
4455 .delete_continuity_record(identity, grant.fencing_token)
4456 .await
4457 .err();
4458 let cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4459 if let Some(delete_error) = delete_error {
4460 return Err(IdentityRuntimeError::Internal(format!(
4461 "{err}{}; tentative continuity cleanup failed: {delete_error}{}",
4462 unregister_error
4463 .as_ref()
4464 .map(|e| format!("; unregister session failed: {e}"))
4465 .unwrap_or_default(),
4466 cleanup_error
4467 .as_ref()
4468 .map(|e| format!("; lease cleanup failed: {e}"))
4469 .unwrap_or_default(),
4470 )));
4471 }
4472 if let Some(cleanup_error) = cleanup_error {
4473 return Err(IdentityRuntimeError::Internal(format!(
4474 "{err}{}; lease cleanup failed: {cleanup_error}",
4475 unregister_error
4476 .as_ref()
4477 .map(|e| format!("; unregister session failed: {e}"))
4478 .unwrap_or_default(),
4479 )));
4480 }
4481 if let Some(unregister_error) = unregister_error {
4482 return Err(IdentityRuntimeError::Internal(format!(
4483 "{err}; unregister session failed: {unregister_error}"
4484 )));
4485 }
4486 return Err(err);
4487 }
4488 let created_session_id = bridge
4489 .create_session(
4490 identity,
4491 &record.agent_runtime_id,
4492 &spec,
4493 &draft,
4494 &record.session_id,
4495 )
4496 .await
4497 .map_err(|err| {
4498 IdentityRuntimeError::Internal(format!("bridge create_session: {err}"))
4499 });
4500 if let Err(err) = &created_session_id {
4501 let detail = err.to_string();
4502 if is_host_rejected_build_error(&detail) {
4503 self.mark_host_rejected_build_park(identity, detail).await;
4504 }
4505 }
4506 match created_session_id {
4507 Ok(session_id) => {
4508 if session_id != provisional_session_id {
4509 abandoned_session_registrations.push(provisional_session_id);
4510 }
4511 record.session_id = session_id;
4512 }
4513 Err(err) => {
4514 let unregister_error = Self::unregister_bridge_session_runtime_states(
4515 bridge.as_ref(),
4516 std::slice::from_ref(&provisional_session_id),
4517 )
4518 .await;
4519 let cleanup_error =
4520 bridge.retire_member(&record.agent_runtime_id).await.err();
4521 let delete_error = self
4522 .continuity_store
4523 .delete_continuity_record(identity, grant.fencing_token)
4524 .await
4525 .err();
4526 let lease_cleanup_error =
4527 self.release_uninstalled_materialize_lease(&grant).await;
4528 if unregister_error.is_some()
4529 || cleanup_error.is_some()
4530 || delete_error.is_some()
4531 || lease_cleanup_error.is_some()
4532 {
4533 return Err(IdentityRuntimeError::Internal(format!(
4534 "{err}{}{}{}{}",
4535 unregister_error
4536 .as_ref()
4537 .map(|e| format!("; unregister session failed: {e}"))
4538 .unwrap_or_default(),
4539 cleanup_error
4540 .as_ref()
4541 .map(|e| format!("; cleanup retire failed: {e}"))
4542 .unwrap_or_default(),
4543 delete_error
4544 .as_ref()
4545 .map(|e| format!("; tentative continuity cleanup failed: {e}"))
4546 .unwrap_or_default(),
4547 lease_cleanup_error
4548 .as_ref()
4549 .map(|e| format!("; lease cleanup failed: {e}"))
4550 .unwrap_or_default(),
4551 )));
4552 }
4553 return Err(err);
4554 }
4555 }
4556 }
4557 record
4558 };
4559
4560 if let Err(err) = self
4561 .continuity_store
4562 .upsert_continuity_record(&record, grant.fencing_token)
4563 .await
4564 {
4565 let unregister_error = if let Some(bridge) = self.bridge.as_ref() {
4566 let mut sessions_to_unregister = abandoned_session_registrations.clone();
4567 sessions_to_unregister.push(record.session_id.clone());
4568 Self::unregister_bridge_session_runtime_states(
4569 bridge.as_ref(),
4570 &sessions_to_unregister,
4571 )
4572 .await
4573 } else {
4574 None
4575 };
4576 let cleanup_error = if let Some(bridge) = self.bridge.as_ref() {
4577 bridge.retire_member(&record.agent_runtime_id).await.err()
4578 } else {
4579 None
4580 };
4581 let restore_error = self
4582 .restore_continuity_after_materialize_failure(
4583 identity,
4584 original_continuity.as_ref(),
4585 &grant,
4586 )
4587 .await;
4588 let lease_cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4589 if unregister_error.is_some()
4590 || cleanup_error.is_some()
4591 || restore_error.is_some()
4592 || lease_cleanup_error.is_some()
4593 {
4594 return Err(IdentityRuntimeError::Internal(format!(
4595 "continuity upsert after materialize: {err}{}{}{}{}",
4596 unregister_error
4597 .as_ref()
4598 .map(|e| format!("; unregister session failed: {e}"))
4599 .unwrap_or_default(),
4600 cleanup_error
4601 .as_ref()
4602 .map(|e| format!("; cleanup retire failed: {e}"))
4603 .unwrap_or_default(),
4604 restore_error
4605 .as_ref()
4606 .map(|e| format!("; continuity rollback failed: {e}"))
4607 .unwrap_or_default(),
4608 lease_cleanup_error
4609 .as_ref()
4610 .map(|e| format!("; lease cleanup failed: {e}"))
4611 .unwrap_or_default(),
4612 )));
4613 }
4614 return Err(IdentityRuntimeError::Internal(format!(
4615 "continuity upsert after materialize: {err}"
4616 )));
4617 }
4618 if let Some(bridge) = self.bridge.as_ref() {
4619 let register_result = bridge
4620 .register_session_runtime_state(
4621 &record.session_id,
4622 identity,
4623 record.generation,
4624 record.checkpoint_version,
4625 grant.fencing_token,
4626 )
4627 .await;
4628 let effective_checkpoint_version = match register_result {
4629 Ok(version) => version,
4630 Err(err) => {
4631 let mut sessions_to_unregister = abandoned_session_registrations.clone();
4632 sessions_to_unregister.push(record.session_id.clone());
4633 let unregister_error = Self::unregister_bridge_session_runtime_states(
4634 bridge.as_ref(),
4635 &sessions_to_unregister,
4636 )
4637 .await;
4638 let cleanup_error = bridge.retire_member(&record.agent_runtime_id).await.err();
4639 let restore_error = self
4640 .restore_continuity_after_materialize_failure(
4641 identity,
4642 original_continuity.as_ref(),
4643 &grant,
4644 )
4645 .await;
4646 let lease_cleanup_error =
4647 self.release_uninstalled_materialize_lease(&grant).await;
4648 return Err(IdentityRuntimeError::Internal(format!(
4649 "bridge register actual session runtime state: {err}{}{}{}{}",
4650 unregister_error
4651 .as_ref()
4652 .map(|e| format!("; unregister session failed: {e}"))
4653 .unwrap_or_default(),
4654 cleanup_error
4655 .as_ref()
4656 .map(|e| format!("; cleanup retire failed: {e}"))
4657 .unwrap_or_default(),
4658 restore_error
4659 .as_ref()
4660 .map(|e| format!("; continuity rollback failed: {e}"))
4661 .unwrap_or_default(),
4662 lease_cleanup_error
4663 .as_ref()
4664 .map(|e| format!("; lease cleanup failed: {e}"))
4665 .unwrap_or_default(),
4666 )));
4667 }
4668 };
4669 record.checkpoint_version = effective_checkpoint_version;
4670 if let Some(err) = Self::unregister_bridge_session_runtime_states(
4671 bridge.as_ref(),
4672 &abandoned_session_registrations,
4673 )
4674 .await
4675 {
4676 let actual_unregister_error = Self::unregister_bridge_session_runtime_states(
4677 bridge.as_ref(),
4678 std::slice::from_ref(&record.session_id),
4679 )
4680 .await;
4681 let unregister_error = actual_unregister_error
4682 .map(|actual_err| format!("{err}; actual session: {actual_err}"))
4683 .unwrap_or(err);
4684 let cleanup_error = bridge.retire_member(&record.agent_runtime_id).await.err();
4685 let restore_error = self
4686 .restore_continuity_after_materialize_failure(
4687 identity,
4688 original_continuity.as_ref(),
4689 &grant,
4690 )
4691 .await;
4692 let lease_cleanup_error = self.release_uninstalled_materialize_lease(&grant).await;
4693 return Err(IdentityRuntimeError::Internal(format!(
4694 "bridge unregister abandoned session runtime state: {unregister_error}{}{}{}",
4695 cleanup_error
4696 .as_ref()
4697 .map(|e| format!("; cleanup retire failed: {e}"))
4698 .unwrap_or_default(),
4699 restore_error
4700 .as_ref()
4701 .map(|e| format!("; continuity rollback failed: {e}"))
4702 .unwrap_or_default(),
4703 lease_cleanup_error
4704 .as_ref()
4705 .map(|e| format!("; lease cleanup failed: {e}"))
4706 .unwrap_or_default(),
4707 )));
4708 }
4709 }
4710
4711 {
4712 let mut entries = self.entries.write().await;
4713 let entry = entries
4714 .get_mut(identity)
4715 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4716 entry.continuity = Some(record.clone());
4717 entry.lease = Some(Self::lease_entry_from_grant(&grant));
4718 entry.state = IdentityLifecycleState::Active;
4719 entry.checkpoint_version = record.checkpoint_version;
4720 }
4721 self.emit_event(
4722 identity,
4723 IdentityEvent::StateChanged {
4724 identity: identity.clone(),
4725 new_state: IdentityLifecycleState::Active,
4726 },
4727 )
4728 .await;
4729 self.clear_materialization_backoff(identity).await;
4730 Ok(record)
4731 }
4732
4733 async fn best_effort_materialize_identity(
4734 &self,
4735 identity: AgentIdentity,
4736 initiator: Option<&AgentIdentity>,
4737 operation: &'static str,
4738 ) -> Option<ContinuityRecord> {
4739 let attempt_lock = self.best_effort_materialization_lock_for(&identity).await;
4740 let _attempt_guard = attempt_lock.lock().await;
4741
4742 if let Some(error) = self.materialization_backoff_error(&identity).await {
4743 tracing::debug!(
4744 identity = %identity,
4745 initiator = initiator.map(ToString::to_string).as_deref(),
4746 error = %error,
4747 "identity best-effort materialization skipped due to materialization backoff"
4748 );
4749 return None;
4750 }
4751
4752 match self.materialize(&identity).await {
4753 Ok(record) => {
4754 self.clear_materialization_backoff(&identity).await;
4755 Some(record)
4756 }
4757 Err(err) => {
4758 tracing::warn!(
4759 identity = %identity,
4760 initiator = initiator.map(ToString::to_string).as_deref(),
4761 error = %err,
4762 "identity best-effort materialization skipped identity after materialization failure"
4763 );
4764 self.record_best_effort_materialization_failure(
4765 &identity, initiator, operation, &err,
4766 )
4767 .await;
4768 None
4769 }
4770 }
4771 }
4772
4773 async fn materialize_all_records(
4774 &self,
4775 ) -> Vec<(
4776 AgentIdentity,
4777 Result<ContinuityRecord, IdentityRuntimeError>,
4778 )> {
4779 let identities = self.registered_identities().await;
4780 stream::iter(identities.into_iter().map(|identity| async move {
4781 let result = self.materialize(&identity).await;
4782 (identity, result)
4783 }))
4784 .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4785 .collect::<Vec<_>>()
4786 .await
4787 }
4788
4789 pub async fn materialize_all(&self) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4795 let identities = self.registered_identities().await;
4796 let records = stream::iter(identities.into_iter().map(|identity| async move {
4797 self.best_effort_materialize_identity(identity, None, "materialize_all")
4798 .await
4799 }))
4800 .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4801 .filter_map(async move |record| record)
4802 .collect::<Vec<_>>()
4803 .await;
4804
4805 let desired_edges = self.desired_peer_edges.read().await.clone();
4806 if !desired_edges.is_empty()
4807 && let Err(err) = self.reconcile_managed_peer_edges(&desired_edges).await
4808 {
4809 tracing::warn!(
4810 error = %err,
4811 "identity materialize_all completed with topology reconcile warning"
4812 );
4813 }
4814
4815 Ok(records)
4816 }
4817
4818 pub async fn materialize_all_required(
4822 &self,
4823 ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4824 let results = self.materialize_all_records().await;
4825 let mut records = Vec::with_capacity(results.len());
4826 let mut failures = Vec::new();
4827
4828 for (identity, result) in results {
4829 match result {
4830 Ok(record) => records.push(record),
4831 Err(err) => failures.push(format!("{identity}: {err}")),
4832 }
4833 }
4834
4835 if !failures.is_empty() {
4836 return Err(IdentityRuntimeError::Internal(format!(
4837 "identity-first required materialization failed for {} identities: {}",
4838 failures.len(),
4839 failures.join("; ")
4840 )));
4841 }
4842
4843 let desired_edges = self.desired_peer_edges.read().await.clone();
4844 if !desired_edges.is_empty() {
4845 self.reconcile_managed_peer_edges(&desired_edges).await?;
4846 }
4847
4848 Ok(records)
4849 }
4850
4851 pub async fn materialize_all_required_tracked(
4853 self: &Arc<Self>,
4854 ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4855 let runtime = Arc::clone(self);
4856 self.run_tracked_foreground(async move { runtime.materialize_all_required().await })
4857 .await
4858 }
4859
4860 pub async fn materialize_reachable_peers(
4863 &self,
4864 identity: &AgentIdentity,
4865 ) -> Result<Vec<ContinuityRecord>, IdentityRuntimeError> {
4866 let peers = self.reachable_peer_identities(identity).await;
4867 let records = stream::iter(peers.into_iter().map(|peer| async move {
4868 self.best_effort_materialize_identity(
4869 peer,
4870 Some(identity),
4871 "materialize_reachable_peers",
4872 )
4873 .await
4874 }))
4875 .buffer_unordered(MANAGED_PEER_RECONCILE_CONCURRENCY)
4876 .filter_map(async move |record| record)
4877 .collect::<Vec<_>>()
4878 .await;
4879
4880 let desired_edges = self.desired_peer_edges.read().await.clone();
4881 if !desired_edges.is_empty()
4882 && let Err(err) = self.reconcile_managed_peer_edges(&desired_edges).await
4883 {
4884 tracing::warn!(
4885 identity = %identity,
4886 error = %err,
4887 "identity peer materialization completed with topology reconcile warning"
4888 );
4889 }
4890
4891 Ok(records)
4892 }
4893
4894 pub async fn subscribe(
4903 &self,
4904 identity: &AgentIdentity,
4905 ) -> Result<broadcast::Receiver<IdentityEvent>, IdentityRuntimeError> {
4906 let channels = self.event_channels.read().await;
4907 let tx = channels
4908 .get(identity)
4909 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4910 Ok(tx.subscribe())
4911 }
4912
4913 pub async fn update_spec(&self, spec: DurableAgentSpec) -> Result<(), IdentityRuntimeError> {
4915 let mut entries = self.entries.write().await;
4916 let entry = entries
4917 .get_mut(&spec.identity)
4918 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
4919 entry.spec = spec;
4920 Ok(())
4921 }
4922
4923 async fn release_broken_lease_locked(
4934 &self,
4935 identity: &AgentIdentity,
4936 ) -> Result<bool, IdentityRuntimeError> {
4937 let pending = {
4938 let mut entries = self.entries.write().await;
4939 let entry = entries
4940 .get(identity)
4941 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4942 if entry.state != IdentityLifecycleState::Broken {
4943 return Ok(false);
4944 }
4945
4946 let existing_pending = entry.pending_lease_release.clone();
4947 let retained_lease = entry.lease.as_ref().map(|lease| LeaseGrant {
4948 identity: identity.clone(),
4949 fencing_token: lease.fencing_token,
4950 ttl: lease.ttl,
4951 });
4952 if let (Some(pending), Some(retained)) =
4953 (existing_pending.as_ref(), retained_lease.as_ref())
4954 && pending.fencing_token != retained.fencing_token
4955 {
4956 return Err(IdentityRuntimeError::Internal(format!(
4957 "Broken identity {identity} retained conflicting fencing tokens {} and {}",
4958 pending.fencing_token, retained.fencing_token
4959 )));
4960 }
4961
4962 let grant = existing_pending.or(retained_lease);
4963 if let Some(grant) = grant.as_ref() {
4964 let entry = entries
4965 .get_mut(identity)
4966 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4967 entry.lease = None;
4968 entry.pending_lease_release = Some(grant.clone());
4969 }
4970 grant
4971 };
4972 let Some(grant) = pending else {
4973 return Ok(false);
4974 };
4975
4976 if let Err(error) = self
4977 .lease_provider
4978 .release_leases(std::slice::from_ref(&grant))
4979 .await
4980 {
4981 self.mark_bootstrap_from_lifecycle(
4982 identity,
4983 IdentityLifecycleState::Broken,
4984 Some(format!("pending lease release retry: {error}")),
4985 );
4986 return Err(IdentityRuntimeError::Lease(error));
4987 }
4988
4989 let mut entries = self.entries.write().await;
4990 let entry = entries
4991 .get_mut(identity)
4992 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
4993 match entry.pending_lease_release.as_ref() {
4994 Some(current) if current.fencing_token == grant.fencing_token => {
4995 entry.pending_lease_release = None;
4996 }
4997 Some(current) => {
4998 return Err(IdentityRuntimeError::Internal(format!(
4999 "pending lease release for {identity} changed from fencing token {} to {} under lifecycle authority",
5000 grant.fencing_token, current.fencing_token
5001 )));
5002 }
5003 None => {
5004 return Err(IdentityRuntimeError::Internal(format!(
5005 "pending lease release for {identity} disappeared under lifecycle authority"
5006 )));
5007 }
5008 }
5009 Ok(true)
5010 }
5011
5012 async fn cleanup_broken_lower_plane_locked(
5024 &self,
5025 identity: &AgentIdentity,
5026 ) -> Result<(), IdentityRuntimeError> {
5027 let continuity = {
5028 let entries = self.entries.read().await;
5029 let entry = entries
5030 .get(identity)
5031 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5032 (entry.state == IdentityLifecycleState::Broken)
5033 .then(|| entry.continuity.clone())
5034 .flatten()
5035 };
5036 let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5037 return Ok(());
5038 };
5039
5040 let retire_error = bridge.retire_member(&record.agent_runtime_id).await.err();
5041 let unregister_error = bridge
5042 .unregister_session_runtime_state(&record.session_id)
5043 .await
5044 .err();
5045 match (retire_error, unregister_error) {
5046 (None, None) => Ok(()),
5047 (retire_error, unregister_error) => Err(IdentityRuntimeError::Internal(format!(
5048 "cleanup Broken lower-plane state for {identity}{}{}",
5049 retire_error
5050 .map(|error| format!("; retire member: {error}"))
5051 .unwrap_or_default(),
5052 unregister_error
5053 .map(|error| format!("; unregister session: {error}"))
5054 .unwrap_or_default()
5055 ))),
5056 }
5057 }
5058
5059 async fn accept_bootstrap_spec_locked(
5060 &self,
5061 spec: DurableAgentSpec,
5062 generation: u64,
5063 ) -> Result<(), IdentityRuntimeError> {
5064 let mut entries = self.entries.write().await;
5065 let entry = entries
5066 .get_mut(&spec.identity)
5067 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
5068 entry.spec = spec;
5069 entry.bootstrap_generation = generation;
5070 Ok(())
5071 }
5072
5073 async fn reconcile_roster_members(
5083 &self,
5084 roster: &[DurableAgentSpec],
5085 generation: u64,
5086 ) -> Result<(), IdentityRuntimeError> {
5087 self.reconcile_roster_members_after_lifecycle_lock(roster, generation, |_| {
5088 std::future::ready(())
5089 })
5090 .await
5091 }
5092
5093 async fn reconcile_roster_members_after_lifecycle_lock<H, F>(
5097 &self,
5098 roster: &[DurableAgentSpec],
5099 generation: u64,
5100 mut after_lifecycle_lock: H,
5101 ) -> Result<(), IdentityRuntimeError>
5102 where
5103 H: FnMut(&AgentIdentity) -> F,
5104 F: Future<Output = ()>,
5105 {
5106 Self::validate_roster_uniqueness(roster)?;
5107 let desired = roster
5108 .iter()
5109 .map(|spec| (spec.identity.clone(), spec.clone()))
5110 .collect::<BTreeMap<_, _>>();
5111 let registered = self.registered_identities().await;
5112
5113 for identity in registered
5114 .iter()
5115 .filter(|identity| !desired.contains_key(*identity))
5116 {
5117 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
5118 let _lifecycle_guard = lifecycle_lock.lock().await;
5119 after_lifecycle_lock(identity).await;
5120 self.cleanup_broken_lower_plane_locked(identity).await?;
5121 self.release_broken_lease_locked(identity).await?;
5122 let state = self
5123 .entries
5124 .read()
5125 .await
5126 .get(identity)
5127 .map(|entry| entry.state);
5128 let Some(state) = state else {
5129 continue;
5130 };
5131 match state {
5132 IdentityLifecycleState::Active => {
5133 self.retire_locked(identity).await?;
5134 }
5135 IdentityLifecycleState::Dormant
5136 | IdentityLifecycleState::Broken
5137 | IdentityLifecycleState::Uninitialized => {
5138 let lease = self
5139 .entries
5140 .read()
5141 .await
5142 .get(identity)
5143 .and_then(|entry| entry.lease.as_ref())
5144 .map(|lease| LeaseGrant {
5145 identity: identity.clone(),
5146 fencing_token: lease.fencing_token,
5147 ttl: lease.ttl,
5148 });
5149 if let Some(lease) = lease {
5150 self.lease_provider
5151 .release_leases(std::slice::from_ref(&lease))
5152 .await
5153 .map_err(IdentityRuntimeError::Lease)?;
5154 }
5155 }
5156 IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended => {
5157 return Err(IdentityRuntimeError::InvalidState {
5158 identity: identity.clone(),
5159 state,
5160 operation: "reconcile_roster_remove",
5161 });
5162 }
5163 }
5164 self.event_channels.write().await.remove(identity);
5165 self.entries.write().await.remove(identity);
5166 }
5167
5168 for (identity, desired_spec) in desired {
5169 let lifecycle_lock = self.lifecycle_lock_for(&identity).await;
5170 let _lifecycle_guard = lifecycle_lock.lock().await;
5171 after_lifecycle_lock(&identity).await;
5172 if !self.entries.read().await.contains_key(&identity) {
5176 continue;
5177 }
5178 let reconciling_broken_entry = {
5179 let entries = self.entries.read().await;
5180 entries
5181 .get(&identity)
5182 .is_some_and(|entry| entry.state == IdentityLifecycleState::Broken)
5183 };
5184 if reconciling_broken_entry {
5185 self.cleanup_broken_lower_plane_locked(&identity).await?;
5186 }
5187 self.release_broken_lease_locked(&identity).await?;
5191 let current = self
5192 .entries
5193 .read()
5194 .await
5195 .get(&identity)
5196 .map(|entry| (entry.spec.clone(), entry.state));
5197 let Some((current_spec, state)) = current else {
5198 continue;
5199 };
5200 if current_spec == desired_spec {
5201 self.accept_bootstrap_spec_locked(desired_spec, generation)
5202 .await?;
5203 continue;
5204 }
5205 if matches!(
5206 state,
5207 IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended
5208 ) {
5209 return Err(IdentityRuntimeError::InvalidState {
5210 identity,
5211 state,
5212 operation: "reconcile_roster_replace",
5213 });
5214 }
5215 if current_spec.profile == desired_spec.profile {
5221 self.accept_bootstrap_spec_locked(desired_spec, generation)
5222 .await?;
5223 continue;
5224 }
5225 match state {
5226 IdentityLifecycleState::Active => {
5227 self.retire_locked(&identity).await?;
5228 }
5229 IdentityLifecycleState::Dormant
5230 | IdentityLifecycleState::Broken
5231 | IdentityLifecycleState::Uninitialized => {}
5232 IdentityLifecycleState::Retiring | IdentityLifecycleState::Suspended => {
5233 return Err(IdentityRuntimeError::InvalidState {
5234 identity,
5235 state,
5236 operation: "reconcile_roster_replace",
5237 });
5238 }
5239 }
5240 self.accept_bootstrap_spec_locked(desired_spec, generation)
5241 .await?;
5242 }
5243
5244 Ok(())
5245 }
5246
5247 pub async fn adopt_roster_spec(
5257 &self,
5258 roster_provider: &Arc<dyn RosterProvider>,
5259 identity: &AgentIdentity,
5260 ) {
5261 self.adopt_roster_spec_with_context(roster_provider, identity, None)
5262 .await;
5263 }
5264
5265 async fn adopt_roster_spec_with_context(
5266 &self,
5267 roster_provider: &Arc<dyn RosterProvider>,
5268 identity: &AgentIdentity,
5269 mob_definition: Option<meerkat_mob::MobDefinition>,
5270 ) {
5271 match roster_provider
5272 .roster(&RosterContext {
5273 mob_definition,
5274 previous_identities: Vec::new(),
5275 })
5276 .await
5277 {
5278 Ok(specs) => {
5279 if let Some(spec) = specs.into_iter().find(|s| &s.identity == identity)
5280 && let Err(err) = self.update_spec(spec).await
5281 {
5282 tracing::warn!(
5283 identity = %identity,
5284 error = %err,
5285 "reset: failed to adopt current roster spec; rebuilding on stored spec",
5286 );
5287 }
5288 }
5289 Err(err) => {
5290 tracing::warn!(
5291 identity = %identity,
5292 error = %err,
5293 "reset: roster provider failed; rebuilding on stored spec",
5294 );
5295 }
5296 }
5297 }
5298
5299 pub async fn update_lease(
5301 &self,
5302 identity: &AgentIdentity,
5303 grant: LeaseGrant,
5304 ) -> Result<(), IdentityRuntimeError> {
5305 let fencing_token = grant.fencing_token;
5306 let mut entries = self.entries.write().await;
5307 let entry = entries
5308 .get_mut(identity)
5309 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5310 entry.lease = Some(LeaseEntry {
5311 fencing_token,
5312 ttl: grant.ttl,
5313 acquired_at: Instant::now(),
5314 });
5315 drop(entries);
5316 self.lease_renewal_notify.notify_one();
5317 self.emit_event(
5318 identity,
5319 IdentityEvent::LeaseUpdated {
5320 identity: identity.clone(),
5321 fencing_token,
5322 },
5323 )
5324 .await;
5325 Ok(())
5326 }
5327
5328 pub async fn mark_lease_lost(
5330 &self,
5331 identity: &AgentIdentity,
5332 ) -> Result<(), IdentityRuntimeError> {
5333 let mut entries = self.entries.write().await;
5334 let entry = entries
5335 .get_mut(identity)
5336 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5337 entry.state = IdentityLifecycleState::Broken;
5338 entry.lease = None;
5339 drop(entries);
5340 self.mark_bootstrap_from_lifecycle(
5341 identity,
5342 IdentityLifecycleState::Broken,
5343 Some("external lease authority was lost".to_string()),
5344 );
5345 self.emit_event(
5346 identity,
5347 IdentityEvent::LeaseLost {
5348 identity: identity.clone(),
5349 },
5350 )
5351 .await;
5352 Ok(())
5353 }
5354
5355 #[allow(dead_code)]
5357 pub(crate) async fn remove(&self, identity: &AgentIdentity) -> Option<IdentityEntry> {
5358 self.event_channels.write().await.remove(identity);
5359 self.entries.write().await.remove(identity)
5360 }
5361
5362 pub async fn set_state(
5364 &self,
5365 identity: &AgentIdentity,
5366 state: IdentityLifecycleState,
5367 ) -> Result<(), IdentityRuntimeError> {
5368 let mut entries = self.entries.write().await;
5369 let entry = entries
5370 .get_mut(identity)
5371 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5372 entry.state = state;
5373 drop(entries);
5374 if state == IdentityLifecycleState::Active {
5375 self.lease_renewal_notify.notify_one();
5376 }
5377 self.emit_event(
5378 identity,
5379 IdentityEvent::StateChanged {
5380 identity: identity.clone(),
5381 new_state: state,
5382 },
5383 )
5384 .await;
5385 Ok(())
5386 }
5387
5388 pub(crate) async fn mark_active_runtime_broken(
5393 &self,
5394 identity: &AgentIdentity,
5395 expected_runtime_id: &str,
5396 expected_fencing_token: u64,
5397 detail: &str,
5398 ) -> Result<bool, IdentityRuntimeError> {
5399 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
5400 let _lifecycle_guard = lifecycle_lock.lock().await;
5401 let changed = {
5402 let mut entries = self.entries.write().await;
5403 let entry = entries
5404 .get_mut(identity)
5405 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5406 let is_current_active = entry.state == IdentityLifecycleState::Active
5407 && entry
5408 .continuity
5409 .as_ref()
5410 .is_some_and(|record| record.agent_runtime_id.as_str() == expected_runtime_id)
5411 && entry
5412 .lease
5413 .as_ref()
5414 .is_some_and(|lease| lease.fencing_token.get() == expected_fencing_token);
5415 if is_current_active {
5416 entry.state = IdentityLifecycleState::Broken;
5419 }
5420 is_current_active
5421 };
5422 if changed {
5423 self.mark_bootstrap_from_lifecycle(
5424 identity,
5425 IdentityLifecycleState::Broken,
5426 Some(detail.to_string()),
5427 );
5428 self.emit_event(
5429 identity,
5430 IdentityEvent::StateChanged {
5431 identity: identity.clone(),
5432 new_state: IdentityLifecycleState::Broken,
5433 },
5434 )
5435 .await;
5436 }
5437 Ok(changed)
5438 }
5439
5440 fn check_lease(entry: &IdentityEntry) -> Result<FencingToken, IdentityRuntimeError> {
5447 match &entry.lease {
5448 Some(lease) if !lease.is_expired() => Ok(lease.fencing_token),
5449 Some(_) => Err(IdentityRuntimeError::LeaseLost(entry.spec.identity.clone())),
5450 None => Err(IdentityRuntimeError::NoActiveLease(
5451 entry.spec.identity.clone(),
5452 )),
5453 }
5454 }
5455
5456 fn lease_entry_from_grant(grant: &LeaseGrant) -> LeaseEntry {
5457 LeaseEntry {
5458 fencing_token: grant.fencing_token,
5459 ttl: grant.ttl,
5460 acquired_at: Instant::now(),
5461 }
5462 }
5463
5464 async fn stage_active_grant(
5469 &self,
5470 identity: &AgentIdentity,
5471 expected_previous: Option<FencingToken>,
5472 grant: &LeaseGrant,
5473 ) -> Result<Option<ContinuityRecord>, IdentityRuntimeError> {
5474 let staged = {
5475 let mut entries = self.entries.write().await;
5476 (|| {
5477 if grant.identity != *identity {
5478 return Err(IdentityRuntimeError::Internal(format!(
5479 "provider returned grant for {} while publishing active authority for {identity}",
5480 grant.identity
5481 )));
5482 }
5483 let entry = entries
5484 .get_mut(identity)
5485 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5486 if entry.state != IdentityLifecycleState::Active {
5487 return Err(IdentityRuntimeError::InvalidState {
5488 identity: identity.clone(),
5489 state: entry.state,
5490 operation: "stage_active_grant",
5491 });
5492 }
5493 let current = entry
5494 .lease
5495 .as_ref()
5496 .ok_or_else(|| IdentityRuntimeError::NoActiveLease(identity.clone()))?;
5497 if let Some(expected) = expected_previous
5498 && current.fencing_token != expected
5499 {
5500 return Err(IdentityRuntimeError::Internal(format!(
5501 "active lease for {identity} changed from fencing token {expected} to {} before renewed authority could be staged",
5502 current.fencing_token
5503 )));
5504 }
5505 if let Some(pending) = entry.pending_lease_release.as_ref() {
5506 return Err(IdentityRuntimeError::Internal(format!(
5507 "active identity {identity} already has pending fencing token {} while staging {}",
5508 pending.fencing_token, grant.fencing_token
5509 )));
5510 }
5511
5512 let continuity = entry.continuity.clone();
5513 entry.state = IdentityLifecycleState::Broken;
5518 entry.lease = None;
5519 entry.pending_lease_release = Some(grant.clone());
5520 Ok(continuity)
5521 })()
5522 };
5523
5524 match staged {
5525 Ok(continuity) => Ok(continuity),
5526 Err(error) => {
5527 if let Err(cleanup_error) = self
5528 .release_or_park_untracked_leases(std::slice::from_ref(grant))
5529 .await
5530 {
5531 return Err(IdentityRuntimeError::Internal(format!(
5532 "{error}; exact grant cleanup failed: {cleanup_error}"
5533 )));
5534 }
5535 Err(error)
5536 }
5537 }
5538 }
5539
5540 async fn fail_staged_active_grant(
5541 &self,
5542 identity: &AgentIdentity,
5543 grant: &LeaseGrant,
5544 detail: String,
5545 ) {
5546 let retained_by_entry = {
5547 let mut entries = self.entries.write().await;
5548 entries.get_mut(identity).is_some_and(|entry| {
5549 if entry
5550 .pending_lease_release
5551 .as_ref()
5552 .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
5553 {
5554 entry.state = IdentityLifecycleState::Broken;
5555 entry.lease = None;
5556 true
5557 } else {
5558 false
5559 }
5560 })
5561 };
5562 if !retained_by_entry {
5563 self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5564 .await;
5565 }
5566 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Broken, Some(detail));
5567 self.emit_event(
5568 identity,
5569 IdentityEvent::StateChanged {
5570 identity: identity.clone(),
5571 new_state: IdentityLifecycleState::Broken,
5572 },
5573 )
5574 .await;
5575 }
5576
5577 async fn suspend_bridge_continuity(
5578 &self,
5579 continuity: Option<&ContinuityRecord>,
5580 ) -> Result<(), IdentityRuntimeError> {
5581 let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5582 return Ok(());
5583 };
5584 bridge
5585 .suspend_session_runtime_state(&record.session_id)
5586 .await
5587 .map_err(|error| {
5588 IdentityRuntimeError::Internal(format!(
5589 "suspend bridge session runtime state before authority rotation: {error}"
5590 ))
5591 })
5592 }
5593
5594 async fn resume_bridge_continuity(
5595 &self,
5596 identity: &AgentIdentity,
5597 continuity: Option<&ContinuityRecord>,
5598 grant: &LeaseGrant,
5599 ) -> Result<(), IdentityRuntimeError> {
5600 let (Some(bridge), Some(record)) = (self.bridge.as_ref(), continuity) else {
5601 return Ok(());
5602 };
5603 bridge
5604 .register_session_runtime_state(
5605 &record.session_id,
5606 identity,
5607 record.generation,
5608 record.checkpoint_version,
5609 grant.fencing_token,
5610 )
5611 .await
5612 .map(|_| ())
5613 .map_err(|error| {
5614 IdentityRuntimeError::Internal(format!(
5615 "resume bridge session runtime state after pre-commit renewal failure: {error}"
5616 ))
5617 })
5618 }
5619
5620 async fn break_existing_active_grant(
5621 &self,
5622 identity: &AgentIdentity,
5623 grant: &LeaseGrant,
5624 detail: String,
5625 ) {
5626 let retained = {
5627 let mut entries = self.entries.write().await;
5628 entries.get_mut(identity).is_some_and(|entry| {
5629 if entry
5630 .lease
5631 .as_ref()
5632 .is_some_and(|lease| lease.fencing_token == grant.fencing_token)
5633 {
5634 entry.state = IdentityLifecycleState::Broken;
5635 entry.lease = None;
5636 entry.pending_lease_release = Some(grant.clone());
5637 true
5638 } else {
5639 false
5640 }
5641 })
5642 };
5643 if !retained {
5644 self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5645 .await;
5646 }
5647 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Broken, Some(detail));
5648 self.emit_event(
5649 identity,
5650 IdentityEvent::StateChanged {
5651 identity: identity.clone(),
5652 new_state: IdentityLifecycleState::Broken,
5653 },
5654 )
5655 .await;
5656 }
5657
5658 async fn commit_staged_active_grant(
5659 &self,
5660 identity: &AgentIdentity,
5661 grant: &LeaseGrant,
5662 continuity: Option<ContinuityRecord>,
5663 ) -> Result<(), IdentityRuntimeError> {
5664 let committed = {
5665 let mut entries = self.entries.write().await;
5666 match entries.get_mut(identity) {
5667 Some(entry)
5668 if entry
5669 .pending_lease_release
5670 .as_ref()
5671 .is_some_and(|pending| pending.fencing_token == grant.fencing_token) =>
5672 {
5673 if let Some(record) = continuity {
5674 entry.checkpoint_version = record.checkpoint_version;
5675 entry.continuity = Some(record);
5676 }
5677 entry.pending_lease_release = None;
5678 entry.lease = Some(Self::lease_entry_from_grant(grant));
5679 entry.state = IdentityLifecycleState::Active;
5680 Ok(())
5681 }
5682 Some(entry) => Err(IdentityRuntimeError::Internal(format!(
5683 "staged fencing token {} for {identity} was replaced before commit (state {:?})",
5684 grant.fencing_token, entry.state
5685 ))),
5686 None => Err(IdentityRuntimeError::UnknownIdentity(identity.clone())),
5687 }
5688 };
5689 if let Err(error) = committed {
5690 self.park_unactivated_lease_releases(std::slice::from_ref(grant))
5691 .await;
5692 return Err(error);
5693 }
5694 Ok(())
5695 }
5696
5697 async fn publish_active_grant(
5698 &self,
5699 identity: &AgentIdentity,
5700 expected_previous: Option<FencingToken>,
5701 grant: &LeaseGrant,
5702 ) -> Result<FencingToken, IdentityRuntimeError> {
5703 let mut continuity = self
5704 .stage_active_grant(identity, expected_previous, grant)
5705 .await?;
5706
5707 if let Err(error) = self.suspend_bridge_continuity(continuity.as_ref()).await {
5712 self.fail_staged_active_grant(identity, grant, error.to_string())
5713 .await;
5714 return Err(error);
5715 }
5716
5717 if let Some(record) = continuity.as_mut() {
5718 if let Err(error) = self
5719 .continuity_store
5720 .upsert_continuity_record(record, grant.fencing_token)
5721 .await
5722 {
5723 let runtime_error = IdentityRuntimeError::Store(error);
5724 self.fail_staged_active_grant(
5725 identity,
5726 grant,
5727 format!("active grant continuity publication failed: {runtime_error}"),
5728 )
5729 .await;
5730 return Err(runtime_error);
5731 }
5732 if let Some(bridge) = self.bridge.as_ref() {
5733 match bridge
5734 .register_session_runtime_state(
5735 &record.session_id,
5736 identity,
5737 record.generation,
5738 record.checkpoint_version,
5739 grant.fencing_token,
5740 )
5741 .await
5742 {
5743 Ok(version) => {
5744 record.checkpoint_version = CheckpointVersion::new(
5745 record.checkpoint_version.get().max(version.get()),
5746 );
5747 }
5748 Err(error) => {
5749 let runtime_error = IdentityRuntimeError::Internal(format!(
5750 "bridge refresh session runtime state after active grant publication: {error}"
5751 ));
5752 let _ = self.suspend_bridge_continuity(Some(record)).await;
5753 self.fail_staged_active_grant(identity, grant, runtime_error.to_string())
5754 .await;
5755 return Err(runtime_error);
5756 }
5757 }
5758 }
5759 }
5760
5761 let session_to_reference = continuity.as_ref().map(|record| record.session_id.clone());
5762 if let Err(error) = self
5763 .commit_staged_active_grant(identity, grant, continuity)
5764 .await
5765 {
5766 if let (Some(bridge), Some(session_id)) =
5767 (self.bridge.as_ref(), session_to_reference.as_ref())
5768 {
5769 let _ = bridge.suspend_session_runtime_state(session_id).await;
5770 }
5771 return Err(error);
5772 }
5773 self.lease_renewal_notify.notify_one();
5774 self.emit_event(
5775 identity,
5776 IdentityEvent::LeaseUpdated {
5777 identity: identity.clone(),
5778 fencing_token: grant.fencing_token,
5779 },
5780 )
5781 .await;
5782 Ok(grant.fencing_token)
5783 }
5784
5785 async fn ensure_active_lease(
5786 &self,
5787 identity: &AgentIdentity,
5788 ) -> Result<FencingToken, IdentityRuntimeError> {
5789 let (grant, continuity) = {
5790 let entries = self.entries.read().await;
5791 let entry = entries
5792 .get(identity)
5793 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5794 let lease = match &entry.lease {
5795 Some(lease) if lease.is_healthy() => return Ok(lease.fencing_token),
5796 Some(lease) => lease,
5797 None => return Err(IdentityRuntimeError::NoActiveLease(identity.clone())),
5798 };
5799 (
5800 LeaseGrant {
5801 identity: identity.clone(),
5802 fencing_token: lease.fencing_token,
5803 ttl: lease.ttl,
5804 },
5805 entry.continuity.clone(),
5806 )
5807 };
5808
5809 if let Err(suspend_error) = self.suspend_bridge_continuity(continuity.as_ref()).await {
5814 if let Err(resume_error) = self
5815 .resume_bridge_continuity(identity, continuity.as_ref(), &grant)
5816 .await
5817 {
5818 self.break_existing_active_grant(
5819 identity,
5820 &grant,
5821 format!("{suspend_error}; bridge rollback failed: {resume_error}"),
5822 )
5823 .await;
5824 return Err(IdentityRuntimeError::Internal(format!(
5825 "{suspend_error}; bridge rollback failed: {resume_error}"
5826 )));
5827 }
5828 return Err(suspend_error);
5829 }
5830
5831 let renewed = match self
5832 .lease_provider
5833 .renew_leases(std::slice::from_ref(&grant))
5834 .await
5835 {
5836 Ok(renewed) => renewed,
5837 Err(error) => {
5838 let runtime_error = IdentityRuntimeError::Lease(error);
5839 if let Err(resume_error) = self
5840 .resume_bridge_continuity(identity, continuity.as_ref(), &grant)
5841 .await
5842 {
5843 self.break_existing_active_grant(
5844 identity,
5845 &grant,
5846 format!("{runtime_error}; bridge rollback failed: {resume_error}"),
5847 )
5848 .await;
5849 return Err(IdentityRuntimeError::Internal(format!(
5850 "{runtime_error}; bridge rollback failed: {resume_error}"
5851 )));
5852 }
5853 return Err(runtime_error);
5854 }
5855 };
5856 let renewed_grant = match renewed.get(identity) {
5857 Some(super::types::LeaseRenewResult::Renewed(grant)) => grant.clone(),
5858 Some(super::types::LeaseRenewResult::Lost { .. }) | None => {
5859 self.mark_lease_lost(identity).await?;
5860 return Err(IdentityRuntimeError::LeaseLost(identity.clone()));
5861 }
5862 };
5863
5864 self.publish_active_grant(identity, Some(grant.fencing_token), &renewed_grant)
5865 .await
5866 }
5867
5868 async fn mark_lifecycle_in_progress(
5869 &self,
5870 identity: &AgentIdentity,
5871 state: IdentityLifecycleState,
5872 ) -> Result<IdentityEntry, IdentityRuntimeError> {
5873 let mut entries = self.entries.write().await;
5874 let entry = entries
5875 .get_mut(identity)
5876 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
5877 let snapshot = entry.clone();
5878 entry.state = state;
5879 entry.lease = None;
5880 drop(entries);
5881 self.mark_bootstrap_materialization_started(identity, None);
5882 Ok(snapshot)
5883 }
5884
5885 async fn restore_entry(&self, identity: &AgentIdentity, entry: IdentityEntry) {
5886 let state = entry.state;
5887 self.entries.write().await.insert(identity.clone(), entry);
5888 self.mark_bootstrap_from_lifecycle(identity, state, None);
5889 }
5890
5891 async fn restore_entry_with_grant(
5892 &self,
5893 identity: &AgentIdentity,
5894 mut entry: IdentityEntry,
5895 grant: &LeaseGrant,
5896 ) {
5897 let restore_live_lease = entry.state == IdentityLifecycleState::Active;
5898 if let Some(record) = entry.continuity.as_ref() {
5899 if let Err(err) = self
5900 .continuity_store
5901 .upsert_continuity_record(record, grant.fencing_token)
5902 .await
5903 {
5904 tracing::warn!(
5905 %identity,
5906 error = %err,
5907 "failed to advance restored continuity fencing token after lifecycle failure"
5908 );
5909 entry.state = IdentityLifecycleState::Broken;
5910 } else if let Some(bridge) = self.bridge.as_ref()
5911 && let Err(err) = bridge
5912 .register_session_runtime_state(
5913 &record.session_id,
5914 identity,
5915 record.generation,
5916 record.checkpoint_version,
5917 grant.fencing_token,
5918 )
5919 .await
5920 {
5921 tracing::warn!(
5922 %identity,
5923 error = %err,
5924 "failed to refresh restored session runtime state after lifecycle failure"
5925 );
5926 entry.state = IdentityLifecycleState::Broken;
5927 }
5928 }
5929 let retain_grant = restore_live_lease;
5934 entry.lease = retain_grant.then(|| Self::lease_entry_from_grant(grant));
5935 if !retain_grant {
5936 entry.pending_lease_release = Some(grant.clone());
5937 match self
5938 .lease_provider
5939 .release_leases(std::slice::from_ref(grant))
5940 .await
5941 {
5942 Ok(()) => entry.pending_lease_release = None,
5943 Err(err) => {
5944 tracing::warn!(
5945 %identity,
5946 error = %err,
5947 "failed to release lifecycle rollback lease for non-active identity; exact grant parked for retry"
5948 );
5949 entry.state = IdentityLifecycleState::Broken;
5950 }
5951 }
5952 }
5953 self.restore_entry(identity, entry).await;
5954 if retain_grant {
5955 self.lease_renewal_notify.notify_one();
5956 }
5957 }
5958
5959 pub(crate) async fn reuse_active_restore_state(
5966 &self,
5967 spec: &DurableAgentSpec,
5968 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
5969 self.ensure_active_lease(&spec.identity).await?;
5975
5976 let mut entries = self.entries.write().await;
5977 let entry = entries
5978 .get_mut(&spec.identity)
5979 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(spec.identity.clone()))?;
5980 if entry.state != IdentityLifecycleState::Active {
5981 return Err(IdentityRuntimeError::InvalidState {
5982 identity: spec.identity.clone(),
5983 state: entry.state,
5984 operation: "reuse active restore state",
5985 });
5986 }
5987 if entry.lease.is_none() {
5988 return Err(IdentityRuntimeError::NoActiveLease(spec.identity.clone()));
5989 }
5990 let record = entry.continuity.clone().ok_or_else(|| {
5991 IdentityRuntimeError::Internal(format!(
5992 "active identity {} has no continuity record",
5993 spec.identity
5994 ))
5995 })?;
5996 entry.spec = spec.clone();
5997 Ok(record)
5998 }
5999
6000 async fn restore_broken_entry_and_release_grant(
6009 &self,
6010 identity: &AgentIdentity,
6011 mut entry: IdentityEntry,
6012 grant: &LeaseGrant,
6013 ) {
6014 entry.state = IdentityLifecycleState::Broken;
6015 entry.lease = None;
6016 entry.pending_lease_release = Some(grant.clone());
6020 self.restore_entry(identity, entry).await;
6021 match self
6022 .lease_provider
6023 .release_leases(std::slice::from_ref(grant))
6024 .await
6025 {
6026 Ok(()) => {
6027 let mut entries = self.entries.write().await;
6028 if let Some(entry) = entries.get_mut(identity)
6029 && entry
6030 .pending_lease_release
6031 .as_ref()
6032 .is_some_and(|pending| pending.fencing_token == grant.fencing_token)
6033 {
6034 entry.pending_lease_release = None;
6035 }
6036 }
6037 Err(err) => {
6038 self.mark_bootstrap_from_lifecycle(
6039 identity,
6040 IdentityLifecycleState::Broken,
6041 Some(format!("pending lease release: {err}")),
6042 );
6043 tracing::warn!(
6044 %identity,
6045 error = %err,
6046 "failed to release lease after lifecycle transaction became Broken; exact grant parked for retry"
6047 );
6048 }
6049 }
6050 }
6051
6052 async fn restore_broken_entry_with_fenced_store(
6053 &self,
6054 identity: &AgentIdentity,
6055 mut entry: IdentityEntry,
6056 grant: &LeaseGrant,
6057 ) {
6058 entry.state = IdentityLifecycleState::Broken;
6059 entry.lease = None;
6060 if let Some(record) = entry.continuity.as_ref()
6061 && let Err(err) = self
6062 .continuity_store
6063 .upsert_continuity_record(record, grant.fencing_token)
6064 .await
6065 {
6066 tracing::warn!(
6067 %identity,
6068 error = %err,
6069 "failed to preserve fenced continuity record for broken identity"
6070 );
6071 }
6072 self.restore_broken_entry_and_release_grant(identity, entry, grant)
6073 .await;
6074 }
6075
6076 async fn restore_broken_entry_from_authoritative_continuity(
6082 &self,
6083 identity: &AgentIdentity,
6084 mut entry: IdentityEntry,
6085 grant: &LeaseGrant,
6086 ) {
6087 let authoritative = self
6088 .continuity_store
6089 .resolve_many(std::slice::from_ref(identity))
6090 .await;
6091 entry.continuity = match authoritative {
6092 Ok(resolved) => match resolved.get(identity) {
6093 Some(super::types::ContinuityResolveState::Ready { record }) => {
6094 Some(record.clone())
6095 }
6096 Some(super::types::ContinuityResolveState::Broken { failure }) => {
6097 failure.record.clone()
6098 }
6099 Some(super::types::ContinuityResolveState::Uninitialized) | None => None,
6100 },
6101 Err(error) => {
6102 tracing::warn!(
6103 %identity,
6104 error = %error,
6105 "failed to resolve authoritative continuity after reset rollback failure"
6106 );
6107 None
6108 }
6109 };
6110 entry.checkpoint_version = entry
6111 .continuity
6112 .as_ref()
6113 .map(|record| record.checkpoint_version)
6114 .unwrap_or(CheckpointVersion::new(0));
6115 self.restore_broken_entry_and_release_grant(identity, entry, grant)
6116 .await;
6117 }
6118
6119 async fn mark_rebind_failure_broken(
6120 &self,
6121 identity: &AgentIdentity,
6122 mut entry: IdentityEntry,
6123 grant: &LeaseGrant,
6124 rebound_record: &ContinuityRecord,
6125 ) {
6126 entry.state = IdentityLifecycleState::Broken;
6127 entry.lease = None;
6128 entry.checkpoint_version = rebound_record.checkpoint_version;
6129 entry.continuity = Some(rebound_record.clone());
6130 if let Err(err) = self
6131 .continuity_store
6132 .upsert_continuity_record(rebound_record, grant.fencing_token)
6133 .await
6134 {
6135 tracing::warn!(
6136 %identity,
6137 session_id = %rebound_record.session_id,
6138 error = %err,
6139 "failed to preserve rebound continuity after live respawn rebind failure"
6140 );
6141 }
6142 self.restore_broken_entry_and_release_grant(identity, entry, grant)
6143 .await;
6144 }
6145
6146 async fn restore_entry_after_reset_bridge_failure(
6147 &self,
6148 identity: &AgentIdentity,
6149 expected_attempt: &ContinuityRecord,
6150 entry: IdentityEntry,
6151 grant: &LeaseGrant,
6152 force_broken: bool,
6153 ) -> Option<ContinuityStoreError> {
6154 let rollback_error = self
6155 .continuity_store
6156 .rollback_continuity_record(
6157 expected_attempt,
6158 entry.continuity.as_ref(),
6159 grant.fencing_token,
6160 )
6161 .await
6162 .err();
6163 if rollback_error.is_some() {
6164 self.restore_broken_entry_from_authoritative_continuity(identity, entry, grant)
6165 .await;
6166 } else if force_broken {
6167 self.restore_broken_entry_and_release_grant(identity, entry, grant)
6168 .await;
6169 } else {
6170 self.restore_entry_with_grant(identity, entry, grant).await;
6171 }
6172 rollback_error
6173 }
6174
6175 async fn restore_continuity_after_materialize_failure(
6176 &self,
6177 identity: &AgentIdentity,
6178 previous: Option<&ContinuityRecord>,
6179 grant: &LeaseGrant,
6180 ) -> Option<ContinuityStoreError> {
6181 match previous {
6182 Some(record) => self
6183 .continuity_store
6184 .upsert_continuity_record(record, grant.fencing_token)
6185 .await
6186 .err(),
6187 None => self
6188 .continuity_store
6189 .delete_continuity_record(identity, grant.fencing_token)
6190 .await
6191 .err(),
6192 }
6193 }
6194
6195 async fn unregister_bridge_session_runtime_states(
6196 bridge: &dyn SessionBridge,
6197 session_ids: &[SessionId],
6198 ) -> Option<String> {
6199 let mut errors = Vec::new();
6200 let mut seen = BTreeSet::new();
6201 for session_id in session_ids {
6202 if !seen.insert(session_id.to_string()) {
6203 continue;
6204 }
6205 if let Err(err) = bridge.unregister_session_runtime_state(session_id).await {
6206 errors.push(format!("{session_id}: {err}"));
6207 }
6208 }
6209 (!errors.is_empty()).then(|| errors.join("; "))
6210 }
6211
6212 async fn advance_existing_continuity_fence(
6213 &self,
6214 identity: &AgentIdentity,
6215 entry: &IdentityEntry,
6216 grant: &LeaseGrant,
6217 ) -> Result<(), IdentityRuntimeError> {
6218 if let Some(record) = entry.continuity.as_ref() {
6219 self.continuity_store
6220 .upsert_continuity_record(record, grant.fencing_token)
6221 .await
6222 .map_err(IdentityRuntimeError::Store)?;
6223 }
6224 let _ = identity;
6225 Ok(())
6226 }
6227
6228 async fn refresh_existing_session_runtime_state(
6229 &self,
6230 identity: &AgentIdentity,
6231 record: &ContinuityRecord,
6232 grant: &LeaseGrant,
6233 ) -> Result<CheckpointVersion, IdentityRuntimeError> {
6234 let Some(bridge) = self.bridge.as_ref() else {
6235 return Ok(record.checkpoint_version);
6236 };
6237 bridge
6238 .register_session_runtime_state(
6239 &record.session_id,
6240 identity,
6241 record.generation,
6242 record.checkpoint_version,
6243 grant.fencing_token,
6244 )
6245 .await
6246 .map_err(|err| {
6247 IdentityRuntimeError::Internal(format!(
6248 "bridge refresh session runtime state: {err}"
6249 ))
6250 })
6251 }
6252
6253 pub async fn send(
6267 &self,
6268 identity: &AgentIdentity,
6269 content: &meerkat_core::ContentInput,
6270 ) -> Result<FencingToken, IdentityRuntimeError> {
6271 self.send_with_mode(identity, content, HandlingMode::Queue)
6272 .await
6273 }
6274
6275 pub async fn send_tracked(
6277 self: &Arc<Self>,
6278 identity: &AgentIdentity,
6279 content: &meerkat_core::ContentInput,
6280 ) -> Result<FencingToken, IdentityRuntimeError> {
6281 self.send_with_mode_tracked(identity, content, HandlingMode::Queue)
6282 .await
6283 }
6284
6285 pub async fn send_with_mode(
6291 &self,
6292 identity: &AgentIdentity,
6293 content: &meerkat_core::ContentInput,
6294 handling_mode: HandlingMode,
6295 ) -> Result<FencingToken, IdentityRuntimeError> {
6296 self.send_with_mode_and_interaction(identity, content, handling_mode, None)
6297 .await
6298 }
6299
6300 pub async fn send_admission_tracked(
6310 self: &Arc<Self>,
6311 identity: &AgentIdentity,
6312 expected_alias: Option<&str>,
6313 content: &meerkat_core::ContentInput,
6314 handling_mode: HandlingMode,
6315 interaction_id: Option<&str>,
6316 ) -> Result<SendAdmission, IdentityRuntimeError> {
6317 let runtime = Arc::clone(self);
6318 let identity = identity.clone();
6319 let expected_alias = expected_alias.map(ToString::to_string);
6320 let content = content.clone();
6321 let interaction_id = interaction_id.map(ToString::to_string);
6322 self.run_tracked_foreground(async move {
6323 runtime
6324 .send_with_mode_and_interaction_with_expected_member_alias(
6325 &identity,
6326 expected_alias.as_deref(),
6327 &content,
6328 handling_mode,
6329 interaction_id.as_deref(),
6330 )
6331 .await
6332 .map(|(fencing_token, completion_baseline)| SendAdmission {
6333 fencing_token,
6334 completion_baseline,
6335 })
6336 })
6337 .await
6338 }
6339
6340 pub async fn send_with_mode_tracked(
6342 self: &Arc<Self>,
6343 identity: &AgentIdentity,
6344 content: &meerkat_core::ContentInput,
6345 handling_mode: HandlingMode,
6346 ) -> Result<FencingToken, IdentityRuntimeError> {
6347 self.send_with_mode_and_interaction_tracked(identity, content, handling_mode, None)
6348 .await
6349 }
6350
6351 pub async fn send_with_mode_and_interaction_tracked(
6353 self: &Arc<Self>,
6354 identity: &AgentIdentity,
6355 content: &meerkat_core::ContentInput,
6356 handling_mode: HandlingMode,
6357 interaction_id: Option<&str>,
6358 ) -> Result<FencingToken, IdentityRuntimeError> {
6359 self.send_admission_tracked(identity, None, content, handling_mode, interaction_id)
6360 .await
6361 .map(|admission| admission.fencing_token)
6362 }
6363
6364 pub async fn send_with_mode_and_interaction_member_alias_tracked(
6369 self: &Arc<Self>,
6370 identity: &AgentIdentity,
6371 expected_alias: &str,
6372 content: &meerkat_core::ContentInput,
6373 handling_mode: HandlingMode,
6374 interaction_id: Option<&str>,
6375 ) -> Result<FencingToken, IdentityRuntimeError> {
6376 self.send_admission_tracked(
6377 identity,
6378 Some(expected_alias),
6379 content,
6380 handling_mode,
6381 interaction_id,
6382 )
6383 .await
6384 .map(|admission| admission.fencing_token)
6385 }
6386
6387 pub async fn send_with_mode_and_interaction(
6394 &self,
6395 identity: &AgentIdentity,
6396 content: &meerkat_core::ContentInput,
6397 handling_mode: HandlingMode,
6398 interaction_id: Option<&str>,
6399 ) -> Result<FencingToken, IdentityRuntimeError> {
6400 self.send_with_mode_and_interaction_with_expected_member_alias(
6401 identity,
6402 None,
6403 content,
6404 handling_mode,
6405 interaction_id,
6406 )
6407 .await
6408 .map(|(token, _)| token)
6409 }
6410
6411 async fn send_with_mode_and_interaction_with_expected_member_alias(
6412 &self,
6413 identity: &AgentIdentity,
6414 expected_alias: Option<&str>,
6415 content: &meerkat_core::ContentInput,
6416 handling_mode: HandlingMode,
6417 interaction_id: Option<&str>,
6418 ) -> Result<(FencingToken, CompletionCursor), IdentityRuntimeError> {
6419 let should_materialize = {
6420 let entries = self.entries.read().await;
6421 let entry = entries
6422 .get(identity)
6423 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6424
6425 if entry.spec.addressability == AgentAddressability::InternalOnly {
6427 return Err(IdentityRuntimeError::NotAddressable(NotAddressable {
6428 identity: identity.clone(),
6429 addressability: entry.spec.addressability,
6430 }));
6431 }
6432 entry.state == IdentityLifecycleState::Dormant
6433 || entry.state == IdentityLifecycleState::Uninitialized
6434 };
6435 if should_materialize {
6436 self.materialize_with_expected_member_alias(identity, expected_alias)
6437 .await?;
6438 }
6439 if handling_mode != HandlingMode::Steer {
6444 if let Some(expected_alias) = expected_alias {
6445 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6446 let _lifecycle_guard = lifecycle_lock.lock().await;
6447 self.ensure_expected_member_alias_current(identity, expected_alias)
6448 .await?;
6449 }
6450 self.materialize_reachable_peers(identity).await?;
6451 }
6452
6453 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6454 let _lifecycle_guard = lifecycle_lock.lock().await;
6455 if let Some(expected_alias) = expected_alias {
6456 self.ensure_expected_member_alias_current(identity, expected_alias)
6457 .await?;
6458 }
6459 {
6460 let entries = self.entries.read().await;
6461 let entry = entries
6462 .get(identity)
6463 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6464 if entry.state != IdentityLifecycleState::Active {
6465 return Err(IdentityRuntimeError::InvalidState {
6466 identity: identity.clone(),
6467 state: entry.state,
6468 operation: "send",
6469 });
6470 }
6471 }
6472
6473 let mut token = self.ensure_active_lease(identity).await?;
6474 let completion_baseline = self.rebase_completion_cursor(identity, token);
6481 let (runtime_id, memory_session_key, memory_generation, bridge_interaction_id) = {
6482 let entries = self.entries.read().await;
6483 let entry = entries
6484 .get(identity)
6485 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6486 (
6487 entry
6488 .continuity
6489 .as_ref()
6490 .map(|c| c.agent_runtime_id.clone()),
6491 entry.continuity.as_ref().map(|c| c.session_id.to_string()),
6493 entry.continuity.as_ref().map(|c| c.generation.get()),
6494 interaction_id_for_delivery(&entry.spec, interaction_id),
6495 )
6496 };
6497 let (content_to_deliver, injected_context) = self
6498 .prepare_member_delivery(
6499 identity,
6500 content,
6501 memory_session_key.as_deref(),
6502 memory_generation,
6503 handling_mode == HandlingMode::Steer,
6504 )
6505 .await?;
6506
6507 if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id) {
6509 let delivered_session_id = bridge
6510 .deliver_with_mode_and_context(
6511 rid,
6512 &content_to_deliver,
6513 &injected_context,
6514 handling_mode,
6515 bridge_interaction_id,
6516 )
6517 .await
6518 .map_err(|e| IdentityRuntimeError::Internal(format!("bridge deliver: {e}")))?;
6519 if let Some(rebound_token) = self
6520 .reconcile_delivered_session_locked(identity, delivered_session_id)
6521 .await?
6522 {
6523 token = rebound_token;
6524 }
6525 }
6526
6527 Ok((token, completion_baseline))
6528 }
6529
6530 pub async fn dispatch(
6544 &self,
6545 identity: &AgentIdentity,
6546 input: &DispatchInput,
6547 ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6548 self.dispatch_with_expected_member_alias(identity, None, input)
6549 .await
6550 .map(|outcome| (outcome.admission.fencing_token, outcome.admission.durable))
6551 }
6552
6553 async fn prepare_member_delivery(
6570 &self,
6571 identity: &AgentIdentity,
6572 content: &meerkat_core::ContentInput,
6573 memory_session_key: Option<&str>,
6574 memory_generation: Option<u64>,
6575 steer: bool,
6576 ) -> Result<(meerkat_core::ContentInput, Vec<meerkat_core::ContentInput>), IdentityRuntimeError>
6577 {
6578 if steer {
6579 return Ok((content.clone(), Vec::new()));
6580 }
6581 match self.agent_memory.read().await.clone() {
6582 Some(injector) => {
6583 if let Some(session_key) = memory_session_key {
6584 injector.note_current_session(identity, session_key);
6585 if let Some(generation) = memory_generation {
6586 injector.note_session_generation(identity, session_key, generation);
6587 }
6588 }
6589 let defanged = injector.defang_inbound(identity, content);
6590 let injected_context = injector
6591 .inject_for_turn(identity, memory_session_key, &defanged)
6592 .await
6593 .map_err(|err| {
6594 IdentityRuntimeError::Internal(format!("agent memory recall: {err}"))
6595 })?;
6596 Ok((defanged, injected_context))
6597 }
6598 None => Ok((content.clone(), Vec::new())),
6599 }
6600 }
6601
6602 async fn dispatch_with_expected_member_alias(
6603 &self,
6604 identity: &AgentIdentity,
6605 expected_alias: Option<&str>,
6606 input: &DispatchInput,
6607 ) -> Result<DispatchOutcome, IdentityRuntimeError> {
6608 let should_materialize = {
6609 let entries = self.entries.read().await;
6610 let entry = entries
6611 .get(identity)
6612 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6613 entry.state == IdentityLifecycleState::Dormant
6614 || entry.state == IdentityLifecycleState::Uninitialized
6615 };
6616 if should_materialize {
6617 self.materialize_with_expected_member_alias(identity, expected_alias)
6618 .await?;
6619 }
6620 if let Some(expected_alias) = expected_alias {
6621 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6622 let _lifecycle_guard = lifecycle_lock.lock().await;
6623 self.ensure_expected_member_alias_current(identity, expected_alias)
6624 .await?;
6625 }
6626 self.materialize_reachable_peers(identity).await?;
6627
6628 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6629 let _lifecycle_guard = lifecycle_lock.lock().await;
6630 if let Some(expected_alias) = expected_alias {
6631 self.ensure_expected_member_alias_current(identity, expected_alias)
6632 .await?;
6633 }
6634 {
6635 let entries = self.entries.read().await;
6636 let entry = entries
6637 .get(identity)
6638 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6639 if entry.state != IdentityLifecycleState::Active {
6640 return Err(IdentityRuntimeError::InvalidState {
6641 identity: identity.clone(),
6642 state: entry.state,
6643 operation: "dispatch",
6644 });
6645 }
6646 }
6647
6648 let mut token = self.ensure_active_lease(identity).await?;
6649 let completion_baseline = self.rebase_completion_cursor(identity, token);
6652 let (is_durable, runtime_id, memory_session_key, memory_generation) = {
6653 let entries = self.entries.read().await;
6654 let entry = entries
6655 .get(identity)
6656 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6657 let is_durable = entry.has_runtime_store;
6659
6660 let runtime_id = entry
6661 .continuity
6662 .as_ref()
6663 .map(|c| c.agent_runtime_id.clone());
6664
6665 (
6666 is_durable,
6667 runtime_id,
6668 entry.continuity.as_ref().map(|c| c.session_id.to_string()),
6669 entry.continuity.as_ref().map(|c| c.generation.get()),
6670 )
6671 };
6672
6673 let (content_to_deliver, injected_context) = self
6678 .prepare_member_delivery(
6679 identity,
6680 &input.content,
6681 memory_session_key.as_deref(),
6682 memory_generation,
6683 false,
6684 )
6685 .await?;
6686 let delivery_identity = match (&input.idempotency_key, &input.correlation_id) {
6693 (None, None) => None,
6694 (Some(idempotency_key), Some(correlation_id)) => {
6695 let app_lane = matches!(
6707 input.origin,
6708 super::types::DispatchOrigin::Connector | super::types::DispatchOrigin::System
6709 );
6710 let canonical = if app_lane {
6711 crate::member_comms_id::canonical_correlation_id(correlation_id.as_str())
6712 } else {
6713 std::borrow::Cow::Borrowed(correlation_id.as_str())
6714 };
6715 if canonical.as_ref() != correlation_id.as_str() {
6716 tracing::info!(
6717 identity = %identity,
6718 canonical_correlation = %canonical,
6719 source_len = correlation_id.as_str().len(),
6720 "app-supplied delivery correlation canonicalized to UUIDv5 \
6721 for bridge admission"
6722 );
6723 }
6724 Some(
6725 meerkat_mob::MobDeliveryIdentity::new(
6726 idempotency_key.as_str(),
6727 canonical.as_ref(),
6728 )
6729 .map_err(|error| {
6730 IdentityRuntimeError::InvalidDeliveryIdentity {
6731 identity: identity.clone(),
6732 detail: error.to_string(),
6733 }
6734 })?,
6735 )
6736 }
6737 (Some(_), None) => {
6738 return Err(IdentityRuntimeError::InvalidDeliveryIdentity {
6739 identity: identity.clone(),
6740 detail: "idempotency key without a correlation id (half-pair)".to_string(),
6741 });
6742 }
6743 (None, Some(_)) => {
6744 return Err(IdentityRuntimeError::InvalidDeliveryIdentity {
6745 identity: identity.clone(),
6746 detail: "correlation id without an idempotency key (half-pair)".to_string(),
6747 });
6748 }
6749 };
6750
6751 let mut dispatched_session_id = None;
6755 if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id) {
6756 let mut delivery =
6757 super::bridge::BridgeDelivery::new(content_to_deliver.clone(), HandlingMode::Queue);
6758 delivery.injected_context = injected_context.clone();
6759 delivery.interaction_id = delivery_identity
6760 .as_ref()
6761 .map(|identity| identity.correlation_id.clone());
6762 delivery.delivery_identity = delivery_identity.clone();
6763 let delivered_session_id = bridge
6764 .deliver_admitted(rid, delivery)
6765 .await
6766 .map_err(|e| IdentityRuntimeError::Internal(format!("bridge dispatch: {e}")))?;
6767 dispatched_session_id = Some(delivered_session_id.clone());
6768 if let Some(rebound_token) = self
6769 .reconcile_delivered_session_locked(identity, delivered_session_id)
6770 .await?
6771 {
6772 token = rebound_token;
6773 }
6774 }
6775
6776 Ok(DispatchOutcome {
6777 admission: DispatchAdmission {
6778 fencing_token: token,
6779 durable: is_durable,
6780 completion_baseline,
6781 },
6782 session_id: dispatched_session_id,
6783 })
6784 }
6785
6786 pub async fn dispatch_tracked(
6788 self: &Arc<Self>,
6789 identity: &AgentIdentity,
6790 input: &DispatchInput,
6791 ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6792 let runtime = Arc::clone(self);
6793 let identity = identity.clone();
6794 let input = input.clone();
6795 self.run_tracked_foreground(async move { runtime.dispatch(&identity, &input).await })
6796 .await
6797 }
6798
6799 pub async fn dispatch_member_alias_tracked(
6802 self: &Arc<Self>,
6803 identity: &AgentIdentity,
6804 expected_alias: &str,
6805 input: &DispatchInput,
6806 ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
6807 self.dispatch_admission_tracked(identity, Some(expected_alias), input)
6808 .await
6809 .map(|admission| (admission.fencing_token, admission.durable))
6810 }
6811
6812 pub async fn dispatch_admission_tracked(
6816 self: &Arc<Self>,
6817 identity: &AgentIdentity,
6818 expected_alias: Option<&str>,
6819 input: &DispatchInput,
6820 ) -> Result<DispatchAdmission, IdentityRuntimeError> {
6821 let runtime = Arc::clone(self);
6822 let identity = identity.clone();
6823 let expected_alias = expected_alias.map(ToString::to_string);
6824 let input = input.clone();
6825 self.run_tracked_foreground(async move {
6826 runtime
6827 .dispatch_with_expected_member_alias(&identity, expected_alias.as_deref(), &input)
6828 .await
6829 .map(|outcome| outcome.admission)
6830 })
6831 .await
6832 }
6833
6834 pub(crate) async fn dispatch_member_alias_with_session_tracked(
6837 self: &Arc<Self>,
6838 identity: &AgentIdentity,
6839 expected_alias: &str,
6840 input: &DispatchInput,
6841 ) -> Result<Option<SessionId>, IdentityRuntimeError> {
6842 let runtime = Arc::clone(self);
6843 let identity = identity.clone();
6844 let expected_alias = expected_alias.to_string();
6845 let input = input.clone();
6846 self.run_tracked_foreground(async move {
6847 runtime
6848 .dispatch_with_expected_member_alias(&identity, Some(&expected_alias), &input)
6849 .await
6850 .map(|outcome| outcome.session_id)
6851 })
6852 .await
6853 }
6854
6855 pub async fn status(
6861 &self,
6862 identity: &AgentIdentity,
6863 ) -> Result<IdentityStatus, IdentityRuntimeError> {
6864 let entries = self.entries.read().await;
6865 let entry = entries
6866 .get(identity)
6867 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
6868
6869 let lease_info = entry.lease.as_ref().map(|l| LeaseInfo {
6870 fencing_token: l.fencing_token,
6871 ttl_remaining: l.ttl_remaining(),
6872 healthy: l.is_healthy(),
6873 });
6874
6875 let continuity_health = Some(ContinuityHealth {
6876 store_reachable: true, durability_policy: self.durability_policy.clone(),
6878 last_checkpoint_version: if entry.checkpoint_version.get() > 0 {
6879 Some(entry.checkpoint_version)
6880 } else {
6881 None
6882 },
6883 });
6884
6885 Ok(IdentityStatus {
6886 identity: identity.clone(),
6887 state: entry.state,
6888 agent_runtime_id: entry
6889 .continuity
6890 .as_ref()
6891 .map(|c| c.agent_runtime_id.clone()),
6892 session_id: entry.continuity.as_ref().map(|c| c.session_id.clone()),
6893 profile: Some(entry.spec.profile.clone()),
6894 runtime_mode: entry.spec.runtime_mode_override,
6895 addressability: entry.spec.addressability,
6896 display_name: entry.spec.display_name.clone(),
6897 labels: entry.spec.labels.clone(),
6898 generation: entry.continuity.as_ref().map(|c| c.generation),
6899 checkpoint_version: if entry.checkpoint_version.get() > 0 {
6900 Some(entry.checkpoint_version)
6901 } else {
6902 None
6903 },
6904 lease: lease_info,
6905 continuity_health,
6906 continuity_unrecoverable: entry.continuity_unrecoverable.clone(),
6907 })
6908 }
6909
6910 pub async fn statuses(&self) -> Vec<IdentityStatus> {
6913 let identities = self
6914 .entries
6915 .read()
6916 .await
6917 .keys()
6918 .cloned()
6919 .collect::<Vec<_>>();
6920 let mut statuses = Vec::with_capacity(identities.len());
6921 for identity in identities {
6922 if let Ok(status) = self.status(&identity).await {
6923 statuses.push(status);
6924 }
6925 }
6926 statuses
6927 }
6928
6929 pub async fn retire(
6935 &self,
6936 identity: &AgentIdentity,
6937 ) -> Result<FencingToken, IdentityRuntimeError> {
6938 self.retire_with_expected_member_alias(identity, None).await
6939 }
6940
6941 async fn retire_with_expected_member_alias(
6942 &self,
6943 identity: &AgentIdentity,
6944 expected_alias: Option<&str>,
6945 ) -> Result<FencingToken, IdentityRuntimeError> {
6946 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
6947 let _lifecycle_guard = lifecycle_lock.lock().await;
6948 if let Some(expected_alias) = expected_alias {
6949 self.ensure_expected_member_alias_current(identity, expected_alias)
6950 .await?;
6951 }
6952 self.retire_locked(identity).await
6953 }
6954
6955 async fn retire_locked(
6956 &self,
6957 identity: &AgentIdentity,
6958 ) -> Result<FencingToken, IdentityRuntimeError> {
6959 self.ensure_active_lease(identity).await?;
6960 let registered_entry = self
6961 .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Retiring)
6962 .await?;
6963 let _previous_token = match Self::check_lease(®istered_entry) {
6964 Ok(token) => token,
6965 Err(err) => {
6966 self.restore_entry(identity, registered_entry).await;
6967 return Err(err);
6968 }
6969 };
6970 let runtime_id = registered_entry
6971 .continuity
6972 .as_ref()
6973 .map(|c| c.agent_runtime_id.clone());
6974 let session_id = registered_entry
6975 .continuity
6976 .as_ref()
6977 .map(|c| c.session_id.clone());
6978
6979 let acquire_result = match self
6980 .lease_provider
6981 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
6982 .await
6983 {
6984 Ok(result) => result,
6985 Err(err) => {
6986 self.restore_entry(identity, registered_entry).await;
6987 return Err(IdentityRuntimeError::Lease(err));
6988 }
6989 };
6990
6991 let grant = match acquire_result.get(identity) {
6992 Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
6993 _ => {
6994 self.restore_entry(identity, registered_entry).await;
6995 return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
6996 }
6997 };
6998 if let Err(err) = self
6999 .advance_existing_continuity_fence(identity, ®istered_entry, &grant)
7000 .await
7001 {
7002 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7003 .await;
7004 return Err(err);
7005 }
7006 if let Some(record) = registered_entry.continuity.as_ref()
7012 && let Err(error) = self
7013 .refresh_existing_session_runtime_state(identity, record, &grant)
7014 .await
7015 {
7016 self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
7017 .await;
7018 return Err(IdentityRuntimeError::Internal(format!(
7019 "bridge refresh session authority before retire: {error}"
7020 )));
7021 }
7022
7023 if let Some(injector) = self.agent_memory.read().await.clone() {
7027 if let Some(session_id) = session_id.as_ref() {
7028 injector
7029 .distill_before_rotation(
7030 identity,
7031 &session_id.to_string(),
7032 crate::memory::distiller::DistillCause::Retire,
7033 )
7034 .await;
7035 }
7036 injector
7039 .note_identity_retired(
7040 identity,
7041 session_id
7042 .as_ref()
7043 .map(std::string::ToString::to_string)
7044 .as_deref(),
7045 "retire",
7046 )
7047 .await;
7048 }
7049
7050 if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id)
7052 && let Err(err) = bridge.retire_member(rid).await
7053 {
7054 self.restore_entry_with_grant(identity, registered_entry, &grant)
7055 .await;
7056 return Err(IdentityRuntimeError::Internal(format!(
7057 "bridge retire: {err}"
7058 )));
7059 }
7060 if let (Some(bridge), Some(session_id)) = (&self.bridge, &session_id)
7061 && let Err(err) = bridge.unregister_session_runtime_state(session_id).await
7062 {
7063 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7064 .await;
7065 return Err(IdentityRuntimeError::Internal(format!(
7066 "bridge unregister retired session: {err}"
7067 )));
7068 }
7069
7070 let release_result = self
7071 .lease_provider
7072 .release_leases(std::slice::from_ref(&grant))
7073 .await;
7074 let (state, bootstrap_error) = match &release_result {
7075 Ok(()) => (IdentityLifecycleState::Retiring, None),
7076 Err(error) => (
7077 IdentityLifecycleState::Broken,
7078 Some(format!("lease release after retire: {error}")),
7079 ),
7080 };
7081 {
7082 let mut entries = self.entries.write().await;
7083 let entry = entries
7084 .get_mut(identity)
7085 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7086 entry.state = state;
7087 entry.lease = None;
7088 entry.pending_lease_release = release_result.as_ref().err().map(|_| grant.clone());
7089 }
7090 self.mark_bootstrap_from_lifecycle(identity, state, bootstrap_error);
7091 release_result.map_err(IdentityRuntimeError::Lease)?;
7092 Ok(grant.fencing_token)
7093 }
7094
7095 pub async fn respawn(
7106 &self,
7107 identity: &AgentIdentity,
7108 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7109 self.respawn_with_expected_member_alias(identity, None)
7110 .await
7111 }
7112
7113 async fn respawn_with_expected_member_alias(
7114 &self,
7115 identity: &AgentIdentity,
7116 expected_alias: Option<&str>,
7117 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7118 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7119 let _lifecycle_guard = lifecycle_lock.lock().await;
7120 if let Some(expected_alias) = expected_alias {
7121 self.ensure_expected_member_alias_current(identity, expected_alias)
7122 .await?;
7123 }
7124 self.respawn_locked(identity).await
7125 }
7126
7127 async fn respawn_locked(
7130 &self,
7131 identity: &AgentIdentity,
7132 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7133 let registered_entry = self
7134 .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7135 .await?;
7136
7137 let acquire_result = match self
7139 .lease_provider
7140 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7141 .await
7142 {
7143 Ok(result) => result,
7144 Err(err) => {
7145 self.restore_entry(identity, registered_entry).await;
7146 return Err(IdentityRuntimeError::Lease(err));
7147 }
7148 };
7149
7150 let grant = match acquire_result.get(identity) {
7151 Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7152 _ => {
7153 self.restore_entry(identity, registered_entry).await;
7154 return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7155 }
7156 };
7157
7158 if let Err(err) = self
7159 .advance_existing_continuity_fence(identity, ®istered_entry, &grant)
7160 .await
7161 {
7162 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7163 .await;
7164 return Err(err);
7165 }
7166
7167 let resolved = match self
7169 .continuity_store
7170 .resolve_many(std::slice::from_ref(identity))
7171 .await
7172 {
7173 Ok(resolved) => resolved,
7174 Err(err) => {
7175 self.restore_entry_with_grant(identity, registered_entry.clone(), &grant)
7176 .await;
7177 return Err(IdentityRuntimeError::Store(err));
7178 }
7179 };
7180
7181 let record = match resolved.get(identity) {
7182 Some(super::types::ContinuityResolveState::Ready { record }) => record.clone(),
7183 Some(super::types::ContinuityResolveState::Broken { failure }) => {
7184 self.restore_entry_with_grant(identity, registered_entry, &grant)
7185 .await;
7186 return Err(IdentityRuntimeError::Internal(format!(
7187 "broken continuity for {identity}: {}",
7188 failure.detail
7189 )));
7190 }
7191 Some(super::types::ContinuityResolveState::Uninitialized) => {
7192 self.restore_entry_with_grant(identity, registered_entry, &grant)
7193 .await;
7194 return Err(IdentityRuntimeError::Internal(format!(
7195 "cannot respawn uninitialized identity {identity}"
7196 )));
7197 }
7198 None => {
7199 self.restore_entry_with_grant(identity, registered_entry, &grant)
7200 .await;
7201 return Err(IdentityRuntimeError::Store(
7202 ContinuityStoreError::NotFound {
7203 identity: identity.clone(),
7204 },
7205 ));
7206 }
7207 };
7208
7209 if let Some(injector) = self.agent_memory.read().await.clone() {
7214 let session_key = record.session_id.to_string();
7215 injector.note_session_generation(identity, &session_key, record.generation.get());
7216 injector
7217 .distill_before_rotation(
7218 identity,
7219 &session_key,
7220 crate::memory::distiller::DistillCause::Respawn,
7221 )
7222 .await;
7223 }
7224
7225 let effective_checkpoint_version = match self
7226 .refresh_existing_session_runtime_state(identity, &record, &grant)
7227 .await
7228 {
7229 Ok(version) => version,
7230 Err(err) => {
7231 self.restore_entry_with_grant(identity, registered_entry, &grant)
7232 .await;
7233 return Err(err);
7234 }
7235 };
7236 let mut record = record;
7237 record.checkpoint_version = effective_checkpoint_version;
7238
7239 let mut entries = self.entries.write().await;
7241 let Some(entry) = entries.get_mut(identity) else {
7242 drop(entries);
7243 if let Err(err) = self
7244 .release_or_park_untracked_leases(std::slice::from_ref(&grant))
7245 .await
7246 {
7247 tracing::warn!(
7248 %identity,
7249 error = %err,
7250 "failed to release lease after respawn entry disappeared; exact grant parked for retry"
7251 );
7252 }
7253 return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7254 };
7255 entry.continuity = Some(record.clone());
7256 entry.lease = Some(LeaseEntry {
7257 fencing_token: grant.fencing_token,
7258 ttl: grant.ttl,
7259 acquired_at: Instant::now(),
7260 });
7261 entry.state = IdentityLifecycleState::Active;
7262 entry.checkpoint_version = record.checkpoint_version;
7263 drop(entries);
7264 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
7265
7266 Ok(record)
7267 }
7268
7269 pub async fn rebind_session_after_live_respawn(
7273 &self,
7274 identity: &AgentIdentity,
7275 session_id: SessionId,
7276 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7277 self.rebind_session_after_live_respawn_with_expected_member_alias(
7278 identity, None, session_id,
7279 )
7280 .await
7281 }
7282
7283 async fn rebind_session_after_live_respawn_with_expected_member_alias(
7284 &self,
7285 identity: &AgentIdentity,
7286 expected_alias: Option<&str>,
7287 session_id: SessionId,
7288 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7289 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7290 let _lifecycle_guard = lifecycle_lock.lock().await;
7291 if let Some(expected_alias) = expected_alias {
7292 self.ensure_expected_member_alias_current(identity, expected_alias)
7293 .await?;
7294 }
7295 self.rebind_session_after_live_respawn_locked(identity, session_id)
7296 .await
7297 }
7298
7299 async fn reconcile_delivered_session_locked(
7300 &self,
7301 identity: &AgentIdentity,
7302 delivered_session_id: SessionId,
7303 ) -> Result<Option<FencingToken>, IdentityRuntimeError> {
7304 let current_session_id = {
7305 let entries = self.entries.read().await;
7306 let entry = entries
7307 .get(identity)
7308 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7309 entry
7310 .continuity
7311 .as_ref()
7312 .map(|record| record.session_id.clone())
7313 };
7314
7315 let Some(current_session_id) = current_session_id else {
7316 return Ok(None);
7317 };
7318 if current_session_id == delivered_session_id {
7319 return Ok(None);
7320 }
7321
7322 tracing::warn!(
7323 %identity,
7324 old_session_id = %current_session_id,
7325 new_session_id = %delivered_session_id,
7326 "identity bridge delivery returned a rotated session; rebinding continuity"
7327 );
7328 self.rebind_session_after_live_respawn_locked(identity, delivered_session_id)
7329 .await?;
7330
7331 let entries = self.entries.read().await;
7332 let entry = entries
7333 .get(identity)
7334 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
7335 Ok(entry.lease.as_ref().map(|lease| lease.fencing_token))
7336 }
7337
7338 async fn rebind_session_after_live_respawn_locked(
7339 &self,
7340 identity: &AgentIdentity,
7341 session_id: SessionId,
7342 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7343 let registered_entry = self
7344 .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7345 .await?;
7346
7347 let acquire_result = match self
7348 .lease_provider
7349 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7350 .await
7351 {
7352 Ok(result) => result,
7353 Err(err) => {
7354 self.restore_entry(identity, registered_entry).await;
7355 return Err(IdentityRuntimeError::Lease(err));
7356 }
7357 };
7358
7359 let grant = match acquire_result.get(identity) {
7360 Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7361 _ => {
7362 self.restore_entry(identity, registered_entry).await;
7363 return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7364 }
7365 };
7366
7367 let mut record = match registered_entry.continuity.as_ref() {
7368 Some(record) => record.clone(),
7369 None => {
7370 self.restore_entry_with_grant(identity, registered_entry, &grant)
7371 .await;
7372 return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7373 }
7374 };
7375 if let Err(err) = self
7376 .advance_existing_continuity_fence(identity, ®istered_entry, &grant)
7377 .await
7378 {
7379 if let Some(bridge) = self.bridge.as_ref()
7380 && let Err(unregister_err) =
7381 bridge.unregister_session_runtime_state(&session_id).await
7382 {
7383 tracing::warn!(
7384 %identity,
7385 session_id = %session_id,
7386 error = %unregister_err,
7387 "failed to unregister rebound session after continuity fence failure"
7388 );
7389 }
7390 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7391 .await;
7392 return Err(err);
7393 }
7394 let previous_session_id = record.session_id.clone();
7395 record.session_id = session_id;
7396
7397 if let Err(err) = self
7398 .continuity_store
7399 .upsert_continuity_record(&record, grant.fencing_token)
7400 .await
7401 {
7402 if let Some(bridge) = self.bridge.as_ref()
7403 && let Err(unregister_err) = bridge
7404 .unregister_session_runtime_state(&record.session_id)
7405 .await
7406 {
7407 tracing::warn!(
7408 %identity,
7409 session_id = %record.session_id,
7410 error = %unregister_err,
7411 "failed to unregister rebound session after continuity upsert failure"
7412 );
7413 }
7414 self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7415 .await;
7416 return Err(IdentityRuntimeError::Store(err));
7417 }
7418
7419 let effective = match self
7426 .continuity_store
7427 .resolve_many(std::slice::from_ref(identity))
7428 .await
7429 {
7430 Ok(effective) => effective,
7431 Err(error) => {
7432 self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7433 .await;
7434 return Err(IdentityRuntimeError::Store(error));
7435 }
7436 };
7437 match effective.get(identity) {
7438 Some(super::types::ContinuityResolveState::Ready {
7439 record: effective_record,
7440 }) if effective_record.session_id == record.session_id
7441 && effective_record.generation == record.generation =>
7442 {
7443 record = effective_record.clone();
7444 }
7445 other => {
7446 self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7447 .await;
7448 return Err(IdentityRuntimeError::Internal(format!(
7449 "rebind continuity read-back did not return the committed session/generation: {other:?}"
7450 )));
7451 }
7452 }
7453
7454 if let Some(bridge) = self.bridge.as_ref() {
7455 match bridge
7456 .register_session_runtime_state(
7457 &record.session_id,
7458 identity,
7459 record.generation,
7460 record.checkpoint_version,
7461 grant.fencing_token,
7462 )
7463 .await
7464 {
7465 Ok(version) => {
7466 record.checkpoint_version =
7467 CheckpointVersion::new(record.checkpoint_version.get().max(version.get()));
7468 }
7469 Err(err) => {
7470 if let Err(unregister_err) = bridge
7471 .unregister_session_runtime_state(&record.session_id)
7472 .await
7473 {
7474 tracing::warn!(
7475 %identity,
7476 session_id = %record.session_id,
7477 error = %unregister_err,
7478 "failed to unregister rebound session after bridge register failure"
7479 );
7480 }
7481 self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7482 .await;
7483 return Err(IdentityRuntimeError::Internal(format!(
7484 "bridge rebind respawned session runtime state: {err}"
7485 )));
7486 }
7487 }
7488 if previous_session_id != record.session_id
7489 && let Err(err) = bridge
7490 .unregister_session_runtime_state(&previous_session_id)
7491 .await
7492 {
7493 tracing::warn!(
7494 %identity,
7495 session_id = %previous_session_id,
7496 error = %err,
7497 "failed to unregister previous session after live respawn rebind"
7498 );
7499 }
7500 }
7501
7502 if let Err(err) = self
7503 .continuity_store
7504 .upsert_continuity_record(&record, grant.fencing_token)
7505 .await
7506 {
7507 if let Some(bridge) = self.bridge.as_ref()
7508 && let Err(unregister_err) = bridge
7509 .unregister_session_runtime_state(&record.session_id)
7510 .await
7511 {
7512 tracing::warn!(
7513 %identity,
7514 session_id = %record.session_id,
7515 error = %unregister_err,
7516 "failed to unregister rebound session after final continuity upsert failure"
7517 );
7518 }
7519 self.mark_rebind_failure_broken(identity, registered_entry, &grant, &record)
7520 .await;
7521 return Err(IdentityRuntimeError::Store(err));
7522 }
7523
7524 self.register(
7525 registered_entry.spec,
7526 IdentityLifecycleState::Active,
7527 Some(record.clone()),
7528 Some(grant),
7529 )
7530 .await;
7531 Ok(record)
7532 }
7533
7534 pub async fn reset(
7545 &self,
7546 identity: &AgentIdentity,
7547 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7548 self.reset_with_expected_member_alias(identity, None).await
7549 }
7550
7551 async fn reset_with_expected_member_alias(
7552 &self,
7553 identity: &AgentIdentity,
7554 expected_alias: Option<&str>,
7555 ) -> Result<ContinuityRecord, IdentityRuntimeError> {
7556 let mut foreground_shutdown = self.foreground_cancel.subscribe();
7557 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
7558 let _lifecycle_guard = lifecycle_lock.lock().await;
7559 if let Some(expected_alias) = expected_alias {
7560 self.ensure_expected_member_alias_current(identity, expected_alias)
7561 .await?;
7562 }
7563 self.adopt_current_roster_spec_for_reset(identity).await;
7566 let registered_entry = self
7567 .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Suspended)
7568 .await?;
7569
7570 let acquire_result = match self
7572 .lease_provider
7573 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
7574 .await
7575 {
7576 Ok(result) => result,
7577 Err(err) => {
7578 self.restore_entry(identity, registered_entry).await;
7579 return Err(IdentityRuntimeError::Lease(err));
7580 }
7581 };
7582
7583 let grant = match acquire_result.get(identity) {
7584 Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
7585 _ => {
7586 self.restore_entry(identity, registered_entry).await;
7587 return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
7588 }
7589 };
7590 if let Err(err) = self
7591 .advance_existing_continuity_fence(identity, ®istered_entry, &grant)
7592 .await
7593 {
7594 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7595 .await;
7596 return Err(err);
7597 }
7598
7599 let resolved = match self
7601 .continuity_store
7602 .resolve_many(std::slice::from_ref(identity))
7603 .await
7604 {
7605 Ok(resolved) => resolved,
7606 Err(err) => {
7607 self.restore_entry_with_grant(identity, registered_entry.clone(), &grant)
7608 .await;
7609 return Err(IdentityRuntimeError::Store(err));
7610 }
7611 };
7612
7613 let current_gen = match resolved.get(identity) {
7614 Some(super::types::ContinuityResolveState::Ready { record }) => record.generation,
7615 Some(super::types::ContinuityResolveState::Uninitialized) => {
7616 ContinuityGeneration::new(0)
7617 }
7618 _ => ContinuityGeneration::new(0),
7619 };
7620
7621 let new_gen = ContinuityGeneration::new(current_gen.get() + 1);
7623 let new_session_id = meerkat_core::types::SessionId::new();
7624 let new_runtime_id = AgentRuntimeId::parse(&format!("rt:{identity}:{}", new_gen.get()))
7625 .map_err(|e| {
7626 IdentityRuntimeError::Internal(format!("failed to mint runtime id: {e}"))
7627 })?;
7628
7629 let new_record = ContinuityRecord {
7630 identity: identity.clone(),
7631 agent_runtime_id: new_runtime_id,
7632 session_id: new_session_id,
7633 generation: new_gen,
7634 checkpoint_version: CheckpointVersion::new(0),
7635 };
7636 let spec = registered_entry.spec.clone();
7637 let mut draft = super::types::AgentBuildDraft {
7638 model: None,
7639 system_prompt: None,
7640 additional_instructions: spec.additional_instructions.clone(),
7641 labels: spec.labels.clone(),
7642 app_context: spec.context.clone(),
7643 external_tools: Vec::new(),
7644 local_external_tools: Default::default(),
7645 provider_params: None,
7646 };
7647 if self.bridge.is_some() {
7648 let active_peers = self.entries.read().await.keys().cloned().collect();
7649 let managed_edges = self.desired_peer_edges.read().await.clone();
7650 let build_context = AgentBuildContext {
7651 identity: identity.clone(),
7652 active_peers,
7653 managed_edges,
7654 runtime_services: self.runtime_services(),
7655 };
7656 if let Some(customizer) = self.customizer.read().await.clone() {
7657 let customize = customizer.customize_build(&build_context, &spec, &mut draft);
7658 tokio::pin!(customize);
7659 let customize_result = if *foreground_shutdown.borrow() {
7660 None
7661 } else {
7662 tokio::select! {
7663 result = &mut customize => Some(result),
7664 changed = foreground_shutdown.changed() => {
7665 if changed.is_ok() && *foreground_shutdown.borrow() {
7666 None
7667 } else {
7668 Some(customize.await)
7669 }
7670 }
7671 }
7672 };
7673 let Some(customize_result) = customize_result else {
7674 self.restore_entry_with_grant(identity, registered_entry, &grant)
7675 .await;
7676 return Err(IdentityRuntimeError::Internal(
7677 "identity reset cancelled during shutdown before session installation"
7678 .to_string(),
7679 ));
7680 };
7681 if let Err(err) = customize_result {
7682 self.restore_entry_with_grant(identity, registered_entry, &grant)
7683 .await;
7684 return Err(IdentityRuntimeError::Internal(format!(
7685 "customizer after reset: {err}"
7686 )));
7687 }
7688 }
7689 }
7690
7691 if let Some(bridge) = &self.bridge {
7693 if let Err(err) = self
7694 .continuity_store
7695 .upsert_continuity_record(&new_record, grant.fencing_token)
7696 .await
7697 {
7698 self.restore_entry_with_grant(identity, registered_entry, &grant)
7699 .await;
7700 return Err(IdentityRuntimeError::Store(err));
7701 }
7702 let provisional_record = new_record.clone();
7703
7704 let old_runtime_id = registered_entry
7705 .continuity
7706 .as_ref()
7707 .map(|c| c.agent_runtime_id.clone());
7708 let old_session_id = registered_entry
7709 .continuity
7710 .as_ref()
7711 .map(|c| c.session_id.clone());
7712
7713 let session_id = bridge
7714 .create_session(
7715 identity,
7716 &new_record.agent_runtime_id,
7717 &spec,
7718 &draft,
7719 &new_record.session_id,
7720 )
7721 .await
7722 .map_err(|e| {
7723 IdentityRuntimeError::Internal(format!(
7724 "bridge create_session after reset: {e}"
7725 ))
7726 });
7727 let session_id = match session_id {
7728 Ok(session_id) => session_id,
7729 Err(err) => {
7730 let cleanup_error = bridge
7731 .retire_member(&new_record.agent_runtime_id)
7732 .await
7733 .err();
7734 let delete_error = self
7735 .restore_entry_after_reset_bridge_failure(
7736 identity,
7737 &provisional_record,
7738 registered_entry.clone(),
7739 &grant,
7740 cleanup_error.is_some(),
7741 )
7742 .await;
7743 if cleanup_error.is_some() || delete_error.is_some() {
7744 return Err(IdentityRuntimeError::Internal(format!(
7745 "{err}{}{}",
7746 cleanup_error
7747 .as_ref()
7748 .map(|e| format!("; cleanup retire failed: {e}"))
7749 .unwrap_or_default(),
7750 delete_error
7751 .as_ref()
7752 .map(|e| format!("; tentative continuity cleanup failed: {e}"))
7753 .unwrap_or_default()
7754 )));
7755 }
7756 return Err(err);
7757 }
7758 };
7759 let mut new_record = new_record;
7761 new_record.session_id = session_id;
7762 tracing::debug!(
7763 identity = %identity,
7764 runtime_id = %new_record.agent_runtime_id,
7765 session_id = %new_record.session_id,
7766 "reset bridge create_session completed",
7767 );
7768
7769 if let Err(err) = self
7770 .continuity_store
7771 .upsert_continuity_record(&new_record, grant.fencing_token)
7772 .await
7773 {
7774 let unregister_error = Self::unregister_bridge_session_runtime_states(
7775 bridge.as_ref(),
7776 std::slice::from_ref(&new_record.session_id),
7777 )
7778 .await;
7779 let cleanup_error = bridge
7780 .retire_member(&new_record.agent_runtime_id)
7781 .await
7782 .err();
7783 let delete_error = self
7784 .restore_entry_after_reset_bridge_failure(
7785 identity,
7786 &provisional_record,
7787 registered_entry.clone(),
7788 &grant,
7789 unregister_error.is_some() || cleanup_error.is_some(),
7790 )
7791 .await;
7792 if unregister_error.is_some() || cleanup_error.is_some() || delete_error.is_some() {
7793 return Err(IdentityRuntimeError::Internal(format!(
7794 "continuity upsert actual session after reset: {err}{}{}{}",
7795 unregister_error
7796 .as_ref()
7797 .map(|e| format!("; unregister session failed: {e}"))
7798 .unwrap_or_default(),
7799 cleanup_error
7800 .as_ref()
7801 .map(|e| format!("; cleanup retire failed: {e}"))
7802 .unwrap_or_default(),
7803 delete_error
7804 .as_ref()
7805 .map(|e| format!("; tentative continuity cleanup failed: {e}"))
7806 .unwrap_or_default(),
7807 )));
7808 }
7809 return Err(IdentityRuntimeError::Store(err));
7810 }
7811
7812 let register_result = bridge
7813 .register_session_runtime_state(
7814 &new_record.session_id,
7815 identity,
7816 new_record.generation,
7817 new_record.checkpoint_version,
7818 grant.fencing_token,
7819 )
7820 .await;
7821 let effective_checkpoint_version = match register_result {
7822 Ok(version) => version,
7823 Err(err) => {
7824 let unregister_error = Self::unregister_bridge_session_runtime_states(
7825 bridge.as_ref(),
7826 std::slice::from_ref(&new_record.session_id),
7827 )
7828 .await;
7829 let cleanup_error = bridge
7830 .retire_member(&new_record.agent_runtime_id)
7831 .await
7832 .err();
7833 let mut detail =
7834 format!("bridge register actual session runtime state after reset: {err}");
7835 if let Some(unregister_error) = unregister_error.as_ref() {
7836 detail
7837 .push_str(&format!("; unregister session failed: {unregister_error}"));
7838 }
7839 if let Some(cleanup_error) = cleanup_error.as_ref() {
7840 detail.push_str(&format!("; cleanup retire failed: {cleanup_error}"));
7841 }
7842 let delete_error = self
7843 .restore_entry_after_reset_bridge_failure(
7844 identity,
7845 &new_record,
7846 registered_entry.clone(),
7847 &grant,
7848 unregister_error.is_some() || cleanup_error.is_some(),
7849 )
7850 .await;
7851 if let Some(delete_error) = delete_error {
7852 return Err(IdentityRuntimeError::Internal(format!(
7853 "{detail}; tentative continuity cleanup failed: {delete_error}"
7854 )));
7855 }
7856 return Err(IdentityRuntimeError::Internal(detail));
7857 }
7858 };
7859 new_record.checkpoint_version = effective_checkpoint_version;
7860 tracing::debug!(
7861 identity = %identity,
7862 runtime_id = %new_record.agent_runtime_id,
7863 session_id = %new_record.session_id,
7864 checkpoint_version = new_record.checkpoint_version.get(),
7865 "reset bridge session runtime state registered",
7866 );
7867
7868 if let Err(err) = self
7869 .continuity_store
7870 .upsert_continuity_record(&new_record, grant.fencing_token)
7871 .await
7872 {
7873 tracing::warn!(
7874 identity = %identity,
7875 runtime_id = %new_record.agent_runtime_id,
7876 session_id = %new_record.session_id,
7877 error = %err,
7878 "reset final continuity upsert failed after bridge materialization; rolling back new generation",
7879 );
7880 let unregister_error = bridge
7881 .unregister_session_runtime_state(&new_record.session_id)
7882 .await
7883 .err();
7884 let cleanup_error = bridge
7885 .retire_member(&new_record.agent_runtime_id)
7886 .await
7887 .err();
7888 let rollback_error = self
7889 .continuity_store
7890 .rollback_continuity_record(
7891 &new_record,
7892 registered_entry.continuity.as_ref(),
7893 grant.fencing_token,
7894 )
7895 .await
7896 .err();
7897 if rollback_error.is_some() {
7898 self.restore_broken_entry_from_authoritative_continuity(
7899 identity,
7900 registered_entry,
7901 &grant,
7902 )
7903 .await;
7904 } else {
7905 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
7906 .await;
7907 }
7908 if unregister_error.is_some() || cleanup_error.is_some() || rollback_error.is_some()
7909 {
7910 return Err(IdentityRuntimeError::Internal(format!(
7911 "continuity upsert after reset: {err}{}{}{}",
7912 unregister_error
7913 .as_ref()
7914 .map(|e| format!("; unregister session failed: {e}"))
7915 .unwrap_or_default(),
7916 cleanup_error
7917 .as_ref()
7918 .map(|e| format!("; cleanup retire failed: {e}"))
7919 .unwrap_or_default(),
7920 rollback_error
7921 .as_ref()
7922 .map(|e| format!("; continuity rollback failed: {e}"))
7923 .unwrap_or_default()
7924 )));
7925 }
7926 return Err(IdentityRuntimeError::Store(err));
7927 }
7928
7929 let mut entries = self.entries.write().await;
7931 let Some(entry) = entries.get_mut(identity) else {
7932 drop(entries);
7933 tracing::warn!(
7934 identity = %identity,
7935 runtime_id = %new_record.agent_runtime_id,
7936 session_id = %new_record.session_id,
7937 "reset entry disappeared after bridge materialization; rolling back new generation",
7938 );
7939 let _ = bridge
7940 .unregister_session_runtime_state(&new_record.session_id)
7941 .await;
7942 let _ = bridge.retire_member(&new_record.agent_runtime_id).await;
7943 if let Err(err) = self
7944 .continuity_store
7945 .rollback_continuity_record(
7946 &new_record,
7947 registered_entry.continuity.as_ref(),
7948 grant.fencing_token,
7949 )
7950 .await
7951 {
7952 tracing::warn!(
7953 %identity,
7954 error = %err,
7955 "failed to roll back continuity after reset entry disappeared"
7956 );
7957 }
7958 if let Err(err) = self
7959 .release_or_park_untracked_leases(std::slice::from_ref(&grant))
7960 .await
7961 {
7962 tracing::warn!(
7963 %identity,
7964 error = %err,
7965 "failed to release lease after reset entry disappeared; exact grant parked for retry"
7966 );
7967 }
7968 return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
7969 };
7970 entry.continuity = Some(new_record.clone());
7971 entry.lease = Some(Self::lease_entry_from_grant(&grant));
7972 entry.state = IdentityLifecycleState::Active;
7973 entry.checkpoint_version = new_record.checkpoint_version;
7974 tracing::debug!(
7975 identity = %identity,
7976 runtime_id = %new_record.agent_runtime_id,
7977 session_id = %new_record.session_id,
7978 "reset completed",
7979 );
7980 drop(entries);
7981
7982 let cleanup_old_runtime_id = old_runtime_id
7989 .as_ref()
7990 .filter(|old_id| *old_id != &new_record.agent_runtime_id)
7991 .cloned();
7992 let cleanup_old_session_id = old_session_id
7993 .as_ref()
7994 .filter(|old_session_id| *old_session_id != &new_record.session_id)
7995 .cloned();
7996 let reset_memory_injector = self.agent_memory.read().await.clone();
7997 let reset_memory_capture = reset_memory_injector.as_ref().and_then(|injector| {
7998 registered_entry.continuity.as_ref().map(|old_continuity| {
7999 PendingResetMemoryCapture {
8000 injector: injector.clone(),
8001 identity: identity.clone(),
8002 session_key: old_continuity.session_id.to_string(),
8003 generation: old_continuity.generation.get(),
8004 }
8005 })
8006 });
8007 let reset_bridge_cleanup = self
8012 .record_old_bridge_cleanup_after_reset(
8013 cleanup_old_runtime_id,
8014 cleanup_old_session_id,
8015 reset_memory_capture.clone(),
8016 )
8017 .await;
8018 if let Some(injector) = reset_memory_injector.as_ref() {
8026 injector.clear_taint_for_identity(identity);
8027 injector.note_session_generation(
8028 identity,
8029 &new_record.session_id.to_string(),
8030 new_record.generation.get(),
8031 );
8032 if let Some(old_continuity) = registered_entry.continuity.as_ref() {
8033 let old_session_key = old_continuity.session_id.to_string();
8034 injector.note_reset_boundary(&old_session_key);
8035 injector.note_session_generation(
8036 identity,
8037 &old_session_key,
8038 old_continuity.generation.get(),
8039 );
8040 }
8041 }
8042
8043 if let Some(cleanup) = reset_bridge_cleanup {
8044 self.spawn_old_bridge_cleanup_after_reset(bridge.clone(), cleanup)
8045 .await;
8046 }
8047 tracing::debug!(
8048 identity = %identity,
8049 runtime_id = %new_record.agent_runtime_id,
8050 session_id = %new_record.session_id,
8051 "reset old bridge cleanup debt recorded after continuity commit",
8052 );
8053 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
8054 return Ok(new_record);
8055 }
8056
8057 if let Err(err) = self
8059 .continuity_store
8060 .upsert_continuity_record(&new_record, grant.fencing_token)
8061 .await
8062 {
8063 self.restore_entry_with_grant(identity, registered_entry, &grant)
8064 .await;
8065 return Err(IdentityRuntimeError::Store(err));
8066 }
8067
8068 let mut entries = self.entries.write().await;
8070 let Some(entry) = entries.get_mut(identity) else {
8071 drop(entries);
8072 if let Err(err) = self
8073 .continuity_store
8074 .rollback_continuity_record(
8075 &new_record,
8076 registered_entry.continuity.as_ref(),
8077 grant.fencing_token,
8078 )
8079 .await
8080 {
8081 tracing::warn!(
8082 %identity,
8083 error = %err,
8084 "failed to roll back continuity after reset validation entry disappeared"
8085 );
8086 }
8087 if let Err(err) = self
8088 .release_or_park_untracked_leases(std::slice::from_ref(&grant))
8089 .await
8090 {
8091 tracing::warn!(
8092 %identity,
8093 error = %err,
8094 "failed to release lease after reset validation entry disappeared; exact grant parked for retry"
8095 );
8096 }
8097 return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
8098 };
8099 entry.continuity = Some(new_record.clone());
8100 entry.lease = Some(Self::lease_entry_from_grant(&grant));
8101 entry.state = IdentityLifecycleState::Active;
8102 entry.checkpoint_version = CheckpointVersion::new(0);
8103 drop(entries);
8104 if let Some(injector) = self.agent_memory.read().await.as_ref() {
8105 injector.clear_taint_for_identity(identity);
8106 if let Some(old_continuity) = registered_entry.continuity.as_ref() {
8108 let old_session_key = old_continuity.session_id.to_string();
8109 injector.note_reset_boundary(&old_session_key);
8110 injector.spawn_rotation_distillation(
8111 identity,
8112 &old_session_key,
8113 crate::memory::distiller::DistillCause::Reset,
8114 );
8115 }
8116 }
8117
8118 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Active, None);
8119 Ok(new_record)
8120 }
8121
8122 pub async fn delete_identity(
8132 &self,
8133 identity: &AgentIdentity,
8134 ) -> Result<(), IdentityRuntimeError> {
8135 self.delete_identity_with_expected_member_alias(identity, None)
8136 .await
8137 }
8138
8139 async fn delete_identity_with_expected_member_alias(
8140 &self,
8141 identity: &AgentIdentity,
8142 expected_alias: Option<&str>,
8143 ) -> Result<(), IdentityRuntimeError> {
8144 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
8145 let _lifecycle_guard = lifecycle_lock.lock().await;
8146 if let Some(expected_alias) = expected_alias {
8147 self.ensure_expected_member_alias_current(identity, expected_alias)
8148 .await?;
8149 }
8150 self.delete_identity_locked(identity).await
8151 }
8152
8153 async fn delete_identity_locked(
8154 &self,
8155 identity: &AgentIdentity,
8156 ) -> Result<(), IdentityRuntimeError> {
8157 let registered_entry = self
8158 .mark_lifecycle_in_progress(identity, IdentityLifecycleState::Retiring)
8159 .await?;
8160 let runtime_id = registered_entry
8161 .continuity
8162 .as_ref()
8163 .map(|c| c.agent_runtime_id.clone());
8164 let session_id = registered_entry
8165 .continuity
8166 .as_ref()
8167 .map(|c| c.session_id.clone());
8168
8169 let acquire_result = match self
8171 .lease_provider
8172 .acquire_leases(std::slice::from_ref(identity), &self.runtime_instance_id)
8173 .await
8174 {
8175 Ok(result) => result,
8176 Err(err) => {
8177 self.restore_entry(identity, registered_entry).await;
8178 return Err(IdentityRuntimeError::Lease(err));
8179 }
8180 };
8181
8182 let grant = match acquire_result.get(identity) {
8183 Some(super::types::LeaseAcquireResult::Acquired(g)) => g.clone(),
8184 _ => {
8185 self.restore_entry(identity, registered_entry).await;
8186 return Err(IdentityRuntimeError::NoActiveLease(identity.clone()));
8187 }
8188 };
8189 if let Err(err) = self
8190 .advance_existing_continuity_fence(identity, ®istered_entry, &grant)
8191 .await
8192 {
8193 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8194 .await;
8195 return Err(err);
8196 }
8197 if let Some(record) = registered_entry.continuity.as_ref()
8203 && let Err(error) = self
8204 .refresh_existing_session_runtime_state(identity, record, &grant)
8205 .await
8206 {
8207 self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
8208 .await;
8209 return Err(IdentityRuntimeError::Internal(format!(
8210 "bridge refresh session authority before delete: {error}"
8211 )));
8212 }
8213
8214 if let Some(injector) = self.agent_memory.read().await.clone() {
8219 if let Some(session_id) = session_id.as_ref() {
8220 let session_key = session_id.to_string();
8221 injector
8222 .distill_before_rotation(
8223 identity,
8224 &session_key,
8225 crate::memory::distiller::DistillCause::Delete,
8226 )
8227 .await;
8228 injector
8235 .drop_orphaned_session_scope(
8236 &session_key,
8237 crate::memory::distiller::DistillCause::Delete,
8238 )
8239 .await;
8240 }
8241 injector
8243 .note_identity_retired(
8244 identity,
8245 session_id
8246 .as_ref()
8247 .map(std::string::ToString::to_string)
8248 .as_deref(),
8249 "delete",
8250 )
8251 .await;
8252 }
8253
8254 if let (Some(bridge), Some(rid)) = (&self.bridge, &runtime_id)
8257 && let Err(err) = bridge.retire_member(rid).await
8258 {
8259 self.restore_entry_with_grant(identity, registered_entry, &grant)
8260 .await;
8261 return Err(IdentityRuntimeError::Internal(format!(
8262 "bridge retire before delete: {err}"
8263 )));
8264 }
8265
8266 if let (Some(bridge), Some(session_id)) = (&self.bridge, &session_id)
8267 && let Some(err) = Self::unregister_bridge_session_runtime_states(
8268 bridge.as_ref(),
8269 std::slice::from_ref(session_id),
8270 )
8271 .await
8272 {
8273 self.restore_broken_entry_with_fenced_store(identity, registered_entry, &grant)
8274 .await;
8275 return Err(IdentityRuntimeError::Internal(format!(
8276 "bridge unregister session before delete: {err}"
8277 )));
8278 }
8279
8280 if let Some(session_id) = session_id.as_ref() {
8293 match self
8294 .continuity_store
8295 .load_session_snapshot(session_id)
8296 .await
8297 {
8298 Ok(Some(snapshot)) => {
8299 let cas_token = serde_json::from_slice::<meerkat_core::Session>(&snapshot.data)
8300 .ok()
8301 .and_then(|current| {
8302 meerkat_core::session_store::session_projection_cas_token(¤t).ok()
8303 });
8304 match cas_token {
8305 Some(token) => match self
8306 .continuity_store
8307 .delete_session_snapshot_if_current_revision(session_id, &token)
8308 .await
8309 {
8310 Ok(true) => {}
8311 Ok(false) => {
8312 tracing::warn!(
8313 identity = %identity,
8314 session_id = %session_id,
8315 "session-scoped snapshot delete unsupported or superseded; \
8316 relying on delete_continuity_record's record-scoped \
8317 deletion - a non-conforming external store may retain a \
8318 resurrectable session body"
8319 );
8320 }
8321 Err(err) => {
8322 self.restore_broken_entry_and_release_grant(
8323 identity,
8324 registered_entry,
8325 &grant,
8326 )
8327 .await;
8328 return Err(IdentityRuntimeError::Store(err));
8329 }
8330 },
8331 None => {
8332 tracing::warn!(
8333 identity = %identity,
8334 session_id = %session_id,
8335 "durable session row cannot be revision-tokened for CAS \
8336 delete; relying on delete_continuity_record's record-scoped \
8337 deletion"
8338 );
8339 }
8340 }
8341 }
8342 Ok(None) => {}
8343 Err(err) => {
8344 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8345 .await;
8346 return Err(IdentityRuntimeError::Store(err));
8347 }
8348 }
8349 }
8350
8351 if let Err(err) = self
8353 .continuity_store
8354 .delete_continuity_record(identity, grant.fencing_token)
8355 .await
8356 {
8357 self.restore_broken_entry_and_release_grant(identity, registered_entry, &grant)
8358 .await;
8359 return Err(IdentityRuntimeError::Store(err));
8360 }
8361
8362 let release_result = self
8367 .lease_provider
8368 .release_leases(std::slice::from_ref(&grant))
8369 .await;
8370
8371 match release_result {
8372 Ok(()) => {
8373 self.event_channels.write().await.remove(identity);
8374 self.entries.write().await.remove(identity);
8375 self.mark_bootstrap_from_lifecycle(identity, IdentityLifecycleState::Dormant, None);
8379 Ok(())
8380 }
8381 Err(error) => {
8382 let mut entries = self.entries.write().await;
8388 let entry = entries
8389 .get_mut(identity)
8390 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8391 entry.state = IdentityLifecycleState::Broken;
8392 entry.continuity = None;
8393 entry.lease = None;
8394 entry.pending_lease_release = Some(grant);
8395 drop(entries);
8396 self.mark_bootstrap_from_lifecycle(
8397 identity,
8398 IdentityLifecycleState::Broken,
8399 Some(format!("lease release after delete: {error}")),
8400 );
8401 Err(IdentityRuntimeError::Lease(error))
8402 }
8403 }
8404 }
8405
8406 pub async fn checkpoint(
8412 &self,
8413 identity: &AgentIdentity,
8414 snapshot: &SessionSnapshot,
8415 ) -> Result<CheckpointVersion, IdentityRuntimeError> {
8416 let lifecycle_lock = self.lifecycle_lock_for(identity).await;
8417 let _lifecycle_guard = lifecycle_lock.lock().await;
8418 {
8419 let entries = self.entries.read().await;
8420 let entry = entries
8421 .get(identity)
8422 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8423 if entry.state != IdentityLifecycleState::Active {
8424 return Err(IdentityRuntimeError::InvalidState {
8425 identity: identity.clone(),
8426 state: entry.state,
8427 operation: "checkpoint",
8428 });
8429 }
8430 }
8431
8432 let token = self.ensure_active_lease(identity).await?;
8433 let (record, new_version) = {
8434 let entries = self.entries.read().await;
8435 let entry = entries
8436 .get(identity)
8437 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8438 let record = entry
8439 .continuity
8440 .as_ref()
8441 .ok_or_else(|| {
8442 IdentityRuntimeError::Internal(format!("no continuity record for {identity}"))
8443 })?
8444 .clone();
8445
8446 let new_version = CheckpointVersion::new(entry.checkpoint_version.get() + 1);
8447 (record, new_version)
8448 };
8449
8450 self.continuity_store
8452 .save_session_snapshot(
8453 identity,
8454 &record.session_id,
8455 record.generation,
8456 new_version,
8457 token,
8458 snapshot,
8459 )
8460 .await?;
8461
8462 {
8464 let mut entries = self.entries.write().await;
8465 if let Some(entry) = entries.get_mut(identity) {
8466 entry.checkpoint_version = new_version;
8467 }
8468 }
8469
8470 self.emit_event(
8471 identity,
8472 IdentityEvent::CheckpointCompleted {
8473 identity: identity.clone(),
8474 version: new_version,
8475 },
8476 )
8477 .await;
8478
8479 Ok(new_version)
8480 }
8481
8482 pub async fn roster_inspect(
8488 &self,
8489 ) -> BTreeMap<AgentIdentity, (DurableAgentSpec, IdentityStatus)> {
8490 let entries = self.entries.read().await;
8491 let mut result = BTreeMap::new();
8492 for (identity, entry) in entries.iter() {
8493 let lease_info = entry.lease.as_ref().map(|l| LeaseInfo {
8494 fencing_token: l.fencing_token,
8495 ttl_remaining: l.ttl_remaining(),
8496 healthy: l.is_healthy(),
8497 });
8498 let continuity_health = Some(ContinuityHealth {
8499 store_reachable: true,
8500 durability_policy: self.durability_policy.clone(),
8501 last_checkpoint_version: if entry.checkpoint_version.get() > 0 {
8502 Some(entry.checkpoint_version)
8503 } else {
8504 None
8505 },
8506 });
8507 let status = IdentityStatus {
8508 identity: identity.clone(),
8509 state: entry.state,
8510 agent_runtime_id: entry
8511 .continuity
8512 .as_ref()
8513 .map(|c| c.agent_runtime_id.clone()),
8514 session_id: entry.continuity.as_ref().map(|c| c.session_id.clone()),
8515 profile: Some(entry.spec.profile.clone()),
8516 runtime_mode: entry.spec.runtime_mode_override,
8517 addressability: entry.spec.addressability,
8518 display_name: entry.spec.display_name.clone(),
8519 labels: entry.spec.labels.clone(),
8520 generation: entry.continuity.as_ref().map(|c| c.generation),
8521 checkpoint_version: if entry.checkpoint_version.get() > 0 {
8522 Some(entry.checkpoint_version)
8523 } else {
8524 None
8525 },
8526 lease: lease_info,
8527 continuity_health,
8528 continuity_unrecoverable: entry.continuity_unrecoverable.clone(),
8529 };
8530 result.insert(identity.clone(), (entry.spec.clone(), status));
8531 }
8532 result
8533 }
8534
8535 pub fn validate_roster_uniqueness(
8541 specs: &[DurableAgentSpec],
8542 ) -> Result<(), IdentityRuntimeError> {
8543 let mut seen = std::collections::BTreeSet::new();
8544 for spec in specs {
8545 if !seen.insert(&spec.identity) {
8546 return Err(IdentityRuntimeError::DuplicateIdentity(
8547 spec.identity.clone(),
8548 ));
8549 }
8550 }
8551 Ok(())
8552 }
8553
8554 #[allow(dead_code)]
8560 pub(crate) async fn entries(&self) -> BTreeMap<AgentIdentity, IdentityEntry> {
8561 self.entries.read().await.clone()
8562 }
8563
8564 pub async fn contains(&self, identity: &AgentIdentity) -> bool {
8566 self.entries.read().await.contains_key(identity)
8567 }
8568
8569 pub async fn is_active(&self, identity: &AgentIdentity) -> bool {
8571 self.entries
8572 .read()
8573 .await
8574 .get(identity)
8575 .is_some_and(|e| e.state == IdentityLifecycleState::Active)
8576 }
8577
8578 fn identity_from_member_alias(alias: &str) -> Option<(AgentIdentity, bool)> {
8579 let alias = crate::member_comms_id::runtime_alias_str(alias);
8580 let alias = alias.as_ref();
8581 alias
8582 .strip_prefix("rt:")
8583 .and_then(|rest| rest.rsplit_once(':'))
8584 .filter(|(identity, generation)| {
8585 !identity.is_empty()
8586 && !generation.is_empty()
8587 && generation.chars().all(|ch| ch.is_ascii_digit())
8588 })
8589 .and_then(|(identity, _)| AgentIdentity::parse(identity).ok())
8590 .map(|identity| (identity, true))
8591 .or_else(|| {
8592 AgentIdentity::parse(alias)
8593 .ok()
8594 .map(|identity| (identity, false))
8595 })
8596 }
8597
8598 pub(crate) fn identity_for_generated_member_alias(alias: &str) -> Option<AgentIdentity> {
8603 Self::identity_from_member_alias(alias).and_then(|(identity, generated)| {
8604 if generated { Some(identity) } else { None }
8605 })
8606 }
8607
8608 pub(crate) async fn identity_for_member_mutation(&self, alias: &str) -> Option<AgentIdentity> {
8612 match Self::identity_from_member_alias(alias) {
8613 Some((identity, true)) => Some(identity),
8614 Some((identity, false)) => self.contains(&identity).await.then_some(identity),
8615 None => None,
8616 }
8617 }
8618
8619 async fn ensure_expected_member_alias_current(
8620 &self,
8621 identity: &AgentIdentity,
8622 expected_alias: &str,
8623 ) -> Result<(), IdentityRuntimeError> {
8624 let canonical_alias = crate::member_comms_id::runtime_alias_str(expected_alias);
8625 let Some((alias_identity, generated_runtime_alias)) =
8626 Self::identity_from_member_alias(canonical_alias.as_ref())
8627 else {
8628 return Err(IdentityRuntimeError::UnknownIdentity(identity.clone()));
8629 };
8630 if alias_identity != *identity {
8631 return Err(IdentityRuntimeError::UnknownIdentity(alias_identity));
8632 }
8633 let entries = self.entries.read().await;
8634 let entry = entries
8635 .get(identity)
8636 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8637 if generated_runtime_alias {
8638 let current = entry
8639 .continuity
8640 .as_ref()
8641 .map(|record| record.agent_runtime_id.clone());
8642 if current.as_ref().map(AgentRuntimeId::as_str) != Some(canonical_alias.as_ref()) {
8643 return Err(IdentityRuntimeError::StaleRuntimeAlias {
8644 identity: identity.clone(),
8645 requested: canonical_alias.into_owned(),
8646 current,
8647 });
8648 }
8649 }
8650 Ok(())
8651 }
8652
8653 pub async fn owned_identity_for_member_alias(&self, alias: &str) -> Option<AgentIdentity> {
8664 let (identity, _) = Self::identity_from_member_alias(alias)?;
8665 self.contains(&identity).await.then_some(identity)
8666 }
8667
8668 pub async fn broken_identities(&self) -> Vec<AgentIdentity> {
8670 self.entries
8671 .read()
8672 .await
8673 .iter()
8674 .filter(|(_, entry)| entry.state == IdentityLifecycleState::Broken)
8675 .map(|(identity, _)| identity.clone())
8676 .collect()
8677 }
8678
8679 pub async fn repairable_broken_identities(&self) -> Vec<AgentIdentity> {
8682 self.entries
8683 .read()
8684 .await
8685 .iter()
8686 .filter(|(_, entry)| {
8687 entry.state == IdentityLifecycleState::Broken
8688 && entry.continuity_unrecoverable.is_none()
8689 && entry
8690 .host_rejected_build_park
8691 .as_ref()
8692 .is_none_or(|park| park.spec_digest != durable_spec_digest(&entry.spec))
8693 })
8694 .map(|(identity, _)| identity.clone())
8695 .collect()
8696 }
8697
8698 pub async fn continuity_unrecoverable(
8700 &self,
8701 identity: &AgentIdentity,
8702 ) -> Option<ContinuityUnrecoverable> {
8703 self.entries
8704 .read()
8705 .await
8706 .get(identity)
8707 .and_then(|entry| entry.continuity_unrecoverable.clone())
8708 }
8709
8710 pub async fn mark_continuity_unrecoverable(
8718 &self,
8719 identity: &AgentIdentity,
8720 reason: String,
8721 ) -> bool {
8722 let marked = {
8723 let mut entries = self.entries.write().await;
8724 match entries.get_mut(identity) {
8725 Some(entry) if entry.state == IdentityLifecycleState::Broken => {
8726 entry.continuity_unrecoverable = Some(ContinuityUnrecoverable {
8727 reason: reason.clone(),
8728 });
8729 true
8730 }
8731 _ => false,
8732 }
8733 };
8734 if marked {
8735 self.mark_bootstrap_from_lifecycle(
8738 identity,
8739 IdentityLifecycleState::Broken,
8740 Some(reason),
8741 );
8742 }
8743 marked
8744 }
8745
8746 pub async fn clear_continuity_unrecoverable(&self, identity: &AgentIdentity) -> bool {
8748 let mut entries = self.entries.write().await;
8749 match entries.get_mut(identity) {
8750 Some(entry) => entry.continuity_unrecoverable.take().is_some(),
8751 None => false,
8752 }
8753 }
8754
8755 pub async fn host_rejected_build_park(
8760 &self,
8761 identity: &AgentIdentity,
8762 ) -> Option<HostRejectedBuildPark> {
8763 let mut entries = self.entries.write().await;
8764 let entry = entries.get_mut(identity)?;
8765 let park = entry.host_rejected_build_park.clone()?;
8766 if park.spec_digest == durable_spec_digest(&entry.spec) {
8767 Some(park)
8768 } else {
8769 entry.host_rejected_build_park = None;
8770 None
8771 }
8772 }
8773
8774 pub(crate) async fn mark_host_rejected_build_park(
8779 &self,
8780 identity: &AgentIdentity,
8781 reason: String,
8782 ) -> bool {
8783 let marked_state = {
8784 let mut entries = self.entries.write().await;
8785 match entries.get_mut(identity) {
8786 Some(entry) => {
8787 entry.host_rejected_build_park = Some(HostRejectedBuildPark {
8788 reason: reason.clone(),
8789 spec_digest: durable_spec_digest(&entry.spec),
8790 });
8791 Some(entry.state)
8792 }
8793 None => None,
8794 }
8795 };
8796 let Some(state) = marked_state else {
8797 return false;
8798 };
8799 tracing::warn!(
8800 identity = %identity,
8801 reason = %reason,
8802 "host deterministically rejected this identity's build; parking (no automatic \
8803 retry) until the roster spec changes"
8804 );
8805 self.mark_bootstrap_from_lifecycle(identity, state, Some(reason));
8810 true
8811 }
8812
8813 pub async fn clear_host_rejected_build_park(&self, identity: &AgentIdentity) -> bool {
8816 let mut entries = self.entries.write().await;
8817 match entries.get_mut(identity) {
8818 Some(entry) => entry.host_rejected_build_park.take().is_some(),
8819 None => false,
8820 }
8821 }
8822
8823 pub(crate) async fn continuity_session_id(
8825 &self,
8826 identity: &AgentIdentity,
8827 ) -> Option<SessionId> {
8828 self.entries
8829 .read()
8830 .await
8831 .get(identity)
8832 .and_then(|entry| entry.continuity.as_ref().map(|c| c.session_id.clone()))
8833 }
8834
8835 pub fn continuity_store(&self) -> &Arc<dyn ContinuityStore> {
8837 &self.continuity_store
8838 }
8839
8840 pub fn lease_provider(&self) -> &Arc<dyn LeaseProvider> {
8842 &self.lease_provider
8843 }
8844
8845 pub fn runtime_instance_id(&self) -> &str {
8847 &self.runtime_instance_id
8848 }
8849
8850 pub fn durability_policy(&self) -> &DurabilityPolicy {
8852 &self.durability_policy
8853 }
8854
8855 pub fn has_runtime_store(&self) -> bool {
8857 self.has_runtime_store
8858 }
8859
8860 pub fn bridge(&self) -> Option<&Arc<dyn SessionBridge>> {
8862 self.bridge.as_ref()
8863 }
8864
8865 pub async fn send_text(
8871 &self,
8872 identity: &AgentIdentity,
8873 text: impl Into<String>,
8874 ) -> Result<FencingToken, IdentityRuntimeError> {
8875 self.send(identity, &meerkat_core::ContentInput::Text(text.into()))
8876 .await
8877 }
8878
8879 pub async fn dispatch_text(
8881 &self,
8882 identity: &AgentIdentity,
8883 text: impl Into<String>,
8884 ) -> Result<(FencingToken, bool), IdentityRuntimeError> {
8885 self.dispatch(identity, &DispatchInput::system(text)).await
8886 }
8887
8888 pub async fn restore_flow(
8890 &self,
8891 roster: &[DurableAgentSpec],
8892 topology_provider: Option<&dyn super::contracts::TopologyProvider>,
8893 customizer: Option<&dyn super::contracts::AgentCustomizer>,
8894 ) -> Result<super::orchestrator::RestoreFlowResult, IdentityRuntimeError> {
8895 super::orchestrator::restore_flow(self, roster, topology_provider, customizer).await
8896 }
8897
8898 pub async fn runtime_id_for(
8900 &self,
8901 identity: &AgentIdentity,
8902 ) -> Result<AgentRuntimeId, IdentityRuntimeError> {
8903 let entries = self.entries.read().await;
8904 let entry = entries
8905 .get(identity)
8906 .ok_or_else(|| IdentityRuntimeError::UnknownIdentity(identity.clone()))?;
8907 entry
8908 .continuity
8909 .as_ref()
8910 .map(|c| c.agent_runtime_id.clone())
8911 .ok_or_else(|| {
8912 IdentityRuntimeError::Internal(format!("no continuity record for {identity}"))
8913 })
8914 }
8915
8916 pub async fn inspect(
8918 &self,
8919 identity: &AgentIdentity,
8920 ) -> Result<super::bridge::MemberInspection, IdentityRuntimeError> {
8921 let runtime_id = self.runtime_id_for(identity).await?;
8922 let bridge = self
8923 .bridge
8924 .as_ref()
8925 .ok_or_else(|| IdentityRuntimeError::Internal("no bridge configured".to_string()))?;
8926 bridge
8927 .inspect_member(&runtime_id)
8928 .await
8929 .map_err(|e| IdentityRuntimeError::Internal(format!("inspect: {e}")))
8930 }
8931
8932 async fn registered_completion_epoch(&self, identity: &AgentIdentity) -> Option<FencingToken> {
8940 let entries = self.entries.read().await;
8941 entries.get(identity).map(|entry| {
8942 entry
8943 .lease
8944 .as_ref()
8945 .map_or_else(|| FencingToken::new(0), |lease| lease.fencing_token)
8946 })
8947 }
8948
8949 fn rebase_completion_cursor(
8952 &self,
8953 identity: &AgentIdentity,
8954 epoch: FencingToken,
8955 ) -> CompletionCursor {
8956 let mut cursors = self
8957 .completion_cursors
8958 .lock()
8959 .unwrap_or_else(std::sync::PoisonError::into_inner);
8960 let cursor = cursors
8961 .entry(identity.clone())
8962 .or_insert_with(|| CompletionCursor::start(epoch));
8963 *cursor = cursor.rebased(epoch);
8964 *cursor
8965 }
8966
8967 fn retained_completion_cursor(&self, identity: &AgentIdentity) -> CompletionCursor {
8971 self.completion_cursors
8972 .lock()
8973 .unwrap_or_else(std::sync::PoisonError::into_inner)
8974 .get(identity)
8975 .copied()
8976 .unwrap_or_default()
8977 }
8978
8979 pub async fn completion_cursor(&self, identity: &AgentIdentity) -> CompletionCursor {
8988 match self.registered_completion_epoch(identity).await {
8989 Some(epoch) => self.rebase_completion_cursor(identity, epoch),
8990 None => self.retained_completion_cursor(identity),
8991 }
8992 }
8993
8994 pub async fn record_turn_completed(&self, identity: &AgentIdentity) -> CompletionCursor {
9007 let epoch = self
9010 .registered_completion_epoch(identity)
9011 .await
9012 .unwrap_or_else(|| self.retained_completion_cursor(identity).epoch);
9013 let mut cursors = self
9014 .completion_cursors
9015 .lock()
9016 .unwrap_or_else(std::sync::PoisonError::into_inner);
9017 let cursor = cursors
9018 .entry(identity.clone())
9019 .or_insert_with(|| CompletionCursor::start(epoch));
9020 *cursor = cursor.rebased(epoch).advanced();
9021 *cursor
9022 }
9023
9024 pub async fn wait_for_completion(
9036 &self,
9037 identity: &AgentIdentity,
9038 baseline: CompletionCursor,
9039 timeout: Duration,
9040 ) -> Result<CompletionCursor, IdentityRuntimeError> {
9041 let deadline = Instant::now() + timeout;
9042 loop {
9043 let cursor = self.completion_cursor(identity).await;
9044 match cursor.progress_since(baseline) {
9045 CompletionProgress::Completed => return Ok(cursor),
9046 CompletionProgress::IncarnationChanged => {
9047 return Err(IdentityRuntimeError::CompletionIncarnationChanged {
9048 identity: identity.clone(),
9049 baseline,
9050 observed: cursor,
9051 });
9052 }
9053 CompletionProgress::Pending => {}
9054 }
9055 let now = Instant::now();
9056 if now >= deadline {
9057 return Err(IdentityRuntimeError::Internal(format!(
9058 "timed out after {}s waiting for a turn past {baseline} on {identity}",
9059 timeout.as_secs_f64()
9060 )));
9061 }
9062 tokio::time::sleep(
9063 COMPLETION_POLL_INTERVAL.min(deadline.saturating_duration_since(now)),
9064 )
9065 .await;
9066 }
9067 }
9068
9069 pub fn default_timeout(&self) -> Duration {
9071 self.default_timeout
9072 }
9073
9074 async fn execute_reset_bridge_cleanup(
9075 bridge: &Arc<dyn SessionBridge>,
9076 cleanup: &PendingResetBridgeCleanup,
9077 ) -> Result<(), BridgeError> {
9078 match (&cleanup.runtime_id, &cleanup.session_id) {
9079 (Some(runtime_id), Some(session_id)) => {
9080 bridge
9081 .retire_reset_superseded_member(runtime_id, session_id)
9082 .await
9083 }
9084 (Some(runtime_id), None) => bridge.retire_member(runtime_id).await,
9085 (None, Some(session_id)) => bridge.unregister_session_runtime_state(session_id).await,
9086 (None, None) => Ok(()),
9087 }
9088 }
9089
9090 async fn clear_reset_bridge_cleanup_if_current(
9091 pending: &RwLock<BTreeMap<String, PendingResetBridgeCleanup>>,
9092 key: &str,
9093 cleanup: &PendingResetBridgeCleanup,
9094 ) {
9095 let mut pending = pending.write().await;
9096 if pending.get(key) == Some(cleanup) {
9097 pending.remove(key);
9098 }
9099 }
9100
9101 async fn run_reset_memory_capture_if_pending(
9102 pending: &RwLock<BTreeMap<String, PendingResetBridgeCleanup>>,
9103 key: &str,
9104 cleanup: &mut PendingResetBridgeCleanup,
9105 ) {
9106 let Some(capture) = cleanup.memory_capture.take() else {
9107 return;
9108 };
9109 capture.run().await;
9110
9111 let mut pending = pending.write().await;
9115 if let Some(current) = pending.get_mut(key)
9116 && current == cleanup
9117 {
9118 current.memory_capture = None;
9119 }
9120 }
9121
9122 async fn record_old_bridge_cleanup_after_reset(
9123 &self,
9124 old_runtime_id: Option<AgentRuntimeId>,
9125 old_session_id: Option<SessionId>,
9126 memory_capture: Option<PendingResetMemoryCapture>,
9127 ) -> Option<PendingResetBridgeCleanup> {
9128 if old_runtime_id.is_none() && old_session_id.is_none() {
9129 return None;
9130 }
9131 let cleanup = PendingResetBridgeCleanup {
9132 runtime_id: old_runtime_id,
9133 session_id: old_session_id,
9134 memory_capture,
9135 };
9136 let key = cleanup.key();
9137 self.pending_reset_bridge_cleanups
9138 .write()
9139 .await
9140 .insert(key, cleanup.clone());
9141 Some(cleanup)
9142 }
9143
9144 async fn spawn_old_bridge_cleanup_after_reset(
9145 &self,
9146 bridge: Arc<dyn SessionBridge>,
9147 mut cleanup: PendingResetBridgeCleanup,
9148 ) {
9149 let key = cleanup.key();
9150 let runtime_instance_id = self.runtime_instance_id.clone();
9151 let pending = self.pending_reset_bridge_cleanups.clone();
9152 let mut tasks = self.reset_bridge_cleanup_tasks.lock().await;
9153 while let Some(result) = tasks.try_join_next() {
9154 if let Err(error) = result {
9155 tracing::error!(
9156 runtime_instance_id = %self.runtime_instance_id,
9157 %error,
9158 "reset bridge cleanup task panicked"
9159 );
9160 }
9161 }
9162 tasks.spawn(async move {
9163 Self::run_reset_memory_capture_if_pending(&pending, &key, &mut cleanup).await;
9164 match Self::execute_reset_bridge_cleanup(&bridge, &cleanup).await {
9165 Ok(()) => {
9166 Self::clear_reset_bridge_cleanup_if_current(&pending, &key, &cleanup).await;
9167 }
9168 Err(error) => tracing::warn!(
9169 runtime_instance_id = %runtime_instance_id,
9170 cleanup_key = %key,
9171 %error,
9172 "reset-superseded bridge cleanup failed; retaining exact shutdown debt",
9173 ),
9174 }
9175 });
9176 }
9177
9178 pub(crate) async fn join_reset_bridge_cleanup_tasks(&self) {
9182 let mut tasks = self.reset_bridge_cleanup_tasks.lock().await;
9183 while let Some(result) = tasks.join_next().await {
9184 if let Err(error) = result {
9185 tracing::error!(
9186 runtime_instance_id = %self.runtime_instance_id,
9187 %error,
9188 "reset bridge cleanup task panicked"
9189 );
9190 }
9191 }
9192 }
9193
9194 pub(crate) async fn drain_pending_reset_bridge_cleanups(
9198 &self,
9199 ) -> Result<usize, IdentityRuntimeError> {
9200 let Some(bridge) = self.bridge.as_ref() else {
9201 return Ok(0);
9202 };
9203 let pending = self
9204 .pending_reset_bridge_cleanups
9205 .read()
9206 .await
9207 .values()
9208 .cloned()
9209 .collect::<Vec<_>>();
9210 let mut completed = 0_usize;
9211 let mut errors = Vec::new();
9212 for mut cleanup in pending {
9213 let key = cleanup.key();
9214 Self::run_reset_memory_capture_if_pending(
9215 &self.pending_reset_bridge_cleanups,
9216 &key,
9217 &mut cleanup,
9218 )
9219 .await;
9220 match Self::execute_reset_bridge_cleanup(bridge, &cleanup).await {
9221 Ok(()) => {
9222 Self::clear_reset_bridge_cleanup_if_current(
9223 &self.pending_reset_bridge_cleanups,
9224 &key,
9225 &cleanup,
9226 )
9227 .await;
9228 completed += 1;
9229 }
9230 Err(error) => errors.push(format!("{key}: {error}")),
9231 }
9232 }
9233 if errors.is_empty() {
9234 Ok(completed)
9235 } else {
9236 Err(IdentityRuntimeError::Internal(format!(
9237 "reset bridge cleanup debt remains: {}",
9238 errors.join("; ")
9239 )))
9240 }
9241 }
9242
9243 pub async fn wait_for_output(
9254 &self,
9255 identity: &AgentIdentity,
9256 timeout: Duration,
9257 ) -> Result<String, IdentityRuntimeError> {
9258 let deadline = Instant::now() + timeout;
9259 loop {
9260 if let Ok(inspection) = self.inspect(identity).await
9261 && let Some(preview) = inspection.output_preview
9262 {
9263 return Ok(preview);
9264 }
9265 if Instant::now() >= deadline {
9266 return Err(IdentityRuntimeError::Internal(format!(
9267 "timed out waiting for output from {identity}"
9268 )));
9269 }
9270 tokio::time::sleep(Duration::from_millis(500)).await;
9271 }
9272 }
9273
9274 pub async fn wait_for_output_containing(
9276 &self,
9277 identity: &AgentIdentity,
9278 needle: &str,
9279 timeout: Duration,
9280 ) -> Result<String, IdentityRuntimeError> {
9281 let deadline = Instant::now() + timeout;
9282 loop {
9283 if let Ok(inspection) = self.inspect(identity).await
9284 && let Some(ref preview) = inspection.output_preview
9285 && preview.contains(needle)
9286 {
9287 return Ok(preview.clone());
9288 }
9289 if Instant::now() >= deadline {
9290 return Err(IdentityRuntimeError::Internal(format!(
9291 "timed out waiting for output containing '{needle}' from {identity}"
9292 )));
9293 }
9294 tokio::time::sleep(Duration::from_millis(500)).await;
9295 }
9296 }
9297}
9298
9299pub async fn wire_cross_mob_by_identity(
9305 local_irt: &IdentityRuntime,
9306 local_identity: &AgentIdentity,
9307 remote_irt: &IdentityRuntime,
9308 remote_identity: &AgentIdentity,
9309 local_unified: &crate::UnifiedRuntime,
9310 remote_mob_id: &str,
9311) -> Result<(), IdentityRuntimeError> {
9312 let local_rt = local_irt.runtime_id_for(local_identity).await?;
9313 let remote_rt = remote_irt.runtime_id_for(remote_identity).await?;
9314 Box::pin(local_unified.wire_cross_mob(local_rt.as_str(), remote_rt.as_str(), remote_mob_id))
9315 .await
9316 .map_err(|e| IdentityRuntimeError::Internal(format!("wire_cross_mob: {e}")))
9317}
9318
9319#[cfg(test)]
9320mod reset_reprofile_tests {
9321 use super::*;
9322 use std::sync::Arc;
9323 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
9324 use tokio::sync::{Mutex as AsyncMutex, RwLock as AsyncRwLock};
9325
9326 use super::super::bridge::{
9327 BridgeDelivery, BridgeError, MemberInspection, ResumeSessionOutcome,
9328 };
9329 use super::super::contracts::RosterProvider;
9330 use super::super::local_lease::LocalLeaseProvider;
9331 use super::super::local_store::LocalContinuityStore;
9332 use super::super::types::{
9333 AgentBuildDraft, ContinuityResolveState, CustomizerError, LeaseAcquireResult, LeaseError,
9334 LeaseRenewResult, RosterError, SessionSnapshot,
9335 };
9336
9337 struct MutableRoster {
9338 specs: AsyncRwLock<Vec<DurableAgentSpec>>,
9339 }
9340
9341 struct LostRenewLeaseProvider {
9342 inner: LocalLeaseProvider,
9343 lose_renewals: AtomicBool,
9344 }
9345
9346 impl Default for LostRenewLeaseProvider {
9347 fn default() -> Self {
9348 Self {
9349 inner: LocalLeaseProvider::new(),
9350 lose_renewals: AtomicBool::new(false),
9351 }
9352 }
9353 }
9354
9355 #[async_trait::async_trait]
9356 impl LeaseProvider for LostRenewLeaseProvider {
9357 async fn acquire_leases(
9358 &self,
9359 identities: &[AgentIdentity],
9360 runtime_instance: &str,
9361 ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
9362 self.inner
9363 .acquire_leases(identities, runtime_instance)
9364 .await
9365 }
9366
9367 async fn renew_leases(
9368 &self,
9369 grants: &[LeaseGrant],
9370 ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
9371 if self.lose_renewals.load(Ordering::SeqCst) {
9372 return Ok(grants
9373 .iter()
9374 .map(|grant| {
9375 (
9376 grant.identity.clone(),
9377 LeaseRenewResult::Lost {
9378 identity: grant.identity.clone(),
9379 },
9380 )
9381 })
9382 .collect());
9383 }
9384 self.inner.renew_leases(grants).await
9385 }
9386
9387 async fn release_leases(&self, grants: &[LeaseGrant]) -> Result<(), LeaseError> {
9388 self.inner.release_leases(grants).await
9389 }
9390 }
9391
9392 struct RecordingReleaseLeaseProvider {
9393 inner: LocalLeaseProvider,
9394 fail_next_release: AtomicBool,
9395 release_attempts: AsyncMutex<Vec<LeaseGrant>>,
9396 }
9397
9398 impl Default for RecordingReleaseLeaseProvider {
9399 fn default() -> Self {
9400 Self {
9401 inner: LocalLeaseProvider::new(),
9402 fail_next_release: AtomicBool::new(false),
9403 release_attempts: AsyncMutex::new(Vec::new()),
9404 }
9405 }
9406 }
9407
9408 impl RecordingReleaseLeaseProvider {
9409 fn fail_next_release(&self) {
9410 self.fail_next_release.store(true, Ordering::SeqCst);
9411 }
9412
9413 async fn release_attempts(&self) -> Vec<LeaseGrant> {
9414 self.release_attempts.lock().await.clone()
9415 }
9416 }
9417
9418 #[async_trait::async_trait]
9419 impl LeaseProvider for RecordingReleaseLeaseProvider {
9420 async fn acquire_leases(
9421 &self,
9422 identities: &[AgentIdentity],
9423 runtime_instance: &str,
9424 ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
9425 self.inner
9426 .acquire_leases(identities, runtime_instance)
9427 .await
9428 }
9429
9430 async fn renew_leases(
9431 &self,
9432 grants: &[LeaseGrant],
9433 ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
9434 self.inner.renew_leases(grants).await
9435 }
9436
9437 async fn release_leases(&self, grants: &[LeaseGrant]) -> Result<(), LeaseError> {
9438 self.release_attempts.lock().await.extend_from_slice(grants);
9439 if self.fail_next_release.swap(false, Ordering::SeqCst) {
9440 return Err(LeaseError::ProviderUnavailable(
9441 "synthetic retained Broken lease release failure".to_string(),
9442 ));
9443 }
9444 self.inner.release_leases(grants).await
9445 }
9446 }
9447
9448 struct GatedResetContinuityStore {
9449 inner: Arc<LocalContinuityStore>,
9450 upsert_calls: AtomicUsize,
9451 fail_on_call: AtomicUsize,
9452 failure_started: Notify,
9453 release_failure: Notify,
9454 }
9455
9456 #[derive(Default)]
9457 struct GatedResetCustomizer {
9458 entered: Notify,
9459 }
9460
9461 impl GatedResetCustomizer {
9462 async fn wait_for_entry(&self) {
9463 self.entered.notified().await;
9464 }
9465 }
9466
9467 #[async_trait::async_trait]
9468 impl AgentCustomizer for GatedResetCustomizer {
9469 async fn customize_build(
9470 &self,
9471 _context: &AgentBuildContext,
9472 _spec: &DurableAgentSpec,
9473 _draft: &mut AgentBuildDraft,
9474 ) -> Result<(), CustomizerError> {
9475 self.entered.notify_one();
9476 futures::future::pending::<()>().await;
9477 Ok(())
9478 }
9479 }
9480
9481 impl GatedResetContinuityStore {
9482 fn new() -> Result<Self, ContinuityStoreError> {
9483 Ok(Self {
9484 inner: Arc::new(LocalContinuityStore::in_memory()?),
9485 upsert_calls: AtomicUsize::new(0),
9486 fail_on_call: AtomicUsize::new(usize::MAX),
9487 failure_started: Notify::new(),
9488 release_failure: Notify::new(),
9489 })
9490 }
9491
9492 fn fail_after_successful_upserts(&self, successful_upserts: usize) {
9493 let current = self.upsert_calls.load(Ordering::SeqCst);
9494 self.fail_on_call
9495 .store(current + successful_upserts + 1, Ordering::SeqCst);
9496 }
9497
9498 async fn wait_for_failure(&self) {
9499 self.failure_started.notified().await;
9500 }
9501
9502 fn release_failure(&self) {
9503 self.release_failure.notify_one();
9504 }
9505 }
9506
9507 #[async_trait::async_trait]
9508 impl ContinuityStore for GatedResetContinuityStore {
9509 async fn resolve_many(
9510 &self,
9511 identities: &[AgentIdentity],
9512 ) -> Result<BTreeMap<AgentIdentity, ContinuityResolveState>, ContinuityStoreError> {
9513 self.inner.resolve_many(identities).await
9514 }
9515
9516 async fn load_session_snapshot(
9517 &self,
9518 session_id: &SessionId,
9519 ) -> Result<Option<SessionSnapshot>, ContinuityStoreError> {
9520 self.inner.load_session_snapshot(session_id).await
9521 }
9522
9523 async fn save_session_snapshot(
9524 &self,
9525 identity: &AgentIdentity,
9526 session_id: &SessionId,
9527 generation: ContinuityGeneration,
9528 version: CheckpointVersion,
9529 fencing_token: FencingToken,
9530 snapshot: &SessionSnapshot,
9531 ) -> Result<(), ContinuityStoreError> {
9532 self.inner
9533 .save_session_snapshot(
9534 identity,
9535 session_id,
9536 generation,
9537 version,
9538 fencing_token,
9539 snapshot,
9540 )
9541 .await
9542 }
9543
9544 async fn upsert_continuity_record(
9545 &self,
9546 record: &ContinuityRecord,
9547 fencing_token: FencingToken,
9548 ) -> Result<(), ContinuityStoreError> {
9549 let call = self.upsert_calls.fetch_add(1, Ordering::SeqCst) + 1;
9550 if self
9551 .fail_on_call
9552 .compare_exchange(call, usize::MAX, Ordering::SeqCst, Ordering::SeqCst)
9553 .is_ok()
9554 {
9555 self.failure_started.notify_one();
9556 self.release_failure.notified().await;
9557 return Err(ContinuityStoreError::Io(
9558 "gated final reset upsert failure".to_string(),
9559 ));
9560 }
9561 self.inner
9562 .upsert_continuity_record(record, fencing_token)
9563 .await
9564 }
9565
9566 async fn rollback_continuity_record(
9567 &self,
9568 expected_attempt: &ContinuityRecord,
9569 previous: Option<&ContinuityRecord>,
9570 fencing_token: FencingToken,
9571 ) -> Result<(), ContinuityStoreError> {
9572 self.inner
9573 .rollback_continuity_record(expected_attempt, previous, fencing_token)
9574 .await
9575 }
9576
9577 async fn delete_continuity_record(
9578 &self,
9579 identity: &AgentIdentity,
9580 fencing_token: FencingToken,
9581 ) -> Result<(), ContinuityStoreError> {
9582 self.inner
9583 .delete_continuity_record(identity, fencing_token)
9584 .await
9585 }
9586 }
9587
9588 impl MutableRoster {
9589 fn new(specs: Vec<DurableAgentSpec>) -> Self {
9590 Self {
9591 specs: AsyncRwLock::new(specs),
9592 }
9593 }
9594
9595 async fn set(&self, specs: Vec<DurableAgentSpec>) {
9596 *self.specs.write().await = specs;
9597 }
9598 }
9599
9600 #[async_trait::async_trait]
9601 impl RosterProvider for MutableRoster {
9602 async fn roster(
9603 &self,
9604 _context: &RosterContext,
9605 ) -> Result<Vec<DurableAgentSpec>, RosterError> {
9606 Ok(self.specs.read().await.clone())
9607 }
9608 }
9609
9610 #[derive(Default)]
9611 struct RecordingBridge {
9612 create_profiles: AsyncMutex<Vec<String>>,
9613 create_delay: Duration,
9614 creates_in_flight: AtomicUsize,
9615 max_creates_in_flight: AtomicUsize,
9616 retired_runtime_ids: AsyncMutex<Vec<String>>,
9617 hanging_retire_runtime_ids: AsyncMutex<BTreeSet<String>>,
9618 failing_register_session_ids: AsyncMutex<BTreeSet<String>>,
9619 failing_unregister_session_ids: AsyncMutex<BTreeSet<String>>,
9620 registered_fencing_tokens: AsyncMutex<Vec<FencingToken>>,
9621 authority_transitions: AsyncMutex<Vec<String>>,
9622 }
9623
9624 impl RecordingBridge {
9625 async fn create_profiles(&self) -> Vec<String> {
9626 self.create_profiles.lock().await.clone()
9627 }
9628
9629 fn max_creates_in_flight(&self) -> usize {
9630 self.max_creates_in_flight.load(Ordering::SeqCst)
9631 }
9632
9633 async fn retired_runtime_ids(&self) -> Vec<String> {
9634 self.retired_runtime_ids.lock().await.clone()
9635 }
9636
9637 async fn hang_retire_for(&self, runtime_id: &AgentRuntimeId) {
9638 self.hanging_retire_runtime_ids
9639 .lock()
9640 .await
9641 .insert(runtime_id.to_string());
9642 }
9643
9644 async fn fail_unregister_for(&self, session_id: &SessionId) {
9645 self.failing_unregister_session_ids
9646 .lock()
9647 .await
9648 .insert(session_id.to_string());
9649 }
9650
9651 async fn allow_unregister_for(&self, session_id: &SessionId) {
9652 self.failing_unregister_session_ids
9653 .lock()
9654 .await
9655 .remove(&session_id.to_string());
9656 }
9657
9658 async fn fail_register_for(&self, session_id: &SessionId) {
9659 self.failing_register_session_ids
9660 .lock()
9661 .await
9662 .insert(session_id.to_string());
9663 }
9664
9665 async fn registered_fencing_tokens(&self) -> Vec<FencingToken> {
9666 self.registered_fencing_tokens.lock().await.clone()
9667 }
9668
9669 async fn authority_transitions(&self) -> Vec<String> {
9670 self.authority_transitions.lock().await.clone()
9671 }
9672 }
9673
9674 #[async_trait::async_trait]
9675 impl SessionBridge for RecordingBridge {
9676 async fn create_session(
9677 &self,
9678 _identity: &AgentIdentity,
9679 _runtime_id: &AgentRuntimeId,
9680 spec: &DurableAgentSpec,
9681 _draft: &AgentBuildDraft,
9682 session_id: &SessionId,
9683 ) -> Result<SessionId, BridgeError> {
9684 let in_flight = self.creates_in_flight.fetch_add(1, Ordering::SeqCst) + 1;
9685 self.max_creates_in_flight
9686 .fetch_max(in_flight, Ordering::SeqCst);
9687 tokio::time::sleep(self.create_delay).await;
9688 self.creates_in_flight.fetch_sub(1, Ordering::SeqCst);
9689 self.create_profiles
9690 .lock()
9691 .await
9692 .push(spec.profile.to_string());
9693 Ok(session_id.clone())
9694 }
9695
9696 async fn resume_session(
9697 &self,
9698 _identity: &AgentIdentity,
9699 _runtime_id: &AgentRuntimeId,
9700 _spec: &DurableAgentSpec,
9701 _draft: &AgentBuildDraft,
9702 _session_id: &SessionId,
9703 _snapshot: &SessionSnapshot,
9704 ) -> Result<ResumeSessionOutcome, BridgeError> {
9705 Err(BridgeError::Mob(
9706 "resume not used in reset test".to_string(),
9707 ))
9708 }
9709
9710 async fn deliver_admitted(
9711 &self,
9712 _runtime_id: &AgentRuntimeId,
9713 _delivery: BridgeDelivery,
9714 ) -> Result<SessionId, BridgeError> {
9715 Err(BridgeError::Mob(
9716 "deliver not used in reset test".to_string(),
9717 ))
9718 }
9719
9720 async fn checkpoint_session(
9721 &self,
9722 _runtime_id: &AgentRuntimeId,
9723 _session_id: &SessionId,
9724 ) -> Result<SessionSnapshot, BridgeError> {
9725 Err(BridgeError::Mob(
9726 "checkpoint not used in reset test".to_string(),
9727 ))
9728 }
9729
9730 async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9731 self.retired_runtime_ids
9732 .lock()
9733 .await
9734 .push(runtime_id.to_string());
9735 if self
9736 .hanging_retire_runtime_ids
9737 .lock()
9738 .await
9739 .contains(runtime_id.as_str())
9740 {
9741 futures::future::pending::<()>().await;
9742 }
9743 Ok(())
9744 }
9745
9746 async fn inspect_member(
9747 &self,
9748 _runtime_id: &AgentRuntimeId,
9749 ) -> Result<MemberInspection, BridgeError> {
9750 Err(BridgeError::Mob(
9751 "inspect not used in reset test".to_string(),
9752 ))
9753 }
9754
9755 async fn register_session_runtime_state(
9756 &self,
9757 session_id: &SessionId,
9758 _identity: &AgentIdentity,
9759 _generation: ContinuityGeneration,
9760 checkpoint_version: CheckpointVersion,
9761 fencing_token: FencingToken,
9762 ) -> Result<CheckpointVersion, BridgeError> {
9763 self.registered_fencing_tokens
9764 .lock()
9765 .await
9766 .push(fencing_token);
9767 self.authority_transitions
9768 .lock()
9769 .await
9770 .push(format!("register:{}", fencing_token.get()));
9771 if self
9772 .failing_register_session_ids
9773 .lock()
9774 .await
9775 .contains(&session_id.to_string())
9776 {
9777 return Err(BridgeError::Mob(
9778 "synthetic live-session rebind failure".to_string(),
9779 ));
9780 }
9781 Ok(checkpoint_version)
9782 }
9783
9784 async fn suspend_session_runtime_state(
9785 &self,
9786 session_id: &SessionId,
9787 ) -> Result<(), BridgeError> {
9788 self.authority_transitions
9789 .lock()
9790 .await
9791 .push(format!("suspend:{session_id}"));
9792 Ok(())
9793 }
9794
9795 async fn unregister_session_runtime_state(
9796 &self,
9797 session_id: &SessionId,
9798 ) -> Result<(), BridgeError> {
9799 if self
9800 .failing_unregister_session_ids
9801 .lock()
9802 .await
9803 .contains(&session_id.to_string())
9804 {
9805 return Err(BridgeError::Mob("old session still draining".to_string()));
9806 }
9807 Ok(())
9808 }
9809 }
9810
9811 #[derive(Default)]
9816 struct HotReloadBridge {
9817 sessions: AsyncMutex<BTreeMap<String, SessionId>>,
9818 retire_calls: AtomicUsize,
9819 resume_calls: AtomicUsize,
9820 }
9821
9822 #[async_trait::async_trait]
9823 impl SessionBridge for HotReloadBridge {
9824 async fn create_session(
9825 &self,
9826 _identity: &AgentIdentity,
9827 runtime_id: &AgentRuntimeId,
9828 _spec: &DurableAgentSpec,
9829 _draft: &AgentBuildDraft,
9830 session_id: &SessionId,
9831 ) -> Result<SessionId, BridgeError> {
9832 self.sessions
9833 .lock()
9834 .await
9835 .insert(runtime_id.to_string(), session_id.clone());
9836 Ok(session_id.clone())
9837 }
9838
9839 async fn resume_session(
9840 &self,
9841 _identity: &AgentIdentity,
9842 _runtime_id: &AgentRuntimeId,
9843 _spec: &DurableAgentSpec,
9844 _draft: &AgentBuildDraft,
9845 _session_id: &SessionId,
9846 _snapshot: &SessionSnapshot,
9847 ) -> Result<ResumeSessionOutcome, BridgeError> {
9848 self.resume_calls.fetch_add(1, Ordering::SeqCst);
9849 Err(BridgeError::Mob(
9850 "same-profile metadata hot reload attempted session resume".to_string(),
9851 ))
9852 }
9853
9854 async fn deliver_admitted(
9855 &self,
9856 runtime_id: &AgentRuntimeId,
9857 _delivery: BridgeDelivery,
9858 ) -> Result<SessionId, BridgeError> {
9859 self.sessions
9860 .lock()
9861 .await
9862 .get(runtime_id.as_str())
9863 .cloned()
9864 .ok_or_else(|| BridgeError::Mob(format!("missing session for {runtime_id}")))
9865 }
9866
9867 async fn checkpoint_session(
9868 &self,
9869 _runtime_id: &AgentRuntimeId,
9870 _session_id: &SessionId,
9871 ) -> Result<SessionSnapshot, BridgeError> {
9872 Err(BridgeError::Mob(
9873 "checkpoint not used in hot-reload test".to_string(),
9874 ))
9875 }
9876
9877 async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9878 self.retire_calls.fetch_add(1, Ordering::SeqCst);
9879 self.sessions.lock().await.remove(runtime_id.as_str());
9880 Ok(())
9881 }
9882
9883 async fn inspect_member(
9884 &self,
9885 _runtime_id: &AgentRuntimeId,
9886 ) -> Result<MemberInspection, BridgeError> {
9887 Err(BridgeError::Mob(
9888 "inspect not used in hot-reload test".to_string(),
9889 ))
9890 }
9891 }
9892
9893 #[derive(Default)]
9897 struct LostCleanupBridge {
9898 members: AsyncMutex<BTreeSet<String>>,
9899 session_runtime_states: AsyncMutex<BTreeSet<String>>,
9900 retire_calls: AtomicUsize,
9901 unregister_calls: AtomicUsize,
9902 resume_collisions: AtomicUsize,
9903 }
9904
9905 impl LostCleanupBridge {
9906 async fn member_count(&self) -> usize {
9907 self.members.lock().await.len()
9908 }
9909
9910 async fn session_runtime_state_count(&self) -> usize {
9911 self.session_runtime_states.lock().await.len()
9912 }
9913 }
9914
9915 async fn force_renewal_lost(
9916 runtime: &IdentityRuntime,
9917 lease_provider: &LostRenewLeaseProvider,
9918 identity: &AgentIdentity,
9919 ) -> Result<(), Box<dyn std::error::Error>> {
9920 lease_provider.lose_renewals.store(true, Ordering::SeqCst);
9921 {
9922 let mut entries = runtime.entries.write().await;
9923 let entry = entries
9924 .get_mut(identity)
9925 .ok_or("identity disappeared before forced renewal")?;
9926 let lease = entry
9927 .lease
9928 .as_mut()
9929 .ok_or("active identity has no lease before forced renewal")?;
9930 lease.ttl = Duration::ZERO;
9931 }
9932 assert!(matches!(
9933 runtime.renew_due_leases_once().await,
9934 Err(IdentityRuntimeError::LeaseLost(lost)) if lost == *identity
9935 ));
9936 Ok(())
9937 }
9938
9939 #[async_trait::async_trait]
9940 impl SessionBridge for LostCleanupBridge {
9941 async fn create_session(
9942 &self,
9943 _identity: &AgentIdentity,
9944 runtime_id: &AgentRuntimeId,
9945 _spec: &DurableAgentSpec,
9946 _draft: &AgentBuildDraft,
9947 session_id: &SessionId,
9948 ) -> Result<SessionId, BridgeError> {
9949 if !self.members.lock().await.insert(runtime_id.to_string()) {
9950 return Err(BridgeError::Mob(format!(
9951 "member collision for {runtime_id}"
9952 )));
9953 }
9954 Ok(session_id.clone())
9955 }
9956
9957 async fn resume_session(
9958 &self,
9959 _identity: &AgentIdentity,
9960 runtime_id: &AgentRuntimeId,
9961 _spec: &DurableAgentSpec,
9962 _draft: &AgentBuildDraft,
9963 session_id: &SessionId,
9964 _snapshot: &SessionSnapshot,
9965 ) -> Result<ResumeSessionOutcome, BridgeError> {
9966 if !self.members.lock().await.insert(runtime_id.to_string()) {
9967 self.resume_collisions.fetch_add(1, Ordering::SeqCst);
9968 return Err(BridgeError::Mob(format!(
9969 "member collision for {runtime_id}"
9970 )));
9971 }
9972 Ok(ResumeSessionOutcome::Resumed {
9973 session_id: session_id.clone(),
9974 })
9975 }
9976
9977 async fn deliver_admitted(
9978 &self,
9979 _runtime_id: &AgentRuntimeId,
9980 _delivery: BridgeDelivery,
9981 ) -> Result<SessionId, BridgeError> {
9982 Err(BridgeError::Mob(
9983 "deliver not used in lost cleanup test".to_string(),
9984 ))
9985 }
9986
9987 async fn checkpoint_session(
9988 &self,
9989 _runtime_id: &AgentRuntimeId,
9990 _session_id: &SessionId,
9991 ) -> Result<SessionSnapshot, BridgeError> {
9992 Err(BridgeError::Mob(
9993 "checkpoint not used in lost cleanup test".to_string(),
9994 ))
9995 }
9996
9997 async fn retire_member(&self, runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
9998 self.retire_calls.fetch_add(1, Ordering::SeqCst);
9999 self.members.lock().await.remove(runtime_id.as_str());
10000 Ok(())
10001 }
10002
10003 async fn inspect_member(
10004 &self,
10005 _runtime_id: &AgentRuntimeId,
10006 ) -> Result<MemberInspection, BridgeError> {
10007 Err(BridgeError::Mob(
10008 "inspect not used in lost cleanup test".to_string(),
10009 ))
10010 }
10011
10012 async fn register_session_runtime_state(
10013 &self,
10014 session_id: &SessionId,
10015 _identity: &AgentIdentity,
10016 _generation: ContinuityGeneration,
10017 checkpoint_version: CheckpointVersion,
10018 _fencing_token: FencingToken,
10019 ) -> Result<CheckpointVersion, BridgeError> {
10020 self.session_runtime_states
10021 .lock()
10022 .await
10023 .insert(session_id.to_string());
10024 Ok(checkpoint_version)
10025 }
10026
10027 async fn unregister_session_runtime_state(
10028 &self,
10029 session_id: &SessionId,
10030 ) -> Result<(), BridgeError> {
10031 self.unregister_calls.fetch_add(1, Ordering::SeqCst);
10032 self.session_runtime_states
10033 .lock()
10034 .await
10035 .remove(&session_id.to_string());
10036 Ok(())
10037 }
10038 }
10039
10040 fn durable_spec(identity: AgentIdentity, profile: &str) -> DurableAgentSpec {
10041 DurableAgentSpec {
10042 identity,
10043 profile: meerkat_mob::ProfileName::from(profile),
10044 addressability: AgentAddressability::Addressable,
10045 display_name: None,
10046 labels: BTreeMap::new(),
10047 context: None,
10048 additional_instructions: Vec::new(),
10049 initial_message: None,
10050 runtime_mode_override: None,
10051 backend: None,
10052 binding: None,
10053 }
10054 }
10055
10056 #[derive(Default)]
10060 struct HostRejectingBridge {
10061 create_attempts: std::sync::atomic::AtomicUsize,
10062 }
10063
10064 impl HostRejectingBridge {
10065 fn attempts(&self) -> usize {
10066 self.create_attempts
10067 .load(std::sync::atomic::Ordering::SeqCst)
10068 }
10069 }
10070
10071 #[async_trait::async_trait]
10072 impl SessionBridge for HostRejectingBridge {
10073 async fn create_session(
10074 &self,
10075 _identity: &AgentIdentity,
10076 _runtime_id: &AgentRuntimeId,
10077 _spec: &DurableAgentSpec,
10078 _draft: &AgentBuildDraft,
10079 _session_id: &SessionId,
10080 ) -> Result<SessionId, BridgeError> {
10081 self.create_attempts
10082 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
10083 Err(BridgeError::Mob(
10084 "spawn_member: callback/build_agent failed: callback error: candidate-mode \
10085 effect gate refused this build"
10086 .to_string(),
10087 ))
10088 }
10089
10090 async fn resume_session(
10091 &self,
10092 _identity: &AgentIdentity,
10093 _runtime_id: &AgentRuntimeId,
10094 _spec: &DurableAgentSpec,
10095 _draft: &AgentBuildDraft,
10096 _session_id: &SessionId,
10097 _snapshot: &SessionSnapshot,
10098 ) -> Result<ResumeSessionOutcome, BridgeError> {
10099 Err(BridgeError::Mob(
10100 "resume not used in host-reject test".to_string(),
10101 ))
10102 }
10103
10104 async fn deliver_admitted(
10105 &self,
10106 _runtime_id: &AgentRuntimeId,
10107 _delivery: BridgeDelivery,
10108 ) -> Result<SessionId, BridgeError> {
10109 Err(BridgeError::Mob(
10110 "deliver not used in host-reject test".to_string(),
10111 ))
10112 }
10113
10114 async fn checkpoint_session(
10115 &self,
10116 _runtime_id: &AgentRuntimeId,
10117 _session_id: &SessionId,
10118 ) -> Result<SessionSnapshot, BridgeError> {
10119 Err(BridgeError::Mob(
10120 "checkpoint not used in host-reject test".to_string(),
10121 ))
10122 }
10123
10124 async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
10125 Ok(())
10126 }
10127 }
10128
10129 #[tokio::test]
10134 async fn host_rejected_build_parks_identity_until_spec_changes()
10135 -> Result<(), Box<dyn std::error::Error>> {
10136 let identity = AgentIdentity::parse("domain:gated")?;
10137 let bridge = Arc::new(HostRejectingBridge::default());
10138 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10139 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10140 lease_provider: Arc::new(LocalLeaseProvider::new()),
10141 runtime_instance_id: "host-reject-park-test".to_string(),
10142 has_runtime_store: true,
10143 durability_policy: DurabilityPolicy::SyncWriteThrough,
10144 bridge: Some(bridge.clone()),
10145 default_timeout: None,
10146 }));
10147 runtime
10148 .register(
10149 durable_spec(identity.clone(), "domain"),
10150 IdentityLifecycleState::Dormant,
10151 None,
10152 None,
10153 )
10154 .await;
10155
10156 assert!(
10159 runtime.materialize(&identity).await.is_err(),
10160 "gated build must fail"
10161 );
10162 assert_eq!(bridge.attempts(), 1);
10163 let park = runtime
10164 .host_rejected_build_park(&identity)
10165 .await
10166 .ok_or("the first host rejection must park the identity")?;
10167 assert!(park.reason.contains("callback error"));
10168
10169 match runtime.materialize(&identity).await {
10174 Err(IdentityRuntimeError::HostRejectedBuild { reason, .. }) => {
10175 assert!(reason.contains("callback error"));
10176 }
10177 other => return Err(format!("expected the typed park, got {other:?}").into()),
10178 }
10179 assert_eq!(
10180 bridge.attempts(),
10181 1,
10182 "a parked identity must not re-ask the host"
10183 );
10184 runtime
10185 .register(
10186 durable_spec(identity.clone(), "domain"),
10187 IdentityLifecycleState::Broken,
10188 None,
10189 None,
10190 )
10191 .await;
10192 assert!(
10193 runtime.repairable_broken_identities().await.is_empty(),
10194 "the repair supervisor must skip a parked identity"
10195 );
10196
10197 let mut changed = durable_spec(identity.clone(), "domain");
10200 changed
10201 .labels
10202 .insert("policy_epoch".to_string(), "2".to_string());
10203 runtime
10204 .register(changed, IdentityLifecycleState::Dormant, None, None)
10205 .await;
10206 assert!(
10207 runtime.host_rejected_build_park(&identity).await.is_none(),
10208 "a spec change must clear the park"
10209 );
10210 assert!(runtime.materialize(&identity).await.is_err());
10211 assert_eq!(
10212 bridge.attempts(),
10213 2,
10214 "a changed spec re-admits exactly one new build attempt"
10215 );
10216 Ok(())
10217 }
10218
10219 #[test]
10220 fn external_delivery_omits_unrepresentable_interaction_id_carrier()
10221 -> Result<(), Box<dyn std::error::Error>> {
10222 let identity = AgentIdentity::parse("target-smoke")?;
10223 let mut external = durable_spec(identity.clone(), "target");
10224 external.backend = Some(meerkat_mob::MobBackendKind::External);
10225 assert_eq!(
10226 interaction_id_for_delivery(&external, Some("interaction-1")),
10227 None
10228 );
10229
10230 let local = durable_spec(identity, "target");
10231 assert_eq!(
10232 interaction_id_for_delivery(&local, Some("interaction-1")),
10233 Some("interaction-1")
10234 );
10235 Ok(())
10236 }
10237
10238 async fn lazy_context_with_broken_retained_lease(
10239 identity: AgentIdentity,
10240 spec: DurableAgentSpec,
10241 roster: Arc<MutableRoster>,
10242 lease_provider: Arc<RecordingReleaseLeaseProvider>,
10243 runtime_instance_id: &str,
10244 ) -> Result<
10245 (
10246 Arc<IdentityRuntime>,
10247 IdentityFirstRuntimeContext,
10248 LeaseGrant,
10249 ),
10250 Box<dyn std::error::Error>,
10251 > {
10252 let continuity_store = Arc::new(LocalContinuityStore::in_memory()?);
10253 let acquired = lease_provider
10254 .acquire_leases(std::slice::from_ref(&identity), runtime_instance_id)
10255 .await?;
10256 let grant = match acquired.get(&identity) {
10257 Some(LeaseAcquireResult::Acquired(grant)) => grant.clone(),
10258 other => return Err(format!("expected retained lease grant, got {other:?}").into()),
10259 };
10260 let record = ContinuityRecord {
10261 identity: identity.clone(),
10262 agent_runtime_id: AgentRuntimeId::parse(&format!("rt:{identity}:0"))?,
10263 session_id: SessionId::new(),
10264 generation: ContinuityGeneration::new(0),
10265 checkpoint_version: CheckpointVersion::new(0),
10266 };
10267 continuity_store
10268 .upsert_continuity_record(&record, grant.fencing_token)
10269 .await?;
10270 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10271 continuity_store,
10272 lease_provider,
10273 runtime_instance_id: runtime_instance_id.to_string(),
10274 has_runtime_store: true,
10275 durability_policy: DurabilityPolicy::SyncWriteThrough,
10276 bridge: None,
10277 default_timeout: None,
10278 }));
10279 runtime
10280 .register(
10281 spec,
10282 IdentityLifecycleState::Active,
10283 Some(record),
10284 Some(grant.clone()),
10285 )
10286 .await;
10287 runtime
10292 .entries
10293 .write()
10294 .await
10295 .get_mut(&identity)
10296 .ok_or("seeded identity disappeared")?
10297 .state = IdentityLifecycleState::Broken;
10298 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
10299 runtime.clone(),
10300 roster,
10301 None,
10302 None,
10303 None,
10304 IdentityBootstrapMode::LazyMaterialize,
10305 );
10306 Ok((runtime, context, grant))
10307 }
10308
10309 async fn active_alias_runtime(
10310 runtime_instance_id: &str,
10311 identity: &str,
10312 ) -> Result<(Arc<IdentityRuntime>, String), Box<dyn std::error::Error>> {
10313 let identity = AgentIdentity::parse(identity)?;
10314 let alias = format!("rt:{identity}:0");
10315 let record = ContinuityRecord {
10316 identity: identity.clone(),
10317 agent_runtime_id: AgentRuntimeId::parse(&alias)?,
10318 session_id: SessionId::new(),
10319 generation: ContinuityGeneration::new(0),
10320 checkpoint_version: CheckpointVersion::new(0),
10321 };
10322 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10323 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10324 lease_provider: Arc::new(LocalLeaseProvider::new()),
10325 runtime_instance_id: runtime_instance_id.to_string(),
10326 has_runtime_store: true,
10327 durability_policy: DurabilityPolicy::SyncWriteThrough,
10328 bridge: None,
10329 default_timeout: None,
10330 }));
10331 runtime
10332 .register(
10333 durable_spec(identity, "domain"),
10334 IdentityLifecycleState::Active,
10335 Some(record),
10336 None,
10337 )
10338 .await;
10339 Ok((runtime, alias))
10340 }
10341
10342 async fn alias_target(
10343 runtime: &Arc<IdentityRuntime>,
10344 alias: &str,
10345 ) -> Result<MemberAliasLifecycleTarget, Box<dyn std::error::Error>> {
10346 runtime
10347 .member_alias_lifecycle_target(alias)
10348 .await?
10349 .ok_or_else(|| {
10350 format!("generated alias did not resolve to lifecycle target: {alias}").into()
10351 })
10352 }
10353
10354 #[tokio::test]
10355 async fn permanent_stream_loss_breaks_only_the_current_active_embodiment()
10356 -> Result<(), Box<dyn std::error::Error>> {
10357 let identity = AgentIdentity::parse("domain:stream-loss")?;
10358 let (runtime, alias) =
10359 active_alias_runtime("stream-loss-runtime", identity.as_str()).await?;
10360 runtime
10361 .update_lease(
10362 &identity,
10363 LeaseGrant {
10364 identity: identity.clone(),
10365 fencing_token: FencingToken::new(7),
10366 ttl: Duration::from_mins(1),
10367 },
10368 )
10369 .await?;
10370
10371 assert!(
10372 runtime
10373 .mark_active_runtime_broken(&identity, &alias, 7, "event stream closed")
10374 .await?
10375 );
10376 assert_eq!(
10377 runtime.status(&identity).await?.state,
10378 IdentityLifecycleState::Broken
10379 );
10380 assert!(
10381 runtime
10382 .entries
10383 .read()
10384 .await
10385 .get(&identity)
10386 .and_then(|entry| entry.lease.as_ref())
10387 .is_some(),
10388 "repair must retain the exact live lease for fenced cleanup"
10389 );
10390 Ok(())
10391 }
10392
10393 #[tokio::test]
10394 async fn stale_stream_loss_does_not_break_replacement_embodiment()
10395 -> Result<(), Box<dyn std::error::Error>> {
10396 let identity = AgentIdentity::parse("domain:replacement")?;
10397 let (runtime, _) = active_alias_runtime("replacement-runtime", identity.as_str()).await?;
10398
10399 assert!(
10400 !runtime
10401 .mark_active_runtime_broken(
10402 &identity,
10403 "rt:domain:replacement:stale",
10404 0,
10405 "stale event stream closed",
10406 )
10407 .await?
10408 );
10409 assert_eq!(
10410 runtime.status(&identity).await?.state,
10411 IdentityLifecycleState::Active
10412 );
10413 Ok(())
10414 }
10415
10416 #[tokio::test]
10417 async fn stale_fence_stream_loss_does_not_break_same_alias_replacement()
10418 -> Result<(), Box<dyn std::error::Error>> {
10419 let identity = AgentIdentity::parse("domain:replacement-fence")?;
10420 let (runtime, alias) =
10421 active_alias_runtime("replacement-fence-runtime", identity.as_str()).await?;
10422 runtime
10423 .update_lease(
10424 &identity,
10425 LeaseGrant {
10426 identity: identity.clone(),
10427 fencing_token: FencingToken::new(8),
10428 ttl: Duration::from_mins(1),
10429 },
10430 )
10431 .await?;
10432
10433 assert!(
10434 !runtime
10435 .mark_active_runtime_broken(&identity, &alias, 7, "old-fence event stream closed",)
10436 .await?
10437 );
10438 assert_eq!(
10439 runtime.status(&identity).await?.state,
10440 IdentityLifecycleState::Active
10441 );
10442 Ok(())
10443 }
10444
10445 #[tokio::test]
10446 async fn compound_alias_targets_sort_opposite_orders_without_deadlock()
10447 -> Result<(), Box<dyn std::error::Error>> {
10448 let (runtime_a, alias_a) = active_alias_runtime("00-alias-runtime", "domain:alpha").await?;
10449 let (runtime_b, alias_b) = active_alias_runtime("01-alias-runtime", "domain:beta").await?;
10450
10451 let held_a = alias_target(&runtime_a, &alias_a)
10456 .await?
10457 .lock
10458 .clone()
10459 .lock_owned()
10460 .await;
10461 let completed = Arc::new(AtomicUsize::new(0));
10462
10463 let forward = tokio::spawn({
10464 let targets = vec![
10465 alias_target(&runtime_a, &alias_a).await?,
10466 alias_target(&runtime_b, &alias_b).await?,
10467 ];
10468 let completed = Arc::clone(&completed);
10469 async move {
10470 IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10471 let completed = Arc::clone(&completed);
10472 async move {
10473 completed.fetch_add(1, Ordering::SeqCst);
10474 Ok(())
10475 }
10476 })
10477 .await
10478 }
10479 });
10480 tokio::time::timeout(Duration::from_secs(1), async {
10481 loop {
10482 if !runtime_a.foreground_operations.lock().await.is_empty() {
10483 break;
10484 }
10485 tokio::task::yield_now().await;
10486 }
10487 })
10488 .await?;
10489
10490 let reverse = tokio::spawn({
10491 let targets = vec![
10492 alias_target(&runtime_b, &alias_b).await?,
10493 alias_target(&runtime_a, &alias_a).await?,
10494 ];
10495 let completed = Arc::clone(&completed);
10496 async move {
10497 IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10498 let completed = Arc::clone(&completed);
10499 async move {
10500 completed.fetch_add(1, Ordering::SeqCst);
10501 Ok(())
10502 }
10503 })
10504 .await
10505 }
10506 });
10507 tokio::time::timeout(Duration::from_secs(1), async {
10508 loop {
10509 if runtime_a.foreground_operations.lock().await.len() >= 2 {
10510 break;
10511 }
10512 tokio::task::yield_now().await;
10513 }
10514 })
10515 .await?;
10516 tokio::task::yield_now().await;
10517
10518 let b_probe = alias_target(&runtime_b, &alias_b)
10519 .await?
10520 .lock
10521 .try_lock_owned()
10522 .map_err(|_| "reverse-order request acquired B before globally-first A")?;
10523 drop(b_probe);
10524 drop(held_a);
10525
10526 tokio::time::timeout(Duration::from_secs(2), forward).await???;
10527 tokio::time::timeout(Duration::from_secs(2), reverse).await???;
10528 assert_eq!(completed.load(Ordering::SeqCst), 2);
10529
10530 runtime_a.close_foreground_operations();
10531 runtime_b.close_foreground_operations();
10532 tokio::time::timeout(Duration::from_secs(2), async {
10533 tokio::join!(
10534 runtime_a.join_foreground_operations(),
10535 runtime_b.join_foreground_operations()
10536 );
10537 })
10538 .await?;
10539 Ok(())
10540 }
10541
10542 #[tokio::test]
10543 async fn dropped_compound_alias_caller_still_completes_operation_boundary()
10544 -> Result<(), Box<dyn std::error::Error>> {
10545 let (runtime_a, alias_a) = active_alias_runtime("00-drop-runtime", "domain:alpha").await?;
10546 let (runtime_b, alias_b) = active_alias_runtime("01-drop-runtime", "domain:beta").await?;
10547 let targets = vec![
10548 alias_target(&runtime_a, &alias_a).await?,
10549 alias_target(&runtime_b, &alias_b).await?,
10550 ];
10551 let entered = Arc::new(tokio::sync::Semaphore::new(0));
10552 let release = Arc::new(tokio::sync::Semaphore::new(0));
10553 let completed = Arc::new(AtomicUsize::new(0));
10554 let caller = tokio::spawn({
10555 let entered = Arc::clone(&entered);
10556 let release = Arc::clone(&release);
10557 let completed = Arc::clone(&completed);
10558 async move {
10559 IdentityRuntime::run_member_alias_targets_operation_tracked(targets, move || {
10560 let entered = Arc::clone(&entered);
10561 let release = Arc::clone(&release);
10562 let completed = Arc::clone(&completed);
10563 async move {
10564 entered.add_permits(1);
10565 release
10566 .acquire()
10567 .await
10568 .map_err(|error| error.to_string())?
10569 .forget();
10570 completed.fetch_add(1, Ordering::SeqCst);
10571 Ok(())
10572 }
10573 })
10574 .await
10575 }
10576 });
10577 tokio::time::timeout(Duration::from_secs(2), entered.acquire())
10578 .await??
10579 .forget();
10580
10581 caller.abort();
10582 match caller.await {
10583 Err(error) => assert!(error.is_cancelled()),
10584 Ok(result) => {
10585 return Err(format!("aborted caller unexpectedly returned: {result:?}").into());
10586 }
10587 }
10588 release.add_permits(1);
10589
10590 runtime_a.close_foreground_operations();
10591 runtime_b.close_foreground_operations();
10592 tokio::time::timeout(Duration::from_secs(2), async {
10593 tokio::join!(
10594 runtime_a.join_foreground_operations(),
10595 runtime_b.join_foreground_operations()
10596 );
10597 })
10598 .await?;
10599 assert_eq!(
10600 completed.load(Ordering::SeqCst),
10601 1,
10602 "runtime-owned compound transaction must reach its boundary"
10603 );
10604 Ok(())
10605 }
10606
10607 #[tokio::test]
10608 async fn either_compound_alias_runtime_shutdown_waits_for_operation_boundary()
10609 -> Result<(), Box<dyn std::error::Error>> {
10610 for shutdown_first_runtime in [true, false] {
10611 let suffix = if shutdown_first_runtime {
10612 "first"
10613 } else {
10614 "second"
10615 };
10616 let (runtime_a, alias_a) =
10617 active_alias_runtime(&format!("00-shutdown-{suffix}"), "domain:alpha").await?;
10618 let (runtime_b, alias_b) =
10619 active_alias_runtime(&format!("01-shutdown-{suffix}"), "domain:beta").await?;
10620 let targets = vec![
10621 alias_target(&runtime_a, &alias_a).await?,
10622 alias_target(&runtime_b, &alias_b).await?,
10623 ];
10624 let entered = Arc::new(tokio::sync::Semaphore::new(0));
10625 let release = Arc::new(tokio::sync::Semaphore::new(0));
10626 let operation = tokio::spawn({
10627 let entered = Arc::clone(&entered);
10628 let release = Arc::clone(&release);
10629 async move {
10630 IdentityRuntime::run_member_alias_targets_operation_tracked(
10631 targets,
10632 move || {
10633 let entered = Arc::clone(&entered);
10634 let release = Arc::clone(&release);
10635 async move {
10636 entered.add_permits(1);
10637 release
10638 .acquire()
10639 .await
10640 .map_err(|error| error.to_string())?
10641 .forget();
10642 Ok(())
10643 }
10644 },
10645 )
10646 .await
10647 }
10648 });
10649 tokio::time::timeout(Duration::from_secs(2), entered.acquire())
10650 .await??
10651 .forget();
10652
10653 let shutting_down = if shutdown_first_runtime {
10654 Arc::clone(&runtime_a)
10655 } else {
10656 Arc::clone(&runtime_b)
10657 };
10658 shutting_down.close_foreground_operations();
10659 let mut shutdown = tokio::spawn(async move {
10660 shutting_down.join_foreground_operations().await;
10661 });
10662 assert!(
10663 tokio::time::timeout(Duration::from_millis(50), &mut shutdown)
10664 .await
10665 .is_err(),
10666 "shutdown of {suffix} participating runtime returned before compound boundary"
10667 );
10668
10669 release.add_permits(1);
10670 tokio::time::timeout(Duration::from_secs(2), &mut shutdown).await??;
10671 tokio::time::timeout(Duration::from_secs(2), operation).await???;
10672
10673 runtime_a.close_foreground_operations();
10674 runtime_b.close_foreground_operations();
10675 tokio::time::timeout(Duration::from_secs(2), async {
10676 tokio::join!(
10677 runtime_a.join_foreground_operations(),
10678 runtime_b.join_foreground_operations()
10679 );
10680 })
10681 .await?;
10682 }
10683 Ok(())
10684 }
10685
10686 #[tokio::test]
10687 async fn restore_flow_bounds_parallel_member_creation() -> Result<(), Box<dyn std::error::Error>>
10688 {
10689 let specs = (0..8)
10690 .map(|index| {
10691 Ok(durable_spec(
10692 AgentIdentity::parse(&format!("domain:restore-{index}"))?,
10693 "domain",
10694 ))
10695 })
10696 .collect::<Result<Vec<_>, Box<dyn std::error::Error>>>()?;
10697 let bridge = Arc::new(RecordingBridge {
10698 create_delay: Duration::from_millis(25),
10699 ..RecordingBridge::default()
10700 });
10701 let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
10702 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10703 lease_provider: Arc::new(LocalLeaseProvider::new()),
10704 runtime_instance_id: "parallel-restore-test".to_string(),
10705 has_runtime_store: true,
10706 durability_policy: DurabilityPolicy::SyncWriteThrough,
10707 bridge: Some(bridge.clone()),
10708 default_timeout: None,
10709 });
10710
10711 let result = super::super::orchestrator::restore_flow(&runtime, &specs, None, None).await?;
10712
10713 assert_eq!(result.outcomes.len(), specs.len());
10714 assert!(
10715 bridge.max_creates_in_flight() > 1,
10716 "member creation should no longer be serial"
10717 );
10718 assert!(
10719 bridge.max_creates_in_flight()
10720 <= super::super::orchestrator::IDENTITY_RESTORE_CONCURRENCY,
10721 "restore concurrency must remain bounded"
10722 );
10723 Ok(())
10724 }
10725
10726 #[tokio::test]
10727 async fn reset_reprofiles_session_from_runtime_configured_roster_provider()
10728 -> Result<(), Box<dyn std::error::Error>> {
10729 let identity = AgentIdentity::parse("domain:security")?;
10730 let roster = Arc::new(MutableRoster::new(vec![durable_spec(
10731 identity.clone(),
10732 "domain",
10733 )]));
10734 let bridge = Arc::new(RecordingBridge::default());
10735 let runtime = Arc::new(
10736 IdentityRuntime::new(IdentityRuntimeConfig {
10737 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
10738 lease_provider: Arc::new(LocalLeaseProvider::new()),
10739 runtime_instance_id: "reset-reprofile-test".to_string(),
10740 has_runtime_store: true,
10741 durability_policy: DurabilityPolicy::SyncWriteThrough,
10742 bridge: Some(bridge.clone()),
10743 default_timeout: None,
10744 })
10745 .with_reset_roster_provider(roster.clone()),
10746 );
10747
10748 super::super::orchestrator::restore_flow(
10749 &runtime,
10750 &roster
10751 .roster(&RosterContext {
10752 mob_definition: None,
10753 previous_identities: Vec::new(),
10754 })
10755 .await?,
10756 None,
10757 None,
10758 )
10759 .await?;
10760 roster
10761 .set(vec![durable_spec(identity.clone(), "security")])
10762 .await;
10763
10764 let record = runtime.reset(&identity).await?;
10765
10766 assert_eq!(record.generation.get(), 1);
10767 assert_eq!(
10768 bridge.create_profiles().await,
10769 vec!["domain".to_string(), "security".to_string()]
10770 );
10771 let status = runtime.status(&identity).await?;
10772 assert_eq!(
10773 status.profile.map(|profile| profile.to_string()).as_deref(),
10774 Some("security")
10775 );
10776 Ok(())
10777 }
10778
10779 #[tokio::test]
10780 async fn shutdown_joins_reset_after_outer_abort_at_final_upsert()
10781 -> Result<(), Box<dyn std::error::Error>> {
10782 let identity = AgentIdentity::parse("domain:security")?;
10783 let roster = Arc::new(MutableRoster::new(vec![durable_spec(
10784 identity.clone(),
10785 "domain",
10786 )]));
10787 let bridge = Arc::new(RecordingBridge::default());
10788 let store = Arc::new(GatedResetContinuityStore::new()?);
10789 let runtime = Arc::new(
10790 IdentityRuntime::new(IdentityRuntimeConfig {
10791 continuity_store: store.clone(),
10792 lease_provider: Arc::new(LocalLeaseProvider::new()),
10793 runtime_instance_id: "tracked-reset-shutdown-test".to_string(),
10794 has_runtime_store: true,
10795 durability_policy: DurabilityPolicy::SyncWriteThrough,
10796 bridge: Some(bridge.clone()),
10797 default_timeout: None,
10798 })
10799 .with_reset_roster_provider(roster.clone()),
10800 );
10801
10802 super::super::orchestrator::restore_flow(
10803 &runtime,
10804 &roster
10805 .roster(&RosterContext {
10806 mob_definition: None,
10807 previous_identities: Vec::new(),
10808 })
10809 .await?,
10810 None,
10811 None,
10812 )
10813 .await?;
10814 let old_record = match store
10815 .resolve_many(std::slice::from_ref(&identity))
10816 .await?
10817 .remove(&identity)
10818 {
10819 Some(ContinuityResolveState::Ready { record }) => record,
10820 other => return Err(format!("expected initial continuity, got {other:?}").into()),
10821 };
10822 store.fail_after_successful_upserts(3);
10823
10824 let outer = tokio::spawn({
10825 let runtime = Arc::clone(&runtime);
10826 let identity = identity.clone();
10827 async move { runtime.reset_tracked(&identity).await }
10828 });
10829 store.wait_for_failure().await;
10830 outer.abort();
10831 match outer.await {
10832 Err(error) => assert!(error.is_cancelled()),
10833 Ok(result) => {
10834 return Err(
10835 format!("outer reset waiter unexpectedly completed: {result:?}").into(),
10836 );
10837 }
10838 }
10839
10840 runtime.close_foreground_operations();
10841 let mut join = tokio::spawn({
10842 let runtime = Arc::clone(&runtime);
10843 async move { runtime.join_foreground_operations().await }
10844 });
10845 assert!(
10846 tokio::time::timeout(Duration::from_millis(50), &mut join)
10847 .await
10848 .is_err(),
10849 "shutdown must wait for the runtime-owned reset transaction"
10850 );
10851 assert!(
10852 bridge.retired_runtime_ids().await.is_empty(),
10853 "rollback cleanup cannot run before the final upsert resolves"
10854 );
10855
10856 store.release_failure();
10857 tokio::time::timeout(Duration::from_secs(2), &mut join)
10858 .await
10859 .map_err(|_| "shutdown did not join reset rollback")??;
10860
10861 let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
10862 assert_eq!(
10863 resolved.get(&identity),
10864 Some(&ContinuityResolveState::Ready {
10865 record: old_record.clone(),
10866 }),
10867 "abandoned reset must roll durable continuity back before shutdown completes"
10868 );
10869 let status = runtime.status(&identity).await?;
10870 assert_eq!(
10871 status.agent_runtime_id.as_ref(),
10872 Some(&old_record.agent_runtime_id)
10873 );
10874 assert_eq!(status.state, IdentityLifecycleState::Broken);
10875 assert_eq!(
10876 bridge.retired_runtime_ids().await,
10877 vec!["rt:domain:security:1".to_string()],
10878 "rollback must retire only the tentative reset generation"
10879 );
10880 Ok(())
10881 }
10882
10883 #[tokio::test]
10884 async fn shutdown_cancels_reset_customizer_and_restores_old_generation()
10885 -> Result<(), Box<dyn std::error::Error>> {
10886 let identity = AgentIdentity::parse("domain:security")?;
10887 let spec = durable_spec(identity.clone(), "domain");
10888 let bridge = Arc::new(RecordingBridge::default());
10889 let store = Arc::new(LocalContinuityStore::in_memory()?);
10890 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10891 continuity_store: store.clone(),
10892 lease_provider: Arc::new(LocalLeaseProvider::new()),
10893 runtime_instance_id: "reset-customizer-shutdown-test".to_string(),
10894 has_runtime_store: true,
10895 durability_policy: DurabilityPolicy::SyncWriteThrough,
10896 bridge: Some(bridge.clone()),
10897 default_timeout: None,
10898 }));
10899
10900 super::super::orchestrator::restore_flow(&runtime, std::slice::from_ref(&spec), None, None)
10901 .await?;
10902 let old_record = match store
10903 .resolve_many(std::slice::from_ref(&identity))
10904 .await?
10905 .remove(&identity)
10906 {
10907 Some(ContinuityResolveState::Ready { record }) => record,
10908 other => return Err(format!("expected initial continuity, got {other:?}").into()),
10909 };
10910
10911 let customizer = Arc::new(GatedResetCustomizer::default());
10912 runtime.set_agent_customizer(Some(customizer.clone())).await;
10913 let outer = tokio::spawn({
10914 let runtime = Arc::clone(&runtime);
10915 let identity = identity.clone();
10916 async move { runtime.reset_tracked(&identity).await }
10917 });
10918 tokio::time::timeout(Duration::from_secs(2), customizer.wait_for_entry())
10919 .await
10920 .map_err(|_| "reset did not enter the gated customizer")?;
10921 outer.abort();
10922 match outer.await {
10923 Err(error) => assert!(error.is_cancelled()),
10924 Ok(result) => {
10925 return Err(
10926 format!("outer reset waiter unexpectedly completed: {result:?}").into(),
10927 );
10928 }
10929 }
10930
10931 runtime.close_foreground_operations();
10932 tokio::time::timeout(Duration::from_secs(2), runtime.join_foreground_operations())
10933 .await
10934 .map_err(|_| "shutdown hung on the reset customizer")?;
10935
10936 let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
10937 assert_eq!(
10938 resolved.get(&identity),
10939 Some(&ContinuityResolveState::Ready {
10940 record: old_record.clone(),
10941 }),
10942 "shutdown cancellation must roll durable continuity back to the old generation"
10943 );
10944 let status = runtime.status(&identity).await?;
10945 assert_eq!(status.state, IdentityLifecycleState::Active);
10946 assert_eq!(
10947 status.agent_runtime_id.as_ref(),
10948 Some(&old_record.agent_runtime_id)
10949 );
10950 assert_eq!(status.session_id.as_ref(), Some(&old_record.session_id));
10951 assert_eq!(bridge.create_profiles().await, vec!["domain".to_string()]);
10952 assert!(
10953 bridge.retired_runtime_ids().await.is_empty(),
10954 "cancellation before session installation must not touch either bridge generation"
10955 );
10956 Ok(())
10957 }
10958
10959 #[tokio::test]
10960 async fn shutdown_cancelled_dormant_reset_releases_temporary_lease()
10961 -> Result<(), Box<dyn std::error::Error>> {
10962 let identity = AgentIdentity::parse("domain:dormant")?;
10963 let bridge = Arc::new(RecordingBridge::default());
10964 let store = Arc::new(LocalContinuityStore::in_memory()?);
10965 let lease_provider = Arc::new(LocalLeaseProvider::new());
10966 let runtime_instance_id = "dormant-reset-customizer-shutdown-test";
10967 let acquired = lease_provider
10968 .acquire_leases(std::slice::from_ref(&identity), runtime_instance_id)
10969 .await?;
10970 let initial_grant = match acquired.get(&identity) {
10971 Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
10972 other => return Err(format!("expected initial lease, got {other:?}").into()),
10973 };
10974 let old_record = ContinuityRecord {
10975 identity: identity.clone(),
10976 agent_runtime_id: AgentRuntimeId::parse("rt:domain:dormant:0")?,
10977 session_id: SessionId::new(),
10978 generation: ContinuityGeneration::new(0),
10979 checkpoint_version: CheckpointVersion::new(0),
10980 };
10981 store
10982 .upsert_continuity_record(&old_record, initial_grant.fencing_token)
10983 .await?;
10984 lease_provider
10985 .release_leases(std::slice::from_ref(&initial_grant))
10986 .await?;
10987 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
10988 continuity_store: store.clone(),
10989 lease_provider: lease_provider.clone(),
10990 runtime_instance_id: runtime_instance_id.to_string(),
10991 has_runtime_store: true,
10992 durability_policy: DurabilityPolicy::SyncWriteThrough,
10993 bridge: Some(bridge.clone()),
10994 default_timeout: None,
10995 }));
10996 runtime
10997 .register(
10998 durable_spec(identity.clone(), "domain"),
10999 IdentityLifecycleState::Dormant,
11000 Some(old_record.clone()),
11001 None,
11002 )
11003 .await;
11004
11005 let customizer = Arc::new(GatedResetCustomizer::default());
11006 runtime.set_agent_customizer(Some(customizer.clone())).await;
11007 let outer = tokio::spawn({
11008 let runtime = Arc::clone(&runtime);
11009 let identity = identity.clone();
11010 async move { runtime.reset_tracked(&identity).await }
11011 });
11012 tokio::time::timeout(Duration::from_secs(2), customizer.wait_for_entry())
11013 .await
11014 .map_err(|_| "dormant reset did not enter the gated customizer")?;
11015 outer.abort();
11016 match outer.await {
11017 Err(error) => assert!(error.is_cancelled()),
11018 Ok(result) => {
11019 return Err(
11020 format!("outer dormant reset unexpectedly completed: {result:?}").into(),
11021 );
11022 }
11023 }
11024 runtime.close_foreground_operations();
11025 tokio::time::timeout(Duration::from_secs(2), runtime.join_foreground_operations())
11026 .await
11027 .map_err(|_| "shutdown hung on the dormant reset customizer")?;
11028
11029 let status = runtime.status(&identity).await?;
11030 assert_eq!(status.state, IdentityLifecycleState::Dormant);
11031 assert!(status.lease.is_none());
11032 let resolved = store.resolve_many(std::slice::from_ref(&identity)).await?;
11033 assert_eq!(
11034 resolved.get(&identity),
11035 Some(&ContinuityResolveState::Ready {
11036 record: old_record.clone(),
11037 })
11038 );
11039 let failover = lease_provider
11040 .acquire_leases(std::slice::from_ref(&identity), "dormant-reset-failover")
11041 .await?;
11042 assert!(
11043 matches!(
11044 failover.get(&identity),
11045 Some(super::super::types::LeaseAcquireResult::Acquired(_))
11046 ),
11047 "shutdown rollback must release the dormant reset lease: {failover:?}"
11048 );
11049 assert!(bridge.create_profiles().await.is_empty());
11050 assert!(bridge.retired_runtime_ids().await.is_empty());
11051 Ok(())
11052 }
11053
11054 #[tokio::test]
11055 async fn tracked_identity_respawn_preserves_authoritative_continuity()
11056 -> Result<(), Box<dyn std::error::Error>> {
11057 let identity = AgentIdentity::parse("domain:durable-respawn")?;
11058 let spec = durable_spec(identity.clone(), "domain");
11059 let bridge = Arc::new(RecordingBridge::default());
11060 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11061 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11062 lease_provider: Arc::new(LocalLeaseProvider::new()),
11063 runtime_instance_id: "durable-respawn".to_string(),
11064 has_runtime_store: true,
11065 durability_policy: DurabilityPolicy::SyncWriteThrough,
11066 bridge: Some(bridge.clone()),
11067 default_timeout: None,
11068 }));
11069 super::super::orchestrator::restore_flow(&runtime, std::slice::from_ref(&spec), None, None)
11070 .await?;
11071
11072 let before = runtime.status(&identity).await?;
11073 let runtime_alias = before
11074 .agent_runtime_id
11075 .clone()
11076 .ok_or("missing durable runtime id")?;
11077 let respawned = runtime
11078 .respawn_identity_in_place_tracked(&identity, Some(runtime_alias.as_str()))
11079 .await?;
11080
11081 assert_eq!(Some(respawned.session_id), before.session_id);
11082 assert_eq!(Some(respawned.agent_runtime_id), before.agent_runtime_id);
11083 assert_eq!(Some(respawned.generation), before.generation);
11084 assert_eq!(
11085 runtime.status(&identity).await?.state,
11086 IdentityLifecycleState::Active
11087 );
11088 assert!(
11089 bridge.retired_runtime_ids().await.is_empty(),
11090 "identity respawn must not retire and recreate the authoritative session"
11091 );
11092 Ok(())
11093 }
11094
11095 #[tokio::test]
11096 async fn stale_alias_materialization_preserves_active_and_dormant_bootstrap_status()
11097 -> Result<(), Box<dyn std::error::Error>> {
11098 let identity = AgentIdentity::parse("domain:bootstrap-alias")?;
11099 let spec = durable_spec(identity.clone(), "domain");
11100 let current_alias = AgentRuntimeId::parse("rt:domain:bootstrap-alias:1")?;
11101 let stale_alias = "rt:domain:bootstrap-alias:0";
11102 let store = Arc::new(LocalContinuityStore::in_memory()?);
11103 let lease_provider = Arc::new(LocalLeaseProvider::new());
11104 let grants = lease_provider
11105 .acquire_leases(std::slice::from_ref(&identity), "bootstrap-alias-status")
11106 .await?;
11107 let grant = match grants.get(&identity) {
11108 Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
11109 other => return Err(format!("expected acquired lease, got {other:?}").into()),
11110 };
11111 let record = ContinuityRecord {
11112 identity: identity.clone(),
11113 agent_runtime_id: current_alias,
11114 session_id: SessionId::new(),
11115 generation: ContinuityGeneration::new(1),
11116 checkpoint_version: CheckpointVersion::new(0),
11117 };
11118 store
11119 .upsert_continuity_record(&record, grant.fencing_token)
11120 .await?;
11121 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11122 continuity_store: store,
11123 lease_provider,
11124 runtime_instance_id: "bootstrap-alias-status".to_string(),
11125 has_runtime_store: true,
11126 durability_policy: DurabilityPolicy::SyncWriteThrough,
11127 bridge: None,
11128 default_timeout: None,
11129 }));
11130 runtime
11131 .register(
11132 spec.clone(),
11133 IdentityLifecycleState::Active,
11134 Some(record),
11135 Some(grant),
11136 )
11137 .await;
11138
11139 let active_generation =
11140 runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11141 runtime
11142 .begin_identity_bootstrap(
11143 active_generation,
11144 IdentityBootstrapMode::LazyMaterialize,
11145 std::slice::from_ref(&spec),
11146 )
11147 .await;
11148 runtime.modify_bootstrap_status(Some(active_generation), |status| {
11149 status.complete = true;
11150 status.refresh_aggregates();
11151 });
11152 let active_before = runtime.identity_bootstrap_status();
11153 assert_eq!(
11154 active_before
11155 .identities
11156 .get(&identity)
11157 .map(|entry| entry.state),
11158 Some(IdentityBootstrapState::Active)
11159 );
11160 assert!(matches!(
11161 runtime
11162 .materialize_with_expected_member_alias(&identity, Some(stale_alias))
11163 .await,
11164 Err(IdentityRuntimeError::StaleRuntimeAlias { .. })
11165 ));
11166 assert_eq!(runtime.identity_bootstrap_status(), active_before);
11167
11168 runtime.retire(&identity).await?;
11169 let dormant_generation =
11170 runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11171 runtime
11172 .begin_identity_bootstrap(
11173 dormant_generation,
11174 IdentityBootstrapMode::LazyMaterialize,
11175 std::slice::from_ref(&spec),
11176 )
11177 .await;
11178 runtime.modify_bootstrap_status(Some(dormant_generation), |status| {
11179 status.complete = true;
11180 status.refresh_aggregates();
11181 });
11182 let dormant_before = runtime.identity_bootstrap_status();
11183 assert_eq!(
11184 dormant_before
11185 .identities
11186 .get(&identity)
11187 .map(|entry| entry.state),
11188 Some(IdentityBootstrapState::Dormant)
11189 );
11190 assert!(matches!(
11191 runtime
11192 .materialize_with_expected_member_alias(&identity, Some(stale_alias))
11193 .await,
11194 Err(IdentityRuntimeError::StaleRuntimeAlias { .. })
11195 ));
11196 assert_eq!(runtime.identity_bootstrap_status(), dormant_before);
11197 Ok(())
11198 }
11199
11200 #[tokio::test]
11201 async fn eager_reconcile_hot_reloads_active_metadata_without_retire_or_resume()
11202 -> Result<(), Box<dyn std::error::Error>> {
11203 let identity = AgentIdentity::parse("identity:luka")?;
11204 let mut initial_spec = durable_spec(identity.clone(), "personal");
11205 initial_spec
11206 .labels
11207 .insert("revision".to_string(), "v1".to_string());
11208 let roster = Arc::new(MutableRoster::new(vec![initial_spec.clone()]));
11209 let bridge = Arc::new(HotReloadBridge::default());
11210 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11211 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11212 lease_provider: Arc::new(LocalLeaseProvider::new()),
11213 runtime_instance_id: "eager-hot-reload-contract".to_string(),
11214 has_runtime_store: true,
11215 durability_policy: DurabilityPolicy::SyncWriteThrough,
11216 bridge: Some(bridge.clone()),
11217 default_timeout: None,
11218 }));
11219 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11220 runtime.clone(),
11221 roster.clone(),
11222 None,
11223 None,
11224 None,
11225 IdentityBootstrapMode::EagerMaterialize,
11226 );
11227 context
11228 .bootstrap_roster(std::slice::from_ref(&initial_spec))
11229 .await?;
11230
11231 let before = runtime.status(&identity).await?;
11232 assert_eq!(before.state, IdentityLifecycleState::Active);
11233 let before_session = before.session_id.clone();
11234 let before_runtime_id = before.agent_runtime_id.clone();
11235 let before_generation = before.generation;
11236 let before_token = before
11237 .lease
11238 .as_ref()
11239 .ok_or("initial active identity must have a lease")?
11240 .fencing_token;
11241
11242 let mut updated_spec = initial_spec;
11243 updated_spec.addressability = AgentAddressability::InternalOnly;
11244 updated_spec
11245 .labels
11246 .insert("timezone".to_string(), "Europe/Stockholm".to_string());
11247 updated_spec
11248 .labels
11249 .insert("revision".to_string(), "v2".to_string());
11250 roster.set(vec![updated_spec]).await;
11251
11252 let result = context.refresh_desired_topology().await?;
11253 assert!(matches!(
11254 result.outcomes.get(&identity),
11255 Some(super::super::orchestrator::RestoreOutcome::Resumed { .. })
11256 ));
11257 let after = runtime.status(&identity).await?;
11258 assert_eq!(after.state, IdentityLifecycleState::Active);
11259 assert_eq!(after.addressability, AgentAddressability::InternalOnly);
11260 assert_eq!(
11261 after.labels.get("timezone").map(String::as_str),
11262 Some("Europe/Stockholm")
11263 );
11264 assert_eq!(after.session_id, before_session);
11265 assert_eq!(after.agent_runtime_id, before_runtime_id);
11266 assert_eq!(after.generation, before_generation);
11267 assert_eq!(
11268 after
11269 .lease
11270 .as_ref()
11271 .ok_or("hot-reloaded identity must retain its lease")?
11272 .fencing_token,
11273 before_token,
11274 "same-profile reconcile must preserve exact live authority"
11275 );
11276 assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 0);
11277 assert_eq!(bridge.resume_calls.load(Ordering::SeqCst), 0);
11278
11279 assert!(matches!(
11280 runtime
11281 .send(
11282 &identity,
11283 &meerkat_core::ContentInput::Text("external".to_string()),
11284 )
11285 .await,
11286 Err(IdentityRuntimeError::NotAddressable(_))
11287 ));
11288 let (dispatch_token, durable) = runtime
11289 .dispatch(&identity, &DispatchInput::system("system notice"))
11290 .await?;
11291 assert_eq!(dispatch_token, before_token);
11292 assert!(durable);
11293 Ok(())
11294 }
11295
11296 #[tokio::test]
11297 async fn lazy_unchanged_reconcile_retained_broken_lease_failure_stays_shutdown_visible()
11298 -> Result<(), Box<dyn std::error::Error>> {
11299 let identity = AgentIdentity::parse("identity:broken-retained-unchanged")?;
11300 let spec = durable_spec(identity.clone(), "personal");
11301 let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11302 let lease_provider = Arc::new(RecordingReleaseLeaseProvider::default());
11303 let (runtime, context, retained_grant) = lazy_context_with_broken_retained_lease(
11304 identity.clone(),
11305 spec,
11306 roster,
11307 lease_provider.clone(),
11308 "broken-retained-unchanged",
11309 )
11310 .await?;
11311
11312 lease_provider.fail_next_release();
11313 let error = match context.refresh_desired_topology().await {
11314 Err(error) => error,
11315 Ok(_) => return Err("failed exact release allowed lazy registration".into()),
11316 };
11317 assert!(
11318 error
11319 .to_string()
11320 .contains("synthetic retained Broken lease release failure")
11321 );
11322 let status = runtime.status(&identity).await?;
11323 assert_eq!(status.state, IdentityLifecycleState::Broken);
11324 assert!(
11325 status.lease.is_none(),
11326 "failed release authority must be pending, never advertised as active"
11327 );
11328 assert_eq!(
11329 runtime
11330 .entries
11331 .read()
11332 .await
11333 .get(&identity)
11334 .and_then(|entry| entry.pending_lease_release.as_ref())
11335 .map(|grant| grant.fencing_token),
11336 Some(retained_grant.fencing_token),
11337 "the exact retained token must survive provider failure"
11338 );
11339 assert_eq!(
11340 lease_provider
11341 .release_attempts()
11342 .await
11343 .iter()
11344 .map(|grant| grant.fencing_token)
11345 .collect::<Vec<_>>(),
11346 vec![retained_grant.fencing_token]
11347 );
11348
11349 assert_eq!(
11350 runtime.release_all_leases_for_shutdown().await?,
11351 1,
11352 "shutdown must retain visibility of the staged exact grant"
11353 );
11354 assert_eq!(
11355 lease_provider
11356 .release_attempts()
11357 .await
11358 .iter()
11359 .map(|grant| grant.fencing_token)
11360 .collect::<Vec<_>>(),
11361 vec![retained_grant.fencing_token, retained_grant.fencing_token],
11362 "shutdown must retry the same exact fencing token"
11363 );
11364 assert!(
11365 runtime
11366 .entries
11367 .read()
11368 .await
11369 .get(&identity)
11370 .is_some_and(|entry| {
11371 entry.lease.is_none() && entry.pending_lease_release.is_none()
11372 })
11373 );
11374
11375 context.refresh_desired_topology().await?;
11376 let recovered = runtime.status(&identity).await?;
11377 assert_eq!(recovered.state, IdentityLifecycleState::Dormant);
11378 assert!(recovered.lease.is_none());
11379 let failover = lease_provider
11380 .acquire_leases(std::slice::from_ref(&identity), "other-runtime")
11381 .await?;
11382 assert!(matches!(
11383 failover.get(&identity),
11384 Some(LeaseAcquireResult::Acquired(_))
11385 ));
11386 Ok(())
11387 }
11388
11389 #[tokio::test]
11390 async fn lazy_profile_replace_releases_retained_broken_lease_before_overwrite()
11391 -> Result<(), Box<dyn std::error::Error>> {
11392 let identity = AgentIdentity::parse("identity:broken-retained-replace")?;
11393 let original = durable_spec(identity.clone(), "personal-v1");
11394 let roster = Arc::new(MutableRoster::new(vec![original.clone()]));
11395 let lease_provider = Arc::new(RecordingReleaseLeaseProvider::default());
11396 let (runtime, context, retained_grant) = lazy_context_with_broken_retained_lease(
11397 identity.clone(),
11398 original,
11399 roster.clone(),
11400 lease_provider.clone(),
11401 "broken-retained-replace",
11402 )
11403 .await?;
11404
11405 roster
11406 .set(vec![durable_spec(identity.clone(), "personal-v2")])
11407 .await;
11408 context.refresh_desired_topology().await?;
11409
11410 let status = runtime.status(&identity).await?;
11411 assert_eq!(status.state, IdentityLifecycleState::Dormant);
11412 assert_eq!(
11413 status.profile.as_ref().map(ToString::to_string).as_deref(),
11414 Some("personal-v2")
11415 );
11416 assert!(status.lease.is_none());
11417 assert_eq!(
11418 lease_provider
11419 .release_attempts()
11420 .await
11421 .iter()
11422 .map(|grant| grant.fencing_token)
11423 .collect::<Vec<_>>(),
11424 vec![retained_grant.fencing_token],
11425 "profile replacement must release the retained exact token before lazy registration"
11426 );
11427 assert!(
11428 runtime
11429 .entries
11430 .read()
11431 .await
11432 .get(&identity)
11433 .is_some_and(|entry| entry.pending_lease_release.is_none())
11434 );
11435 let failover = lease_provider
11436 .acquire_leases(std::slice::from_ref(&identity), "replacement-failover")
11437 .await?;
11438 assert!(matches!(
11439 failover.get(&identity),
11440 Some(LeaseAcquireResult::Acquired(_))
11441 ));
11442 Ok(())
11443 }
11444
11445 #[tokio::test]
11446 async fn lease_lost_then_roster_remove_cleans_lower_plane_before_dropping_entry()
11447 -> Result<(), Box<dyn std::error::Error>> {
11448 let identity = AgentIdentity::parse("identity:lost-remove")?;
11449 let spec = durable_spec(identity.clone(), "personal");
11450 let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11451 let bridge = Arc::new(LostCleanupBridge::default());
11452 let lease_provider = Arc::new(LostRenewLeaseProvider::default());
11453 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11454 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11455 lease_provider: lease_provider.clone(),
11456 runtime_instance_id: "lost-remove-cleanup".to_string(),
11457 has_runtime_store: true,
11458 durability_policy: DurabilityPolicy::SyncWriteThrough,
11459 bridge: Some(bridge.clone()),
11460 default_timeout: None,
11461 }));
11462 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11463 runtime.clone(),
11464 roster.clone(),
11465 None,
11466 None,
11467 None,
11468 IdentityBootstrapMode::EagerMaterialize,
11469 );
11470 context
11471 .bootstrap_roster(std::slice::from_ref(&spec))
11472 .await?;
11473 assert_eq!(bridge.member_count().await, 1);
11474 assert_eq!(bridge.session_runtime_state_count().await, 1);
11475
11476 force_renewal_lost(&runtime, &lease_provider, &identity).await?;
11477 assert_eq!(
11478 runtime.status(&identity).await?.state,
11479 IdentityLifecycleState::Broken
11480 );
11481 assert_eq!(bridge.member_count().await, 1);
11482 let unregisters_after_lost = bridge.unregister_calls.load(Ordering::SeqCst);
11483
11484 roster.set(Vec::new()).await;
11485 context.refresh_desired_topology().await?;
11486 assert!(!runtime.contains(&identity).await);
11487 assert_eq!(
11488 bridge.member_count().await,
11489 0,
11490 "stale member survived removal"
11491 );
11492 assert_eq!(
11493 bridge.session_runtime_state_count().await,
11494 0,
11495 "stale session authority survived removal"
11496 );
11497 assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 1);
11498 assert_eq!(
11499 bridge.unregister_calls.load(Ordering::SeqCst),
11500 unregisters_after_lost + 1,
11501 "roster removal must idempotently unregister after Lost"
11502 );
11503 Ok(())
11504 }
11505
11506 #[tokio::test]
11507 async fn lease_lost_then_profile_replace_cleans_old_member_before_resume()
11508 -> Result<(), Box<dyn std::error::Error>> {
11509 let identity = AgentIdentity::parse("identity:lost-replace")?;
11510 let original = durable_spec(identity.clone(), "personal-v1");
11511 let roster = Arc::new(MutableRoster::new(vec![original.clone()]));
11512 let bridge = Arc::new(LostCleanupBridge::default());
11513 let lease_provider = Arc::new(LostRenewLeaseProvider::default());
11514 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11515 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11516 lease_provider: lease_provider.clone(),
11517 runtime_instance_id: "lost-replace-cleanup".to_string(),
11518 has_runtime_store: true,
11519 durability_policy: DurabilityPolicy::SyncWriteThrough,
11520 bridge: Some(bridge.clone()),
11521 default_timeout: None,
11522 }));
11523 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11524 runtime.clone(),
11525 roster.clone(),
11526 None,
11527 None,
11528 None,
11529 IdentityBootstrapMode::EagerMaterialize,
11530 );
11531 context
11532 .bootstrap_roster(std::slice::from_ref(&original))
11533 .await?;
11534 force_renewal_lost(&runtime, &lease_provider, &identity).await?;
11535 let unregisters_after_lost = bridge.unregister_calls.load(Ordering::SeqCst);
11536
11537 let replacement = durable_spec(identity.clone(), "personal-v2");
11538 roster.set(vec![replacement]).await;
11539 context.refresh_desired_topology().await?;
11540
11541 let status = runtime.status(&identity).await?;
11542 assert_eq!(status.state, IdentityLifecycleState::Active);
11543 assert_eq!(
11544 status.profile.as_ref().map(ToString::to_string).as_deref(),
11545 Some("personal-v2")
11546 );
11547 assert_eq!(bridge.retire_calls.load(Ordering::SeqCst), 1);
11548 assert_eq!(
11549 bridge.unregister_calls.load(Ordering::SeqCst),
11550 unregisters_after_lost + 1,
11551 "profile replacement must idempotently unregister before resume"
11552 );
11553 assert_eq!(
11554 bridge.resume_collisions.load(Ordering::SeqCst),
11555 0,
11556 "replacement encountered the stale lower-plane alias"
11557 );
11558 assert_eq!(bridge.member_count().await, 1);
11559 assert_eq!(bridge.session_runtime_state_count().await, 1);
11560 Ok(())
11561 }
11562
11563 #[tokio::test]
11564 async fn foreground_materialization_completion_cannot_overwrite_newer_lazy_reconcile_status()
11565 -> Result<(), Box<dyn std::error::Error>> {
11566 let identity = AgentIdentity::parse("domain:foreground-generation")?;
11567 let mut v1 = durable_spec(identity.clone(), "domain");
11568 v1.labels
11569 .insert("roster_revision".to_string(), "v1".to_string());
11570 let mut v2 = v1.clone();
11571 v2.profile = meerkat_mob::ProfileName::from("replacement");
11572 v2.labels
11573 .insert("roster_revision".to_string(), "v2".to_string());
11574
11575 let roster = Arc::new(MutableRoster::new(vec![v1.clone()]));
11576 let bridge = Arc::new(RecordingBridge::default());
11577 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11578 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11579 lease_provider: Arc::new(LocalLeaseProvider::new()),
11580 runtime_instance_id: "foreground-generation-fence".to_string(),
11581 has_runtime_store: true,
11582 durability_policy: DurabilityPolicy::SyncWriteThrough,
11583 bridge: Some(bridge.clone()),
11584 default_timeout: None,
11585 }));
11586 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11587 runtime.clone(),
11588 roster.clone(),
11589 None,
11590 None,
11591 None,
11592 IdentityBootstrapMode::LazyMaterialize,
11593 );
11594 context.bootstrap_roster(std::slice::from_ref(&v1)).await?;
11595 let (v1_generation, initial) = runtime.identity_bootstrap_status_with_generation();
11596 assert!(initial.complete);
11597 assert!(!initial.ready);
11598 assert_eq!(
11599 initial.identities.get(&identity).map(|entry| entry.state),
11600 Some(IdentityBootstrapState::Dormant)
11601 );
11602
11603 let completion_entered = Arc::new(Notify::new());
11608 let release_completion = Arc::new(Notify::new());
11609 let materialize = tokio::spawn({
11610 let runtime = runtime.clone();
11611 let identity = identity.clone();
11612 let completion_entered = completion_entered.clone();
11613 let release_completion = release_completion.clone();
11614 async move {
11615 runtime
11616 .materialize_with_expected_member_alias_after_inner(
11617 &identity,
11618 None,
11619 async move {
11620 completion_entered.notify_one();
11621 release_completion.notified().await;
11622 },
11623 )
11624 .await
11625 }
11626 });
11627 completion_entered.notified().await;
11628 assert_eq!(
11629 runtime.status(&identity).await?.state,
11630 IdentityLifecycleState::Active
11631 );
11632
11633 roster.set(vec![v2]).await;
11634 context.refresh_desired_topology().await?;
11635 let after_v2 = runtime.identity_bootstrap_status();
11636 assert!(after_v2.complete);
11637 assert!(!after_v2.ready);
11638 assert_eq!(
11639 after_v2.identities.get(&identity).map(|entry| entry.state),
11640 Some(IdentityBootstrapState::Dormant)
11641 );
11642 let lifecycle = runtime.status(&identity).await?;
11643 assert_eq!(lifecycle.state, IdentityLifecycleState::Dormant);
11644 assert_eq!(
11645 lifecycle.labels.get("roster_revision").map(String::as_str),
11646 Some("v2")
11647 );
11648 assert_eq!(bridge.retired_runtime_ids().await.len(), 1);
11649
11650 release_completion.notify_one();
11653 materialize.await??;
11654 assert_eq!(runtime.identity_bootstrap_status(), after_v2);
11655 assert_eq!(
11656 runtime.status(&identity).await?.state,
11657 IdentityLifecycleState::Dormant
11658 );
11659 assert_ne!(
11660 runtime.identity_bootstrap_status_with_generation().0,
11661 v1_generation
11662 );
11663 Ok(())
11664 }
11665
11666 #[tokio::test]
11667 async fn foreground_materialization_uses_generation_of_reconcile_that_wins_lifecycle_lock()
11668 -> Result<(), Box<dyn std::error::Error>> {
11669 let identity = AgentIdentity::parse("domain:foreground-reconcile-first")?;
11670 let spec = durable_spec(identity.clone(), "domain");
11671 let roster = Arc::new(MutableRoster::new(vec![spec.clone()]));
11672 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11673 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11674 lease_provider: Arc::new(LocalLeaseProvider::new()),
11675 runtime_instance_id: "foreground-reconcile-first".to_string(),
11676 has_runtime_store: true,
11677 durability_policy: DurabilityPolicy::SyncWriteThrough,
11678 bridge: None,
11679 default_timeout: None,
11680 }));
11681 let context = IdentityFirstRuntimeContext::new_with_bootstrap_mode(
11682 runtime.clone(),
11683 roster,
11684 None,
11685 None,
11686 None,
11687 IdentityBootstrapMode::LazyMaterialize,
11688 );
11689 context
11690 .bootstrap_roster(std::slice::from_ref(&spec))
11691 .await?;
11692 let first_generation = runtime.identity_bootstrap_status_with_generation().0;
11693
11694 let next_generation =
11700 runtime.begin_identity_bootstrap_pending(IdentityBootstrapMode::LazyMaterialize);
11701 assert_ne!(next_generation, first_generation);
11702 runtime
11703 .begin_identity_bootstrap(
11704 next_generation,
11705 IdentityBootstrapMode::LazyMaterialize,
11706 std::slice::from_ref(&spec),
11707 )
11708 .await;
11709
11710 let reconcile_holds_lock = Arc::new(Notify::new());
11711 let release_reconcile = Arc::new(Notify::new());
11712 let paused = Arc::new(AtomicBool::new(false));
11713 let reconcile = tokio::spawn({
11714 let runtime = runtime.clone();
11715 let spec = spec.clone();
11716 let reconcile_holds_lock = reconcile_holds_lock.clone();
11717 let release_reconcile = release_reconcile.clone();
11718 let paused = paused.clone();
11719 async move {
11720 runtime
11721 .reconcile_roster_members_after_lifecycle_lock(
11722 std::slice::from_ref(&spec),
11723 next_generation,
11724 move |_| {
11725 let should_pause = !paused.swap(true, Ordering::SeqCst);
11726 let reconcile_holds_lock = reconcile_holds_lock.clone();
11727 let release_reconcile = release_reconcile.clone();
11728 async move {
11729 if should_pause {
11730 reconcile_holds_lock.notify_one();
11731 release_reconcile.notified().await;
11732 }
11733 }
11734 },
11735 )
11736 .await
11737 }
11738 });
11739 reconcile_holds_lock.notified().await;
11740
11741 let materialize = tokio::spawn({
11742 let runtime = runtime.clone();
11743 let identity = identity.clone();
11744 async move { runtime.materialize(&identity).await }
11745 });
11746 tokio::task::yield_now().await;
11748 release_reconcile.notify_one();
11749 reconcile.await??;
11750 materialize.await??;
11751
11752 let (published_generation, status) = runtime.identity_bootstrap_status_with_generation();
11753 assert_eq!(published_generation, next_generation);
11754 assert_eq!(
11755 status.identities.get(&identity).map(|entry| entry.state),
11756 Some(IdentityBootstrapState::Active),
11757 "G+1 materialization completion was misbound to the older pass"
11758 );
11759 let entries = runtime.entries.read().await;
11760 assert_eq!(
11761 entries
11762 .get(&identity)
11763 .map(|entry| entry.bootstrap_generation),
11764 Some(next_generation),
11765 "unchanged roster acceptance must still stamp the new generation"
11766 );
11767 Ok(())
11768 }
11769
11770 #[tokio::test]
11771 async fn stale_runtime_alias_preflight_cannot_mutate_new_generation()
11772 -> Result<(), Box<dyn std::error::Error>> {
11773 let identity = AgentIdentity::parse("domain:alias-race")?;
11774 let old_alias = "rt:domain:alias-race:0";
11775 let store = Arc::new(LocalContinuityStore::in_memory()?);
11776 let lease_provider = Arc::new(LocalLeaseProvider::new());
11777 let acquired = lease_provider
11778 .acquire_leases(std::slice::from_ref(&identity), "alias-race-test")
11779 .await?;
11780 let grant = match acquired.get(&identity) {
11781 Some(super::super::types::LeaseAcquireResult::Acquired(grant)) => grant.clone(),
11782 other => return Err(format!("expected initial lease, got {other:?}").into()),
11783 };
11784 let old_record = ContinuityRecord {
11785 identity: identity.clone(),
11786 agent_runtime_id: AgentRuntimeId::parse(old_alias)?,
11787 session_id: SessionId::new(),
11788 generation: ContinuityGeneration::new(0),
11789 checkpoint_version: CheckpointVersion::new(0),
11790 };
11791 store
11792 .upsert_continuity_record(&old_record, grant.fencing_token)
11793 .await?;
11794 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11795 continuity_store: store,
11796 lease_provider,
11797 runtime_instance_id: "alias-race-test".to_string(),
11798 has_runtime_store: true,
11799 durability_policy: DurabilityPolicy::SyncWriteThrough,
11800 bridge: None,
11801 default_timeout: None,
11802 }));
11803 runtime
11804 .register(
11805 durable_spec(identity.clone(), "domain"),
11806 IdentityLifecycleState::Active,
11807 Some(old_record),
11808 Some(grant),
11809 )
11810 .await;
11811
11812 let preflight_identity = runtime
11813 .owned_identity_for_member_alias(old_alias)
11814 .await
11815 .ok_or("old alias did not pass ownership preflight")?;
11816 let new_record = runtime.reset_tracked(&identity).await?;
11817 assert_eq!(new_record.generation, ContinuityGeneration::new(1));
11818
11819 let respawn_error = match runtime
11820 .respawn_member_alias_tracked(&preflight_identity, old_alias)
11821 .await
11822 {
11823 Ok(_) => return Err("old alias respawned the replacement generation".into()),
11824 Err(error) => error,
11825 };
11826 assert!(matches!(
11827 respawn_error,
11828 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11829 if requested == old_alias
11830 ));
11831 let content = meerkat_core::ContentInput::Text("stale alias delivery".to_string());
11832 let send_error = match runtime
11833 .send_with_mode_and_interaction_member_alias_tracked(
11834 &preflight_identity,
11835 old_alias,
11836 &content,
11837 HandlingMode::Queue,
11838 None,
11839 )
11840 .await
11841 {
11842 Ok(_) => return Err("old alias delivered to the replacement generation".into()),
11843 Err(error) => error,
11844 };
11845 assert!(matches!(
11846 send_error,
11847 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11848 if requested == old_alias
11849 ));
11850 let dispatch = DispatchInput {
11851 content,
11852 origin: super::super::types::DispatchOrigin::System,
11853 correlation_id: None,
11854 idempotency_key: None,
11855 };
11856 let dispatch_error = match runtime
11857 .dispatch_member_alias_tracked(&preflight_identity, old_alias, &dispatch)
11858 .await
11859 {
11860 Ok(_) => return Err("old alias dispatched to the replacement generation".into()),
11861 Err(error) => error,
11862 };
11863 assert!(matches!(
11864 dispatch_error,
11865 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11866 if requested == old_alias
11867 ));
11868 let rebind_error = match runtime
11869 .rebind_session_after_live_respawn_member_alias_tracked(
11870 &preflight_identity,
11871 old_alias,
11872 SessionId::new(),
11873 )
11874 .await
11875 {
11876 Ok(_) => return Err("old alias rebound the replacement generation".into()),
11877 Err(error) => error,
11878 };
11879 assert!(matches!(
11880 rebind_error,
11881 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11882 if requested == old_alias
11883 ));
11884 let retire_error = match runtime
11885 .retire_member_alias_tracked(&preflight_identity, old_alias)
11886 .await
11887 {
11888 Ok(_) => return Err("old alias retired the replacement generation".into()),
11889 Err(error) => error,
11890 };
11891 assert!(matches!(
11892 retire_error,
11893 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11894 if requested == old_alias
11895 ));
11896 let reset_error = match runtime
11897 .reset_member_alias_tracked(&preflight_identity, old_alias)
11898 .await
11899 {
11900 Ok(_) => return Err("old alias reset the replacement generation".into()),
11901 Err(error) => error,
11902 };
11903 assert!(matches!(
11904 reset_error,
11905 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11906 if requested == old_alias
11907 ));
11908 let delete_error = match runtime
11909 .delete_identity_member_alias_tracked(&preflight_identity, old_alias)
11910 .await
11911 {
11912 Ok(()) => return Err("old alias deleted the replacement generation".into()),
11913 Err(error) => error,
11914 };
11915 assert!(matches!(
11916 delete_error,
11917 IdentityRuntimeError::StaleRuntimeAlias { ref requested, .. }
11918 if requested == old_alias
11919 ));
11920 let status = runtime.status(&identity).await?;
11921 assert_eq!(status.state, IdentityLifecycleState::Active);
11922 assert_eq!(status.generation, Some(ContinuityGeneration::new(1)));
11923 assert_eq!(
11924 status.agent_runtime_id.as_ref(),
11925 Some(&new_record.agent_runtime_id)
11926 );
11927 Ok(())
11928 }
11929
11930 #[tokio::test]
11931 async fn reset_reprofiles_session_from_identity_first_context_roster_provider()
11932 -> Result<(), Box<dyn std::error::Error>> {
11933 let identity = AgentIdentity::parse("domain:security")?;
11934 let roster = Arc::new(MutableRoster::new(vec![durable_spec(
11935 identity.clone(),
11936 "domain",
11937 )]));
11938 let bridge = Arc::new(RecordingBridge::default());
11939 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
11940 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11941 lease_provider: Arc::new(LocalLeaseProvider::new()),
11942 runtime_instance_id: "reset-reprofile-context-test".to_string(),
11943 has_runtime_store: true,
11944 durability_policy: DurabilityPolicy::SyncWriteThrough,
11945 bridge: Some(bridge.clone()),
11946 default_timeout: None,
11947 }));
11948 let context =
11949 IdentityFirstRuntimeContext::new(runtime.clone(), roster.clone(), None, None, None);
11950
11951 context.refresh_desired_topology().await?;
11952 roster
11953 .set(vec![durable_spec(identity.clone(), "security")])
11954 .await;
11955
11956 let record = runtime.reset(&identity).await?;
11957
11958 assert_eq!(record.generation.get(), 1);
11959 assert_eq!(
11960 bridge.create_profiles().await,
11961 vec!["domain".to_string(), "security".to_string()]
11962 );
11963 let status = runtime.status(&identity).await?;
11964 assert_eq!(
11965 status.profile.map(|profile| profile.to_string()).as_deref(),
11966 Some("security")
11967 );
11968 Ok(())
11969 }
11970
11971 #[tokio::test]
11972 async fn reset_records_exact_cleanup_debt_without_waiting_for_hung_old_retire()
11973 -> Result<(), Box<dyn std::error::Error>> {
11974 let identity = AgentIdentity::parse("domain:security")?;
11975 let roster = Arc::new(MutableRoster::new(vec![durable_spec(
11976 identity.clone(),
11977 "domain",
11978 )]));
11979 let bridge = Arc::new(RecordingBridge::default());
11980 let runtime = Arc::new(
11981 IdentityRuntime::new(IdentityRuntimeConfig {
11982 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
11983 lease_provider: Arc::new(LocalLeaseProvider::new()),
11984 runtime_instance_id: "reset-skips-old-retire-test".to_string(),
11985 has_runtime_store: true,
11986 durability_policy: DurabilityPolicy::SyncWriteThrough,
11987 bridge: Some(bridge.clone()),
11988 default_timeout: Some(Duration::from_millis(50)),
11989 })
11990 .with_reset_roster_provider(roster.clone()),
11991 );
11992
11993 super::super::orchestrator::restore_flow(
11994 &runtime,
11995 &roster
11996 .roster(&RosterContext {
11997 mob_definition: None,
11998 previous_identities: Vec::new(),
11999 })
12000 .await?,
12001 None,
12002 None,
12003 )
12004 .await?;
12005
12006 let old_runtime_id = AgentRuntimeId::parse("rt:domain:security:0")?;
12007 bridge.hang_retire_for(&old_runtime_id).await;
12008 roster
12009 .set(vec![durable_spec(identity.clone(), "security")])
12010 .await;
12011
12012 let record = tokio::time::timeout(Duration::from_secs(1), runtime.reset(&identity))
12013 .await
12014 .map_err(|_| "reset timed out waiting for old generation retirement")??;
12015
12016 assert_eq!(record.generation.get(), 1);
12017 assert_eq!(
12018 bridge.create_profiles().await,
12019 vec!["domain".to_string(), "security".to_string()]
12020 );
12021 tokio::time::timeout(Duration::from_secs(1), async {
12022 loop {
12023 if bridge
12024 .retired_runtime_ids()
12025 .await
12026 .contains(&old_runtime_id.to_string())
12027 {
12028 break;
12029 }
12030 tokio::task::yield_now().await;
12031 }
12032 })
12033 .await
12034 .map_err(|_| "reset cleanup task never attempted the old generation")?;
12035 assert!(
12036 runtime
12037 .pending_reset_bridge_cleanups
12038 .read()
12039 .await
12040 .values()
12041 .any(|cleanup| cleanup.runtime_id.as_ref() == Some(&old_runtime_id)),
12042 "hung cleanup must retain the exact old runtime/session debt for shutdown"
12043 );
12044 let status = runtime.status(&identity).await?;
12045 assert_eq!(
12046 status.profile.map(|profile| profile.to_string()).as_deref(),
12047 Some("security")
12048 );
12049 Ok(())
12050 }
12051
12052 #[tokio::test]
12053 async fn reset_cleanup_failure_stays_retryable_after_new_generation()
12054 -> Result<(), Box<dyn std::error::Error>> {
12055 let identity = AgentIdentity::parse("domain:security")?;
12056 let roster = Arc::new(MutableRoster::new(vec![durable_spec(
12057 identity.clone(),
12058 "domain",
12059 )]));
12060 let bridge = Arc::new(RecordingBridge::default());
12061 let runtime = Arc::new(
12062 IdentityRuntime::new(IdentityRuntimeConfig {
12063 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12064 lease_provider: Arc::new(LocalLeaseProvider::new()),
12065 runtime_instance_id: "reset-unregister-failure-test".to_string(),
12066 has_runtime_store: true,
12067 durability_policy: DurabilityPolicy::SyncWriteThrough,
12068 bridge: Some(bridge.clone()),
12069 default_timeout: Some(Duration::from_millis(50)),
12070 })
12071 .with_reset_roster_provider(roster.clone()),
12072 );
12073
12074 super::super::orchestrator::restore_flow(
12075 &runtime,
12076 &roster
12077 .roster(&RosterContext {
12078 mob_definition: None,
12079 previous_identities: Vec::new(),
12080 })
12081 .await?,
12082 None,
12083 None,
12084 )
12085 .await?;
12086
12087 let old_status = runtime.status(&identity).await?;
12088 let Some(old_session_id) = old_status.session_id else {
12089 return Err("initial session id missing".into());
12090 };
12091 bridge.fail_unregister_for(&old_session_id).await;
12092 roster
12093 .set(vec![durable_spec(identity.clone(), "security")])
12094 .await;
12095
12096 let record = tokio::time::timeout(Duration::from_secs(1), runtime.reset(&identity))
12097 .await
12098 .map_err(|_| "reset timed out waiting for old session unregister cleanup")??;
12099
12100 assert_eq!(record.generation.get(), 1);
12101 assert_eq!(
12102 bridge.create_profiles().await,
12103 vec!["domain".to_string(), "security".to_string()]
12104 );
12105 runtime.join_reset_bridge_cleanup_tasks().await;
12106 assert_eq!(
12107 runtime.pending_reset_bridge_cleanups.read().await.len(),
12108 1,
12109 "failed unregister must retain exact cleanup debt"
12110 );
12111 bridge.allow_unregister_for(&old_session_id).await;
12112 assert_eq!(runtime.drain_pending_reset_bridge_cleanups().await?, 1);
12113 assert!(
12114 runtime
12115 .pending_reset_bridge_cleanups
12116 .read()
12117 .await
12118 .is_empty()
12119 );
12120 let status = runtime.status(&identity).await?;
12121 assert_eq!(
12122 status.profile.map(|profile| profile.to_string()).as_deref(),
12123 Some("security")
12124 );
12125 Ok(())
12126 }
12127
12128 #[tokio::test]
12129 async fn active_renewal_suspends_bridge_before_publishing_replacement_authority()
12130 -> Result<(), Box<dyn std::error::Error>> {
12131 struct RotatingLeaseProvider;
12132
12133 #[async_trait::async_trait]
12134 impl LeaseProvider for RotatingLeaseProvider {
12135 async fn acquire_leases(
12136 &self,
12137 _identities: &[AgentIdentity],
12138 _runtime_instance: &str,
12139 ) -> Result<BTreeMap<AgentIdentity, LeaseAcquireResult>, LeaseError> {
12140 Ok(BTreeMap::new())
12141 }
12142
12143 async fn renew_leases(
12144 &self,
12145 grants: &[LeaseGrant],
12146 ) -> Result<BTreeMap<AgentIdentity, LeaseRenewResult>, LeaseError> {
12147 Ok(grants
12148 .iter()
12149 .map(|grant| {
12150 let renewed = LeaseGrant {
12151 identity: grant.identity.clone(),
12152 fencing_token: FencingToken::new(grant.fencing_token.get() + 1),
12153 ttl: Duration::from_mins(5),
12154 };
12155 (grant.identity.clone(), LeaseRenewResult::Renewed(renewed))
12156 })
12157 .collect())
12158 }
12159
12160 async fn release_leases(&self, _grants: &[LeaseGrant]) -> Result<(), LeaseError> {
12161 Ok(())
12162 }
12163 }
12164
12165 let identity = AgentIdentity::parse("domain:renewal-barrier")?;
12166 let runtime_instance = "active-renewal-barrier-test";
12167 let store = Arc::new(LocalContinuityStore::in_memory()?);
12168 let lease = Arc::new(RotatingLeaseProvider);
12169 let bridge = Arc::new(RecordingBridge::default());
12170 let first = LeaseGrant {
12171 identity: identity.clone(),
12172 fencing_token: FencingToken::new(1),
12173 ttl: Duration::ZERO,
12174 };
12175 let record = ContinuityRecord {
12176 identity: identity.clone(),
12177 agent_runtime_id: AgentRuntimeId::parse("rt:domain:renewal-barrier:0")?,
12178 session_id: SessionId::new(),
12179 generation: ContinuityGeneration::new(0),
12180 checkpoint_version: CheckpointVersion::new(0),
12181 };
12182 store
12183 .upsert_continuity_record(&record, first.fencing_token)
12184 .await?;
12185 bridge
12186 .register_session_runtime_state(
12187 &record.session_id,
12188 &identity,
12189 record.generation,
12190 record.checkpoint_version,
12191 first.fencing_token,
12192 )
12193 .await?;
12194 let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12195 continuity_store: store,
12196 lease_provider: lease,
12197 runtime_instance_id: runtime_instance.to_string(),
12198 has_runtime_store: true,
12199 durability_policy: DurabilityPolicy::SyncWriteThrough,
12200 bridge: Some(bridge.clone()),
12201 default_timeout: None,
12202 });
12203 runtime
12204 .register(
12205 durable_spec(identity.clone(), "domain"),
12206 IdentityLifecycleState::Active,
12207 Some(record.clone()),
12208 Some(first.clone()),
12209 )
12210 .await;
12211
12212 let renewed_token = runtime.ensure_active_lease(&identity).await?;
12213 assert!(renewed_token > first.fencing_token);
12214 assert_eq!(
12215 bridge.authority_transitions().await,
12216 vec![
12217 format!("register:{}", first.fencing_token.get()),
12218 format!("suspend:{}", record.session_id),
12219 format!("suspend:{}", record.session_id),
12220 format!("register:{}", renewed_token.get()),
12221 ],
12222 "the old bridge authority must be quiesced before replacement publication"
12223 );
12224 assert_eq!(
12225 runtime
12226 .status(&identity)
12227 .await?
12228 .lease
12229 .map(|lease| lease.fencing_token),
12230 Some(renewed_token)
12231 );
12232 Ok(())
12233 }
12234
12235 #[tokio::test]
12236 async fn active_grant_bridge_failure_parks_rotated_token_for_exact_shutdown_release()
12237 -> Result<(), Box<dyn std::error::Error>> {
12238 let identity = AgentIdentity::parse("domain:bridge-fence")?;
12239 let runtime_instance = "active-grant-bridge-failure-test";
12240 let store = Arc::new(LocalContinuityStore::in_memory()?);
12241 let lease = Arc::new(LocalLeaseProvider::new());
12242 let bridge = Arc::new(RecordingBridge::default());
12243 let first = match lease
12244 .acquire_leases(std::slice::from_ref(&identity), runtime_instance)
12245 .await?
12246 .remove(&identity)
12247 {
12248 Some(LeaseAcquireResult::Acquired(grant)) => grant,
12249 other => return Err(format!("initial lease was not acquired: {other:?}").into()),
12250 };
12251 let record = ContinuityRecord {
12252 identity: identity.clone(),
12253 agent_runtime_id: AgentRuntimeId::parse("rt:domain:bridge-fence:0")?,
12254 session_id: SessionId::new(),
12255 generation: ContinuityGeneration::new(0),
12256 checkpoint_version: CheckpointVersion::new(0),
12257 };
12258 store
12259 .upsert_continuity_record(&record, first.fencing_token)
12260 .await?;
12261 let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12262 continuity_store: store,
12263 lease_provider: lease.clone(),
12264 runtime_instance_id: runtime_instance.to_string(),
12265 has_runtime_store: true,
12266 durability_policy: DurabilityPolicy::SyncWriteThrough,
12267 bridge: Some(bridge.clone()),
12268 default_timeout: None,
12269 });
12270 runtime
12271 .register(
12272 durable_spec(identity.clone(), "domain"),
12273 IdentityLifecycleState::Active,
12274 Some(record.clone()),
12275 Some(first.clone()),
12276 )
12277 .await;
12278
12279 let rotated = match lease
12280 .renew_leases(std::slice::from_ref(&first))
12281 .await?
12282 .remove(&identity)
12283 {
12284 Some(LeaseRenewResult::Renewed(grant)) => grant,
12285 other => return Err(format!("lease did not rotate: {other:?}").into()),
12286 };
12287 runtime
12288 .publish_active_grant(&identity, None, &rotated)
12289 .await?;
12290 {
12291 let entries = runtime.entries.read().await;
12292 let entry = entries
12293 .get(&identity)
12294 .ok_or("identity entry disappeared after active grant refresh")?;
12295 assert_eq!(entry.state, IdentityLifecycleState::Active);
12296 assert_eq!(
12297 entry.lease.as_ref().map(|lease| lease.fencing_token),
12298 Some(rotated.fencing_token)
12299 );
12300 }
12301
12302 let failed_grant = match lease
12303 .renew_leases(std::slice::from_ref(&rotated))
12304 .await?
12305 .remove(&identity)
12306 {
12307 Some(LeaseRenewResult::Renewed(grant)) => grant,
12308 other => return Err(format!("second lease did not rotate: {other:?}").into()),
12309 };
12310 bridge.fail_register_for(&record.session_id).await;
12311 let error = match runtime
12312 .publish_active_grant(&identity, None, &failed_grant)
12313 .await
12314 {
12315 Ok(_) => return Err("bridge publication failure unexpectedly succeeded".into()),
12316 Err(error) => error,
12317 };
12318 assert!(
12319 error
12320 .to_string()
12321 .contains("synthetic live-session rebind failure")
12322 );
12323 assert_eq!(
12324 bridge.registered_fencing_tokens().await,
12325 vec![rotated.fencing_token, failed_grant.fencing_token],
12326 "every active refresh must project the provider-committed token into the bridge"
12327 );
12328 {
12329 let entries = runtime.entries.read().await;
12330 let entry = entries
12331 .get(&identity)
12332 .ok_or("identity entry disappeared after bridge failure")?;
12333 assert_eq!(entry.state, IdentityLifecycleState::Broken);
12334 assert!(entry.lease.is_none());
12335 assert_eq!(
12336 entry
12337 .pending_lease_release
12338 .as_ref()
12339 .map(|grant| grant.fencing_token),
12340 Some(failed_grant.fencing_token)
12341 );
12342 }
12343
12344 assert_eq!(runtime.release_all_leases_for_shutdown().await?, 1);
12345 let failover = lease
12346 .acquire_leases(std::slice::from_ref(&identity), "bridge-failure-failover")
12347 .await?;
12348 assert!(matches!(
12349 failover.get(&identity),
12350 Some(LeaseAcquireResult::Acquired(_))
12351 ));
12352 Ok(())
12353 }
12354}
12355
12356#[cfg(test)]
12357mod lease_renewal_backoff_tests {
12358 use super::*;
12359
12360 #[test]
12364 fn lease_renewal_failure_backoff_grows_and_caps() {
12365 let max = Duration::from_mins(1);
12366 assert_eq!(
12367 lease_renewal_failure_backoff(0, max),
12368 LEASE_RENEWAL_FAILURE_BACKOFF_BASE
12369 );
12370 assert_eq!(
12371 lease_renewal_failure_backoff(1, max),
12372 LEASE_RENEWAL_FAILURE_BACKOFF_BASE * 2
12373 );
12374 assert_eq!(lease_renewal_failure_backoff(6, max), max);
12375 assert_eq!(lease_renewal_failure_backoff(99, max), max);
12377 assert!(lease_renewal_failure_backoff(2, max) > lease_renewal_failure_backoff(1, max));
12378 }
12379}
12380
12381#[cfg(test)]
12382mod continuity_repair_supervisor_tests {
12383 use super::*;
12384 use crate::identity_first::bridge::{
12385 BridgeDelivery, BridgeError, ResumeSessionOutcome, SessionBridge,
12386 };
12387 use crate::identity_first::types::{AgentBuildDraft, SessionSnapshot};
12388 use crate::identity_first::{LocalContinuityStore, LocalLeaseProvider, MutableRosterProvider};
12389
12390 struct IdenticallyWedgedResumeBridge {
12396 resume_attempts: std::sync::atomic::AtomicUsize,
12397 }
12398
12399 impl IdenticallyWedgedResumeBridge {
12400 fn attempts(&self) -> usize {
12401 self.resume_attempts
12402 .load(std::sync::atomic::Ordering::SeqCst)
12403 }
12404 }
12405
12406 #[async_trait::async_trait]
12407 impl SessionBridge for IdenticallyWedgedResumeBridge {
12408 async fn create_session(
12409 &self,
12410 _identity: &AgentIdentity,
12411 _runtime_id: &AgentRuntimeId,
12412 _spec: &DurableAgentSpec,
12413 _draft: &AgentBuildDraft,
12414 _session_id: &SessionId,
12415 ) -> Result<SessionId, BridgeError> {
12416 Err(BridgeError::Mob(
12417 "create not expected: the identity resolves Ready".to_string(),
12418 ))
12419 }
12420
12421 async fn resume_session(
12422 &self,
12423 _identity: &AgentIdentity,
12424 _runtime_id: &AgentRuntimeId,
12425 _spec: &DurableAgentSpec,
12426 _draft: &AgentBuildDraft,
12427 _session_id: &SessionId,
12428 _snapshot: &SessionSnapshot,
12429 ) -> Result<ResumeSessionOutcome, BridgeError> {
12430 self.resume_attempts
12431 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
12432 Err(BridgeError::Mob(
12433 "collision retire blocked: disposal precondition holds (wedged for park test)"
12434 .to_string(),
12435 ))
12436 }
12437
12438 async fn deliver_admitted(
12439 &self,
12440 _runtime_id: &AgentRuntimeId,
12441 _delivery: BridgeDelivery,
12442 ) -> Result<SessionId, BridgeError> {
12443 Err(BridgeError::Mob("deliver not used".to_string()))
12444 }
12445
12446 async fn checkpoint_session(
12447 &self,
12448 _runtime_id: &AgentRuntimeId,
12449 _session_id: &SessionId,
12450 ) -> Result<SessionSnapshot, BridgeError> {
12451 Err(BridgeError::Mob("checkpoint not used".to_string()))
12452 }
12453
12454 async fn retire_member(&self, _runtime_id: &AgentRuntimeId) -> Result<(), BridgeError> {
12455 Ok(())
12456 }
12457 }
12458
12459 fn park_test_spec(identity: AgentIdentity) -> DurableAgentSpec {
12460 DurableAgentSpec {
12461 identity,
12462 profile: meerkat_mob::ProfileName::from("domain"),
12463 addressability: crate::identity_first::AgentAddressability::Addressable,
12464 display_name: None,
12465 labels: BTreeMap::new(),
12466 context: None,
12467 additional_instructions: Vec::new(),
12468 initial_message: None,
12469 runtime_mode_override: None,
12470 backend: None,
12471 binding: None,
12472 }
12473 }
12474
12475 #[tokio::test]
12480 async fn repair_loop_parks_typed_after_three_byte_identical_failures()
12481 -> Result<(), Box<dyn std::error::Error>> {
12482 let identity = AgentIdentity::parse("domain:wedged")?;
12483 let spec = park_test_spec(identity.clone());
12484 let session_id = SessionId::new();
12485 let record = ContinuityRecord {
12486 identity: identity.clone(),
12487 agent_runtime_id: AgentRuntimeId::parse(&format!("rt:{identity}:0"))?,
12488 session_id: session_id.clone(),
12489 generation: ContinuityGeneration::new(1),
12490 checkpoint_version: CheckpointVersion::new(1),
12491 };
12492 let continuity_store = Arc::new(LocalContinuityStore::in_memory()?);
12493 continuity_store
12494 .upsert_continuity_record(&record, FencingToken::new(1))
12495 .await?;
12496 let bridge = Arc::new(IdenticallyWedgedResumeBridge {
12497 resume_attempts: std::sync::atomic::AtomicUsize::new(0),
12498 });
12499 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
12500 continuity_store,
12501 lease_provider: Arc::new(LocalLeaseProvider::new()),
12502 runtime_instance_id: "identical-failure-park-test".to_string(),
12503 has_runtime_store: true,
12504 durability_policy: DurabilityPolicy::SyncWriteThrough,
12505 bridge: Some(bridge.clone()),
12506 default_timeout: None,
12507 }));
12508 runtime
12509 .register(
12510 spec.clone(),
12511 IdentityLifecycleState::Broken,
12512 Some(record),
12513 None,
12514 )
12515 .await;
12516 let context = Arc::new(IdentityFirstRuntimeContext::new(
12517 Arc::clone(&runtime),
12518 Arc::new(MutableRosterProvider::new(vec![spec])),
12519 None,
12520 None,
12521 None,
12522 ));
12523 let task = context.spawn_tracked_broken_identity_repair_task(ContinuityRepairPolicy {
12524 initial_backoff: Duration::from_millis(10),
12525 max_backoff: Duration::from_millis(10),
12526 });
12527
12528 let deadline = Instant::now() + Duration::from_secs(30);
12530 let park = loop {
12531 if let Some(park) = runtime.continuity_unrecoverable(&identity).await {
12532 break park;
12533 }
12534 if Instant::now() > deadline {
12535 task.cancel_and_join().await;
12536 return Err("repair loop never parked the identically-failing identity".into());
12537 }
12538 tokio::time::sleep(Duration::from_millis(20)).await;
12539 };
12540 assert!(
12541 park.reason.contains("byte-identical"),
12542 "park reason must name the bounded-identical-retry cause: {}",
12543 park.reason
12544 );
12545 assert!(
12546 park.reason
12547 .contains("disposal precondition holds (wedged for park test)"),
12548 "park reason must carry the blocking failure verbatim: {}",
12549 park.reason
12550 );
12551 let attempts_at_park = bridge.attempts();
12552 assert_eq!(
12553 attempts_at_park, 3,
12554 "the destructive repair must run exactly the bounded attempt count"
12555 );
12556
12557 tokio::time::sleep(Duration::from_millis(200)).await;
12559 assert_eq!(
12560 bridge.attempts(),
12561 attempts_at_park,
12562 "a parked identity must not be re-repaired on the timer"
12563 );
12564 assert!(
12565 runtime
12566 .repairable_broken_identities()
12567 .await
12568 .iter()
12569 .all(|id| id != &identity),
12570 "a parked identity must leave the repairable set"
12571 );
12572 task.cancel_and_join().await;
12573 Ok(())
12574 }
12575
12576 #[tokio::test]
12577 async fn repair_loop_exits_when_supervisor_sender_is_dropped()
12578 -> Result<(), Box<dyn std::error::Error>> {
12579 let runtime = Arc::new(IdentityRuntime::new(IdentityRuntimeConfig {
12580 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12581 lease_provider: Arc::new(LocalLeaseProvider::new()),
12582 runtime_instance_id: "dropped-repair-supervisor-test".to_string(),
12583 has_runtime_store: true,
12584 durability_policy: DurabilityPolicy::SyncWriteThrough,
12585 bridge: None,
12586 default_timeout: None,
12587 }));
12588 let context = Arc::new(IdentityFirstRuntimeContext::new(
12589 runtime,
12590 Arc::new(MutableRosterProvider::new(Vec::new())),
12591 None,
12592 None,
12593 None,
12594 ));
12595 let TrackedContinuityRepairTask { cancel, join } = context
12596 .spawn_tracked_broken_identity_repair_task(ContinuityRepairPolicy {
12597 initial_backoff: Duration::from_mins(1),
12598 max_backoff: Duration::from_mins(1),
12599 });
12600
12601 drop(cancel);
12604 tokio::time::timeout(Duration::from_millis(100), join)
12605 .await
12606 .map_err(|_| "repair loop spun after its cancellation channel closed")??;
12607 Ok(())
12608 }
12609}
12610
12611#[cfg(test)]
12612mod foreground_shutdown_tests {
12613 use super::*;
12614 use crate::identity_first::{LocalContinuityStore, LocalLeaseProvider};
12615
12616 #[test]
12617 fn foreground_shutdown_value_is_retained_for_late_subscribers()
12618 -> Result<(), ContinuityStoreError> {
12619 let runtime = IdentityRuntime::new(IdentityRuntimeConfig {
12620 continuity_store: Arc::new(LocalContinuityStore::in_memory()?),
12621 lease_provider: Arc::new(LocalLeaseProvider::new()),
12622 runtime_instance_id: "late-foreground-cancel-test".to_string(),
12623 has_runtime_store: true,
12624 durability_policy: DurabilityPolicy::SyncWriteThrough,
12625 bridge: None,
12626 default_timeout: None,
12627 });
12628
12629 assert_eq!(runtime.foreground_cancel.receiver_count(), 0);
12630 runtime.close_foreground_operations();
12631 let receiver = runtime.foreground_cancel.subscribe();
12632 assert!(
12633 *receiver.borrow(),
12634 "a task subscribed after close must still observe shutdown"
12635 );
12636 Ok(())
12637 }
12638}