use super::disposal::{
AbortOnError, BulkBestEffort, DisposalContext, DisposalReport, DisposalStep, ErrorPolicy,
WarnAndContinue,
};
use super::flow_frame_engine::FlowFrameLoopStorePlan;
use super::mob_member_lifecycle_projection::{
CanonicalMemberSnapshotMaterial, MobMemberLifecycleInput, MobMemberLifecycleProjection,
kickoff_snapshot_from_machine_state,
};
use super::mob_runtime_bridge_authority::{MobRuntimeBridgeAuthority, MobRuntimeBridgeEffect};
use super::provision_guard::PendingProvision;
use super::scope_gate::{RoutedMobCommand, ScopeAdmission};
use super::terminalization::{FlowFailureCause, TerminalizationOutcome, TerminalizationTarget};
use super::transaction::LifecycleRollback;
use super::*;
#[cfg(not(target_arch = "wasm32"))]
type ActorCommandFuture<'a, T> =
std::pin::Pin<Box<dyn std::future::Future<Output = T> + Send + 'a>>;
#[cfg(target_arch = "wasm32")]
type ActorCommandFuture<'a, T> = std::pin::Pin<Box<dyn std::future::Future<Output = T> + 'a>>;
// Build each command arm's future inside its own monomorphized frame.
//
// A debug build reserves a stack slot for every local in a function and does
// not reuse those slots across match arms, so constructing every handler
// future inline in `dispatch_command_boxed` reserved the SUM of their sizes on
// entry even though exactly one arm ever runs. Measured on the mob spawn path:
// 1322 KB of a 2 MB production worker stack consumed by that prologue alone,
// against a whole-chain peak near 1.8 MB. Linux reserves larger frames than
// macOS arm64, which is why `tools_full_with_explicit_auth_binding_can_spawn_
// within_production_stack_budget` overflowed on CI while passing locally.
//
// Taking a thunk moves construction into this function, so the dispatch frame
// holds only the small closures and the transient peak becomes the largest
// single handler future instead of their sum.
#[cfg(not(target_arch = "wasm32"))]
#[inline(never)]
fn boxed_arm_future<'a, T, F, Fut>(make: F) -> ActorCommandFuture<'a, T>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T> + Send + 'a,
{
Box::pin(make())
}
#[cfg(target_arch = "wasm32")]
#[inline(never)]
fn boxed_arm_future<'a, T, F, Fut>(make: F) -> ActorCommandFuture<'a, T>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T> + 'a,
{
Box::pin(make())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ActorLoopControl {
ProceedBoundary,
SkipBoundary,
BreakActor,
}
// Expand the command match synchronously so each arm constructs and erases
// its own async future. A single async block around the whole match recreates
// the aggregate command-loop poll frame this boundary exists to avoid.
macro_rules! boxed_actor_dispatch {
($cmd:expr, {
$($(#[$meta:meta])* $pattern:pat => $body:block)*
@control
$($(#[$control_meta:meta])* $control_pattern:pat => $control_body:block)*
}) => {{
match $cmd {
$(
$(#[$meta])*
$pattern => {
let future: ActorCommandFuture<'_, ActorLoopControl> =
boxed_arm_future(move || async move {
$body
ActorLoopControl::ProceedBoundary
});
future
}
)*
$(
$(#[$control_meta])*
$control_pattern => {
let future: ActorCommandFuture<'_, ActorLoopControl> =
boxed_arm_future(move || async move $control_body);
future
}
)*
}
}};
}
use crate::control_policy::{
CommandAuthority, CommandAuthorityKind, MemberOperatorExecutionFence, MobControlPrincipal,
ResolvedControlPolicy, ScopeDenial,
};
use crate::event::MobEvent;
use crate::generated::protocol_mob_destroying_session_ingress::MobDestroyingSessionIngressObligation;
use crate::ids::{AgentIdentity, AgentRuntimeId, RespawnTopologyPeerId, StepId};
use crate::machines::mob_machine as mob_dsl;
use crate::run::{MobMachineFlowAuthorityToken, MobMachineFlowRunCommand, MobRunStatus, flow_run};
#[cfg(target_arch = "wasm32")]
use crate::tokio;
use futures::FutureExt;
use futures::stream::{FuturesUnordered, StreamExt};
use meerkat_core::comms::{
CommsTrustMutation, CommsTrustMutationAuthority, CommsTrustMutationResult, PeerAddress,
PeerLifecycleKind, PeerName, PeerRoute, SendError, TrustedPeerDescriptor,
};
use meerkat_core::time_compat::{Duration, Instant, SystemTime};
use meerkat_machine_kernels::generated::mob::command_capabilities as generated_mob_command_capabilities;
use sha2::{Digest as _, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
pub(super) const RETIRE_LOCAL_TRUST_CLEANUP_CONCURRENCY: usize = 32;
const STARTUP_FAILURE_AUTONOMOUS_STOP_POLL_INTERVAL: Duration = Duration::from_millis(25);
const STARTUP_FAILURE_AUTONOMOUS_STOP_DEADLINE: Duration = Duration::from_secs(10);
/// A status projection is observational and must never hold the single mob
/// actor behind a slow or wedged session-runtime read. Unknown progress is a
/// truthful result; delaying lifecycle commands is not.
const MEMBER_PROGRESS_OBSERVATION_TIMEOUT: Duration = Duration::from_millis(250);
/// Actor-owned task termination policy at a `JoinError` boundary.
///
/// This is deliberately typed instead of inferred from log text: callers use
/// `requires_fail_stop` to decide whether an effect may have escaped without a
/// completion and therefore whether this actor incarnation may continue.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ActorTaskJoinPanicDisposition {
AmbiguousEffectFailStop,
RetryableIdempotentCleanup,
TeardownTerminal,
}
impl ActorTaskJoinPanicDisposition {
const fn as_str(self) -> &'static str {
match self {
Self::AmbiguousEffectFailStop => "ambiguous_effect_fail_stop",
Self::RetryableIdempotentCleanup => "retryable_idempotent_cleanup",
Self::TeardownTerminal => "teardown_terminal",
}
}
const fn requires_fail_stop(self) -> bool {
matches!(self, Self::AmbiguousEffectFailStop)
}
}
/// Recover one panic from a Tokio join boundary without rendering the raw
/// `JoinError`: its panic payload can be unbounded, multiline, or secret.
///
/// Callers classify cancellation separately. Tokio currently has only
/// cancellation and panic `JoinError`s, but the non-panic fallback remains
/// bounded and explicit so a future runtime extension cannot reintroduce a
/// raw/error-string logging path.
fn actor_task_join_panic_error(
context: &'static str,
disposition: ActorTaskJoinPanicDisposition,
error: tokio::task::JoinError,
) -> MobError {
let task_id = actor_task_join_error_task_id(&error);
#[cfg(not(target_arch = "wasm32"))]
let detail = if error.is_panic() {
let payload = error.into_panic();
super::panic_capture::panic_payload_detail(payload.as_ref())
} else {
"non-panic abnormal task termination".to_string()
};
// `tokio_with_wasm` exposes cancellation but not Tokio's panic payload or
// task-id API. A non-cancelled join failure is still classified and
// fail-closed by the caller, but there are no recoverable bytes to render.
#[cfg(target_arch = "wasm32")]
let detail = {
let _ = error;
"panic payload unavailable at wasm task join boundary".to_string()
};
tracing::error!(
context,
task_id = %task_id,
disposition = disposition.as_str(),
panic = %detail,
"actor-owned task panicked; payload recovered, sanitized, and classified"
);
MobError::Internal(format!(
"{context} task {task_id} panicked; disposition={}: {detail}",
disposition.as_str()
))
}
#[cfg(not(target_arch = "wasm32"))]
fn actor_task_join_error_is_panic(error: &tokio::task::JoinError) -> bool {
error.is_panic()
}
#[cfg(target_arch = "wasm32")]
fn actor_task_join_error_is_panic(error: &tokio::task::JoinError) -> bool {
!error.is_cancelled()
}
#[cfg(not(target_arch = "wasm32"))]
fn actor_task_join_error_task_id(error: &tokio::task::JoinError) -> String {
error.id().to_string()
}
#[cfg(target_arch = "wasm32")]
fn actor_task_join_error_task_id(_error: &tokio::task::JoinError) -> String {
"unavailable".to_string()
}
/// The identity lease is deliberately short and bounded. Reconciliation
/// renews it while work remains queued; a crashed actor therefore leaves no
/// durable process-lifecycle authority behind for the replacement actor to
/// recover.
const IDENTITY_RECONCILE_LEASE_TTL_MS: u64 = crate::identity::IDENTITY_LEASE_MAX_TTL_MS;
/// Cross-process repair net only. Ordinary reconciliation is causally driven
/// by startup discovery, actor-owned completions, and the exact in-memory
/// backoff deadline below. SQLite has no authoritative cross-process identity
/// notification yet, so retain one deliberately slow PAGE rather than opening
/// a connection and decoding every intent once per second forever.
const IDENTITY_RECONCILE_SAFETY_PAGE_INTERVAL: Duration = Duration::from_secs(5 * 60);
/// Hard per-wake work bound for the cross-process safety net. Large fleets are
/// visited over successive keyset pages; no timer wake materializes all
/// identities.
const IDENTITY_RECONCILE_SAFETY_PAGE_LIMIT: usize = 64;
/// Exponential actuation-backoff schedule for identities whose member
/// actuation keeps failing. Before this existed, the "Backoff" disposition
/// was a status label over an immediate requeue: the terminal-completion
/// requeue plus the 25ms tick re-ran full session provisioning at up to 40Hz
/// per identity, forever (2026-07-29 incident: a fleet of identities against
/// a closed callback transport burned a core with zero durable progress).
const IDENTITY_RECONCILE_BACKOFF_BASE: Duration = Duration::from_millis(250);
const IDENTITY_RECONCILE_BACKOFF_CAP: Duration = Duration::from_secs(60);
/// A valid row can predate the actor's narrow production support. Refuse it
/// visibly without claiming a lease or mutating any observed realization.
fn identity_reconciliation_slice_intent_rejection(
record: &crate::identity::IdentityIntentRecord,
) -> Option<String> {
use crate::identity::{
DesiredExecution, DesiredSessionAuthorityPolicy, IdentityIntent, IdentityRetirementPlan,
};
let IdentityIntent::Present {
session, member, ..
} = &record.intent
else {
return Some(
"absent-intent cleanup is outside the retained sealed-intent reconciliation slice"
.to_string(),
);
};
if session.authority_policy != DesiredSessionAuthorityPolicy::RequireExisting {
return Some(
"session creation is outside the retained sealed-intent reconciliation slice"
.to_string(),
);
}
if member.initial_delivery.is_some() {
return Some(
"initial delivery is outside the retained sealed-intent reconciliation slice"
.to_string(),
);
}
if !matches!(member.execution(), DesiredExecution::ControllingSession) {
return Some(
"non-controlling execution is outside the retained sealed-intent reconciliation slice"
.to_string(),
);
}
match &record.retirement_plan {
IdentityRetirementPlan::Targets { .. } => None,
IdentityRetirementPlan::NoKnownRealization => {
Some("the sealed intent is missing its exact realization custody plan".to_string())
}
}
}
fn identity_reconciliation_slice_unsupported_obligation(
decision: crate::identity::IdentityReconcileDecision,
) -> bool {
use crate::identity::IdentityReconcileDecision;
matches!(
decision,
IdentityReconcileDecision::AcquireLease
| IdentityReconcileDecision::SealRetirementProven
| IdentityReconcileDecision::SealSessionCreationConsumed
| IdentityReconcileDecision::EnsureSessionAuthority
| IdentityReconcileDecision::AwaitExternalBindingCeremony
| IdentityReconcileDecision::EnsureExternalBindingReceipt
| IdentityReconcileDecision::EnsureExternalBinding
| IdentityReconcileDecision::EnsureInitialDeliveryReceipt
| IdentityReconcileDecision::EnsureInitialDelivery
| IdentityReconcileDecision::AwaitInitialDelivery
| IdentityReconcileDecision::ReconcileWiring
| IdentityReconcileDecision::RetireMemberMaterialization
| IdentityReconcileDecision::RetireRuntimeRegistration
| IdentityReconcileDecision::ReleaseSessionAuthority
| IdentityReconcileDecision::Tombstoned
)
}
/// Volatile, actor-run-scoped witness that one identity's domain session was
/// read under one stable store-issued boundary while the pass classified
/// `Converged` under the exact sealed intent authority recorded here. While
/// that intent and boundary stay unchanged, steady-state passes substitute the
/// cached observation for the session-document read. This is never restart
/// authority: a cold actor starts with an empty map and re-observes each
/// identity through the store boundary.
#[derive(Debug, Clone)]
pub(super) struct IdentityConvergedSessionWitness {
intent_revision: u64,
intent_digest: String,
authority_digest: String,
observation: crate::identity::IdentitySessionObservation,
persisted_authority: crate::identity::IdentitySessionStoreAuthority,
}
/// Map threaded through the actor run loop; see
/// [`IdentityConvergedSessionWitness`].
pub(super) type IdentityConvergedSessionWitnesses =
BTreeMap<AgentIdentity, IdentityConvergedSessionWitness>;
/// Disposition of one reconcile pass attempt.
enum IdentityReconcilePassDisposition {
/// The pass settled; `true` requeues the identity for the next tick.
Outcome(bool),
/// The pass classified something other than `Converged` over a cached
/// session witness. The witness was discarded; the caller must re-run
/// the pass with a fresh session-document observation before any status
/// projection or actuation.
ReverifySession,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum IdentitySessionLoadErrorClass {
Malformed,
Unavailable,
}
/// Classify a session read failure by its typed cause. Durable corruption and
/// unsupported persistence are permanent observations; transport, I/O, and
/// unknown failures remain retryable. Display text is diagnostic only.
fn identity_session_load_error_class(
error: &meerkat_core::service::SessionError,
) -> IdentitySessionLoadErrorClass {
use meerkat_core::SessionStoreError;
use meerkat_core::service::SessionError;
match error {
SessionError::PersistenceDisabled | SessionError::Unsupported(_) => {
IdentitySessionLoadErrorClass::Malformed
}
SessionError::Store(source) => match source.downcast_ref::<SessionStoreError>() {
Some(SessionStoreError::Serialization(_) | SessionStoreError::Corrupted(_)) => {
IdentitySessionLoadErrorClass::Malformed
}
Some(
SessionStoreError::Io(_)
| SessionStoreError::NotFound(_)
| SessionStoreError::MonotonicityViolation { .. }
| SessionStoreError::TranscriptContinuityViolation { .. }
| SessionStoreError::TranscriptRevisionConflict { .. }
| SessionStoreError::InvalidTranscriptRewrite { .. }
| SessionStoreError::Internal(_),
)
| None => IdentitySessionLoadErrorClass::Unavailable,
},
// Forked/unverifiable durable evidence is a permanent corruption
// verdict; a tail held for reconciliation — or a recovery refused by
// a still-live conflicting runtime — is a temporary hold the identity
// machine may retry after the hold clears.
SessionError::DurableEvidenceQuarantined { .. } => IdentitySessionLoadErrorClass::Malformed,
SessionError::NotFound { .. }
| SessionError::Busy { .. }
| SessionError::CompactionDisabled
| SessionError::NotRunning { .. }
| SessionError::Agent(_)
| SessionError::DurableTailHeldForRecovery { .. }
| SessionError::DurableTailRecoveryRefused { .. }
| SessionError::FailedWithData { .. } => IdentitySessionLoadErrorClass::Unavailable,
}
}
fn identity_session_error_detail(context: &'static str, error: &dyn std::fmt::Display) -> String {
let detail = error.to_string();
let detail = detail.trim();
if detail.is_empty() {
context.to_string()
} else {
format!("{context}: {detail}")
}
}
fn identity_session_observation_from_load_error(
error: &meerkat_core::service::SessionError,
) -> Result<crate::identity::IdentitySessionObservation, crate::identity::IdentityIntentError> {
let detail = identity_session_error_detail("persisted session read failed", error);
match identity_session_load_error_class(error) {
IdentitySessionLoadErrorClass::Malformed => {
crate::identity::IdentitySessionObservation::malformed_unversioned(detail)
}
IdentitySessionLoadErrorClass::Unavailable => Ok(
crate::identity::IdentitySessionObservation::unavailable(detail),
),
}
}
/// Pure observation mapper for the actor's persisted-session read boundary.
/// Only a successful, explicit `None` is absence. A loaded document is first
/// paired with the stable store-issued authority observed around the read.
/// Session bytes remain domain evidence only and can never authenticate
/// themselves.
fn identity_session_observation_from_persisted_load(
desired: &crate::identity::DesiredSessionTarget,
loaded: Result<Option<meerkat_core::Session>, meerkat_core::service::SessionError>,
authority: Option<crate::identity::IdentitySessionStoreAuthority>,
) -> Result<crate::identity::IdentitySessionObservation, crate::identity::IdentityIntentError> {
use crate::identity::{IdentitySessionObservation, IdentityTargetObservationVersion};
let session = match loaded {
Ok(Some(session)) => session,
Ok(None) => {
if let Some(authority) = authority {
let version = authority.observation_version()?;
return IdentitySessionObservation::ambiguous_divergence(
version.clone(),
IdentityTargetObservationVersion::Version { version },
"store-issued session authority exists but the resume seam returned no session",
);
}
return IdentitySessionObservation::missing(format!(
"session-absent:{}",
desired.session_id
));
}
Err(error) => return identity_session_observation_from_load_error(&error),
};
if session.id() != &desired.session_id {
let bytes = match serde_json::to_vec(&session) {
Ok(bytes) => bytes,
Err(error) => {
return IdentitySessionObservation::malformed_unversioned(
identity_session_error_detail(
"mismatched persisted session could not be serialized for diagnostic evidence",
&error,
),
);
}
};
let evidence_digest = MobActor::identity_evidence_digest(&bytes);
return IdentitySessionObservation::ambiguous_divergence(
evidence_digest.clone(),
IdentityTargetObservationVersion::Version {
version: evidence_digest,
},
"persisted session document identity does not match the desired session",
);
}
let Some(authority) = authority else {
return Ok(IdentitySessionObservation::unavailable(format!(
"persisted session '{}' has no store-issued committed authority",
desired.session_id
)));
};
let authority_version = authority.observation_version()?;
match IdentitySessionObservation::matching(desired, &session, authority) {
Ok(observation) => Ok(observation),
Err(error) => IdentitySessionObservation::ambiguous_divergence(
authority_version.clone(),
IdentityTargetObservationVersion::Version {
version: authority_version,
},
error.to_string(),
),
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod identity_session_observation_tests {
use super::{
IdentitySessionLoadErrorClass, identity_session_load_error_class,
identity_session_observation_from_persisted_load,
};
use crate::identity::{
DesiredSessionAuthorityPolicy, DesiredSessionTarget, IdentityAuthorityCondition,
IdentityExternalCeremonyCondition, IdentityExternalTrustCondition,
IdentityInitialDeliveryCondition, IdentityLeaseCondition, IdentityReceiptCondition,
IdentityReconcileDecision, IdentityReconcileFacts, IdentityResourceCondition,
IdentitySessionCondition, IdentitySessionStoreAuthority, classify_identity_reconciliation,
};
use meerkat_core::service::SessionError;
use meerkat_core::{
Session, SessionGeneration, SessionLineageId, SessionStoreError, types::SessionId,
};
fn desired_for(session: &Session) -> DesiredSessionTarget {
DesiredSessionTarget {
session_id: session.id().clone(),
lineage_id: SessionLineageId::for_session(session.id()),
lineage_generation: SessionGeneration::INITIAL,
authority_policy: DesiredSessionAuthorityPolicy::RequireExisting,
}
}
fn session_and_authority() -> (DesiredSessionTarget, Session, IdentitySessionStoreAuthority) {
let session = Session::with_id(SessionId::new());
let desired = desired_for(&session);
let authority = IdentitySessionStoreAuthority::whole_blob_for_test(
session.id().clone(),
1,
"row-sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
);
(desired, session, authority)
}
fn present_facts(session: IdentitySessionCondition) -> IdentityReconcileFacts {
IdentityReconcileFacts {
intent: IdentityAuthorityCondition::PresentRequireExisting,
lease: IdentityLeaseCondition::HeldByCurrentIncarnation,
external_binding_required: false,
initial_delivery_required: false,
session_creation_receipt: IdentityReceiptCondition::NotRequired,
retirement_receipt: IdentityReceiptCondition::NotRequired,
session,
runtime: IdentityResourceCondition::Missing,
member: IdentityResourceCondition::Missing,
external_binding_receipt: IdentityReceiptCondition::NotRequired,
external_trust: IdentityExternalTrustCondition::NotRequired,
external_ceremony: IdentityExternalCeremonyCondition::NotRequired,
initial_delivery_receipt: IdentityReceiptCondition::NotRequired,
initial_delivery: IdentityInitialDeliveryCondition::NotRequired,
wiring: IdentityResourceCondition::Missing,
}
}
#[test]
fn typed_session_load_error_table_never_launders_faults_into_missing() {
let session_id = SessionId::new();
let cases = vec![
(
SessionError::Store(Box::new(SessionStoreError::Serialization(
"invalid persisted JSON".to_string(),
))),
IdentitySessionLoadErrorClass::Malformed,
),
(
SessionError::Store(Box::new(SessionStoreError::Corrupted(session_id.clone()))),
IdentitySessionLoadErrorClass::Malformed,
),
(
SessionError::PersistenceDisabled,
IdentitySessionLoadErrorClass::Malformed,
),
(
SessionError::Unsupported("durable session reads".to_string()),
IdentitySessionLoadErrorClass::Malformed,
),
(
SessionError::Store(Box::new(SessionStoreError::Io(std::io::Error::other(
"database temporarily unavailable",
)))),
IdentitySessionLoadErrorClass::Unavailable,
),
(
SessionError::Store(Box::new(std::io::Error::other("unknown transport wrapper"))),
IdentitySessionLoadErrorClass::Unavailable,
),
(
SessionError::NotFound {
id: session_id.clone(),
},
IdentitySessionLoadErrorClass::Unavailable,
),
(
SessionError::Busy {
id: session_id.clone(),
},
IdentitySessionLoadErrorClass::Unavailable,
),
];
let desired = DesiredSessionTarget {
session_id,
lineage_id: SessionLineageId::for_session(&SessionId::new()),
lineage_generation: SessionGeneration::INITIAL,
authority_policy: DesiredSessionAuthorityPolicy::RequireExisting,
};
for (error, expected) in cases {
assert_eq!(identity_session_load_error_class(&error), expected);
let observation =
identity_session_observation_from_persisted_load(&desired, Err(error), None)
.expect("typed load failure must produce a total observation");
let expected_condition = match expected {
IdentitySessionLoadErrorClass::Malformed => IdentitySessionCondition::Malformed,
IdentitySessionLoadErrorClass::Unavailable => IdentitySessionCondition::Unavailable,
};
assert_eq!(observation.condition(), expected_condition);
assert_eq!(
observation
.target_precondition()
.expect("load failure target precondition"),
None,
);
assert_eq!(
observation.malformed_unversioned_detail().is_some(),
expected == IdentitySessionLoadErrorClass::Malformed,
);
}
let missing = identity_session_observation_from_persisted_load(&desired, Ok(None), None)
.expect("explicit successful absence must be observable");
assert_eq!(missing.condition(), IdentitySessionCondition::Missing);
}
#[test]
fn loaded_session_matches_only_with_exact_store_issued_authority() {
let (desired, session, authority) = session_and_authority();
let observation = identity_session_observation_from_persisted_load(
&desired,
Ok(Some(session)),
Some(authority.clone()),
)
.expect("store-authorized session must remain a total observation");
assert_eq!(observation.condition(), IdentitySessionCondition::Matching);
assert_eq!(observation.store_authority(), Some(&authority));
}
#[test]
fn body_without_authority_is_unavailable_and_wrong_identity_is_ambiguous() {
let session_without_authority = Session::with_id(SessionId::new());
let unavailable = identity_session_observation_from_persisted_load(
&desired_for(&session_without_authority),
Ok(Some(session_without_authority)),
None,
)
.expect("body without authority must remain observable");
assert_eq!(
unavailable.condition(),
IdentitySessionCondition::Unavailable,
);
let (mut desired, session, authority) = session_and_authority();
desired.session_id = SessionId::new();
let mismatch = identity_session_observation_from_persisted_load(
&desired,
Ok(Some(session)),
Some(authority),
)
.expect("desired session mismatch must remain observable");
assert_eq!(
mismatch.condition(),
IdentitySessionCondition::AmbiguousDivergence,
);
}
#[test]
fn actor_mapper_drives_permanent_corruption_to_block_and_io_to_backoff() {
let desired = DesiredSessionTarget {
session_id: SessionId::new(),
lineage_id: SessionLineageId::for_session(&SessionId::new()),
lineage_generation: SessionGeneration::INITIAL,
authority_policy: DesiredSessionAuthorityPolicy::RequireExisting,
};
let permanent = identity_session_observation_from_persisted_load(
&desired,
Err(SessionError::Store(Box::new(
SessionStoreError::Serialization("persisted garbage".to_string()),
))),
None,
)
.expect("permanent corruption observation");
let transient = identity_session_observation_from_persisted_load(
&desired,
Err(SessionError::Store(Box::new(SessionStoreError::Io(
std::io::Error::other("database locked"),
)))),
None,
)
.expect("transient I/O observation");
assert_eq!(
classify_identity_reconciliation(present_facts(permanent.condition())),
IdentityReconcileDecision::RepairBlocked,
);
assert_eq!(
classify_identity_reconciliation(present_facts(transient.condition())),
IdentityReconcileDecision::Backoff,
);
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum IdentityMemberActuationDisposition {
Applied,
Conflict {
detail: String,
},
RepairBlocked {
detail: String,
},
Backoff {
detail: String,
},
/// The process-local callback transport is closed, so every in-process
/// build attempt fails instantly and can never recover without a new
/// declaration or a process restart. Retrying is pure waste (2026-07-29
/// incident); the identity is parked instead.
ParkedTransportClosed {
detail: String,
},
}
/// Exact durable intent observation that owns one volatile scheduling
/// disposition.
///
/// Backoff/park state may suppress work, so identity alone is insufficient:
/// an old completion must never suppress a replacement declaration. Invalid
/// rows bind to their exact evidence digest; an unavailable read can receive
/// a bounded retry delay but can never be permanently parked.
#[derive(Debug, Clone, PartialEq, Eq)]
enum IdentityReconcileAuthorityKey {
Exact {
intent_revision: u64,
intent_digest: String,
authority_digest: String,
},
Missing,
Unsupported {
evidence_digest: String,
},
Malformed {
evidence_digest: String,
},
Unavailable,
}
impl IdentityReconcileAuthorityKey {
fn from_intent(record: &crate::identity::IdentityIntentRecord) -> Self {
Self::Exact {
intent_revision: record.intent_revision,
intent_digest: record.intent_digest.clone(),
authority_digest: record.authority_digest.clone(),
}
}
fn from_permit(permit: &crate::identity::IdentityActuationPermit) -> Self {
Self::Exact {
intent_revision: permit.intent_revision,
intent_digest: permit.intent_digest.clone(),
authority_digest: permit.intent_authority_digest.clone(),
}
}
fn from_observation(
observation: &crate::identity::IdentityStoredObservation<
crate::identity::IdentityIntentRecord,
>,
) -> Self {
match observation {
crate::identity::IdentityStoredObservation::Valid(record) => Self::from_intent(record),
crate::identity::IdentityStoredObservation::Missing => Self::Missing,
crate::identity::IdentityStoredObservation::Unsupported {
evidence_digest, ..
} => Self::Unsupported {
evidence_digest: evidence_digest.clone(),
},
crate::identity::IdentityStoredObservation::Malformed {
evidence_digest, ..
} => Self::Malformed {
evidence_digest: evidence_digest.clone(),
},
}
}
const fn intent_revision(&self) -> Option<u64> {
match self {
Self::Exact {
intent_revision, ..
} => Some(*intent_revision),
Self::Missing
| Self::Unsupported { .. }
| Self::Malformed { .. }
| Self::Unavailable => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct IdentityReconcileCompletionAuthority {
intent: IdentityReconcileAuthorityKey,
lease_epoch: Option<u64>,
}
impl IdentityReconcileCompletionAuthority {
fn from_permit(permit: &crate::identity::IdentityActuationPermit) -> Self {
Self {
intent: IdentityReconcileAuthorityKey::from_permit(permit),
lease_epoch: Some(permit.lease_epoch),
}
}
}
/// Volatile per-identity actuation-backoff record. Scheduling custody only —
/// never persisted machine state; a cold actor incarnation restarts with a
/// clean slate and re-observes desired state.
#[derive(Debug, Clone)]
pub(super) struct IdentityReconcileBackoffState {
authority: IdentityReconcileAuthorityKey,
consecutive_failures: u32,
next_attempt_at: Instant,
/// The deadline wake already put this identity under actor custody.
/// Excluding it from the next-deadline calculation prevents an expired
/// deadline from becoming an always-ready select branch while a
/// materialization completion is still in flight.
retry_enqueued: bool,
}
/// Volatile per-identity terminal block, bound to the exact durable intent
/// observation that produced it. A changed declaration/row invalidates the
/// block; a stale completion cannot reinstall it.
#[derive(Debug, Clone)]
pub(super) struct IdentityReconcileParkState {
authority: IdentityReconcileAuthorityKey,
}
#[derive(Debug, Clone)]
pub(super) struct IdentityReconcileFailureState {
authority: IdentityReconcileAuthorityKey,
detail: String,
}
fn identity_reconcile_backoff_delay(consecutive_failures: u32) -> Duration {
// 250ms base, doubling per consecutive failure, capped at 60s. The shift
// is clamped well below the cap-saturating exponent so it cannot overflow.
let exponent = consecutive_failures.saturating_sub(1).min(16);
IDENTITY_RECONCILE_BACKOFF_BASE
.saturating_mul(1u32 << exponent)
.min(IDENTITY_RECONCILE_BACKOFF_CAP)
}
/// Bound repeated retry diagnostics without making a persistent incident
/// silent. Attempt one is visible immediately; identical retries then report
/// only at power-of-two checkpoints while durable convergence status retains
/// the latest full detail.
fn identity_reconcile_failure_should_log(consecutive_failures: u32) -> bool {
consecutive_failures.is_power_of_two()
}
fn identity_actuation_error_disposition(error: &MobError) -> IdentityMemberActuationDisposition {
if let MobError::MemberProvisionFailed {
cause: crate::MemberProvisionFailureCause::CallbackTransportClosed { detail },
} = error
{
return IdentityMemberActuationDisposition::ParkedTransportClosed {
detail: detail.clone(),
};
}
let MobError::StorageError(source) = error else {
return IdentityMemberActuationDisposition::Backoff {
detail: error.to_string(),
};
};
match source.downcast_ref::<crate::store::MobStoreError>() {
Some(crate::store::MobStoreError::CasConflict(detail)) => {
IdentityMemberActuationDisposition::Conflict {
detail: detail.clone(),
}
}
Some(crate::store::MobStoreError::IdentityAuthorityBlocked { detail, .. }) => {
IdentityMemberActuationDisposition::RepairBlocked {
detail: detail.clone(),
}
}
Some(crate::store::MobStoreError::WriteFailed(detail)) => {
IdentityMemberActuationDisposition::Backoff {
detail: detail.clone(),
}
}
Some(other) => IdentityMemberActuationDisposition::Backoff {
detail: other.to_string(),
},
None => IdentityMemberActuationDisposition::Backoff {
detail: error.to_string(),
},
}
}
fn identity_member_actuation_disposition(
result: &Result<super::handle::MemberSpawnReceipt, MobError>,
) -> IdentityMemberActuationDisposition {
match result {
Ok(_) => IdentityMemberActuationDisposition::Applied,
Err(error) => identity_actuation_error_disposition(error),
}
}
#[cfg(test)]
mod identity_recovery_slice_tests {
use super::{
IDENTITY_RECONCILE_BACKOFF_BASE, IDENTITY_RECONCILE_BACKOFF_CAP,
IdentityMemberActuationDisposition, identity_actuation_error_disposition,
identity_member_actuation_disposition, identity_reconcile_backoff_delay,
identity_reconcile_failure_should_log,
identity_reconciliation_slice_unsupported_obligation,
};
use crate::identity::IdentityReconcileDecision;
use crate::{MobError, store::MobStoreError};
#[test]
fn nonempty_wiring_remains_a_typed_unsupported_obligation() {
assert!(identity_reconciliation_slice_unsupported_obligation(
IdentityReconcileDecision::ReconcileWiring
));
}
#[test]
fn member_actuation_store_outcomes_preserve_causal_disposition() {
let conflict = Err::<super::super::handle::MemberSpawnReceipt, _>(MobError::from(
MobStoreError::CasConflict("stale member observation".to_string()),
));
assert!(matches!(
identity_member_actuation_disposition(&conflict),
IdentityMemberActuationDisposition::Conflict { ref detail }
if detail == "stale member observation"
));
let repair_blocked = Err::<super::super::handle::MemberSpawnReceipt, _>(MobError::from(
MobStoreError::IdentityAuthorityBlocked {
evidence_digest: Some(format!("sha256:{}", "c".repeat(64))),
detail: "unsafe member evidence".to_string(),
},
));
assert!(matches!(
identity_member_actuation_disposition(&repair_blocked),
IdentityMemberActuationDisposition::RepairBlocked { ref detail }
if detail == "unsafe member evidence"
));
}
#[test]
fn member_actuation_parks_only_typed_terminal_transport_cause() {
let typed =
Err::<super::super::handle::MemberSpawnReceipt, _>(MobError::MemberProvisionFailed {
cause: crate::MemberProvisionFailureCause::CallbackTransportClosed {
detail: "stdio callback bridge reached EOF".to_string(),
},
});
assert!(matches!(
identity_member_actuation_disposition(&typed),
IdentityMemberActuationDisposition::ParkedTransportClosed { ref detail }
if detail == "stdio callback bridge reached EOF"
));
let prose = Err::<super::super::handle::MemberSpawnReceipt, _>(MobError::Internal(
"callback transport closed".to_string(),
));
assert!(matches!(
identity_member_actuation_disposition(&prose),
IdentityMemberActuationDisposition::Backoff { ref detail }
if detail == "internal error: callback transport closed"
));
}
#[test]
fn every_generic_actuation_error_gets_a_typed_bounded_retry_disposition() {
assert!(matches!(
identity_actuation_error_disposition(&MobError::Internal("transient".to_string())),
IdentityMemberActuationDisposition::Backoff { ref detail }
if detail == "internal error: transient"
));
assert_eq!(
identity_reconcile_backoff_delay(0),
IDENTITY_RECONCILE_BACKOFF_BASE
);
assert_eq!(
identity_reconcile_backoff_delay(1),
IDENTITY_RECONCILE_BACKOFF_BASE
);
assert_eq!(
identity_reconcile_backoff_delay(u32::MAX),
IDENTITY_RECONCILE_BACKOFF_CAP
);
}
#[test]
fn identity_reconcile_failure_logging_uses_bounded_repeat_checkpoints() {
let logged = (1..=17)
.filter(|attempt| identity_reconcile_failure_should_log(*attempt))
.collect::<Vec<_>>();
assert_eq!(logged, vec![1, 2, 4, 8, 16]);
assert!(!identity_reconcile_failure_should_log(0));
assert!(!identity_reconcile_failure_should_log(u32::MAX));
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum HostOrphanReleaseReservation {
InFlight,
ReleasedAwaitingFreshOmission,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct PlacedTrackedInputKey {
agent_identity: String,
host_id: String,
generation: u64,
fence_token: u64,
input_id: String,
}
impl PlacedTrackedInputKey {
fn completion(obligation: &crate::event::PlacedCompletionObligationEvent) -> Self {
Self {
agent_identity: obligation.agent_identity.to_string(),
host_id: obligation.host_id.clone(),
generation: obligation.generation.get(),
fence_token: obligation.fence_token.get(),
input_id: obligation.input_id.clone(),
}
}
fn remote_turn(obligation: &crate::event::RemoteTurnObligationEvent) -> Self {
Self {
agent_identity: obligation.agent_identity.to_string(),
host_id: obligation.host_id.clone(),
generation: obligation.generation.get(),
fence_token: obligation.fence_token.get(),
input_id: obligation.input_id.clone(),
}
}
fn kickoff(obligation: &crate::event::PlacedKickoffObligationEvent) -> Self {
Self {
agent_identity: obligation.agent_identity.to_string(),
host_id: obligation.host_id.clone(),
generation: obligation.generation.get(),
fence_token: obligation.fence_token.get(),
input_id: obligation.input_id.clone(),
}
}
}
#[derive(Debug, Default)]
pub(super) struct PlacedCompletionDurableIndex {
used_tracked_inputs: BTreeMap<PlacedTrackedInputKey, PlacedTrackedInputOwner>,
// Private remote-turn intents are the replay-complete authority. Keep
// their TurnKeys separately so an actor-local index rebuild after an
// ambiguous public append cannot forget a key whose Record projection is
// still missing.
private_remote_turns: BTreeMap<PlacedTrackedInputKey, crate::event::RemoteTurnObligationEvent>,
completion_phase_events: BTreeMap<(u64, u8), MobEventKind>,
completion_sequences: BTreeMap<u64, crate::event::PlacedCompletionObligationEvent>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum PlacedTrackedInputOwner {
RemoteTurn(crate::event::RemoteTurnObligationEvent),
Kickoff(crate::event::PlacedKickoffObligationEvent),
Completion(crate::event::PlacedCompletionObligationEvent),
}
impl PlacedCompletionDurableIndex {
pub(super) fn recover(
events: &[crate::event::MobEvent],
mob_id: &MobId,
) -> Result<Self, MobError> {
let mut index = Self::default();
let mut durable_records = BTreeSet::<PlacedTrackedInputKey>::new();
for event in events.iter().filter(|event| &event.mob_id == mob_id) {
if matches!(event.kind, MobEventKind::MobReset) {
// Used host keys remain reserved across reset; phase sequence
// idempotency is scoped to the current machine epoch.
index.completion_phase_events.clear();
index.completion_sequences.clear();
continue;
}
let record_key = match &event.kind {
MobEventKind::RemoteTurnObligationRecorded { obligation } => {
Some(PlacedTrackedInputKey::remote_turn(obligation))
}
MobEventKind::PlacedKickoffObligationRecorded { obligation } => {
Some(PlacedTrackedInputKey::kickoff(obligation))
}
MobEventKind::PlacedCompletionObligationRecorded { obligation } => {
Some(PlacedTrackedInputKey::completion(obligation))
}
_ => None,
};
if let Some(record_key) = record_key
&& !durable_records.insert(record_key.clone())
{
return Err(MobError::Internal(format!(
"exact placed host TurnKey '{}' has multiple durable Record carriers",
record_key.input_id
)));
}
index.observe(&event.kind)?;
}
Ok(index)
}
pub(super) fn recover_with_private_remote_turns(
events: &[crate::event::MobEvent],
mob_id: &MobId,
private_remote_turns: &[crate::event::RemoteTurnObligationEvent],
) -> Result<Self, MobError> {
let mut index = Self::recover(events, mob_id)?;
for obligation in private_remote_turns {
index.observe_private_remote_turn(obligation.clone())?;
}
Ok(index)
}
fn observe_private_remote_turn(
&mut self,
obligation: crate::event::RemoteTurnObligationEvent,
) -> Result<(), MobError> {
let key = PlacedTrackedInputKey::remote_turn(&obligation);
self.observe_tracked_input(
key.clone(),
PlacedTrackedInputOwner::RemoteTurn(obligation.clone()),
)?;
match self.private_remote_turns.get(&key) {
Some(existing) if existing != &obligation => Err(MobError::Internal(format!(
"exact placed host TurnKey '{}' has conflicting private remote-turn owners",
key.input_id
))),
Some(_) => Ok(()),
None => {
self.private_remote_turns.insert(key, obligation);
Ok(())
}
}
}
fn rebuild_from_events(
&mut self,
events: &[crate::event::MobEvent],
mob_id: &MobId,
) -> Result<(), MobError> {
let private_remote_turns = self
.private_remote_turns
.values()
.cloned()
.collect::<Vec<_>>();
*self = Self::recover_with_private_remote_turns(events, mob_id, &private_remote_turns)?;
Ok(())
}
fn phase(kind: &MobEventKind) -> Option<(u8, &crate::event::PlacedCompletionObligationEvent)> {
match kind {
MobEventKind::PlacedCompletionObligationRecorded { obligation } => {
Some((0, obligation))
}
MobEventKind::PlacedCompletionCancellationRequested { obligation } => {
Some((1, obligation))
}
MobEventKind::PlacedCompletionOutcomeResolved { obligation, .. } => {
Some((2, obligation))
}
MobEventKind::PlacedCompletionOutcomeClosed { obligation, .. } => Some((3, obligation)),
MobEventKind::PlacedCompletionOutcomeAcknowledged { obligation } => {
Some((4, obligation))
}
MobEventKind::PlacedCompletionOutcomeDisposed { obligation } => Some((5, obligation)),
_ => None,
}
}
fn observe(&mut self, kind: &MobEventKind) -> Result<(), MobError> {
let remote_turn = match kind {
MobEventKind::RemoteTurnObligationRecorded { obligation }
| MobEventKind::RemoteTurnOutcomeResolved { obligation }
| MobEventKind::RemoteTurnOutcomeAcknowledged { obligation }
| MobEventKind::RemoteTurnOutcomeDisposed { obligation }
| MobEventKind::StepTargetCompleted {
remote_turn_obligation: Some(obligation),
..
}
| MobEventKind::StepTargetFailed {
remote_turn_obligation: Some(obligation),
..
} => Some(obligation),
_ => None,
};
if let Some(obligation) = remote_turn {
self.observe_tracked_input(
PlacedTrackedInputKey::remote_turn(obligation),
PlacedTrackedInputOwner::RemoteTurn(obligation.clone()),
)?;
}
if let MobEventKind::PlacedKickoffObligationRecorded { obligation } = kind {
self.observe_tracked_input(
PlacedTrackedInputKey::kickoff(obligation),
PlacedTrackedInputOwner::Kickoff(obligation.clone()),
)?;
}
let Some((phase, obligation)) = Self::phase(kind) else {
return Ok(());
};
if phase == 0 {
self.observe_tracked_input(
PlacedTrackedInputKey::completion(obligation),
PlacedTrackedInputOwner::Completion(obligation.clone()),
)?;
}
if let Some(existing) = self.completion_sequences.get(&obligation.dispatch_sequence)
&& existing != obligation
{
return Err(MobError::Internal(format!(
"placed completion durable index has conflicting sequence {}",
obligation.dispatch_sequence
)));
}
let key = (obligation.dispatch_sequence, phase);
if let Some(existing) = self.completion_phase_events.get(&key) {
if existing == kind {
return Err(MobError::Internal(format!(
"placed completion durable index has duplicate phase {phase} sequence {}",
obligation.dispatch_sequence
)));
}
return Err(MobError::Internal(format!(
"placed completion durable index has conflicting phase {phase} sequence {}",
obligation.dispatch_sequence
)));
}
self.validate_new_phase(obligation.dispatch_sequence, phase)?;
self.completion_sequences
.insert(obligation.dispatch_sequence, obligation.clone());
self.completion_phase_events.insert(key, kind.clone());
Ok(())
}
fn observe_tracked_input(
&mut self,
key: PlacedTrackedInputKey,
owner: PlacedTrackedInputOwner,
) -> Result<(), MobError> {
match self.used_tracked_inputs.get(&key) {
Some(existing) if existing == &owner => Ok(()),
Some(_) => Err(MobError::Internal(format!(
"exact placed host TurnKey '{}' has multiple durable owners",
key.input_id
))),
None => {
self.used_tracked_inputs.insert(key, owner);
Ok(())
}
}
}
fn reserve_fresh_tracked_input(
&mut self,
key: PlacedTrackedInputKey,
owner: PlacedTrackedInputOwner,
) -> Result<(), MobError> {
if self.used_tracked_inputs.contains_key(&key) {
return Err(MobError::PlacedInteractionIdAlreadyUsed {
interaction_id: key.input_id,
});
}
self.used_tracked_inputs.insert(key, owner);
Ok(())
}
fn has_phase(&self, dispatch_sequence: u64, phase: u8) -> bool {
self.completion_phase_events
.contains_key(&(dispatch_sequence, phase))
}
fn validate_new_phase(&self, dispatch_sequence: u64, phase: u8) -> Result<(), MobError> {
let missing_record = phase != 0 && !self.has_phase(dispatch_sequence, 0);
let branch_conflict = match phase {
// Record must be the first durable carrier in the chain.
0 => (1..=5).any(|other| self.has_phase(dispatch_sequence, other)),
// Cancellation is meaningful only before a terminal carrier.
1 => (2..=5).any(|other| self.has_phase(dispatch_sequence, other)),
// Resolve and Close are mutually exclusive terminal authorities.
2 => [3, 4, 5]
.into_iter()
.any(|other| self.has_phase(dispatch_sequence, other)),
3 => [2, 4, 5]
.into_iter()
.any(|other| self.has_phase(dispatch_sequence, other)),
// ACK may only follow Resolve and cannot follow Close/Dispose.
4 => {
!self.has_phase(dispatch_sequence, 2)
|| [3, 5]
.into_iter()
.any(|other| self.has_phase(dispatch_sequence, other))
}
// Exact release may dispose Pending or Resolved custody, but not
// an already closed or ACK-confirmed chain.
5 => [3, 4]
.into_iter()
.any(|other| self.has_phase(dispatch_sequence, other)),
_ => true,
};
if missing_record || branch_conflict {
return Err(MobError::Internal(format!(
"placed completion sequence {dispatch_sequence} has an illegal, predecessor-free, or cross-terminal phase {phase}"
)));
}
Ok(())
}
fn desired_present(&self, desired: &MobEventKind) -> Result<bool, MobError> {
let (phase, obligation) = Self::phase(desired).ok_or_else(|| {
MobError::Internal("non-completion event queried in completion index".to_string())
})?;
if let Some(existing) = self.completion_sequences.get(&obligation.dispatch_sequence)
&& existing != obligation
{
return Err(MobError::Internal(format!(
"placed completion sequence {} conflicts with durable index",
obligation.dispatch_sequence
)));
}
let present = match self
.completion_phase_events
.get(&(obligation.dispatch_sequence, phase))
{
Some(existing) if existing == desired => Ok(true),
Some(_) => Err(MobError::Internal(format!(
"placed completion phase {phase} sequence {} conflicts with durable index",
obligation.dispatch_sequence
))),
None => {
self.validate_new_phase(obligation.dispatch_sequence, phase)?;
Ok(false)
}
}?;
Ok(present)
}
fn require_exact(&self, desired: &MobEventKind) -> Result<(), MobError> {
if self.desired_present(desired)? {
Ok(())
} else {
let (_, obligation) = Self::phase(desired).ok_or_else(|| {
MobError::Internal("non-completion event queried in completion index".to_string())
})?;
Err(MobError::Internal(format!(
"placed completion sequence {} has no exact durable replay carrier",
obligation.dispatch_sequence
)))
}
}
}
/// Lightweight handle for a spawned autonomous initial turn.
///
/// The `JoinHandle` is for abort on stop/dispose.
pub(super) struct InitialTurnHandle {
handle: tokio::task::JoinHandle<()>,
}
impl InitialTurnHandle {
/// Abort the in-flight initial turn and await its task so it has fully
/// stopped before teardown proceeds. The cancellation `JoinError` is
/// expected and intentionally ignored.
async fn abort_and_join(self) {
self.handle.abort();
if let Err(error) = self.handle.await
&& actor_task_join_error_is_panic(&error)
{
let _ = actor_task_join_panic_error(
"autonomous initial turn teardown",
ActorTaskJoinPanicDisposition::TeardownTerminal,
error,
);
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct AutonomousStopInterruptIncarnation {
member_ref: MemberRef,
expected_member: Option<super::bridge_protocol::BridgeMemberIncarnation>,
}
pub(super) struct AutonomousStopInterruptTask {
incarnation: AutonomousStopInterruptIncarnation,
result_rx: oneshot::Receiver<Result<(), MobError>>,
}
/// Typed flag set recording which physical wiring side(s) of a peer-only edge
/// were successfully installed (local recipient, peer recipient, or both).
///
/// Drives best-effort rollback compensation: the wire rollback unwires only the
/// sides that were installed, and the unwire rollback rewires only the sides
/// that had been torn down. Replaces a hand-rolled `&[&str]` of `"local"` /
/// `"peer"` literals re-classified by `.contains(...)`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct WiringSides {
local: bool,
peer: bool,
}
impl WiringSides {
/// No sides installed/torn down.
fn empty() -> Self {
Self {
local: false,
peer: false,
}
}
/// Only the local recipient side.
fn local() -> Self {
Self {
local: true,
peer: false,
}
}
/// Only the peer recipient side.
fn peer() -> Self {
Self {
local: false,
peer: true,
}
}
/// Both recipient sides.
fn both() -> Self {
Self {
local: true,
peer: true,
}
}
fn has_local(self) -> bool {
self.local
}
fn has_peer(self) -> bool {
self.peer
}
}
enum SubmitWorkDispatchCompletion {
Completed,
AwaitTurnAdmission {
operation_id: Option<meerkat_core::ops::OperationId>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
completion_tx: Option<oneshot::Sender<Result<(), MobError>>>,
llm_identity_applied_tx: Option<super::handle::MemberTurnLlmIdentityAppliedSender>,
/// `Some(identity)` iff the target is machine-placed on a member
/// host: a typed delivery failure then fires the W-D.2 revival
/// trigger back onto the actor.
placed_identity: Option<AgentIdentity>,
/// Exact MobMachine-owned incarnation for a placed ingress-accepted
/// delivery. The host must never receive an unscoped
/// `expected_member: None` payload for a placed member.
placed_incarnation: Option<super::bridge_protocol::BridgeMemberIncarnation>,
/// Canonical non-nil retry-stable transport/idempotency id, distinct
/// from the optional caller transcript identity.
placed_input_id: Option<String>,
},
AwaitTurnCompletion {
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
/// See [`Self::AwaitTurnAdmission::placed_identity`].
placed_identity: Option<AgentIdentity>,
/// Exact MobMachine-owned incarnation for the placed completion
/// delivery. This keeps non-command callers on the explicit remote
/// transport instead of falling through `start_turn`'s Some=>local
/// provisioner arm.
placed_incarnation: Option<super::bridge_protocol::BridgeMemberIncarnation>,
/// Canonical non-nil interaction/input id shared by the waiter,
/// runtime transcript identity, host dedup key, and durable cleanup.
placed_input_id: Option<String>,
/// Machine-owned Record that was durably committed before the
/// authoritative SubmitWork ingress transition.
placed_completion_obligation: Option<crate::event::PlacedCompletionObligationEvent>,
/// Exact active pump context captured before Record/SubmitWork.
placed_completion_context: Option<super::event_pump::RemoteCompletionContext>,
},
}
impl SubmitWorkDispatchCompletion {
fn kind(&self) -> &'static str {
match self {
Self::Completed => "Completed",
Self::AwaitTurnAdmission { .. } => "AwaitTurnAdmission",
Self::AwaitTurnCompletion { .. } => "AwaitTurnCompletion",
}
}
}
struct SubmitWorkDispatchRequest {
content: ContentInput,
/// Optional ordinary System message authored for this exact work turn.
system_prompt: Option<String>,
/// Host-attached injected context riding with the work content.
/// Deliverable on queue-mode turn-driven dispatch (local and remote);
/// autonomous inbox delivery and steer dispatch reject it fail-closed.
injected_context: Vec<ContentInput>,
/// Host-supplied interaction identity for the delivered turn. Stamped
/// into `RuntimeTurnMetadata.transcript_identity` on turn-driven
/// dispatch and onto the injected inbox event on autonomous dispatch,
/// so the committed transcript messages persist the id the host's live
/// interaction frames carry (mobkit ask-15 addendum).
interaction_id: Option<meerkat_core::interaction::InteractionId>,
objective_id: Option<meerkat_core::interaction::ObjectiveId>,
handling_mode: meerkat_core::types::HandlingMode,
external_delivery_identity: Option<crate::store::MobExternalDeliveryIdentity>,
turn_metadata: Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
event_tx:
Option<tokio::sync::mpsc::Sender<meerkat_core::EventEnvelope<meerkat_core::AgentEvent>>>,
completion_tx: Option<oneshot::Sender<Result<(), MobError>>>,
llm_identity_applied_tx: Option<super::handle::MemberTurnLlmIdentityAppliedSender>,
ack_mode: crate::mob_machine::SubmitWorkAckMode,
operation_id: Option<meerkat_core::ops::OperationId>,
/// Present only for a placed TurnCompleted dispatch. Its durable Record
/// already exists; dispatch may realize but must never mint custody.
placed_completion_obligation: Option<crate::event::PlacedCompletionObligationEvent>,
/// Exact active pump lease captured during pre-Record preflight.
placed_completion_context: Option<super::event_pump::RemoteCompletionContext>,
}
struct PreparedPlacedCompletionWait {
expected_member: super::bridge_protocol::BridgeMemberIncarnation,
input_id: String,
waiter: oneshot::Receiver<super::event_pump::RemoteInteractionTerminal>,
obligation: crate::event::PlacedCompletionObligationEvent,
handle: MobHandle,
identity: AgentIdentity,
}
/// Canonical turn-metadata carrier for submit-work delivery: render metadata
/// and the host-supplied transcript interaction identity travel together;
/// an empty pair stays `None` so metadata-less requests keep their shape.
fn submit_work_turn_metadata(
turn_metadata: Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
interaction_id: Option<meerkat_core::interaction::InteractionId>,
objective_id: Option<meerkat_core::interaction::ObjectiveId>,
) -> Result<Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>, MobError> {
let mut merged = turn_metadata.unwrap_or_default();
if interaction_id.is_some() || objective_id.is_some() {
merged
.merge(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
transcript_identity: meerkat_core::types::TranscriptMessageIdentity {
interaction_id,
run_id: None,
objective_id,
},
..Default::default()
},
)
.map_err(|conflict| {
MobError::Internal(format!(
"submit-work turn metadata conflict on `{}`: {}",
conflict.field, conflict.reason
))
})?;
}
Ok((!merged.is_empty()).then_some(merged))
}
fn submit_work_runtime_semantics(
handling_mode: meerkat_core::types::HandlingMode,
turn_metadata: Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
external_delivery_identity: Option<&crate::store::MobExternalDeliveryIdentity>,
) -> meerkat_core::service::StartTurnRuntimeSemantics {
let semantics =
meerkat_core::service::StartTurnRuntimeSemantics::new(handling_mode, None, turn_metadata);
match external_delivery_identity {
Some(identity) => {
semantics.with_input_identity(meerkat_core::service::StartTurnInputIdentity {
idempotency_key: identity.idempotency_key.clone(),
correlation_id: identity.correlation_id.clone(),
})
}
None => semantics,
}
}
fn unsupported_turn_metadata_fields(
metadata: &meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata,
handling_mode: meerkat_core::types::HandlingMode,
allow_render_metadata: bool,
allow_interaction_id: bool,
allow_transient_turn_context: bool,
) -> Vec<&'static str> {
let mut fields = Vec::new();
if metadata
.handling_mode
.is_some_and(|metadata_mode| metadata_mode != handling_mode)
{
fields.push("handling_mode");
}
if metadata.skill_references.is_some() {
fields.push("skill_references");
}
if metadata.turn_tool_overlay.is_some() {
fields.push("turn_tool_overlay");
}
if metadata.additional_instructions.is_some() {
fields.push("additional_instructions");
}
if !metadata.system_prompts.is_empty() {
fields.push("system_prompts");
}
if !allow_transient_turn_context && metadata.transient_turn_context.is_some() {
fields.push("transient_turn_context");
}
if metadata.model.is_some() {
fields.push("model");
}
if metadata.provider.is_some() {
fields.push("provider");
}
if metadata.self_hosted_server_id.is_some() {
fields.push("self_hosted_server_id");
}
if metadata.provider_params.is_some() {
fields.push("provider_params");
}
if metadata.auth_binding.is_some() {
fields.push("auth_binding");
}
if metadata.keep_alive.is_some() {
fields.push("keep_alive");
}
if !allow_render_metadata && metadata.render_metadata.is_some() {
fields.push("render_metadata");
}
if metadata.execution_kind.is_some() {
fields.push("execution_kind");
}
if metadata.peer_response_terminal_apply_intent.is_some() {
fields.push("peer_response_terminal_apply_intent");
}
if !allow_interaction_id && metadata.transcript_identity.interaction_id.is_some() {
fields.push("transcript_identity.interaction_id");
}
if metadata.transcript_identity.run_id.is_some() {
fields.push("transcript_identity.run_id");
}
fields
}
fn validate_member_turn_carriers(
entry: &RosterEntry,
remotely_hosted: bool,
handling_mode: meerkat_core::types::HandlingMode,
has_external_delivery_identity: bool,
turn_metadata: Option<&meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
has_event_sender: bool,
has_completion_sender: bool,
) -> Result<(), MobError> {
let peer_only = remotely_hosted
|| matches!(
&entry.member_ref,
crate::event::MemberRef::BackendPeer {
session_id: None,
..
}
);
let autonomous = entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if has_external_delivery_identity && !remotely_hosted && (autonomous || peer_only) {
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: if autonomous {
"stable external input identity is not representable on autonomous inbox delivery"
} else {
"stable external input identity is not representable on the legacy peer-only lane"
}
.to_string(),
});
}
if (has_event_sender || has_completion_sender) && (peer_only || autonomous) {
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: if peer_only {
"tracked turn event streams are not supported for remotely hosted or peer-only members"
} else {
"tracked turn event streams are not supported by autonomous inbox delivery"
}
.to_string(),
});
}
let Some(metadata) = turn_metadata else {
return Ok(());
};
if metadata
.handling_mode
.is_some_and(|metadata_mode| metadata_mode != handling_mode)
{
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "turn metadata handling_mode must match the explicit member turn handling mode"
.to_string(),
});
}
if metadata.transient_turn_context.is_some()
&& handling_mode == meerkat_core::types::HandlingMode::Steer
{
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "transient turn context requires a queued executor turn".to_string(),
});
}
let unsupported = if peer_only {
unsupported_turn_metadata_fields(metadata, handling_mode, false, false, true)
} else if autonomous {
unsupported_turn_metadata_fields(metadata, handling_mode, true, true, false)
} else {
Vec::new()
};
if unsupported.is_empty() {
return Ok(());
}
Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: format!(
"turn metadata fields are not representable on this member delivery path: {}",
unsupported.join(", ")
),
})
}
fn placed_submit_interaction_id(
supplied: Option<meerkat_core::interaction::InteractionId>,
) -> Result<meerkat_core::interaction::InteractionId, MobError> {
let interaction_id = supplied.unwrap_or_else(|| {
meerkat_core::interaction::InteractionId(meerkat_core::time_compat::new_uuid_v7())
});
if interaction_id.0.is_nil() {
return Err(MobError::InvalidPlacedInteractionId {
interaction_id: interaction_id.0.to_string(),
});
}
Ok(interaction_id)
}
fn placed_turn_supplied_interaction_id(
req: &meerkat_core::service::StartTurnRequest,
) -> Result<Option<meerkat_core::interaction::InteractionId>, MobError> {
let interaction_id = req
.runtime
.turn_metadata
.as_ref()
.and_then(|metadata| metadata.transcript_identity.interaction_id);
if interaction_id.is_some_and(|interaction_id| interaction_id.0.is_nil()) {
return Err(MobError::InvalidPlacedInteractionId {
interaction_id: uuid::Uuid::nil().to_string(),
});
}
Ok(interaction_id)
}
#[cfg(test)]
fn placed_turn_test_request(
interaction_id: Option<meerkat_core::interaction::InteractionId>,
) -> meerkat_core::service::StartTurnRequest {
meerkat_core::service::StartTurnRequest {
injected_context: Vec::new(),
prompt: ContentInput::Text("test".to_string()),
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::runtime_metadata(
submit_work_turn_metadata(None, interaction_id, None)
.expect("test turn metadata should merge")
.unwrap_or_default(),
),
}
}
#[cfg(test)]
#[test]
fn placed_turn_correlation_reuses_supplied_transcript_interaction_id() {
let supplied =
meerkat_core::interaction::InteractionId(uuid::Uuid::from_u128(0x1234_5678_9abc));
let req = placed_turn_test_request(Some(supplied));
assert_eq!(
placed_turn_supplied_interaction_id(&req).expect("supplied interaction id is valid"),
Some(supplied)
);
}
#[cfg(test)]
#[test]
fn placed_turn_admission_preserves_absence_while_completion_mints_and_both_reject_nil() {
assert_eq!(
placed_turn_supplied_interaction_id(&placed_turn_test_request(None))
.expect("absent admission identity remains absent"),
None
);
let minted = placed_submit_interaction_id(None)
.expect("absent completion interaction id mints correlation");
assert!(!minted.0.is_nil());
let nil = meerkat_core::interaction::InteractionId(uuid::Uuid::nil());
assert!(placed_turn_supplied_interaction_id(&placed_turn_test_request(Some(nil))).is_err());
assert!(placed_submit_interaction_id(Some(nil)).is_err());
}
/// Read the machine-owned kickoff objective without allowing a corrupt
/// persisted UUID to degrade into an unscoped turn.
fn machine_kickoff_objective_id(
state: &mob_dsl::MobMachineState,
agent_identity: &AgentIdentity,
) -> Result<Option<meerkat_core::interaction::ObjectiveId>, MobError> {
state
.member_kickoff_objective_ids
.get(&mob_dsl::AgentIdentity::from_domain(agent_identity))
.map(|raw| {
uuid::Uuid::parse_str(raw)
.map(meerkat_core::interaction::ObjectiveId)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine stored invalid kickoff objective id '{raw}' for '{agent_identity}': {error}"
))
})
})
.transpose()
}
fn required_machine_kickoff_objective_id(
state: &mob_dsl::MobMachineState,
agent_identity: &AgentIdentity,
) -> Result<meerkat_core::interaction::ObjectiveId, MobError> {
machine_kickoff_objective_id(state, agent_identity)?.ok_or_else(|| {
MobError::Internal(format!(
"MobMachine has no kickoff objective id for autonomous member '{agent_identity}'"
))
})
}
fn machine_kickoff_turn_metadata(
state: &mob_dsl::MobMachineState,
agent_identity: &AgentIdentity,
) -> Result<Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>, MobError> {
submit_work_turn_metadata(
None,
None,
Some(required_machine_kickoff_objective_id(
state,
agent_identity,
)?),
)
}
#[cfg(feature = "runtime-adapter")]
fn kickoff_resolution_input(
agent_identity: &AgentIdentity,
outcome: meerkat_runtime::completion::CompletionOutcome,
) -> mob_dsl::MobMachineInput {
let member_id = mob_dsl::AgentIdentity::from_domain(agent_identity);
match outcome {
meerkat_runtime::completion::CompletionOutcome::Completed(_)
| meerkat_runtime::completion::CompletionOutcome::CompletedWithoutResult => {
mob_dsl::MobMachineInput::KickoffResolveStarted { member_id }
}
meerkat_runtime::completion::CompletionOutcome::CallbackPending { .. }
| meerkat_runtime::completion::CompletionOutcome::CallbackBatchPending { .. } => {
mob_dsl::MobMachineInput::KickoffResolveCallbackPending { member_id }
}
meerkat_runtime::completion::CompletionOutcome::Cancelled => {
mob_dsl::MobMachineInput::KickoffCancelRequested { member_id }
}
meerkat_runtime::completion::CompletionOutcome::Abandoned { reason: error, .. }
| meerkat_runtime::completion::CompletionOutcome::AbandonedWithError {
reason: error,
..
}
| meerkat_runtime::completion::CompletionOutcome::RuntimeTerminated {
reason: error, ..
} => mob_dsl::MobMachineInput::KickoffResolveFailed { member_id, error },
meerkat_runtime::completion::CompletionOutcome::CompletedWithFinalizationFailure {
error,
} => mob_dsl::MobMachineInput::KickoffResolveFailed {
member_id,
// Runtime completion deliberately withholds the non-durable result;
// kickoff must preserve that fail-closed contract too.
error: error
.detail
.unwrap_or_else(|| "turn finalization failed".to_string()),
},
}
}
fn unresolved_kickoff_member_ids(
state: &mob_dsl::MobMachineState,
) -> std::collections::BTreeSet<String> {
// CallbackPending is an interaction terminal and the kickoff completion
// handle is one-shot. Preserve the phase for diagnostics, but do not keep
// the barrier open for a continuation this lifecycle does not own.
state
.member_kickoff_pending
.iter()
.chain(state.member_kickoff_starting.iter())
.map(|id| id.0.clone())
.collect()
}
#[derive(Debug, Clone)]
enum SubmitWorkIngressAuthority {
Runtime {
agent_runtime_id: mob_dsl::AgentRuntimeId,
fence_token: mob_dsl::FenceToken,
generation: Option<mob_dsl::Generation>,
session_id: mob_dsl::SessionId,
work_id: mob_dsl::WorkId,
origin: mob_dsl::WorkOrigin,
},
PeerRuntime {
agent_runtime_id: mob_dsl::AgentRuntimeId,
fence_token: mob_dsl::FenceToken,
generation: Option<mob_dsl::Generation>,
work_id: mob_dsl::WorkId,
origin: mob_dsl::WorkOrigin,
},
}
impl SubmitWorkIngressAuthority {
fn from_transition(
transition: &mob_dsl::MobMachineTransition,
expected_runtime_id: &mob_dsl::AgentRuntimeId,
expected_fence_token: mob_dsl::FenceToken,
expected_generation: mob_dsl::Generation,
expected_work_id: &mob_dsl::WorkId,
expected_origin: mob_dsl::WorkOrigin,
) -> Result<Self, MobError> {
let mut resolved = None;
for effect in transition.effects() {
let authority = match effect {
mob_dsl::MobMachineEffect::RequestRuntimeIngress {
agent_runtime_id,
fence_token,
generation,
session_id,
work_id,
origin,
} => Self::Runtime {
agent_runtime_id: agent_runtime_id.clone(),
fence_token: *fence_token,
generation: *generation,
session_id: session_id.clone(),
work_id: work_id.clone(),
origin: *origin,
},
mob_dsl::MobMachineEffect::RequestPeerRuntimeIngress {
agent_runtime_id,
fence_token,
generation,
work_id,
origin,
} => Self::PeerRuntime {
agent_runtime_id: agent_runtime_id.clone(),
fence_token: *fence_token,
generation: *generation,
work_id: work_id.clone(),
origin: *origin,
},
_ => continue,
};
authority.verify_payload(
expected_runtime_id,
expected_fence_token,
expected_generation,
expected_work_id,
expected_origin,
)?;
if resolved.replace(authority).is_some() {
return Err(MobError::Internal(
"MobMachine SubmitWork emitted multiple generated runtime ingress authorities"
.to_string(),
));
}
}
resolved.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted SubmitWork but emitted no generated runtime ingress authority"
.to_string(),
)
})
}
fn variant(&self) -> &'static str {
match self {
Self::Runtime { .. } => "RequestRuntimeIngress",
Self::PeerRuntime { .. } => "RequestPeerRuntimeIngress",
}
}
fn agent_runtime_id(&self) -> &mob_dsl::AgentRuntimeId {
match self {
Self::Runtime {
agent_runtime_id, ..
}
| Self::PeerRuntime {
agent_runtime_id, ..
} => agent_runtime_id,
}
}
fn fence_token(&self) -> mob_dsl::FenceToken {
match self {
Self::Runtime { fence_token, .. } | Self::PeerRuntime { fence_token, .. } => {
*fence_token
}
}
}
fn generation(&self) -> Option<mob_dsl::Generation> {
match self {
Self::Runtime { generation, .. } | Self::PeerRuntime { generation, .. } => *generation,
}
}
fn work_id(&self) -> &mob_dsl::WorkId {
match self {
Self::Runtime { work_id, .. } | Self::PeerRuntime { work_id, .. } => work_id,
}
}
fn origin(&self) -> mob_dsl::WorkOrigin {
match self {
Self::Runtime { origin, .. } | Self::PeerRuntime { origin, .. } => *origin,
}
}
fn verify_payload(
&self,
expected_runtime_id: &mob_dsl::AgentRuntimeId,
expected_fence_token: mob_dsl::FenceToken,
expected_generation: mob_dsl::Generation,
expected_work_id: &mob_dsl::WorkId,
expected_origin: mob_dsl::WorkOrigin,
) -> Result<(), MobError> {
if self.agent_runtime_id() != expected_runtime_id
|| self.fence_token() != expected_fence_token
|| self.generation() != Some(expected_generation)
|| self.work_id() != expected_work_id
|| self.origin() != expected_origin
{
return Err(MobError::Internal(format!(
"generated {} authority did not match admitted SubmitWork payload",
self.variant()
)));
}
Ok(())
}
fn verify_member_ref(&self, member_ref: &MemberRef, context: &str) -> Result<(), MobError> {
match self {
Self::Runtime { session_id, .. } => {
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Err(MobError::Internal(format!(
"{context} requires a session-bound member for generated {} authority",
self.variant()
)));
};
let projected_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
if &projected_session_id != session_id {
return Err(MobError::Internal(format!(
"{context} generated {} authority session does not match MobMachine session binding",
self.variant()
)));
}
Ok(())
}
Self::PeerRuntime { .. } => match member_ref {
MemberRef::BackendPeer {
session_id: None, ..
} => Ok(()),
_ => Err(MobError::Internal(format!(
"{context} requires a peer-only member for generated {} authority",
self.variant()
))),
},
}
}
fn is_peer_runtime(&self) -> bool {
matches!(self, Self::PeerRuntime { .. })
}
}
// Sized for real mob-scale startup/shutdown fan-out (50+ members).
#[cfg(not(target_arch = "wasm32"))]
const MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS: usize = 64;
const MAX_LIFECYCLE_NOTIFICATION_TASKS: usize = 16;
const MAX_PARALLEL_PEER_RETIRE_NOTIFICATIONS: usize = 64;
const MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS: usize = 8;
#[cfg(not(test))]
pub(super) const MAX_PENDING_PEER_DELIVERIES: usize = 1024;
#[cfg(test)]
pub(super) const MAX_PENDING_PEER_DELIVERIES: usize = 4;
pub(super) fn advance_rotating_cursor(
item_count: usize,
rotation: usize,
last_selected_offset: Option<usize>,
) -> usize {
if item_count == 0 {
return 0;
}
match last_selected_offset {
Some(offset) => (rotation + offset + 1) % item_count,
None => (rotation + 1) % item_count,
}
}
pub(super) fn lifecycle_origin_fenced(state: &mob_dsl::MobMachineState) -> bool {
state.placed_completion_lifecycle_quiescing
|| state.placed_completion_lifecycle_intent.is_some()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum AutonomousStopPhase {
WaitForPlacedKickoffCleanup,
DriveInterrupts,
}
fn autonomous_stop_phase(pending_kickoffs: usize, resolved_kickoffs: usize) -> AutonomousStopPhase {
if pending_kickoffs != 0 || resolved_kickoffs != 0 {
AutonomousStopPhase::WaitForPlacedKickoffCleanup
} else {
AutonomousStopPhase::DriveInterrupts
}
}
fn disposal_uses_host_release_authority(host_owned: bool) -> bool {
host_owned
}
#[cfg(test)]
pub(super) static SPAWN_PROVISIONED_COMMAND_DELAY_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
pub(super) static IDENTITY_RECONCILE_COMPLETION_REQUEUES: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
pub(super) static IDENTITY_RECONCILE_REPLY_DELIVERY_FAILURES: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
pub(super) static IDENTITY_RECONCILE_SESSION_DOCUMENT_LOADS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
static FAIL_SESSION_INGRESS_DETACH_FOR_SESSION: std::sync::Mutex<Option<SessionId>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
pub(super) fn fail_session_ingress_detach_for_test(session_id: &SessionId) {
if let Ok(mut target) = FAIL_SESSION_INGRESS_DETACH_FOR_SESSION.lock() {
*target = Some(session_id.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_destroy_member_archive_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_destroy_remote_archive_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_remote_supervisor_revoked_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
/// Render a mob member's comms (peer) routing name from typed components
/// through the single owner [`meerkat_core::MemberCommsName`].
///
/// The components here are already typed mob slugs (`MobId`/`ProfileName`/
/// `AgentIdentity`), so construction normally succeeds; the join is owned by
/// `MemberCommsName::Display` rather than an inline `format!`. A component that
/// fails the slug rule fails closed as a typed [`MobError::MemberCommsName`]
/// rather than reconstructing the raw `mob_id/role/member` join that the single
/// owner already rejected.
pub(super) fn render_member_comms_name(
mob_id: &str,
role: &str,
member: &str,
) -> Result<String, MobError> {
Ok(meerkat_core::MemberCommsName::new(mob_id, role, member)?.to_string())
}
fn observed_runtime_id(signal: &mob_dsl::MobMachineSignal) -> Option<&mob_dsl::AgentRuntimeId> {
match signal {
mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id, ..
}
| mob_dsl::MobMachineSignal::ObserveRuntimeRetired {
agent_runtime_id, ..
}
| mob_dsl::MobMachineSignal::ObserveRuntimeDestroyed {
agent_runtime_id, ..
} => Some(agent_runtime_id),
_ => None,
}
}
fn foreign_runtime_observation<'a>(
state: &mob_dsl::MobMachineState,
signal: &'a mob_dsl::MobMachineSignal,
) -> Option<&'a mob_dsl::AgentRuntimeId> {
let agent_runtime_id = observed_runtime_id(signal)?;
(!state.live_runtime_ids.contains(agent_runtime_id)).then_some(agent_runtime_id)
}
/// Bridge budget for one `InstallPeerTrust`/`RemovePeerTrust` route-install
/// send (ADJ-3/ADJ-4 precedent: same 60s materialize budget, exactly one
/// resend of the same payload on TIMEOUT only — a delivered rejection
/// leaves the obligation for the event-driven retry lanes).
const ROUTE_INSTALL_BRIDGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
pub(super) const EXACT_EVENT_RECONCILE_PAGE_SIZE: usize = 64;
const EXACT_EVENT_RECONCILE_READ_ATTEMPTS: usize = 2;
/// Reconcile an append error that may have happened after the write. The
/// cursor floor distinguishes this operation from an identical structural
/// event in an earlier wire/unwire cycle.
pub(super) async fn reconcile_exact_mob_event_after_cursor(
events: &(dyn MobEventStore + 'static),
mob_id: &MobId,
cursor_floor: u64,
desired: &MobEventKind,
) -> Result<Option<MobEvent>, MobError> {
// Freeze the scan horizon after the append returned. This both bounds the
// reconciliation under concurrent writers and avoids mistaking a later
// identical operation for the one whose acknowledgement was lost.
let scan_ceiling = {
let mut last_error = None;
let mut ceiling = None;
for attempt in 1..=EXACT_EVENT_RECONCILE_READ_ATTEMPTS {
match events.latest_cursor().await {
Ok(cursor) => {
ceiling = Some(cursor);
break;
}
Err(error) => {
last_error = Some(error);
if attempt < EXACT_EVENT_RECONCILE_READ_ATTEMPTS {
tracing::warn!(
mob_id = %mob_id,
attempt,
"exact event reconciliation latest-cursor read failed transiently; retrying"
);
tokio::task::yield_now().await;
}
}
}
}
match ceiling {
Some(cursor) => cursor,
None => {
return Err(MobError::Internal(format!(
"exact event reconciliation could not freeze a cursor ceiling after {EXACT_EVENT_RECONCILE_READ_ATTEMPTS} attempts: {}",
last_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown store error".to_string())
)));
}
}
};
if scan_ceiling <= cursor_floor {
return Ok(None);
}
let mut after_cursor = cursor_floor;
while after_cursor < scan_ceiling {
// Retry the SAME page after one transient read failure. The frozen
// ceiling above is never recaptured: an event appended while this
// retry is in flight cannot authorize the earlier uncertain append.
let page = {
let mut last_error = None;
let mut page = None;
for attempt in 1..=EXACT_EVENT_RECONCILE_READ_ATTEMPTS {
match events
.poll(after_cursor, EXACT_EVENT_RECONCILE_PAGE_SIZE)
.await
{
Ok(events) => {
page = Some(events);
break;
}
Err(error) => {
last_error = Some(error);
if attempt < EXACT_EVENT_RECONCILE_READ_ATTEMPTS {
tracing::warn!(
mob_id = %mob_id,
after_cursor,
scan_ceiling,
attempt,
"exact event reconciliation page read failed transiently; retrying the same page"
);
tokio::task::yield_now().await;
}
}
}
}
match page {
Some(events) => events,
None => {
return Err(MobError::Internal(format!(
"exact event reconciliation could not read the page after cursor {after_cursor} below ceiling {scan_ceiling} after {EXACT_EVENT_RECONCILE_READ_ATTEMPTS} attempts: {}",
last_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown store error".to_string())
)));
}
}
};
if page.is_empty() {
return Err(MobError::Internal(format!(
"exact event reconciliation made no progress below cursor ceiling {scan_ceiling} (stuck after {after_cursor})"
)));
}
let page_start = after_cursor;
for event in page {
if event.cursor <= after_cursor {
return Err(MobError::Internal(format!(
"exact event reconciliation received non-increasing cursor {} after {}",
event.cursor, after_cursor
)));
}
if event.cursor > scan_ceiling {
break;
}
after_cursor = event.cursor;
if event.mob_id == *mob_id && event.kind == *desired {
return Ok(Some(event));
}
}
if after_cursor <= page_start {
return Err(MobError::Internal(format!(
"exact event reconciliation page did not advance beyond cursor {page_start}"
)));
}
}
Ok(None)
}
/// Host revoke may synchronously quiesce and archive every materialized
/// member on the target daemon. It is deliberately wider than an ordinary
/// bridge command while remaining bounded for retry convergence.
const HOST_REVOKE_BRIDGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
#[derive(Clone)]
enum WiringEndpoint {
Local {
entry: Box<RosterEntry>,
comms: Arc<dyn CoreCommsRuntime>,
spec: TrustedPeerDescriptor,
comms_name: String,
},
PeerOnly {
spec: TrustedPeerDescriptor,
binding: crate::RuntimeBinding,
},
/// Member placed on a bound remote host (multi-host §10.4). Peer
/// material is MACHINE-owned: `member_peer_endpoints[identity]`, the
/// ack-recorded pubkey @ host acceptor address. Trust for the remote
/// side rides a `RouteInstallObligation`, never a direct runtime write.
Placed {
identity: AgentIdentity,
host: mob_dsl::HostId,
spec: TrustedPeerDescriptor,
},
}
impl WiringEndpoint {
fn spec(&self) -> &TrustedPeerDescriptor {
match self {
Self::Local { spec, .. } | Self::PeerOnly { spec, .. } | Self::Placed { spec, .. } => {
spec
}
}
}
}
#[derive(Clone)]
struct LocalBatchWiringEndpoint {
comms: Arc<dyn CoreCommsRuntime>,
spec: TrustedPeerDescriptor,
removal_key: String,
}
/// Batch wiring endpoint classification (multi-host §10.4): local
/// session-backed members keep today's in-process trust lane; placed
/// members carry their machine peer material and their installs ride the
/// route-install obligation lane.
#[derive(Clone)]
enum BatchWiringEndpoint {
Local(LocalBatchWiringEndpoint),
Placed {
spec: TrustedPeerDescriptor,
removal_key: String,
},
}
impl BatchWiringEndpoint {
fn spec(&self) -> &TrustedPeerDescriptor {
match self {
Self::Local(local) => &local.spec,
Self::Placed { spec, .. } => spec,
}
}
fn removal_key(&self) -> &str {
match self {
Self::Local(local) => &local.removal_key,
Self::Placed { removal_key, .. } => removal_key,
}
}
}
/// Per-mob registration view into the process-scoped controlling acceptor.
/// The shared config owns the single D1 listener; this state remembers only
/// identities registered by this actor so unwire/shutdown cannot remove a
/// different mob's rows.
#[cfg(not(target_arch = "wasm32"))]
pub(super) struct ControllingAcceptorState {
config: super::builder::ControllingAcceptorConfig,
registered: BTreeMap<String, super::builder::ControllingAcceptorRegistrationLease>,
}
#[cfg(not(target_arch = "wasm32"))]
impl ControllingAcceptorState {
pub(super) fn new(config: super::builder::ControllingAcceptorConfig) -> Self {
Self {
config,
registered: BTreeMap::new(),
}
}
/// Publish current session material and retain the exact lease owned by
/// this actor. This deliberately does not treat durable pubkey presence as
/// idempotency: session revival preserves the key while replacing inbox.
pub(super) async fn refresh_registration(
&mut self,
logical_owner: String,
registration: super::builder::MemberAcceptorRegistration,
) -> Result<String, MobError> {
let pubkey_string = registration.pubkey.to_pubkey_string();
let lease = self.config.register(logical_owner, registration).await?;
let advertised = lease.advertised_address.clone();
self.registered.insert(pubkey_string, lease);
Ok(advertised)
}
/// Remove this actor's exact registration lease for `pubkey`. If the
/// process registry has already advanced to a replacement lease, the
/// shared compare-remove is a no-op. Retain the actor lease when removal
/// itself fails so lifecycle retry can redrive the same cleanup.
pub(super) async fn remove_registration(
&mut self,
pubkey: &meerkat_comms::PubKey,
) -> Result<(), MobError> {
let pubkey_string = pubkey.to_pubkey_string();
let Some(lease) = self.registered.remove(&pubkey_string) else {
return Ok(());
};
if let Err(error) = self.config.remove(&lease).await {
self.registered.insert(pubkey_string, lease);
return Err(error);
}
Ok(())
}
#[cfg(test)]
pub(super) fn registration_token(&self, pubkey: &str) -> Option<Arc<()>> {
self.registered
.get(pubkey)
.map(|lease| Arc::clone(&lease.token))
}
/// Remove this actor's projections. The process-level listener remains
/// live for sibling mobs that share the same config.
pub(super) async fn shutdown(self) {
for lease in self.registered.into_values() {
if let Err(error) = self.config.remove(&lease).await {
tracing::warn!(%error, "failed to remove controlling acceptor row at mob shutdown");
}
}
}
}
struct PeerMessageDeliveryPlan {
from: AgentIdentity,
to: AgentIdentity,
sender_comms: Arc<dyn CoreCommsRuntime>,
command: CommsCommand,
}
type PeerDeliveryId = u64;
pub(super) struct PeerDeliveryCompletion {
id: PeerDeliveryId,
}
pub(super) struct PeerDeliveryInflight {
from: AgentIdentity,
to: AgentIdentity,
cancel_token: tokio_util::sync::CancellationToken,
/// Tokio task identity captured from the `AbortHandle` returned by
/// `JoinSet::spawn`. A panicked task returns no `PeerDeliveryCompletion`,
/// so this is the only exact bridge back to the logical delivery id.
#[cfg(not(target_arch = "wasm32"))]
task_id: tokio::task::Id,
}
struct SupervisorPrivateTrustInstall {
peer_id: String,
epoch: u64,
removal_key: String,
}
/// Typed failure outcome from installing a session's supervisor private trust.
///
/// Carries the underlying [`MobError`] plus a *typed* discriminant recording
/// whether the compensating cleanup that removes the just-attempted ("new")
/// supervisor trust also failed. The supervisor-activation rollback path keys
/// on `new_trust_cleanup_failed` directly instead of string-matching the
/// formatted error message — the cleanup verdict is a structured fact, not a
/// substring of human-readable text.
#[derive(Debug)]
struct SupervisorPrivateTrustInstallError {
error: MobError,
new_trust_cleanup_failed: bool,
}
impl SupervisorPrivateTrustInstallError {
/// A failure where the attempted-new-trust cleanup compensation succeeded
/// (or no cleanup was required).
fn without_cleanup_failure(error: MobError) -> Self {
Self {
error,
new_trust_cleanup_failed: false,
}
}
/// A failure where the attempted-new-trust cleanup compensation *also*
/// failed, leaving residual supervisor trust that the activation rollback
/// must surface.
///
/// Only reachable on the runtime-adapter publish path; without that feature
/// the install routine fails earlier and no cleanup is attempted.
#[cfg(feature = "runtime-adapter")]
fn with_failed_new_trust_cleanup(error: MobError) -> Self {
Self {
error,
new_trust_cleanup_failed: true,
}
}
}
impl std::fmt::Display for SupervisorPrivateTrustInstallError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.error, f)
}
}
impl From<MobError> for SupervisorPrivateTrustInstallError {
fn from(error: MobError) -> Self {
Self::without_cleanup_failure(error)
}
}
impl From<SupervisorPrivateTrustInstallError> for MobError {
fn from(value: SupervisorPrivateTrustInstallError) -> Self {
value.error
}
}
struct SupervisorAuthorityActivationError {
error: MobError,
rollback_succeeded: bool,
pending_authority_recorded: bool,
rollback_error: Option<String>,
}
struct SupervisorAuthorityLoad {
durable: crate::store::SupervisorAuthorityRecord,
}
struct SupervisorPendingRotationPersistence {
pending_authority_recorded: bool,
persisted_record: Option<crate::store::SupervisorAuthorityRecord>,
}
struct PendingSupervisorRotationPersistenceRequest<'a> {
current: &'a crate::store::SupervisorAuthorityRecord,
expected_durable: &'a crate::store::SupervisorAuthorityRecord,
pending: &'a crate::store::SupervisorAuthorityRecord,
operation_id: super::bridge_protocol::SupervisorRotationOperationId,
accepted_peer_ids: &'a BTreeSet<String>,
active_peer_ids: &'a BTreeSet<String>,
member_targets: &'a BTreeMap<String, super::bridge_protocol::BridgePeerSpec>,
}
#[cfg(feature = "runtime-adapter")]
struct SupervisorPrivateTrustRevokeRequest<'a> {
adapter: &'a meerkat_runtime::MeerkatMachine,
session_id: &'a SessionId,
comms: &'a dyn CoreCommsRuntime,
peer_id: String,
epoch: u64,
removal_key: String,
allow_absent_pending: bool,
}
struct PreparedSupervisorAuthorityPersistence {
transition: PreparedDslTransition,
authority: crate::store::SupervisorAuthorityPersistenceAuthority,
}
struct PreparedSupervisorAuthorityDeletion {
transition: PreparedDslTransition,
authority: crate::store::SupervisorAuthorityDeletionAuthority,
}
struct PreparedDslInput {
authority: mob_dsl::MobMachinePreparedAuthority,
effects: Vec<mob_dsl::MobMachineEffect>,
phase_changed: bool,
}
struct PreparedDslTransition {
authority: mob_dsl::MobMachinePreparedAuthority,
transition: mob_dsl::MobMachineTransition,
}
pub(super) fn placed_kickoff_snapshot_for_structural_event(
state: &mob_dsl::MobMachineState,
agent_identity: &AgentIdentity,
timestamp_hint: Option<&crate::roster::MobMemberKickoffSnapshot>,
now: SystemTime,
) -> Result<crate::roster::MobMemberKickoffSnapshot, MobError> {
let mut kickoff = kickoff_snapshot_from_machine_state(
agent_identity.as_str(),
state,
timestamp_hint,
)
.ok_or_else(|| {
MobError::Internal(format!(
"prepared placed-kickoff transition produced no kickoff projection for '{agent_identity}'"
))
})?;
// Structural carriers include the public kickoff projection. Exact
// machine replays must therefore reuse its existing timestamp or the
// otherwise-identical retry would append a second Record/terminal pair
// that cold recovery correctly rejects as conflicting custody. A real
// lifecycle change mints a fresh presentation timestamp.
if timestamp_hint.as_ref().is_none_or(|existing| {
existing.objective_id != kickoff.objective_id
|| existing.phase != kickoff.phase
|| existing.error != kickoff.error
}) {
kickoff.updated_at = now;
}
Ok(kickoff)
}
enum PlacedKickoffStructuralReplay<'a> {
Record {
obligation: &'a crate::event::PlacedKickoffObligationEvent,
},
Resolved {
obligation: &'a crate::event::PlacedKickoffObligationEvent,
outcome: &'a crate::event::PlacedKickoffHostOutcomeEvent,
},
RejectedNoEffect {
obligation: &'a crate::event::PlacedKickoffObligationEvent,
error: &'a str,
},
}
impl PlacedKickoffStructuralReplay<'_> {
fn obligation(&self) -> &crate::event::PlacedKickoffObligationEvent {
match self {
Self::Record { obligation }
| Self::Resolved { obligation, .. }
| Self::RejectedNoEffect { obligation, .. } => obligation,
}
}
fn input_id(&self) -> &str {
match self {
Self::Record { obligation }
| Self::Resolved { obligation, .. }
| Self::RejectedNoEffect { obligation, .. } => &obligation.input_id,
}
}
fn carrier_name(&self) -> &'static str {
match self {
Self::Record { .. } => "PlacedKickoffObligationRecorded",
Self::Resolved { .. } => "PlacedKickoffOutcomeResolved",
Self::RejectedNoEffect { .. } => "PlacedKickoffRejectedNoEffect",
}
}
}
fn existing_placed_kickoff_structural_carrier(
events: &[MobEvent],
mob_id: &MobId,
expected: PlacedKickoffStructuralReplay<'_>,
) -> Result<Option<crate::roster::MobMemberKickoffSnapshot>, MobError> {
let mob_events = events
.iter()
.filter(|event| &event.mob_id == mob_id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
let epoch = &mob_events[epoch_start..];
let expected_turn_key = PlacedTrackedInputKey::kickoff(expected.obligation());
let mut exact_match = None;
for (index, event) in epoch.iter().enumerate() {
let (obligation, embedded_kickoff, exact) = match (&expected, &event.kind) {
(
PlacedKickoffStructuralReplay::Record {
obligation: expected,
},
MobEventKind::PlacedKickoffObligationRecorded { obligation },
) if PlacedTrackedInputKey::kickoff(obligation) == expected_turn_key => {
(obligation, None, obligation == *expected)
}
(
PlacedKickoffStructuralReplay::Resolved {
obligation: expected,
outcome: expected_outcome,
},
MobEventKind::PlacedKickoffOutcomeResolved {
obligation,
outcome,
kickoff,
},
) if PlacedTrackedInputKey::kickoff(obligation) == expected_turn_key => (
obligation,
Some(kickoff),
obligation == *expected && outcome == *expected_outcome,
),
(
PlacedKickoffStructuralReplay::RejectedNoEffect {
obligation: expected,
error: expected_error,
},
MobEventKind::PlacedKickoffRejectedNoEffect {
obligation,
error,
kickoff,
},
) if PlacedTrackedInputKey::kickoff(obligation) == expected_turn_key => (
obligation,
Some(kickoff),
obligation == *expected && error == *expected_error,
),
_ => continue,
};
if !exact {
return Err(MobError::Internal(format!(
"{} replay for input '{}' conflicts with its durable carrier",
expected.carrier_name(),
expected.input_id()
)));
}
let paired = epoch.get(index + 1).ok_or_else(|| {
MobError::Internal(format!(
"{} replay for input '{}' has no paired kickoff projection",
expected.carrier_name(),
expected.input_id()
))
})?;
let MobEventKind::MemberKickoffUpdated { member, kickoff } = &paired.kind else {
return Err(MobError::Internal(format!(
"{} replay for input '{}' is not immediately followed by its kickoff projection",
expected.carrier_name(),
expected.input_id()
)));
};
let pair_exact = member == &obligation.agent_identity
&& match embedded_kickoff {
Some(embedded) => kickoff == embedded,
None => {
kickoff.objective_id == Some(obligation.objective_id)
&& kickoff.phase == crate::roster::MobMemberKickoffPhase::Starting
&& kickoff.error.is_none()
}
};
if !pair_exact {
return Err(MobError::Internal(format!(
"{} replay for input '{}' has a mismatched kickoff projection",
expected.carrier_name(),
expected.input_id()
)));
}
if exact_match.replace(kickoff.clone()).is_some() {
return Err(MobError::Internal(format!(
"{} replay for input '{}' found duplicate exact durable carriers",
expected.carrier_name(),
expected.input_id()
)));
}
}
Ok(exact_match)
}
fn exact_structural_batch_present(
events: &[MobEvent],
mob_id: &MobId,
desired: &[MobEventKind],
) -> Result<bool, MobError> {
let Some(structural_head) = desired.first() else {
return Ok(true);
};
let mob_events = events
.iter()
.filter(|event| &event.mob_id == mob_id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
let epoch = &mob_events[epoch_start..];
if epoch
.windows(desired.len())
.any(|window| window.iter().map(|event| &event.kind).eq(desired.iter()))
{
return Ok(true);
}
// The first event is the correlation-scoped structural carrier. Its
// presence without the exact following batch proves a partial or
// conflicting write. Trailing projection events are deliberately not
// unique: cancellation can first persist `MemberKickoffUpdated(Cancelled)`
// and later pair that same projection with the authenticated host outcome.
if epoch.iter().any(|event| &event.kind == structural_head) {
return Err(MobError::Internal(
"durable structural batch is partial or out of order in the current mob epoch"
.to_string(),
));
}
Ok(false)
}
struct BatchWireTrustApplication {
edge: mob_dsl::WiringEdge,
identity: AgentIdentity,
peer_id: String,
comms: Arc<dyn CoreCommsRuntime>,
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
}
struct BatchWireTrustRollback {
edge: mob_dsl::WiringEdge,
identity: AgentIdentity,
peer_id: String,
comms: Arc<dyn CoreCommsRuntime>,
}
struct RetireTrustCleanupPlan {
retiring_comms: Option<Arc<dyn CoreCommsRuntime>>,
retiring_spec: Option<TrustedPeerDescriptor>,
machine_wired_peer_identities: BTreeSet<AgentIdentity>,
trust_unwire_authority_by_peer: BTreeMap<AgentIdentity, CommsTrustMutationAuthority>,
historical_trust_unwire_authorities_by_peer:
BTreeMap<AgentIdentity, Vec<(String, CommsTrustMutationAuthority)>>,
}
impl RetireTrustCleanupPlan {
fn empty() -> Self {
Self {
retiring_comms: None,
retiring_spec: None,
machine_wired_peer_identities: BTreeSet::new(),
trust_unwire_authority_by_peer: BTreeMap::new(),
historical_trust_unwire_authorities_by_peer: BTreeMap::new(),
}
}
fn has_peers(&self) -> bool {
!self.machine_wired_peer_identities.is_empty()
}
}
#[derive(Debug, Clone)]
enum WireTrustAuthority {
GraphAdded(MemberTrustHandoff),
RepairRequested(MemberTrustHandoff),
ExternalGraphAdded(CommsTrustMutationAuthority),
ExternalRepairRequested(CommsTrustMutationAuthority),
}
impl WireTrustAuthority {
fn dsl_added(&self) -> bool {
matches!(self, Self::GraphAdded(_) | Self::ExternalGraphAdded(_))
}
fn is_repair(&self) -> bool {
matches!(
self,
Self::RepairRequested(_) | Self::ExternalRepairRequested(_)
)
}
fn member_handoff(&self) -> Result<&MemberTrustHandoff, MobError> {
match self {
Self::GraphAdded(handoff) | Self::RepairRequested(handoff) => Ok(handoff),
Self::ExternalGraphAdded(_) | Self::ExternalRepairRequested(_) => {
Err(MobError::WiringError(
"external peer authority does not carry member peer handoff".to_string(),
))
}
}
}
fn external_authority(&self) -> Result<&CommsTrustMutationAuthority, MobError> {
match self {
Self::ExternalGraphAdded(authority) | Self::ExternalRepairRequested(authority) => {
Ok(authority)
}
Self::GraphAdded(_) | Self::RepairRequested(_) => Err(MobError::WiringError(
"member trust authority does not carry external peer authority".to_string(),
)),
}
}
}
#[derive(Debug, Clone)]
struct MemberTrustHandoff {
edge: mob_dsl::WiringEdge,
authority: MemberTrustAuthority,
operation: MemberTrustOperation,
}
#[derive(Debug, Clone)]
enum MemberTrustAuthority {
Wiring(crate::generated::protocol_mob_member_trust_wiring::MobMemberTrustWiringObligation),
Unwiring(
crate::generated::protocol_mob_member_trust_unwiring::MobMemberTrustUnwiringObligation,
),
Repair(crate::generated::protocol_mob_member_trust_wiring::MobMemberTrustWiringObligation),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MemberTrustOperation {
Wiring,
Unwiring,
Repair,
}
impl MemberTrustHandoff {
fn peer_id_for(&self, identity: &AgentIdentity) -> Result<&str, MobError> {
let identity = mob_dsl::AgentIdentity::from_domain(identity);
let actual = match &self.authority {
MemberTrustAuthority::Wiring(obligation) | MemberTrustAuthority::Repair(obligation) => {
if obligation.edge().a == identity {
Some(obligation.a_peer_id().0.as_str())
} else if obligation.edge().b == identity {
Some(obligation.b_peer_id().0.as_str())
} else {
None
}
}
MemberTrustAuthority::Unwiring(obligation) => {
if obligation.edge().a == identity {
Some(obligation.a_peer_id().0.as_str())
} else if obligation.edge().b == identity {
Some(obligation.b_peer_id().0.as_str())
} else {
None
}
}
};
actual.ok_or_else(|| {
MobError::WiringError(format!(
"generated member trust obligation does not cover '{identity:?}'"
))
})
}
fn require_peer_id_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
) -> Result<(), MobError> {
let actual = self.peer_id_for(identity)?;
if actual == expected_peer_id {
Ok(())
} else {
Err(MobError::WiringError(format!(
"generated member trust handoff peer id '{actual}' does not match resolved peer id '{expected_peer_id}' for '{identity}'"
)))
}
}
fn wiring_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Wiring(obligation) => {
crate::generated::protocol_mob_member_trust_wiring::wiring_authority_for_identity_with_live_authority(
obligation,
identity.as_str(),
expected_peer_id,
live_authority,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Repair(_) | MemberTrustAuthority::Unwiring(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot wire trust".to_string(),
))
}
}
}
fn unwiring_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Unwiring(obligation) => {
crate::generated::protocol_mob_member_trust_unwiring::unwiring_authority_for_identity(
obligation,
identity.as_str(),
expected_peer_id,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Repair(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot unwire trust".to_string(),
))
}
}
}
fn repair_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Repair(obligation) => {
crate::generated::protocol_mob_member_trust_wiring::repair_authority_for_identity_with_live_authority(
obligation,
identity.as_str(),
expected_peer_id,
live_authority,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Unwiring(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot repair trust".to_string(),
))
}
}
}
fn add_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match self.operation {
MemberTrustOperation::Wiring => {
self.wiring_authority_for(identity, expected_peer_id, live_authority)
}
MemberTrustOperation::Repair => {
self.repair_authority_for(identity, expected_peer_id, live_authority)
}
MemberTrustOperation::Unwiring => Err(MobError::WiringError(
"generated member unwiring handoff cannot add trust".to_string(),
)),
}
}
}
/// Resolve the runtime binding for a spawn request.
///
/// `RuntimeBinding` takes precedence over the legacy `backend` tag. When neither
/// is provided, resolves from profile/definition defaults. `External` without
/// a concrete `RuntimeBinding` is an error — you cannot spawn an external
/// member without declaring the real process identity.
fn resolve_binding(
binding: Option<crate::RuntimeBinding>,
backend: Option<crate::MobBackendKind>,
profile_backend: Option<crate::MobBackendKind>,
definition_default: crate::MobBackendKind,
agent_identity: &AgentIdentity,
) -> Result<crate::RuntimeBinding, MobError> {
if let Some(b) = binding {
return Ok(b);
}
let kind = backend.or(profile_backend).unwrap_or(definition_default);
match kind {
crate::MobBackendKind::Session => Ok(crate::RuntimeBinding::Session),
crate::MobBackendKind::External => Err(MobError::WiringError(format!(
"external backend requires explicit RuntimeBinding for '{agent_identity}'"
))),
}
}
fn normalize_runtime_mode_for_binding(
runtime_mode: crate::MobRuntimeMode,
binding: &crate::RuntimeBinding,
) -> crate::MobRuntimeMode {
match binding {
crate::RuntimeBinding::External { .. } => crate::MobRuntimeMode::TurnDriven,
// Placed members support BOTH modes (plan §7.3): the forced-
// TurnDriven normalization stays External-only.
crate::RuntimeBinding::Session | crate::RuntimeBinding::HostMaterialized { .. } => {
runtime_mode
}
}
}
fn with_spawn_budget_limits(
mut req: meerkat_core::service::CreateSessionRequest,
limits: Option<meerkat_core::BudgetLimits>,
) -> meerkat_core::service::CreateSessionRequest {
if let Some(limits) = limits {
req.build
.get_or_insert_with(meerkat_core::service::SessionBuildOptions::default)
.budget_limits = Some(limits);
}
req
}
pub(super) fn admit_bridge_session_for_spawn(
req: &mut meerkat_core::service::CreateSessionRequest,
) -> SessionId {
let build = req
.build
.get_or_insert_with(meerkat_core::service::SessionBuildOptions::default);
if let Some(session) = build.resume_session.as_ref() {
return session.id().clone();
}
let session_id = SessionId::new();
build.resume_session = Some(meerkat_core::session::Session::with_id(session_id.clone()));
session_id
}
/// Project a DSL `MobPhase` into the shell `MobState` enum. Used by
/// `MobActor::state()` and by the `MobCommand::QueryPhase` reply so that
/// external `MobHandle::status()` callers observe the same DSL-authority
/// value the actor uses internally (dogma #1, #13, #17).
fn project_dsl_phase(phase: mob_dsl::MobPhase) -> MobState {
match phase {
mob_dsl::MobPhase::Running => MobState::Running,
mob_dsl::MobPhase::Stopped => MobState::Stopped,
mob_dsl::MobPhase::Completed => MobState::Completed,
mob_dsl::MobPhase::Destroyed => MobState::Destroyed,
}
}
/// Row #314: derive the machine-owned external-member rebind capability from a
/// member's `MemberRef` bootstrap proof. A backend peer with a non-empty
/// bootstrap token can be supervisor-rebound; session-backed members and peers
/// without a usable token cannot.
pub(super) fn external_member_rebind_capability_from_member_ref(
member_ref: &MemberRef,
) -> mob_dsl::ExternalMemberRebindCapability {
match member_ref {
MemberRef::BackendPeer {
bootstrap_token, ..
} if bootstrap_token
.as_ref()
.is_some_and(|token| !token.is_empty()) =>
{
mob_dsl::ExternalMemberRebindCapability::Available
}
_ => mob_dsl::ExternalMemberRebindCapability::Unavailable,
}
}
/// Extract the pure wire runtime-state observation for MobMachine terminality
/// classification. This is a faithful 1:1 projection of the observed
/// `BridgeMemberRuntimeState`; the terminal/non-terminal verdict is decided by
/// MobMachine, not here. `BridgeMemberRuntimeState` is `#[non_exhaustive]`, so
/// an unrecognized wire state fails closed rather than being silently coerced
/// to a (non-terminal) default.
fn remote_member_runtime_observed_state(
state: super::bridge_protocol::BridgeMemberRuntimeState,
) -> Result<mob_dsl::MobRemoteMemberRuntimeObservedState, MobError> {
use super::bridge_protocol::BridgeMemberRuntimeState as Wire;
use mob_dsl::MobRemoteMemberRuntimeObservedState as Observed;
Ok(match state {
Wire::Initializing => Observed::Initializing,
Wire::Idle => Observed::Idle,
Wire::Attached => Observed::Attached,
Wire::Running => Observed::Running,
Wire::Retired => Observed::Retired,
Wire::Stopped => Observed::Stopped,
Wire::Destroyed => Observed::Destroyed,
other => {
return Err(MobError::Internal(format!(
"unrecognized remote-member runtime wire state `{other}` cannot be classified for terminality"
)));
}
})
}
/// Map a typed wire bridge rejection cause onto the MobMachine observation
/// enum. The wire cause is `#[non_exhaustive]`; any future variant the mob does
/// not yet understand maps to `Internal`, which MobMachine classifies as
/// `FatalBubbleUp` — failing closed (no recovery) on an unrecognized cause.
fn mob_bridge_rejection_cause(
cause: super::bridge_protocol::BridgeRejectionCause,
) -> mob_dsl::MobBridgeRejectionCause {
use super::bridge_protocol::BridgeRejectionCause as Wire;
use mob_dsl::MobBridgeRejectionCause as Mob;
match cause {
Wire::NotBound => Mob::NotBound,
Wire::StaleSupervisor => Mob::StaleSupervisor,
Wire::SenderMismatch => Mob::SenderMismatch,
Wire::AlreadyBound => Mob::AlreadyBound,
Wire::InvalidBootstrapToken => Mob::InvalidBootstrapToken,
Wire::UnsupportedProtocolVersion => Mob::UnsupportedProtocolVersion,
Wire::InvalidSupervisorSpec => Mob::InvalidSupervisorSpec,
Wire::InvalidPeerSpec => Mob::InvalidPeerSpec,
Wire::AddressMismatch => Mob::AddressMismatch,
Wire::Unsupported => Mob::Unsupported,
Wire::Internal => Mob::Internal,
// Fail closed: an unknown future wire cause is treated as a hard
// (fatal) rejection that no rebind can recover.
_ => Mob::Internal,
}
}
/// Render forked conversation rows as a text context block for the new
/// member. ONE renderer over the ONE page row vocabulary
/// (`WireSessionMessage`, DEC-P6E-20): the local fork branch projects its
/// domain page through `WireMemberHistoryPageBody::try_from_history_page` and
/// the remote branch decodes the bridge page — both render here, so local
/// and remote fork context are byte-identical by construction.
fn render_fork_context(
source_member_id: &AgentIdentity,
rows: &[meerkat_contracts::wire::WireHistoryRow],
) -> String {
use meerkat_contracts::wire::{
WireAssistantBlock, WireContentBlock, WireContentInput, WireSessionMessage,
WireToolResultContent,
};
fn content_input_text(content: &WireContentInput) -> String {
match content {
WireContentInput::Text(text) => text.clone(),
WireContentInput::Blocks(blocks) => blocks
.iter()
.filter_map(|block| match block {
WireContentBlock::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join(""),
}
}
let mut lines = Vec::new();
lines.push(format!(
"[Forked conversation context from member '{source_member_id}']"
));
for row in rows {
match &row.0 {
WireSessionMessage::System { content, .. } => {
lines.push(format!("[system]: {content}"));
}
WireSessionMessage::SystemNotice { body, .. } => {
lines.push(format!(
"[system_notice]: {}",
body.clone().unwrap_or_default()
));
}
WireSessionMessage::User { content, .. } => {
lines.push(format!("[user]: {}", content_input_text(content)));
}
WireSessionMessage::BlockAssistant { blocks, .. } => {
// Both `Text` (display) and `Transcript` (spoken) lanes
// project to the rendered text stream; supervisor sees the
// assistant's full visible output regardless of lane.
let text: String = blocks
.iter()
.filter_map(|block| match block {
WireAssistantBlock::Text { text, .. }
| WireAssistantBlock::Transcript { text, .. } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("");
if !text.is_empty() {
lines.push(format!("[assistant]: {text}"));
}
}
WireSessionMessage::ToolResults { results, .. } => {
for result in results {
let text: String = match &result.content {
WireToolResultContent::Text(text) => text.clone(),
WireToolResultContent::Blocks(blocks) => blocks
.iter()
.filter_map(|block| match block {
WireContentBlock::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join(""),
};
if !text.is_empty() {
let preview = if text.len() > 200 {
// Find a valid UTF-8 char boundary at or before byte 200
let end = text
.char_indices()
.map(|(i, _)| i)
.take_while(|&i| i <= 200)
.last()
.unwrap_or(0);
format!("{}...", &text[..end])
} else {
text
};
lines.push(format!("[tool_result({})]: {preview}", result.tool_use_id));
}
}
}
}
}
lines.push("[End of forked context]".to_string());
lines.join("\n")
}
/// Local branch of the placement-switched member history read (DEC-P6E-20/21):
/// the SAME serving semantics as the member host's `ReadMemberHistory` arm
/// (tail addressing, per-page cap) over the local session service, projected
/// through THE shared wire page projection.
async fn local_member_history_page(
session_service: &dyn super::session_service::MobSessionService,
session_id: &SessionId,
generation: u64,
from_index: Option<u64>,
limit: Option<u32>,
) -> Result<super::member_history_proxy::MemberHistoryPageDomain, MobError> {
const LOCAL_HISTORY_PAGE_MAX: usize = 256;
let requested_limit =
limit.map(|value| usize::try_from(value).unwrap_or(LOCAL_HISTORY_PAGE_MAX));
let effective_limit = requested_limit
.unwrap_or(LOCAL_HISTORY_PAGE_MAX)
.clamp(1, LOCAL_HISTORY_PAGE_MAX);
let offset = match (from_index, requested_limit) {
(Some(index), _) => usize::try_from(index).unwrap_or(usize::MAX),
(None, Some(tail_limit)) => {
let view = session_service.read(session_id).await.map_err(|error| {
MobError::Internal(format!("failed to read member session metadata: {error}"))
})?;
view.state
.message_count
.saturating_sub(tail_limit.clamp(1, LOCAL_HISTORY_PAGE_MAX))
}
(None, None) => 0,
};
let page = meerkat_core::service::SessionServiceHistoryExt::read_history(
session_service,
session_id,
meerkat_core::service::SessionHistoryQuery {
offset,
limit: Some(effective_limit),
},
)
.await
.map_err(|error| MobError::Internal(format!("failed to read member history: {error}")))?;
Ok(super::member_history_proxy::MemberHistoryPageDomain {
generation,
page: meerkat_contracts::wire::WireMemberHistoryPageBody::try_from_history_page(&page)
.map_err(|error| {
MobError::Internal(format!(
"failed to project local member history page: {error}"
))
})?,
// The local branch IS the placement switch's local arm (ADJ-P7-2):
// the page was served from the controlling host's own store.
placement: None,
provenance: meerkat_contracts::wire::WireProjectionProvenance::ControllingHostVerified,
})
}
pub(super) struct PendingSpawn {
pub(super) profile_name: ProfileName,
pub(super) agent_identity: AgentIdentity,
pub(super) admitted_bridge_session_id: SessionId,
pub(super) prompt: ContentInput,
pub(super) initial_turn_prompt: Option<ContentInput>,
/// Identity reconciliation resumes an already authoritative transcript.
/// Starting its autonomous runtime must not manufacture a fresh kickoff
/// prompt when the sealed intent has no pending initial delivery.
pub(super) suppress_autonomous_initial_prompt: bool,
/// Target-local identity authority for the final structural member CAS.
/// The built-in member store revalidates this after provisioning; it is
/// volatile scheduling custody, never recovered machine state.
pub(super) identity_member_permit: Option<crate::identity::IdentityActuationPermit>,
pub(super) runtime_mode: crate::MobRuntimeMode,
pub(super) labels: std::collections::BTreeMap<String, String>,
pub(super) owner_bridge_session_id: Option<SessionId>,
pub(super) auto_wire_parent: bool,
/// Peer wiring to restore after respawn completes.
pub(super) restore_wiring: Option<RestoreWiringPlan>,
/// Exact old incarnation for a respawn replacement. Operation identity is
/// independent of whether the saved graph is empty: a zero-edge placed
/// respawn still owns a preservation hold that every terminal path must
/// resolve or durably abandon.
respawn_origin: Option<RespawnOrigin>,
/// Effective profile override from `SpawnTooling::Profile` resolution.
/// Persisted in the roster so respawn/restore can use it.
pub(super) effective_profile_override: Option<crate::profile::Profile>,
/// Field-scoped model override reapplied over the current role profile.
pub(super) effective_model_override: Option<String>,
/// Objective causality inherited from the spawning turn.
pub(super) objective_id: Option<meerkat_core::interaction::ObjectiveId>,
/// Per-spawn external-tool overlay carried to the finalize commit so the
/// actor retention map is updated in the same block as the roster insert.
pub(super) per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
pub(super) authorized_profile_material: AuthorizedSpawnProfileMaterial,
pub(super) continuity_intent: super::handle::SpawnContinuityIntent,
pub(super) progress: Arc<std::sync::Mutex<PendingSpawnProgress>>,
/// External peer id whose recipient trust the provisioner installs ahead
/// of bind/authorize terminality during this provision. Recorded as a
/// MobMachine `pending_recipient_trust` obligation at enqueue; resolved
/// (provision succeeded — terminality confirmed) or rolled back
/// (provision failed — the provisioner already removed newly installed
/// trust) when the provision result lands. `None` for session-backed
/// members, which install no bridge recipient trust.
pub(super) pending_recipient_trust_peer_id: Option<String>,
/// Honest spawn-exec observations threaded into the ladder open
/// (DEC-P3-10; replaces the former hardwired-false block).
pub(super) observations: SpawnExecObservations,
/// Remote (host-materialized) spawn facts opened at enqueue; `None`
/// for every local/external path.
pub(super) remote: Option<Box<RemoteSpawnExec>>,
/// Enqueue instant for per-spawn outcome logging. Volatile scheduling
/// custody, never recovered machine state.
pub(super) enqueued_at: Instant,
pub(super) reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
}
/// Provisioning material that is cheap enough to prepare while the actor owns
/// command execution, or a durable-resume recipe whose snapshot load and agent
/// config build must run after the machine has reserved the pending spawn.
///
/// The deferred variant is intentionally session-backed only. External
/// bindings establish recipient-trust obligations before provisioning, so
/// widening their pre-provision cancellation window needs separate machine
/// semantics rather than sharing this optimization implicitly.
enum SpawnProvisionInput {
Ready(ProvisionMemberRequest),
DeferredResume(Box<DeferredResumeProvision>),
}
struct DeferredResumeProvision {
definition: Arc<MobDefinition>,
profile_name: ProfileName,
agent_identity: AgentIdentity,
profile: crate::profile::Profile,
external_tools: Option<Arc<dyn AgentToolDispatcher>>,
context: Option<serde_json::Value>,
labels: Option<std::collections::BTreeMap<String, String>>,
additional_instructions: Option<Vec<String>>,
shell_env: Option<std::collections::HashMap<String, String>>,
inherited_tool_filter: Option<meerkat_core::InheritedToolVisibilityAuthority>,
tool_access_policy: Option<meerkat_core::ops::ToolAccessPolicy>,
system_prompt_override: Option<super::handle::SpawnSystemPromptOverride>,
resume_id: SessionId,
prompt: ContentInput,
budget_limits: Option<meerkat_core::BudgetLimits>,
keep_alive: bool,
default_llm_client: Option<Arc<dyn LlmClient>>,
binding: crate::RuntimeBinding,
peer_name: String,
owner_bridge_session_id: Option<SessionId>,
ops_registry: Option<Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>>,
generated_self_owned_operation_owner: Option<SessionId>,
}
impl SpawnProvisionInput {
fn binding(&self) -> &crate::RuntimeBinding {
match self {
Self::Ready(request) => &request.binding,
Self::DeferredResume(deferred) => &deferred.binding,
}
}
fn admitted_bridge_session_id(&mut self) -> SessionId {
match self {
Self::Ready(request) => admit_bridge_session_for_spawn(&mut request.create_session),
Self::DeferredResume(deferred) => deferred.resume_id.clone(),
}
}
fn install_generated_self_owned_operation_owner(
&mut self,
generated_owner: &AuthorizedMobSpawnStart,
) -> Result<(), MobError> {
match self {
Self::Ready(request) => {
MobActor::apply_generated_self_owned_operation_owner(request, generated_owner)
}
Self::DeferredResume(deferred) => {
if matches!(deferred.binding, crate::RuntimeBinding::Session)
&& deferred.owner_bridge_session_id.is_none()
&& deferred.ops_registry.is_none()
{
deferred.generated_self_owned_operation_owner =
Some(generated_owner.owner_session_id().clone());
}
Ok(())
}
}
}
async fn into_request(
self,
session_service: Arc<dyn MobSessionService>,
) -> Result<ProvisionMemberRequest, MobError> {
match self {
Self::Ready(request) => Ok(request),
Self::DeferredResume(deferred) => deferred.into_request(session_service).await,
}
}
}
impl DeferredResumeProvision {
async fn into_request(
self: Box<Self>,
session_service: Arc<dyn MobSessionService>,
) -> Result<ProvisionMemberRequest, MobError> {
let Self {
definition,
profile_name,
agent_identity,
profile,
external_tools,
context,
labels,
additional_instructions,
shell_env,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
resume_id,
prompt,
budget_limits,
keep_alive,
default_llm_client,
binding,
peer_name,
owner_bridge_session_id,
ops_registry,
generated_self_owned_operation_owner,
} = *self;
let stored_session = session_service
.materialize_session_for_resume(&resume_id)
.await
.map_err(MobError::from)?
.into_session_or_error(&resume_id)?;
let mut config = build::build_resumed_agent_config(build::BuildResumedAgentConfigParams {
base: build::BuildAgentConfigParams {
mob_id: &definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &definition,
external_tools,
context,
labels,
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
},
expected_session_id: &resume_id,
resumed_session: stored_session,
})
.await?;
config.keep_alive = keep_alive;
if let Some(client) = default_llm_client {
config.llm_client_override = Some(client);
}
let create_session = build::to_create_session_request(&config, prompt);
let mut request = ProvisionMemberRequest {
create_session: with_spawn_budget_limits(create_session, budget_limits),
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
binding,
peer_name,
owner_bridge_session_id,
ops_registry,
generated_self_owned_operation_owner,
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
};
let materialized_session_id = admit_bridge_session_for_spawn(&mut request.create_session);
if materialized_session_id != resume_id {
return Err(MobError::Internal(format!(
"durable resume preparation changed session id from '{resume_id}' to '{materialized_session_id}'"
)));
}
Ok(request)
}
}
/// Incarnation-scoped MobMachine verdict for the pending-spawn mechanics of a
/// public retire request. This is a local mirror of exactly one generated
/// structural effect; it carries no independently derived roster truth.
#[derive(Debug, Clone, PartialEq, Eq)]
enum RetirePendingSpawnVerdict {
CancelCommittedIncarnation {
agent_runtime_id: mob_dsl::AgentRuntimeId,
generation: mob_dsl::Generation,
pending_spawn_session_id: mob_dsl::SessionId,
},
CommittedIncarnationWithoutPendingSpawn {
agent_runtime_id: mob_dsl::AgentRuntimeId,
generation: mob_dsl::Generation,
},
PreservePendingSpawnForAbsentIdentity,
}
#[derive(Debug, Default)]
pub(super) struct PendingSpawnProgress {
pub(super) bridge_session_id: Option<meerkat_core::types::SessionId>,
pub(super) operation_id: Option<meerkat_core::ops::OperationId>,
}
/// Honest spawn-exec observations (DEC-P3-10): a pure function over the
/// destructured spec + resolved profile, computed ONCE at the spawn
/// destructure and threaded as a value into every `BeginSpawnExec`
/// construction. The machine DECIDES; these are shell observations only.
#[derive(Clone, Debug, Default)]
pub(super) struct SpawnExecObservations {
/// EXPLICIT resolved `profile.tools.workgraph`, captured BEFORE
/// `open_profile_tool_categories_for_inherited_filter` (ADJ-6: an
/// inherited-open forced value never sets it).
pub(super) workgraph_required: bool,
pub(super) rust_bundles_present: bool,
pub(super) per_spawn_external_tools_present: bool,
pub(super) host_surface_mcp_allowlist_present: bool,
pub(super) inherited_tool_filter_present: bool,
pub(super) shell_env_present: bool,
pub(super) mcp_stdio_env_present: bool,
pub(super) mcp_http_headers_present: bool,
pub(super) memory_required: bool,
pub(super) mcp_required: bool,
/// Remote resume target (launch `Resume` + placement); `None` locally.
pub(super) resume_session_id: Option<SessionId>,
}
/// Facts for a REMOTE (host-materialized) spawn opened at enqueue time.
/// The machine already holds the tuple from `BeginSpawnExecRemote`; the
/// finalize path re-uses it (never re-mints).
#[derive(Clone, Debug)]
pub(super) struct RemoteSpawnExec {
pub(super) placed_spawn_id: crate::ids::PlacedSpawnId,
pub(super) placement: mob_dsl::HostId,
pub(super) generation: crate::ids::Generation,
pub(super) fence_token: crate::ids::FenceToken,
pub(super) resolved_spec_digest: String,
/// Exact pending carrier durably inserted before bridge dispatch. This
/// witness is retained across the round-trip so lost reads cannot weaken
/// commit or cleanup into identity-only mutations.
pub(super) pending_carrier: crate::store::MobPlacedSpawnCarrierRecord,
}
/// Remote finalize carrier: the enqueue-time facts plus the transport-
/// validated materialize ack.
struct RemoteSpawnFinalize {
placed_spawn_id: crate::ids::PlacedSpawnId,
#[allow(dead_code)]
placement: mob_dsl::HostId,
resolved_spec_digest: String,
pending_carrier: crate::store::MobPlacedSpawnCarrierRecord,
ack: Box<super::provisioner::MaterializedMemberAck>,
}
/// Result of the remote pre-commit failure choreography. Recipient-trust
/// rollback is legal only when the exact remote attempt has been released (or
/// proven absent) and its durable carrier cleanup completed.
struct RemoteSpawnFailureOutcome {
error: MobError,
cleanup_completed: bool,
}
fn observe_spawn_exec_facts(
profile: &crate::profile::Profile,
explicit_workgraph: bool,
per_spawn_external_tools_present: bool,
shell_env: Option<&std::collections::HashMap<String, String>>,
inherited_tool_filter_present: bool,
resume_session_id: Option<SessionId>,
) -> SpawnExecObservations {
let mcp_stdio_env_present = profile.tools.mcp_servers.iter().any(|server| {
matches!(
&server.transport,
meerkat_core::mcp_config::McpTransportConfig::Stdio(stdio) if !stdio.env.is_empty()
)
});
let mcp_http_headers_present = profile.tools.mcp_servers.iter().any(|server| {
matches!(
&server.transport,
meerkat_core::mcp_config::McpTransportConfig::Http(http) if !http.headers.is_empty()
)
});
SpawnExecObservations {
workgraph_required: explicit_workgraph,
rust_bundles_present: !profile.tools.rust_bundles.is_empty(),
per_spawn_external_tools_present,
host_surface_mcp_allowlist_present: !profile.tools.mcp.is_empty(),
inherited_tool_filter_present,
shell_env_present: shell_env.is_some_and(|env| !env.is_empty()),
mcp_stdio_env_present,
mcp_http_headers_present,
memory_required: profile.tools.memory,
mcp_required: !profile.tools.mcp_servers.is_empty(),
resume_session_id,
}
}
/// DEC-P3-7: the ONE owner of `RecordMemberMaterializationFailure.kind`
/// strings — the wire cause's serde tag for typed rejects, reserved shell
/// classes for transport faults. Total: an unclassifiable error carries its
/// failure class, never an empty/absent kind.
fn materialization_failure_kind(error: &MobError) -> String {
match error {
MobError::BridgeCommandRejected { cause, .. } => bridge_rejection_cause_tag(cause),
MobError::BridgeRequestTimedOut { .. } => "materialize_timeout".to_string(),
MobError::CommsError(_) => "materialize_send_failed".to_string(),
other => format!("unclassified:{:?}", other.failure_class()),
}
}
/// Snake-case serde tag of a typed wire rejection cause (e.g.
/// `model_unresolvable`, `stale_fence`). Uses the cause's own serde
/// rendering so the string can never drift from the wire vocabulary.
fn bridge_rejection_cause_tag(cause: &super::bridge_protocol::BridgeRejectionCause) -> String {
match serde_json::to_value(cause) {
Ok(serde_json::Value::String(tag)) => tag,
Ok(serde_json::Value::Object(map)) => map
.get("cause")
.and_then(serde_json::Value::as_str)
.map(str::to_string)
.unwrap_or_else(|| format!("unclassified_cause:{cause:?}")),
_ => format!("unclassified_cause:{cause:?}"),
}
}
#[must_use = "mob spawn start authority must be consumed by pending spawn insertion"]
struct AuthorizedMobSpawnStart {
generated_can_start: generated_mob_command_capabilities::CommandPlanKind,
generated_owner: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
session_id: SessionId,
}
#[must_use = "started mob spawn authority must be consumed by PendingSpawnLineage insertion"]
struct AuthorizedMobSpawnStarted {
generated_owner: generated_mob_command_capabilities::CommandPlanKind,
generated_started: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
session_id: SessionId,
}
#[must_use = "mob spawn completion authority must be observed by pending spawn completion"]
struct AuthorizedMobSpawnCompleted {
generated_plan: generated_mob_command_capabilities::CommandPlanKind,
generated_effect: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
}
impl AuthorizedMobSpawnStart {
fn owner_session_id(&self) -> &SessionId {
debug_assert_eq!(
self.generated_can_start,
generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn
);
debug_assert_eq!(
self.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
&self.session_id
}
fn start(self, pending: &PendingSpawn) -> Result<AuthorizedMobSpawnStarted, MobError> {
debug_assert_eq!(
self.generated_can_start,
generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn
);
debug_assert_eq!(
self.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
if pending.agent_identity != self.agent_identity
|| pending.admitted_bridge_session_id != self.session_id
{
return Err(MobError::Internal(format!(
"MobMachine StageSpawn authority for '{}' / '{}' did not match pending spawn '{}' / '{}'",
self.agent_identity,
self.session_id,
pending.agent_identity,
pending.admitted_bridge_session_id
)));
}
Ok(AuthorizedMobSpawnStarted {
generated_owner: self.generated_owner,
generated_started: generated_mob_command_capabilities::CommandPlanKind::SpawnStarted,
agent_identity: self.agent_identity,
session_id: self.session_id,
})
}
}
#[derive(Clone, Debug)]
pub(super) struct PendingSpawnCleanupAnchor {
spawn_ticket: u64,
agent_identity: AgentIdentity,
session_id: meerkat_core::types::SessionId,
operation_id: meerkat_core::ops::OperationId,
reason: String,
/// Exact MobMachine incarnation whose public retire authorized this
/// pending-spawn cleanup. Generic lifecycle drains (stop/destroy) have no
/// committed member incarnation and leave this absent.
retire_incarnation: Option<RetirePendingSpawnCleanupIncarnation>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct RetirePendingSpawnCleanupIncarnation {
agent_runtime_id: mob_dsl::AgentRuntimeId,
generation: mob_dsl::Generation,
pending_spawn_session_id: mob_dsl::SessionId,
}
#[derive(Clone, Debug, Default)]
pub(super) struct RestoreWiringPlan {
local_peers: Vec<AgentIdentity>,
external_peers: Vec<TrustedPeerDescriptor>,
}
#[derive(Clone, Debug)]
struct RespawnOrigin {
old_runtime_id: AgentRuntimeId,
old_fence_token: crate::ids::FenceToken,
}
struct RespawnSnapshot {
profile_name: ProfileName,
runtime_mode: crate::MobRuntimeMode,
labels: std::collections::BTreeMap<String, String>,
old_runtime_id: crate::ids::AgentRuntimeId,
old_fence_token: crate::ids::FenceToken,
restore_wiring: RestoreWiringPlan,
/// Runtime binding extracted from the old roster entry's member_ref.
/// Preserves real external identity across respawns.
binding: crate::RuntimeBinding,
/// Effective profile override persisted in the roster.
/// Used on respawn to avoid re-resolving from the definition.
effective_profile_override: Option<crate::profile::Profile>,
effective_model_override: Option<String>,
/// The old member is already in a partial-retire state and respawn should
/// retry cleanup instead of re-admitting the original Respawn transition.
cleanup_retry: bool,
}
/// Actor-side respawn handler outcome. Local respawns complete inline;
/// placed respawns DEFER completion: the remote spawn lane finalizes through
/// the actor mailbox (`handle_spawn_provisioned_batch` →
/// `finalize_spawn_from_pending`), so awaiting the replacement receipt ON
/// the actor task would deadlock the loop. The command dispatcher awaits the
/// deferred receipt on a detached task instead.
enum RespawnProgress {
/// The replacement committed inline; the receipt is final.
Completed(super::handle::MemberRespawnReceipt),
/// Placed replacement enqueued on the remote spawn lane; completion is
/// awaited OFF the actor and the receipt is built from the committed
/// roster entry.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
DeferredPlaced {
identity: AgentIdentity,
old_fence_token: crate::ids::FenceToken,
spawn_reply_rx: oneshot::Receiver<Result<super::handle::MemberSpawnReceipt, MobError>>,
},
}
struct FinalizeSpawnOutcome {
receipt: super::handle::MemberSpawnReceipt,
failed_restore_peer_ids: Vec<RespawnTopologyPeerId>,
}
/// Heap-allocated carrier for the spawn-finalize parameters.
///
/// `finalize_spawn_from_pending` was a single ~620-line async fn whose state
/// machine compiled to a ~38KiB future. During fleet restore many of these
/// futures were polled while nested under `MobBuilder::reconcile_resume` and
/// the `MobActor::run` task frame, and the deep comms-drain stage could
/// overflow tokio's 2MiB worker stack. Boxing the parameters here keeps the
/// thin driver frame at a single pointer, and the admit/activate split (each
/// `Box::pin`ned) ensures the pre-commit stage frame is dropped before the
/// post-commit activation work — including the comms drain — is polled.
struct SpawnFinalizeCtx {
profile_name: ProfileName,
agent_identity: AgentIdentity,
generation: crate::ids::Generation,
fence_token: crate::ids::FenceToken,
runtime_mode: crate::MobRuntimeMode,
prompt: ContentInput,
initial_turn_prompt: Option<ContentInput>,
suppress_autonomous_initial_prompt: bool,
identity_member_permit: Option<crate::identity::IdentityActuationPermit>,
labels: std::collections::BTreeMap<String, String>,
operation_id: meerkat_core::ops::OperationId,
owner_bridge_session_id: Option<SessionId>,
auto_wire_parent: bool,
restore_wiring: Option<RestoreWiringPlan>,
effective_profile_override: Option<crate::profile::Profile>,
effective_model_override: Option<String>,
objective_id: Option<meerkat_core::interaction::ObjectiveId>,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
authorized_profile_material: AuthorizedSpawnProfileMaterial,
continuity_intent: super::handle::SpawnContinuityIntent,
observations: SpawnExecObservations,
remote: Option<Box<RemoteSpawnFinalize>>,
}
/// Facts produced by the spawn-admit phase (`BeginSpawnExec` →
/// `CommitSpawnMembership` → `provision.commit()`) that the activation phase
/// (`CommitSpawnActivation`) needs to finish wiring the member into the roster
/// and runtime.
struct SpawnAdmitted {
member_ref: MemberRef,
session_origin: super::provisioner::ProvisionSessionOrigin,
agent_runtime_id: crate::ids::AgentRuntimeId,
is_replacing: bool,
/// Exact endpoint observed for this runtime generation before the
/// write-ahead spawn event. Carried verbatim into activation so the live
/// MobMachine registration cannot diverge from durable replay authority.
member_peer_endpoint: Option<TrustedPeerDescriptor>,
transport_public_key: Option<String>,
}
/// Cross-phase state for the split `finalize_spawn_activate`.
///
/// The activation body was one ~630-line async fn; at opt-level=0 LLVM does
/// no stack-slot coloring, so its merged poll frame reserved a slot for
/// every local in every branch (~700 KiB) and sat on the actor task beneath
/// the whole spawn ladder. The sequential phase helpers
/// (`activate_spawn_membership` → `activate_spawn_wiring` →
/// `activate_spawn_kickoff` → `activate_spawn_commit`) keep only the active
/// phase's frame on the 2 MiB production worker stack; this carrier threads
/// the spawn facts between them.
struct SpawnActivateState {
profile_name: ProfileName,
agent_identity: AgentIdentity,
generation: crate::ids::Generation,
fence_token: crate::ids::FenceToken,
runtime_mode: crate::MobRuntimeMode,
prompt: ContentInput,
initial_turn_prompt: Option<ContentInput>,
suppress_autonomous_initial_prompt: bool,
identity_fenced_member: bool,
labels: std::collections::BTreeMap<String, String>,
operation_id: meerkat_core::ops::OperationId,
owner_bridge_session_id: Option<SessionId>,
auto_wire_parent: bool,
restore_wiring: Option<RestoreWiringPlan>,
effective_profile_override: Option<crate::profile::Profile>,
effective_model_override: Option<String>,
objective_id: Option<meerkat_core::interaction::ObjectiveId>,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
remote: Option<Box<RemoteSpawnFinalize>>,
member_ref: MemberRef,
session_origin: super::provisioner::ProvisionSessionOrigin,
agent_runtime_id: crate::ids::AgentRuntimeId,
is_replacing: bool,
member_peer_endpoint: Option<TrustedPeerDescriptor>,
transport_public_key: Option<String>,
identity: crate::ids::AgentIdentity,
dsl_identity: mob_dsl::AgentIdentity,
/// Filled by the wiring phase; read by the kickoff-phase rollbacks.
planned_wiring_targets: Vec<AgentIdentity>,
/// Filled by the wiring phase; read by the kickoff-phase rollbacks.
wired_spawn_targets: Vec<AgentIdentity>,
/// Filled by the wiring phase; surfaced through the finalize outcome.
failed_restore_peer_ids: Vec<RespawnTopologyPeerId>,
/// Set by the kickoff phase when placed activation already committed.
spawn_activation_committed: bool,
}
impl SpawnActivateState {
fn admit(ctx: Box<SpawnFinalizeCtx>, admitted: SpawnAdmitted) -> Self {
let SpawnFinalizeCtx {
profile_name,
agent_identity,
generation,
fence_token,
runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
labels,
operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material: _,
continuity_intent: _,
observations: _,
remote,
} = *ctx;
let identity_fenced_member = identity_member_permit.is_some();
// Identity convergence owns wiring as a separate generated obligation
// with its own target-local permit. Once MemberSpawned committed under
// an identity member permit, legacy role/parent/respawn wiring must not
// smuggle a second structural write into the member obligation.
let auto_wire_parent = auto_wire_parent && !identity_fenced_member;
let restore_wiring = if identity_fenced_member {
None
} else {
restore_wiring
};
let SpawnAdmitted {
member_ref,
session_origin,
agent_runtime_id,
is_replacing,
member_peer_endpoint,
transport_public_key,
} = admitted;
let identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
Self {
profile_name,
agent_identity,
generation,
fence_token,
runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_fenced_member,
labels,
operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
remote,
member_ref,
session_origin,
agent_runtime_id,
is_replacing,
member_peer_endpoint,
transport_public_key,
identity,
dsl_identity,
planned_wiring_targets: Vec::new(),
wired_spawn_targets: Vec::new(),
failed_restore_peer_ids: Vec::new(),
spawn_activation_committed: false,
}
}
}
struct FailedSpawnRollback<'a> {
generation: crate::ids::Generation,
profile_name: &'a ProfileName,
member_ref: &'a MemberRef,
operation_id: &'a meerkat_core::ops::OperationId,
session_origin: super::provisioner::ProvisionSessionOrigin,
successful_wiring_targets: &'a [AgentIdentity],
planned_wiring_targets: &'a [AgentIdentity],
}
struct RespawnTopologyRestoreResolution {
result: mob_dsl::RespawnTopologyRestoreResultKind,
failed_peer_ids: Vec<RespawnTopologyPeerId>,
}
#[cfg(not(target_arch = "wasm32"))]
struct RemoteDestroyOutcome {
identity: AgentIdentity,
force_destroyed: bool,
orphaned: bool,
errors: Vec<String>,
}
struct RuntimeMetadataSnapshot {
supervisor: Option<crate::store::SupervisorAuthorityRecord>,
external_binding_overlays: Vec<crate::store::ExternalBindingOverlayRecord>,
member_operator_requests: Vec<crate::store::MobMemberOperatorRequestRecord>,
}
// Not `Copy`: `BridgeRejectionCause` carries payload variants since V4.
#[derive(Debug, Clone, PartialEq, Eq)]
enum ExpectedRevokeCleanupFailure {
BridgeRejected {
cause: super::bridge_protocol::BridgeRejectionCause,
},
}
// ---------------------------------------------------------------------------
// MobActor
// ---------------------------------------------------------------------------
/// Allocate above every machine/public or private durable sequence. Receipt
/// rows are included even when a crash left the matching intent cleanup
/// incomplete: such an orphan may not advance custody, but its numeric key is
/// permanently burned and must never be reused.
pub(super) async fn next_remote_turn_dispatch_sequence(
run_store: &Arc<dyn MobRunStore>,
mob_id: &MobId,
machine_highwater: u64,
) -> Result<u64, MobError> {
let mut durable_highwater = machine_highwater;
for run in run_store.list_runs(mob_id, None).await? {
for intent in run_store.list_remote_turn_intents(&run.run_id).await? {
durable_highwater = durable_highwater.max(intent.obligation.dispatch_sequence);
}
for receipt in run_store.list_remote_turn_receipts(&run.run_id).await? {
durable_highwater = durable_highwater.max(receipt.obligation.dispatch_sequence);
}
}
durable_highwater
.checked_add(1)
.ok_or_else(|| MobError::Internal("remote-turn dispatch sequence exhausted".to_string()))
}
fn optional_route_session_matches_machine(
route_session_id: Option<&SessionId>,
machine_session_id: &str,
) -> bool {
route_session_id.is_none_or(|session_id| session_id.to_string() == machine_session_id)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct PendingHostBindAnchor {
pub operation_id: String,
pub request: crate::event::RemoteHostBindRequestEvent,
pub confirmed_authority: Option<crate::store::MobHostAuthorityRecord>,
}
fn host_bind_request_matches_confirmed_authority(
request: &crate::event::RemoteHostBindRequestEvent,
authority: &crate::store::MobHostAuthorityRecord,
) -> bool {
request.host_id == authority.host_id
&& request.peer_id == authority.peer_id
&& request.signing_key == authority.signing_key
&& request.endpoint == authority.endpoint
&& request.authority_epoch == authority.authority_epoch
&& request.binding_generation == authority.binding_generation
&& authority.bind_phase == crate::store::MobHostBindPhaseRecord::Bound
}
fn host_bind_terminal_matches_request(
request: &crate::event::RemoteHostBindRequestEvent,
host_id: &str,
authority_epoch: u64,
binding_generation: u64,
) -> bool {
request.host_id == host_id
&& request.authority_epoch == authority_epoch
&& request.binding_generation == binding_generation
}
/// Reconstruct unfinished host-bind operations from one mob epoch. Phase
/// ordering is itself validated: an ACK confirmation cannot precede its
/// exact Started tuple, and only a confirmed operation can Complete. Exact
/// duplicate rows are idempotent; one operation id naming different tuples
/// is durable corruption.
pub(super) fn pending_host_bind_anchors(
events: &[MobEvent],
) -> Result<Vec<PendingHostBindAnchor>, MobError> {
let mut started = BTreeMap::<String, crate::event::RemoteHostBindRequestEvent>::new();
let mut confirmed = BTreeMap::<String, crate::store::MobHostAuthorityRecord>::new();
let mut completed = BTreeSet::<String>::new();
let mut aborted = BTreeSet::<String>::new();
for event in events {
match &event.kind {
MobEventKind::RemoteHostBindStarted {
operation_id,
request,
} => {
if uuid::Uuid::parse_str(operation_id).is_err() {
return Err(MobError::Internal(format!(
"host bind event carries invalid operation id '{operation_id}'"
)));
}
if request.host_id.is_empty()
|| request.peer_id != request.host_id
|| request.endpoint.is_empty()
|| request.binding_generation == 0
{
return Err(MobError::Internal(format!(
"host bind Started '{operation_id}' carries an invalid descriptor tuple"
)));
}
if confirmed.contains_key(operation_id)
|| completed.contains(operation_id)
|| aborted.contains(operation_id)
{
return Err(MobError::Internal(format!(
"host bind operation '{operation_id}' regressed to Started after a later phase"
)));
}
if let Some(existing) = started.insert(operation_id.clone(), request.clone())
&& &existing != request
{
return Err(MobError::Internal(format!(
"host bind operation '{operation_id}' aliases two Started tuples"
)));
}
}
MobEventKind::RemoteHostBindConfirmed {
operation_id,
authority,
} => {
let request = started.get(operation_id).ok_or_else(|| {
MobError::Internal(format!(
"host bind confirmation '{operation_id}' has no prior Started anchor"
))
})?;
if !host_bind_request_matches_confirmed_authority(request, authority) {
return Err(MobError::Internal(format!(
"host bind confirmation '{operation_id}' drifts from its Started tuple"
)));
}
if aborted.contains(operation_id) || completed.contains(operation_id) {
return Err(MobError::Internal(format!(
"host bind confirmation '{operation_id}' follows a terminal phase"
)));
}
if let Some(existing) = confirmed.insert(operation_id.clone(), authority.clone())
&& &existing != authority
{
return Err(MobError::Internal(format!(
"host bind operation '{operation_id}' aliases two Confirmed authorities"
)));
}
}
MobEventKind::RemoteHostBindCompleted {
operation_id,
host_id,
authority_epoch,
binding_generation,
} => {
let request = started.get(operation_id).ok_or_else(|| {
MobError::Internal(format!(
"host bind completion '{operation_id}' has no prior Started anchor"
))
})?;
let authority = confirmed.get(operation_id).ok_or_else(|| {
MobError::Internal(format!(
"host bind completion '{operation_id}' has no prior Confirmed anchor"
))
})?;
if !host_bind_terminal_matches_request(
request,
host_id,
*authority_epoch,
*binding_generation,
) || !host_bind_request_matches_confirmed_authority(request, authority)
{
return Err(MobError::Internal(format!(
"host bind completion '{operation_id}' drifts from its anchored authority"
)));
}
if aborted.contains(operation_id) {
return Err(MobError::Internal(format!(
"host bind completion '{operation_id}' follows a no-effect abort"
)));
}
completed.insert(operation_id.clone());
}
MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id,
host_id,
authority_epoch,
binding_generation,
} => {
let request = started.get(operation_id).ok_or_else(|| {
MobError::Internal(format!(
"host bind no-effect abort '{operation_id}' has no prior Started anchor"
))
})?;
if !host_bind_terminal_matches_request(
request,
host_id,
*authority_epoch,
*binding_generation,
) {
return Err(MobError::Internal(format!(
"host bind no-effect abort '{operation_id}' drifts from its Started tuple"
)));
}
if confirmed.contains_key(operation_id) || completed.contains(operation_id) {
return Err(MobError::Internal(format!(
"host bind no-effect abort '{operation_id}' follows remote confirmation"
)));
}
aborted.insert(operation_id.clone());
}
_ => {}
}
}
let mut pending = Vec::new();
let mut pending_hosts = BTreeMap::<String, String>::new();
for (operation_id, request) in started {
if completed.contains(&operation_id) || aborted.contains(&operation_id) {
continue;
}
if let Some(existing) = pending_hosts.insert(request.host_id.clone(), operation_id.clone())
{
return Err(MobError::Internal(format!(
"host '{}' has multiple unfinished bind operations '{existing}' and '{operation_id}'",
request.host_id
)));
}
pending.push(PendingHostBindAnchor {
confirmed_authority: confirmed.remove(&operation_id),
operation_id,
request,
});
}
Ok(pending)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct PendingHostRevokeAnchor {
pub operation_id: String,
pub host_id: String,
pub epoch: u64,
pub binding_generation: u64,
pub confirmed: bool,
}
/// Reconstruct unfinished host-revoke operations from one mob epoch. A
/// Started row is the retry intent; Confirmed is the authenticated remote
/// terminal; Completed closes the local authority transition. Operation ids
/// keep repeated revoke/rebind cycles at the same supervisor epoch disjoint.
pub(super) fn pending_host_revoke_anchors(
events: &[MobEvent],
) -> Result<Vec<PendingHostRevokeAnchor>, MobError> {
// operation_id -> (host_id, epoch, binding_generation, monotone phase)
// phase: 1=Started, 2=Confirmed, 3=Completed.
let mut operations = BTreeMap::<String, (String, u64, u64, u8)>::new();
for event in events {
let (operation_id, host_id, epoch, binding_generation, next_phase) = match &event.kind {
MobEventKind::RemoteHostRevokeStarted {
operation_id,
host_id,
epoch,
binding_generation,
} => (operation_id, host_id, *epoch, *binding_generation, 1),
MobEventKind::RemoteHostRevokeConfirmed {
operation_id,
host_id,
epoch,
binding_generation,
} => (operation_id, host_id, *epoch, *binding_generation, 2),
MobEventKind::RemoteHostRevokeCompleted {
operation_id,
host_id,
epoch,
binding_generation,
} => (operation_id, host_id, *epoch, *binding_generation, 3),
_ => continue,
};
if uuid::Uuid::parse_str(operation_id).is_err() {
return Err(MobError::Internal(format!(
"host revoke event carries invalid operation id '{operation_id}'"
)));
}
if host_id.is_empty() || binding_generation == 0 {
return Err(MobError::Internal(format!(
"host revoke operation '{operation_id}' carries an empty host or zero binding generation"
)));
}
let tuple = (host_id.clone(), epoch, binding_generation);
match operations.get_mut(operation_id) {
None if next_phase == 1 => {
operations.insert(
operation_id.clone(),
(host_id.clone(), epoch, binding_generation, 1),
);
}
None => {
return Err(MobError::Internal(format!(
"host revoke operation '{operation_id}' phase {next_phase} has no prior Started anchor"
)));
}
Some((existing_host, existing_epoch, existing_generation, current_phase)) => {
let existing_tuple = (existing_host.clone(), *existing_epoch, *existing_generation);
if existing_tuple != tuple {
return Err(MobError::Internal(format!(
"host revoke operation '{operation_id}' aliases two host bindings"
)));
}
if next_phase == *current_phase {
continue;
}
if next_phase != current_phase.saturating_add(1) {
return Err(MobError::Internal(format!(
"host revoke operation '{operation_id}' has non-monotone phase {current_phase}->{next_phase}"
)));
}
*current_phase = next_phase;
}
}
}
let mut pending = Vec::new();
let mut pending_hosts = BTreeMap::<String, String>::new();
for (operation_id, (host_id, epoch, binding_generation, phase)) in operations {
if phase == 3 {
continue;
}
if let Some(existing) = pending_hosts.insert(host_id.clone(), operation_id.clone()) {
return Err(MobError::Internal(format!(
"host '{host_id}' has multiple unfinished revoke operations '{existing}' and '{operation_id}'"
)));
}
pending.push(PendingHostRevokeAnchor {
confirmed: phase == 2,
operation_id,
host_id,
epoch,
binding_generation,
});
}
Ok(pending)
}
fn require_host_authority_anchors_clear_for_supervisor_rotation(
pending_binds: &[PendingHostBindAnchor],
pending_revokes: &[PendingHostRevokeAnchor],
) -> Result<(), MobError> {
require_host_authority_anchors_clear("rotate supervisor", pending_binds, pending_revokes)
}
pub(super) fn validate_host_authority_anchor_disjointness(
pending_binds: &[PendingHostBindAnchor],
pending_revokes: &[PendingHostRevokeAnchor],
) -> Result<(), MobError> {
let bind_hosts = pending_binds
.iter()
.map(|anchor| anchor.request.host_id.as_str())
.collect::<BTreeSet<_>>();
if let Some(revoke) = pending_revokes
.iter()
.find(|anchor| bind_hosts.contains(anchor.host_id.as_str()))
{
return Err(MobError::Internal(format!(
"host '{}' has overlapping unfinished bind and revoke authority operations",
revoke.host_id
)));
}
Ok(())
}
fn validate_host_authority_anchor_temporal_order(events: &[MobEvent]) -> Result<(), MobError> {
#[derive(Clone, Copy, PartialEq, Eq)]
enum Family {
Bind,
Revoke,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum GenerationState {
Binding,
ReusableAfterAbort,
Bound,
Revoking,
Revoked,
}
let mut open = BTreeMap::<String, (Family, String, u64)>::new();
let mut highwater = BTreeMap::<String, (u64, GenerationState)>::new();
for event in events {
match &event.kind {
MobEventKind::RemoteHostBindStarted {
operation_id,
request,
} => {
if open.get(&request.host_id)
== Some(&(
Family::Bind,
operation_id.clone(),
request.binding_generation,
))
{
continue;
}
if let Some((_, existing, _)) = open.get(&request.host_id) {
return Err(MobError::Internal(format!(
"host '{}' bind operation '{operation_id}' overlaps unfinished authority operation '{existing}'",
request.host_id
)));
}
let (prior, prior_state) = highwater
.get(&request.host_id)
.copied()
.unwrap_or((0, GenerationState::Revoked));
if request.binding_generation < prior
|| (request.binding_generation == prior
&& prior_state != GenerationState::ReusableAfterAbort)
{
return Err(MobError::Internal(format!(
"host '{}' bind operation '{operation_id}' generation {} does not advance prior authority highwater {prior}",
request.host_id, request.binding_generation
)));
}
highwater.insert(
request.host_id.clone(),
(request.binding_generation, GenerationState::Binding),
);
open.insert(
request.host_id.clone(),
(
Family::Bind,
operation_id.clone(),
request.binding_generation,
),
);
}
MobEventKind::RemoteHostBindCompleted {
operation_id,
host_id,
binding_generation,
..
} => {
let closed_current = open.get(host_id)
== Some(&(Family::Bind, operation_id.clone(), *binding_generation));
if closed_current {
open.remove(host_id);
if highwater.get(host_id).map(|(generation, _)| *generation)
== Some(*binding_generation)
{
highwater.insert(
host_id.clone(),
(*binding_generation, GenerationState::Bound),
);
}
}
}
MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id,
host_id,
binding_generation,
..
} => {
let closed_current = open.get(host_id)
== Some(&(Family::Bind, operation_id.clone(), *binding_generation));
if closed_current {
open.remove(host_id);
if highwater.get(host_id).map(|(generation, _)| *generation)
== Some(*binding_generation)
{
highwater.insert(
host_id.clone(),
(*binding_generation, GenerationState::ReusableAfterAbort),
);
}
}
}
MobEventKind::RemoteHostRevokeStarted {
operation_id,
host_id,
binding_generation,
..
} => {
if open.get(host_id)
== Some(&(Family::Revoke, operation_id.clone(), *binding_generation))
{
continue;
}
if let Some((_, existing, _)) = open.get(host_id) {
return Err(MobError::Internal(format!(
"host '{host_id}' revoke operation '{operation_id}' overlaps unfinished authority operation '{existing}'"
)));
}
if let Some((current_generation, current_state)) = highwater.get(host_id).copied()
&& (current_generation != *binding_generation
|| current_state != GenerationState::Bound)
{
return Err(MobError::Internal(format!(
"host '{host_id}' revoke operation '{operation_id}' generation {binding_generation} does not target the current bound authority generation {current_generation}"
)));
}
// A history created before bind anchors existed may have no
// local bind highwater. Admit its first revoke as a legacy
// unknown, then record it so stale/duplicate revokes fail.
highwater.insert(
host_id.clone(),
(*binding_generation, GenerationState::Revoking),
);
open.insert(
host_id.clone(),
(Family::Revoke, operation_id.clone(), *binding_generation),
);
}
MobEventKind::RemoteHostRevokeCompleted {
operation_id,
host_id,
binding_generation,
..
} => {
let closed_current = open.get(host_id)
== Some(&(Family::Revoke, operation_id.clone(), *binding_generation));
if closed_current {
open.remove(host_id);
highwater.insert(
host_id.clone(),
(*binding_generation, GenerationState::Revoked),
);
}
}
_ => {}
}
}
Ok(())
}
/// Validate every durable mob epoch before recovery writes. Unfinished remote
/// authority operations may survive a process restart, but never cross a
/// lifecycle boundary which prevents their recovery choreography from being
/// driven. Reset would erase the retry anchor; completion or destroy would
/// replay a terminal phase and strand it while a delayed host terminal can
/// still take effect.
pub(super) fn validate_host_authority_anchor_epoch_boundaries(
events: &[MobEvent],
) -> Result<(), MobError> {
#[derive(Clone, Copy)]
enum LifecycleMode {
Active,
Destroying,
Terminal(&'static str),
}
fn is_bind_anchor_event(kind: &MobEventKind) -> bool {
matches!(
kind,
MobEventKind::RemoteHostBindStarted { .. }
| MobEventKind::RemoteHostBindConfirmed { .. }
| MobEventKind::RemoteHostBindCompleted { .. }
| MobEventKind::RemoteHostBindAbortedNoEffect { .. }
)
}
fn is_revoke_anchor_event(kind: &MobEventKind) -> bool {
matches!(
kind,
MobEventKind::RemoteHostRevokeStarted { .. }
| MobEventKind::RemoteHostRevokeConfirmed { .. }
| MobEventKind::RemoteHostRevokeCompleted { .. }
)
}
fn validate_segment(events: &[MobEvent], action: Option<&str>) -> Result<(), MobError> {
let binds = pending_host_bind_anchors(events)?;
let revokes = pending_host_revoke_anchors(events)?;
validate_host_authority_anchor_temporal_order(events)?;
validate_host_authority_anchor_disjointness(&binds, &revokes)?;
if let Some(action) = action {
require_host_authority_anchors_clear(action, &binds, &revokes)?;
}
Ok(())
}
let mut segment_start = 0usize;
let mut mode = LifecycleMode::Active;
for (boundary_index, event) in events.iter().enumerate() {
if is_bind_anchor_event(&event.kind) {
match mode {
LifecycleMode::Active => {}
LifecycleMode::Destroying => {
return Err(MobError::Internal(
"host bind authority event appears after MobDestroying; destroy may open only new revoke operations"
.to_string(),
));
}
LifecycleMode::Terminal(marker) => {
return Err(MobError::Internal(format!(
"host bind authority event appears after terminal {marker} without an intervening MobReset"
)));
}
}
} else if is_revoke_anchor_event(&event.kind)
&& let LifecycleMode::Terminal(marker) = mode
{
return Err(MobError::Internal(format!(
"host revoke authority event appears after terminal {marker} without an intervening MobReset"
)));
}
let action = match event.kind {
MobEventKind::MobReset => "cross MobReset epoch boundary",
MobEventKind::MobCompleted => "cross MobCompleted terminal boundary",
MobEventKind::MobDestroying => "cross MobDestroying terminal boundary",
MobEventKind::MobDestroyStorageFinalizing => {
"cross MobDestroyStorageFinalizing terminal boundary"
}
_ => continue,
};
validate_segment(&events[segment_start..boundary_index], Some(action))?;
mode = match event.kind {
MobEventKind::MobReset => LifecycleMode::Active,
MobEventKind::MobDestroying => LifecycleMode::Destroying,
MobEventKind::MobCompleted => LifecycleMode::Terminal("MobCompleted"),
MobEventKind::MobDestroyStorageFinalizing => {
LifecycleMode::Terminal("MobDestroyStorageFinalizing")
}
_ => {
return Err(MobError::Internal(
"host authority boundary classifier produced a non-boundary event".to_string(),
));
}
};
// MobDestroying starts a distinct destroy-owned revoke segment. This
// prevents a late pre-destroy duplicate terminal from masquerading as
// cleanup authority and makes Finalizing require that new segment to
// close completely.
segment_start = boundary_index + 1;
}
let tail_action = match mode {
LifecycleMode::Active | LifecycleMode::Destroying => None,
LifecycleMode::Terminal(marker) => Some(marker),
};
validate_segment(&events[segment_start..], tail_action)
}
fn require_host_authority_anchors_clear(
action: &str,
pending_binds: &[PendingHostBindAnchor],
pending_revokes: &[PendingHostRevokeAnchor],
) -> Result<(), MobError> {
if let Some(anchor) = pending_binds.first() {
return Err(MobError::Internal(format!(
"cannot {action} while host '{}' bind operation '{}' is unfinished at authority epoch {} generation {}",
anchor.request.host_id,
anchor.operation_id,
anchor.request.authority_epoch,
anchor.request.binding_generation,
)));
}
if let Some(anchor) = pending_revokes.first() {
return Err(MobError::Internal(format!(
"cannot {action} while host '{}' revoke operation '{}' is unfinished at authority epoch {} generation {}",
anchor.host_id, anchor.operation_id, anchor.epoch, anchor.binding_generation,
)));
}
Ok(())
}
#[cfg(test)]
mod placed_route_session_tests {
use super::optional_route_session_matches_machine;
#[test]
fn sessionless_roster_route_defers_to_machine_authority() {
assert!(optional_route_session_matches_machine(
None,
"machine-session"
));
}
#[test]
fn mismatched_present_roster_session_fails_closed() {
let route = meerkat_core::SessionId::new();
assert!(!optional_route_session_matches_machine(
Some(&route),
"different-machine-session"
));
}
}
/// The actor that processes mob commands sequentially.
///
/// Owns all mutable state. Runs in a dedicated tokio task.
/// All mutations go through here; reads bypass via shared `Arc` state.
struct ExplicitResumeMemberRebuild {
entry: RosterEntry,
member_ref: MemberRef,
bridge_session_id: SessionId,
requires_materialization: bool,
repoints_session_binding: bool,
recovered_peer_endpoint: Option<TrustedPeerDescriptor>,
}
pub(super) struct MobActor {
pub(super) definition: Arc<MobDefinition>,
pub(super) roster: Arc<RwLock<RosterAuthority>>,
pub(super) events: Arc<dyn MobEventStore>,
pub(super) placed_completion_durable_index: Arc<std::sync::Mutex<PlacedCompletionDurableIndex>>,
pub(super) run_store: Arc<dyn MobRunStore>,
pub(super) provisioner: Arc<dyn MobProvisioner>,
pub(super) flow_engine: FlowEngine,
/// Whether this mob's definition declares an orchestrator.
/// Gates orchestrator-specific transitions and notification fan-out.
pub(super) has_orchestrator: bool,
/// Whether an explicit stopped -> running resume should enqueue the
/// informational orchestrator turn after the durable transition commits.
pub(super) notify_orchestrator_on_resume: bool,
/// Flow-run EXECUTION-HANDLE registries — NOT semantic authority (#210).
///
/// `run_tasks` (JoinHandles), `run_cancel_tokens` (CancellationTokens +
/// owning FlowId), and `flow_streams` (per-run event senders) are the
/// actor's local execution resources for in-flight flow runs. They carry
/// NO semantic verdict: the active-run lifecycle (admission to run/cancel,
/// run-known/no-active-runs gating) is owned by the MobMachine
/// (`run_status`), driven through `apply_command_admission`. No production
/// branch may read these maps' presence to decide a semantic outcome
/// BEFORE machine admission — they are consulted only to perform the
/// EXECUTION (abort the task, fire the token, drop the stream) after the
/// machine has already ruled. They are reconciled against the machine and
/// against each other by `ensure_flow_tracker_alignment` /
/// `flow_tracker_alignment_violation`; the only read of their cardinality
/// is the `#[cfg(test)]`-gated `machine_active_run_count` snapshot.
pub(super) run_tasks: BTreeMap<RunId, tokio::task::JoinHandle<()>>,
pub(super) run_cancel_tokens: BTreeMap<RunId, (tokio_util::sync::CancellationToken, FlowId)>,
pub(super) flow_streams:
Arc<tokio::sync::Mutex<BTreeMap<RunId, mpsc::Sender<meerkat_core::ScopedAgentEvent>>>>,
pub(super) command_tx: mpsc::Sender<RoutedMobCommand>,
/// Late-bound embedder barrier shared by every handle for this mob,
/// including member-session operator tools created by the actor.
pub(super) flow_target_provisioner:
Arc<std::sync::RwLock<Option<super::handle::FlowTargetProvisioner>>>,
pub(super) tool_bundles: BTreeMap<String, Arc<dyn AgentToolDispatcher>>,
pub(super) default_llm_client: Option<Arc<dyn LlmClient>>,
pub(super) retired_event_index: Arc<RwLock<HashSet<String>>>,
/// Same-process idempotency index for the durable retirement-start
/// carrier. Restart recovery replays that carrier into MobMachine's
/// Retiring marker; this index covers the narrower append-before-machine-
/// commit retry window without scanning the full shared event log on each
/// ordinary retire.
pub(super) retirement_started_event_index: Arc<RwLock<HashSet<String>>>,
/// Exact retirement-start carriers that authorize preserving desired
/// machine topology across the old incarnation's terminal gap. This is a
/// subset of `retirement_started_event_index`; replay rebuilds both so an
/// automatic cold-recovery retry cannot downgrade a respawn retirement to
/// ordinary topology cleanup.
pub(super) preserved_respawn_topology_event_index: Arc<RwLock<HashSet<String>>>,
pub(super) autonomous_initial_turns:
Arc<tokio::sync::Mutex<BTreeMap<AgentIdentity, InitialTurnHandle>>>,
/// Volatile execution only. Durable Stop intent and kickoff cancellation
/// live in MobMachine/event custody; these receivers let a command retry
/// observe bounded off-actor interrupts without serially blocking the
/// actor on a blackholed member host.
pub(super) autonomous_stop_interrupts: BTreeMap<AgentIdentity, AutonomousStopInterruptTask>,
/// Same-process exact successes so one slow host does not make command
/// retries re-interrupt already-quiesced peers. Cold replay may safely
/// repeat the exact fenced interrupt.
pub(super) autonomous_stop_interrupted:
BTreeMap<AgentIdentity, AutonomousStopInterruptIncarnation>,
/// Rotating admission cursor for the bounded off-actor interrupt window.
pub(super) autonomous_stop_interrupt_cursor: usize,
pub(super) next_spawn_ticket: u64,
/// Monotonically increasing fence token counter.
/// Each spawn/respawn/reset issues a strictly newer token.
/// Uses `AtomicU64` so `&self` methods (batch finalization) can issue tokens.
pub(super) next_fence_token: std::sync::atomic::AtomicU64,
pub(super) pending_spawns: PendingSpawnLineage,
pub(super) pending_spawn_cleanup_anchors: BTreeMap<u64, PendingSpawnCleanupAnchor>,
pub(super) edge_locks: Arc<super::edge_locks::EdgeLockRegistry>,
pub(super) lifecycle_tasks: tokio::task::JoinSet<Result<(), MobError>>,
/// First completed lifecycle-delivery fault not yet surfaced to a
/// lifecycle caller. The actor loop observes JoinErrors immediately so an
/// ambiguous effect can fail-stop before another command, but ordinary
/// typed delivery failures retain their pre-existing next-call custody.
pub(super) pending_lifecycle_delivery_error: Option<MobError>,
/// Actor-owned detached I/O that must never outlive this actor instance.
///
/// Pending-spawn, flow-run, and autonomous-initial-turn tasks have their
/// own keyed ownership tables; the two wasm Send shims are immediately
/// awaited by their caller. Every other task spawned from actor code
/// (live read/close I/O, host probes/releases, detached replies, and placed
/// turn delivery) lives here so fail-stop and every other actor exit can
/// abort and join it before publishing command-channel closure. Mutating
/// live Open/Control work uses `member_live_mutation_tasks` below.
pub(super) actor_io_tasks: tokio::task::JoinSet<()>,
/// Mutating member-live operations (Open/Control) have their own actor-owned
/// lane. Lifecycle admission drains this set before publishing its durable
/// work-origin fence, so an operation admitted while Running linearizes
/// before Stop/Complete/Reset/Destroy instead of completing through it.
/// Close/Status remain in `actor_io_tasks`: they are cleanup/read-only and
/// must stay usable while lifecycle reconciliation is in progress.
pub(super) member_live_mutation_tasks: tokio::task::JoinSet<MemberLiveMutationCompletion>,
/// Exact successful Opens whose channel could not yet be handed to a
/// current caller (stale incarnation or dropped reply). The JoinSet is
/// only execution custody; this table is the actor-owned cleanup
/// obligation. A failed close remains here and is retried, never reduced
/// to a warning and forgotten.
pub(super) member_live_open_cleanup_obligations: BTreeMap<u64, MemberLiveOpenCleanupObligation>,
/// Tickets currently represented by one task in
/// `member_live_mutation_tasks`; prevents parallel close attempts for the
/// same exact channel while still allowing every obligation to retry.
pub(super) member_live_open_cleanup_inflight: BTreeSet<u64>,
pub(super) next_member_live_open_cleanup_ticket: u64,
/// Exact detached orphan releases awaiting typed completion absorption.
/// This is volatile mechanical custody only: restart clears it and lets
/// durable HostStatus truth drive a retry.
pub(super) orphan_release_reservations:
BTreeMap<super::state::HostOrphanReleaseKey, HostOrphanReleaseReservation>,
/// Volatile binding-incarnation fence for each host identity. Supervisor
/// epochs are not sufficient here: revoke followed by a fresh bind may
/// reuse the same authority epoch while changing the host route/process.
/// Actor restart drops both this map and every detached release task.
pub(super) host_binding_incarnations: BTreeMap<mob_dsl::HostId, u64>,
/// Last authenticated host-process incarnation observed through
/// `HostStatus` or a successful member-events page. This actor-local
/// cache is only a recovery trigger: MobMachine wiring facts remain the
/// sole source of route intent.
pub(super) host_runtime_incarnations:
BTreeMap<mob_dsl::HostId, super::bridge_protocol::BridgeHostRuntimeIncarnation>,
pub(super) next_peer_delivery_ticket: PeerDeliveryId,
pub(super) peer_delivery_tasks: tokio::task::JoinSet<PeerDeliveryCompletion>,
pub(super) peer_delivery_inflight: BTreeMap<PeerDeliveryId, PeerDeliveryInflight>,
pub(super) peer_delivery_permits: Arc<tokio::sync::Semaphore>,
pub(super) session_service: Arc<dyn MobSessionService>,
#[cfg(feature = "runtime-adapter")]
pub(super) runtime_adapter: Option<Arc<meerkat_runtime::MeerkatMachine>>,
pub(super) restore_diagnostics:
Arc<RwLock<HashMap<AgentIdentity, super::handle::RestoreFailureDiagnostic>>>,
/// Per bridge-session shell lifecycle locks for #37 live-materialization
/// revival. These serialize the observation/rebuild mechanics around a
/// machine-owned member binding without copying live-session truth into the
/// MobMachine.
pub(super) member_revival_locks:
Arc<tokio::sync::Mutex<HashMap<SessionId, Arc<tokio::sync::Mutex<()>>>>>,
pub(super) runtime_metadata: Arc<dyn crate::store::MobRuntimeMetadataStore>,
/// Sole desired-state authority for identity intents, leases, and custody.
pub(super) identity: Arc<dyn crate::store::MobIdentityStore>,
/// Built-in atomic join between identity authority and the structural
/// member event target. Split custom stores cannot provide this guarantee.
pub(super) identity_member: Option<Arc<dyn crate::store::MobIdentityMemberStore>>,
/// Replaceable output-only identity convergence diagnostics.
pub(super) identity_status: Arc<dyn crate::store::MobIdentityStatusStore>,
/// Volatile scheduling only. Desired state and progress are always
/// re-observed from their owning stores before one obligation executes.
pub(super) identity_reconcile_queue: VecDeque<AgentIdentity>,
pub(super) identity_reconcile_enqueued: BTreeSet<AgentIdentity>,
/// Volatile keyset position for the bounded cross-process safety scan.
/// `None` starts a new pass; reaching the end resets it to `None`.
pub(super) identity_reconcile_safety_cursor: Option<crate::store::IdentityIntentScanCursor>,
/// Volatile actuator diagnostics only. They are never classifier input or
/// restart authority; the next cold process simply re-observes.
pub(super) identity_reconcile_failures:
Arc<RwLock<BTreeMap<AgentIdentity, IdentityReconcileFailureState>>>,
/// Volatile per-identity actuation-backoff deadlines. The actor arms one
/// timer for the earliest deadline and enqueues only due identities;
/// startup discovery and the slow cross-process safety scan also refuse
/// early re-actuation. Success (or changed durable authority) resets it.
/// Scheduling custody only, never persisted machine state — guards the
/// 2026-07-29 hot-retry incident where every failure requeued immediately.
pub(super) identity_reconcile_backoff: BTreeMap<AgentIdentity, IdentityReconcileBackoffState>,
/// Identities parked after a fatal in-process actuation failure (closed
/// callback transport). Parked identities are skipped by causal/deadline
/// admission and the safety scan; a changed durable intent or a process
/// restart clears the park. Volatile scheduling custody only.
pub(super) identity_reconcile_parked: BTreeMap<AgentIdentity, IdentityReconcileParkState>,
/// Once-per-distinct-payload panic log gate for spawn provisioning tasks
/// (see `panic_capture` for the 2026-07-29 incident WHY): the
/// identity-reconcile requeue can re-run a panicking provision
/// indefinitely, so the recovered payload is logged on transition, never
/// per iteration.
pub(super) spawn_panic_log_ledger: Arc<super::panic_capture::SpawnPanicLogLedger>,
/// Stable logical controller id plus this process-local incarnation.
/// Neither value is checkpoint content authority.
pub(super) identity_reconcile_holder_id: String,
pub(super) identity_reconcile_incarnation_id: String,
pub(super) supervisor_bridge: Arc<super::MobSupervisorBridge>,
/// Phase 6b (ADJ-P6B-1): the LOCAL-branch live gateway — the ONE
/// extracted live pipeline behind the session-id-addressed
/// `MemberLiveHost` seam, injected via
/// `MobBuilder::with_member_live_host` (composition over
/// configuration). `None` (the default) = this process composes no
/// live transport; local live verbs typed-reject
/// `LiveTransportUnavailable` with zero cost.
pub(super) member_live_host: Option<Arc<dyn meerkat_runtime::member_live::MemberLiveHost>>,
/// Member event pump manager (§7.4 phase 6, DEC-P6E-11): owns one
/// detached `PollMemberEvents` loop per placed member. The actor derives
/// pump material from machine facts + the roster's shell transport
/// composition; taps are read directly off this shared handle.
pub(super) member_event_pumps: Arc<super::event_pump::MemberEventPumpManager>,
/// The mob's ONE remote flow ticket registry (DEC-P6F-6), the same Arc
/// the flow executor arms and the event pump feeds. The actor touches it
/// only at member-lifetime boundaries: `drop_lane` when disposal stops a
/// member's pump for good, and `note_member_rematerializing` strictly
/// before a placed respawn's remote release (T-F6 attribution).
pub(super) remote_flow_tickets: Arc<super::remote_flow_ticket::RemoteFlowTicketRegistry>,
/// Controlling-side member-operator upcall responder (DEC-U3), spawned
/// next to the actor and shut down whenever the actor's run loop exits;
/// the responder handle's `Drop` abort is the final backstop.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
pub(super) upcall_responder: Option<super::upcall_responder::MobUpcallResponderHandle>,
pub(super) spawn_policy: Arc<super::spawn_policy::SpawnPolicyService>,
pub(super) dsl_authority: mob_dsl::MobMachineAuthority,
pub(super) dsl_topology_epoch: Arc<std::sync::atomic::AtomicU64>,
pub(super) dsl_authority_owner_token: Arc<dyn std::any::Any + Send + Sync>,
/// Read-only MobMachine state projection for handle-side status/list
/// surfaces. The actor is the sole writer; handles can borrow the latest
/// state without enqueueing behind long shell cleanup work.
pub(super) machine_state_watch_tx: tokio::sync::watch::Sender<mob_dsl::MobMachineState>,
pub(super) reachability_observations: Arc<super::handle::ReachabilityObservations>,
/// Terminal-phase projection for external observers. Written by the
/// actor after every DSL phase transition and once more right before
/// the actor task exits. `MobHandle::status()` falls back to this
/// `watch` receiver when the command channel has closed (actor has
/// exited post-Shutdown/Destroy). The watch is an explicit dogma-#13
/// projection: the actor owns the sole writer, external handles hold
/// read-only receivers, and the source of truth remains the DSL
/// authority inside the actor.
pub(super) phase_watch_tx: tokio::sync::watch::Sender<MobState>,
pub(super) default_external_tools_provider: Option<crate::ExternalToolsProvider>,
/// Fresh process-local dispatcher services for identity reconciliation.
/// The provider is never persisted and receives only an exact sealed
/// intent key; it cannot mutate portable desired material.
pub(super) identity_local_external_tools_provider:
Option<Arc<dyn super::IdentityLocalExternalToolsProvider>>,
/// Per-spawn external-tool overlays (`SpawnMemberSpec.external_tools`)
/// retained for the member's lifetime so machine-authorized revival
/// recomposes the same dispatcher stack the spawn used. Entries are
/// written only in the spawn/respawn commit path (provisioning failure
/// never inserts), replaced or cleared by respawn replacement semantics,
/// and removed at disposal. RwLock: `dispose_remove_from_roster` runs on
/// `&self`.
pub(super) per_spawn_external_tools:
tokio::sync::RwLock<BTreeMap<AgentIdentity, Arc<dyn AgentToolDispatcher>>>,
/// R3 case-3 base-prompt seam for placed spawns (ADJ-2). `None` ⇒
/// case-3 remote spawns fail typed at spec compile.
pub(super) spawn_base_prompt_source:
Option<Arc<dyn super::spec_compiler::SpawnBasePromptSource>>,
pub(super) spawn_member_customizer: Option<Arc<dyn super::SpawnMemberCustomizer>>,
pub(super) realm_profile_store: Option<Arc<dyn crate::store::RealmProfileStore>>,
/// Typed composition binding for the `meerkat_mob_seam` composition
/// (wave-c C-6p). Every routed effect emitted by the mob DSL travels
/// through `CompositionDispatcher::dispatch` via this binding rather
/// than through direct peer / provisioner calls. Construction uses
/// `CompositionBinding::Standalone` by default (test / ephemeral
/// path); production surface assembly swaps in
/// `CompositionBinding::Wired(Arc<dyn CompositionDispatcher<...>>)`.
pub(super) composition_binding: super::composition::MobCompositionBinding,
/// Routed seam-effects queued by sync `apply_dsl_input` /
/// `apply_dsl_signal` calls. Drained by `flush_routed_effects` at
/// every command-loop boundary and after each command handler; the
/// first dispatch failure surfaces as a typed `MobError`, no silent
/// drops.
pub(super) pending_routed_effects: Vec<super::composition::MobSeamEffect>,
pub(super) destroy_cleanup_active: bool,
/// Poison latch for an append whose commit state could not be determined
/// after the bounded exact-cursor read retries. Once set, the common
/// command boundary crash-quiesces volatile producers and terminates the
/// actor before any routed effect or queued command can run. Only cold
/// durable replay may resolve the uncertainty.
/// A durable mutation may have committed but could not be classified
/// exactly. The loop quiesces and terminates before accepting another
/// command; cold recovery is the only authority that may continue.
pub(super) durable_uncertainty_fail_stop: bool,
/// Narrow reply-suppression latch for a respawn whose exact topology
/// abandonment marker could not be made durable. Other fail-stop paths
/// preserve their established typed-error reply before quiescence; this
/// one must expose only channel closure so an undurable abandonment is
/// never laundered into a definitive respawn/lifecycle result.
pub(super) respawn_topology_reply_withheld: bool,
/// Controlling-side reverse-lane acceptor (ADJ-P4-2). `None` means a
/// mixed-host route that needs a local reverse lane fails closed instead
/// of publishing an undialable process-local endpoint.
#[cfg(not(target_arch = "wasm32"))]
pub(super) controlling_acceptor: Option<ControllingAcceptorState>,
}
struct AuthorizedPeerOnlyBind {
peer: TrustedPeerDescriptor,
response: super::bridge_protocol::BridgeBindResponse,
}
enum ActorBoundaryFlushDisposition {
Drained,
RetryAfterMachineClosure(MobError),
Fatal(MobError),
}
/// One resolved dispatch lane for a live verb (phase 6b, DEC-P6B-C3):
/// placement is the CALLER's machine fact; the capability gate has already
/// ruled by the time a lane exists. Both lanes dispatch on detached tasks.
enum MemberLiveLane {
Placed {
peer: TrustedPeerDescriptor,
expected_member: super::bridge_protocol::BridgeMemberIncarnation,
},
Local {
session_id: SessionId,
generation: u64,
},
}
/// Exact dispatch authority retained until one detached mutating live effect
/// has been reconciled on the actor. The placed form keeps the route as well as
/// the full incarnation so a late successful Open can be closed exactly even
/// after the roster has moved on.
#[derive(Clone, Debug)]
pub(super) enum MemberLiveMutationTarget {
Placed {
peer: TrustedPeerDescriptor,
expected_member: super::bridge_protocol::BridgeMemberIncarnation,
},
Local {
session_id: SessionId,
generation: u64,
},
}
pub(super) enum MemberLiveMutationCompletion {
Open {
agent_identity: AgentIdentity,
target: MemberLiveMutationTarget,
result: Result<super::bridge_protocol::LiveOpenResult, MobError>,
ambiguous: bool,
reply_tx: oneshot::Sender<Result<super::state::MemberLiveOpenDelivery, MobError>>,
},
Control {
agent_identity: AgentIdentity,
target: MemberLiveMutationTarget,
result: Result<super::bridge_protocol::BridgeLiveControlOutcome, MobError>,
ambiguous_effect: bool,
reply_tx:
oneshot::Sender<Result<super::bridge_protocol::BridgeLiveControlOutcome, MobError>>,
},
OpenCleanup {
ticket: u64,
durable_persistence_confirmed: bool,
result: Result<(), MobError>,
panicked: bool,
},
OpenDeliveryAck {
ticket: u64,
confirmation: Option<oneshot::Sender<Result<(), MobError>>>,
},
}
pub(super) struct MemberLiveOpenCleanupObligation {
durable_record: crate::store::MobMemberLiveCleanupRecord,
durable_persistence_confirmed: bool,
agent_identity: AgentIdentity,
target: MemberLiveMutationTarget,
channel_id: String,
reason: String,
attempts: u32,
last_error: Option<String>,
/// A panic is an invariant failure, not a transient transport verdict.
/// Retain durable custody but do not execute the same faulting code again
/// in this actor incarnation. Cold recovery reconstructs the obligation
/// with this flag clear after repaired code can take over.
panic_quarantined: bool,
/// The last attempt panicked before durable custody was confirmed. Such
/// an obligation remains retryable, but only on the panic-specific slow
/// cadence rather than the ordinary transport retry cadence.
panic_backoff: bool,
/// True only while a successful result is in the no-await handle
/// handoff. This phase must not transiently fence commands that the
/// acknowledged caller issues immediately after receiving the channel.
delivery_pending_ack: bool,
caller_acknowledged: bool,
delivery_confirmation: Option<oneshot::Sender<Result<(), MobError>>>,
terminal_reply: Option<(
oneshot::Sender<Result<super::state::MemberLiveOpenDelivery, MobError>>,
MobError,
)>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MemberLivePanicDisposition {
TerminalFailure,
AmbiguousEffect,
QuarantinedCleanup,
BackoffUntilDurable,
}
impl MemberLivePanicDisposition {
const fn as_str(self) -> &'static str {
match self {
Self::TerminalFailure => "terminal_failure",
Self::AmbiguousEffect => "ambiguous_effect",
Self::QuarantinedCleanup => "quarantined_cleanup",
Self::BackoffUntilDurable => "backoff_until_durable",
}
}
}
#[derive(Clone, Copy)]
enum MemberLiveReconcileMode {
Background,
Lifecycle,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RemoteTurnCustodyPhase {
Pending,
Committed,
Resolved,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct RemoteTurnCustodyCleanup {
obligation: mob_dsl::RemoteTurnObligation,
phase: RemoteTurnCustodyPhase,
}
/// Exact machine-owned residency tuple whose remote-turn custody may be
/// disposed. Keeping the fencing atoms together prevents cleanup callers from
/// accidentally mixing facts from different member or host incarnations.
#[derive(Debug, Clone, Copy)]
struct ExactRemoteTurnResidency<'a> {
agent_identity: &'a AgentIdentity,
host_id: &'a mob_dsl::HostId,
host_binding_generation: u64,
member_session_id: &'a mob_dsl::SessionId,
generation: u64,
fence_token: u64,
}
impl MobActor {
fn peer_only_member_control_error(
runtime_mode: crate::MobRuntimeMode,
action: &str,
) -> MobError {
MobError::UnsupportedForMode {
mode: runtime_mode,
reason: format!("{action} is not supported for peer-only members in phase 1"),
}
}
fn peer_only_spec_from_parts(
peer_id: &str,
address: &str,
context: &'static str,
pubkey: [u8; 32],
) -> Result<TrustedPeerDescriptor, MobError> {
let peer_name = address
.strip_prefix("inproc://")
.map(|value| value.split('?').next().unwrap_or(value).to_string())
.unwrap_or_else(|| format!("mob_member/backend_peer/{peer_id}"));
let result = TrustedPeerDescriptor::unsigned_with_pubkey(
peer_name,
peer_id.to_string(),
pubkey,
address.to_string(),
);
result.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid peer-only runtime spec: {error}"
))
})
}
/// Resolve the exact generation-bound endpoint that will be registered
/// during spawn activation, without publishing operation readiness.
///
/// This pure observation runs before the write-ahead `MemberSpawned`
/// append. Activation later publishes and registers this same descriptor,
/// making disagreement between the journaled and active endpoints
/// unrepresentable.
async fn resolve_spawn_member_peer_material(
&self,
member_ref: &MemberRef,
profile_name: &ProfileName,
identity: &AgentIdentity,
) -> Result<(Option<TrustedPeerDescriptor>, Option<String>), MobError> {
if let Some(session_id) = member_ref.bridge_session_id() {
let Some(runtime) = self.session_service.comms_runtime(session_id).await else {
return Ok((None, None));
};
let Some(public_key_bytes) = runtime.public_key_bytes() else {
return Ok((None, None));
};
let comms_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
identity.as_str(),
)?;
let peer_id = runtime.peer_id().ok_or_else(|| {
MobError::WiringError(format!(
"spawn endpoint for '{identity}' has a signing key but no peer id"
))
})?;
TrustedPeerDescriptor::validate_pubkey_for_peer_id(peer_id, &public_key_bytes)
.map_err(|error| {
MobError::WiringError(format!(
"invalid spawn endpoint for '{identity}': {error}"
))
})?;
let name = PeerName::new(runtime.comms_name().unwrap_or_else(|| comms_name.clone()))
.map_err(|error| {
MobError::WiringError(format!(
"invalid spawn endpoint name for '{identity}': {error}"
))
})?;
let address = PeerAddress::parse(
runtime
.advertised_address()
.unwrap_or_else(|| format!("inproc://{comms_name}")),
)
.map_err(|error| {
MobError::WiringError(format!(
"invalid spawn endpoint address for '{identity}': {error}"
))
})?;
let endpoint = TrustedPeerDescriptor {
peer_id,
name,
address,
pubkey: public_key_bytes,
};
let public_key = meerkat_comms::PubKey::new(public_key_bytes).to_pubkey_string();
return Ok((Some(endpoint), Some(public_key)));
}
if let MemberRef::BackendPeer {
peer_id,
address,
pubkey,
..
} = member_ref
{
return Ok((
Some(Self::peer_only_spec_from_parts(
peer_id,
address,
"resolve_spawn_member_peer_material",
*pubkey,
)?),
None,
));
}
Ok((None, None))
}
fn peer_only_spec_for_binding(
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
match binding {
crate::RuntimeBinding::External {
peer_id,
address,
pubkey,
..
} => Self::peer_only_spec_from_parts(peer_id, address, context, *pubkey),
crate::RuntimeBinding::Session => Err(MobError::Internal(format!(
"{context}: peer-only runtime spec requested for session binding"
))),
crate::RuntimeBinding::HostMaterialized { .. } => Err(MobError::Internal(format!(
"{context}: peer material is machine-owned for placed members; read member_peer_endpoints"
))),
}
}
fn peer_only_spec_from_member_endpoint(
endpoint: &mob_dsl::MemberPeerEndpoint,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
TrustedPeerDescriptor::unsigned_with_pubkey(
endpoint.name.0.clone(),
endpoint.peer_id.0.clone(),
endpoint.signing_key.0,
endpoint.address.0.clone(),
)
.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid MobMachine member peer endpoint authority: {error}"
))
})
}
fn member_peer_rebind_endpoint_from_transition(
transition: &mob_dsl::MobMachineTransition,
agent_identity: &mob_dsl::AgentIdentity,
context: &'static str,
) -> Result<mob_dsl::MemberPeerEndpoint, MobError> {
let mut authorized = None;
for effect in transition.effects() {
if let mob_dsl::MobMachineEffect::MemberPeerRebindAuthorized {
agent_identity: effect_identity,
peer_endpoint,
..
} = effect
{
if effect_identity != agent_identity {
continue;
}
if authorized.replace(peer_endpoint.clone()).is_some() {
return Err(MobError::WiringError(format!(
"{context}: duplicate generated member peer rebind authority for '{}'",
agent_identity.0
)));
}
}
}
authorized.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: missing generated member peer rebind authority for '{}'",
agent_identity.0
))
})
}
fn trusted_peer_removal_key(peer: &TrustedPeerDescriptor) -> String {
peer.peer_id.to_string()
}
fn supervisor_publish_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_publish::publish_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn supervisor_publish_cleanup_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_publish::cleanup_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn supervisor_revoke_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_revoke::revoke_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn unexpected_trust_mutation_result(
operation: &'static str,
result: CommsTrustMutationResult,
) -> SendError {
SendError::Internal(format!(
"{operation} returned unexpected trust mutation result: {result:?}"
))
}
async fn bind_generated_mob_trust_owner_for_authority(
&self,
comms: &(dyn CoreCommsRuntime + '_),
authority: &CommsTrustMutationAuthority,
) -> Result<(), SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn bind_generated_mob_trust_owner_for_authority_with_token(
comms: &(dyn CoreCommsRuntime + '_),
authority: &CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
if authority.is_mob_machine_source() {
comms
.install_generated_mob_trust_owner(Arc::clone(owner_token))
.await?;
}
Ok(())
}
async fn apply_trusted_peer_add(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<(), SendError> {
self.apply_trusted_peer_add_report(comms, peer, authority)
.await?;
Ok(())
}
async fn apply_trusted_peer_add_report(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
Self::apply_trusted_peer_add_with_owner_token_report(
comms,
peer,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn apply_trusted_peer_add_with_owner_token(
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
Self::apply_trusted_peer_add_with_owner_token_report(comms, peer, authority, owner_token)
.await?;
Ok(())
}
async fn apply_trusted_peer_add_with_owner_token_report(
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<bool, SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
&authority,
owner_token,
)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::AddTrustedPeer { peer, authority })
.await?
{
CommsTrustMutationResult::Added { created } => Ok(created),
result => Err(Self::unexpected_trust_mutation_result(
"add trusted peer",
result,
)),
}
}
async fn apply_trusted_peer_remove(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
Self::apply_trusted_peer_remove_with_owner_token(
comms,
peer_id,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn apply_trusted_peer_remove_with_owner_token(
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<bool, SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
&authority,
owner_token,
)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::RemoveTrustedPeer { peer_id, authority })
.await?
{
CommsTrustMutationResult::Removed { removed } => Ok(removed),
result => Err(Self::unexpected_trust_mutation_result(
"remove trusted peer",
result,
)),
}
}
async fn apply_private_trusted_peer_add(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<(), SendError> {
self.bind_generated_mob_trust_owner_for_authority(comms, &authority)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::AddPrivateTrustedPeer { peer, authority })
.await?
{
CommsTrustMutationResult::Added { .. } => Ok(()),
result => Err(Self::unexpected_trust_mutation_result(
"add private trusted peer",
result,
)),
}
}
async fn apply_private_trusted_peer_remove(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
self.bind_generated_mob_trust_owner_for_authority(comms, &authority)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::RemovePrivateTrustedPeer {
peer_id,
authority,
})
.await?
{
CommsTrustMutationResult::Removed { removed } => Ok(removed),
result => Err(Self::unexpected_trust_mutation_result(
"remove private trusted peer",
result,
)),
}
}
fn supervisor_spec_for_authority(
mob_id: &crate::MobId,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<TrustedPeerDescriptor, MobError> {
let participant_name = format!("{mob_id}/__mob_supervisor__");
let public_key = authority.keypair().public_key();
TrustedPeerDescriptor::unsigned_with_pubkey(
participant_name.clone(),
authority.public_peer_id.clone(),
*public_key.as_bytes(),
format!("inproc://{participant_name}"),
)
.map_err(|error| MobError::WiringError(format!("invalid supervisor spec: {error}")))
}
async fn install_supervisor_private_trust_for_session(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
previous_private_trust_removal_key: Option<&str>,
) -> Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError> {
let authority = self.supervisor_bridge.authority().await;
let spec = Self::supervisor_spec_for_authority(&self.definition.id, &authority)?;
Box::pin(self.install_supervisor_private_trust_for_session_authority(
session_id,
comms,
&authority,
spec,
None,
previous_private_trust_removal_key,
))
.await
}
#[cfg(feature = "runtime-adapter")]
async fn realize_supervisor_private_trust_revoke(
&self,
request: SupervisorPrivateTrustRevokeRequest<'_>,
) -> Result<bool, MobError> {
let SupervisorPrivateTrustRevokeRequest {
adapter,
session_id,
comms,
peer_id,
epoch,
removal_key,
allow_absent_pending,
} = request;
let revoke_transition = match adapter
.stage_supervisor_revoke(session_id, peer_id.clone(), epoch)
.await
{
Ok(transition) => transition,
Err(_) if allow_absent_pending => return Ok(false),
Err(error) => {
return Err(MobError::WiringError(format!(
"previous supervisor private trust revoke rejected for session '{session_id}': {error}"
)));
}
};
let revoke_freshness = adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
.map_err(|error| {
MobError::WiringError(format!(
"previous supervisor private trust revoke freshness unavailable for session '{session_id}': {error}"
))
})?;
let revoke_obligation =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
&revoke_transition,
revoke_freshness,
)
.into_iter()
.find(|obligation| obligation.peer_id() == &peer_id && obligation.epoch() == epoch)
.ok_or_else(|| {
MobError::WiringError(format!(
"previous supervisor private trust revoke for session '{session_id}' produced no generated revoke obligation"
))
})?;
if let Err(error) = self
.apply_private_trusted_peer_remove(
comms,
removal_key,
Self::supervisor_revoke_authority(&revoke_obligation)
.map_err(MobError::WiringError)?,
)
.await
{
let feedback = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
revoke_obligation.peer_id().clone(),
revoke_obligation.epoch(),
error.to_string(),
)
.await;
let mut reason = format!(
"previous supervisor private trust removal failed for session '{session_id}': {error}"
);
if let Err(feedback_error) = feedback {
reason.push_str(&format!("; revoke feedback failed: {feedback_error}"));
}
return Err(MobError::WiringError(reason));
}
adapter
.stage_supervisor_trust_revoked(
session_id,
revoke_obligation.peer_id().clone(),
revoke_obligation.epoch(),
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"previous supervisor private trust revoke feedback rejected for session '{session_id}': {error}"
))
})?;
Ok(true)
}
async fn install_supervisor_private_trust_for_session_authority(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
authority: &crate::store::SupervisorAuthorityRecord,
spec: TrustedPeerDescriptor,
previous_authority: Option<&crate::store::SupervisorAuthorityRecord>,
previous_private_trust_removal_key: Option<&str>,
) -> Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError> {
#[cfg(feature = "runtime-adapter")]
let Some(adapter) = self.runtime_adapter.as_ref() else {
return Err(MobError::Internal(format!(
"cannot publish supervisor private trust for session '{session_id}': runtime adapter unavailable"
))
.into());
};
#[cfg(not(feature = "runtime-adapter"))]
let _ = session_id;
#[cfg(not(feature = "runtime-adapter"))]
{
return Err(MobError::Internal(
"cannot publish supervisor private trust without runtime adapter".to_string(),
)
.into());
}
#[cfg(feature = "runtime-adapter")]
{
use meerkat_runtime::protocol_supervisor_trust_publish;
adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust local endpoint rejected for session '{session_id}': {error}"
))
})?;
let next_name = spec.name.as_str().to_owned();
let next_peer_id = spec.peer_id.as_str().to_owned();
let next_address = spec.address.to_string();
let next_signing_public_key =
meerkat_runtime::comms_drain::encode_supervisor_signing_public_key(spec.pubkey);
let next_epoch = authority.epoch;
let previous = adapter.supervisor_binding(session_id).await;
let already_bound = matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
name,
peer_id,
address,
signing_public_key,
epoch,
} if name == &next_name
&& peer_id == &next_peer_id
&& address == &next_address
&& signing_public_key == &next_signing_public_key
&& *epoch == next_epoch
);
let previous_peer_is_different = matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { peer_id, .. }
if peer_id != &next_peer_id
);
if matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound
) && let Some(previous_authority) = previous_authority
{
// A prior activation attempt may have staged the old binding's
// durable revoke but failed the router removal. The generated
// machine intentionally remains Unbound+RevokePending, so a
// blind BindSupervisor retry is rejected. Rematerialize and
// discharge that exact old peer/epoch obligation first. If the
// binding is simply fresh-Unbound there is no pending revoke;
// the guarded retry is absent and normal bind proceeds.
let _ = self
.realize_supervisor_private_trust_revoke(SupervisorPrivateTrustRevokeRequest {
adapter: adapter.as_ref(),
session_id,
comms: comms.as_ref(),
peer_id: previous_authority.public_peer_id.clone(),
epoch: previous_authority.epoch,
removal_key: previous_private_trust_removal_key
.map(str::to_string)
.unwrap_or_else(|| previous_authority.public_peer_id.clone()),
allow_absent_pending: true,
})
.await?;
}
if previous_peer_is_different {
let meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
peer_id: previous_peer_id,
epoch: previous_epoch,
..
} = &previous
else {
return Err(MobError::Internal(
"supervisor replacement classifier selected an unbound predecessor"
.to_string(),
)
.into());
};
let previous_peer_id = previous_peer_id.clone();
let previous_epoch = *previous_epoch;
let previous_removal_key = previous_private_trust_removal_key
.map(str::to_string)
.unwrap_or_else(|| previous_peer_id.clone());
self.realize_supervisor_private_trust_revoke(SupervisorPrivateTrustRevokeRequest {
adapter: adapter.as_ref(),
session_id,
comms: comms.as_ref(),
peer_id: previous_peer_id,
epoch: previous_epoch,
removal_key: previous_removal_key,
allow_absent_pending: false,
})
.await?;
}
let stage_transition = if already_bound {
adapter
.stage_supervisor_trust_publish_request(
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publish request rejected for session '{session_id}': {error}"
))
})?
} else if previous_peer_is_different {
Self::stage_supervisor_bind_for_private_trust(
adapter,
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust bind rejected for session '{session_id}': {error}"
))
})?
} else {
match &previous {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => {
Self::stage_supervisor_bind_for_private_trust(
adapter,
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust bind rejected for session '{session_id}': {error}"
))
})?
}
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { .. } => {
adapter
.stage_supervisor_authorize(
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust rotation rejected for session '{session_id}': {error}"
))
})?
}
_ => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' saw an unknown supervisor binding variant"
))
.into());
}
}
};
let publish_freshness = adapter
.supervisor_trust_publish_freshness_authority(session_id)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publish freshness unavailable for session '{session_id}': {error}"
))
})?;
let obligations = protocol_supervisor_trust_publish::extract_obligations_with_freshness(
&stage_transition,
publish_freshness,
);
let publish_obligation = match obligations.as_slice() {
[obligation] => obligation.clone(),
[] => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' produced no generated publish obligation"
))
.into());
}
_ => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' produced multiple generated publish obligations"
))
.into());
}
};
if publish_obligation.name() != &next_name
|| publish_obligation.peer_id() != &next_peer_id
|| publish_obligation.address() != &next_address
|| publish_obligation.signing_public_key().as_deref()
!= Some(next_signing_public_key.as_str())
|| publish_obligation.epoch() != next_epoch
{
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' generated obligation did not match the staged supervisor binding"
))
.into());
}
let publish_spec =
meerkat_runtime::comms_drain::trusted_peer_descriptor_from_supervisor_publish_obligation(
&publish_obligation,
)
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' generated invalid trust descriptor: {error}"
))
})?;
let publish_peer_id = publish_obligation.peer_id().clone();
let publish_epoch = publish_obligation.epoch();
let publish_removal_key = Self::trusted_peer_removal_key(&publish_spec);
let publish_cleanup_authority =
Self::supervisor_publish_cleanup_authority(&publish_obligation)
.map_err(MobError::WiringError)?;
let rollback_binding = previous.clone();
if let Err(error) = self
.apply_private_trusted_peer_add(
comms.as_ref(),
publish_spec.clone(),
Self::supervisor_publish_authority(&publish_obligation)
.map_err(MobError::WiringError)?,
)
.await
{
let _ = adapter
.stage_supervisor_trust_publish_failed(
session_id,
publish_peer_id.clone(),
publish_epoch,
error.to_string(),
)
.await;
let new_trust_cleanup_failed = if !already_bound {
self.cleanup_supervisor_private_trust_publish_attempt(
session_id,
comms,
publish_cleanup_authority.clone(),
publish_removal_key.clone(),
"failed to clean up supervisor private trust after publish add failure",
)
.await
.is_err()
} else {
false
};
let rollback = if already_bound {
Ok(())
} else {
self.rollback_supervisor_private_trust_binding(
adapter,
session_id,
comms,
&rollback_binding,
&publish_peer_id,
publish_epoch,
)
.await
};
let mut reason = format!(
"supervisor private trust publication failed for session '{session_id}': {error}"
);
if let Err(rollback_error) = rollback {
reason.push_str(&format!("; rollback failed: {rollback_error}"));
}
let error = MobError::WiringError(reason);
return Err(if new_trust_cleanup_failed {
SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)
} else {
SupervisorPrivateTrustInstallError::without_cleanup_failure(error)
});
}
if let Err(error) = Self::stage_supervisor_trust_published_for_private_trust(
adapter,
session_id,
publish_peer_id.clone(),
publish_epoch,
)
.await
{
let new_trust_cleanup_failed = if !already_bound {
self.cleanup_supervisor_private_trust_publish_attempt(
session_id,
comms,
publish_cleanup_authority,
publish_removal_key.clone(),
"failed to clean up supervisor private trust after rejected publish ack",
)
.await
.is_err()
} else {
false
};
let rollback = if already_bound {
Ok(())
} else {
self.rollback_supervisor_private_trust_binding(
adapter,
session_id,
comms,
&rollback_binding,
&publish_peer_id,
publish_epoch,
)
.await
};
let mut reason = format!(
"supervisor private trust publication ack rejected for session '{session_id}': {error}"
);
if let Err(rollback_error) = rollback {
reason.push_str(&format!("; rollback failed: {rollback_error}"));
}
let error = MobError::WiringError(reason);
return Err(if new_trust_cleanup_failed {
SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)
} else {
SupervisorPrivateTrustInstallError::without_cleanup_failure(error)
});
}
Ok(SupervisorPrivateTrustInstall {
peer_id: next_peer_id,
epoch: next_epoch,
removal_key: publish_removal_key,
})
}
}
#[cfg(feature = "runtime-adapter")]
async fn stage_supervisor_trust_published_for_private_trust(
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
peer_id: String,
epoch: u64,
) -> Result<(), meerkat_runtime::meerkat_machine::SupervisorBindingStageError> {
#[cfg(target_arch = "wasm32")]
{
let adapter = Arc::clone(adapter);
let session_id = session_id.clone();
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let result = adapter
.stage_supervisor_trust_published(&session_id, peer_id, epoch)
.await;
let _ = reply_tx.send(result);
});
reply_rx.await.map_err(|_| {
meerkat_runtime::meerkat_machine::SupervisorBindingStageError::SessionRegistryBusy
})?
}
#[cfg(not(target_arch = "wasm32"))]
{
adapter
.stage_supervisor_trust_published(session_id, peer_id, epoch)
.await
}
}
#[cfg(feature = "runtime-adapter")]
async fn stage_supervisor_bind_for_private_trust(
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
name: String,
peer_id: String,
address: String,
signing_public_key: String,
epoch: u64,
) -> Result<
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineTransition,
meerkat_runtime::meerkat_machine::SupervisorBindingStageError,
> {
#[cfg(target_arch = "wasm32")]
{
let adapter = Arc::clone(adapter);
let session_id = session_id.clone();
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let result = adapter
.stage_supervisor_bind(
&session_id,
name,
peer_id,
address,
signing_public_key,
epoch,
)
.await;
let _ = reply_tx.send(result);
});
reply_rx.await.map_err(|_| {
meerkat_runtime::meerkat_machine::SupervisorBindingStageError::SessionRegistryBusy
})?
}
#[cfg(not(target_arch = "wasm32"))]
{
adapter
.stage_supervisor_bind(
session_id,
name,
peer_id,
address,
signing_public_key,
epoch,
)
.await
}
}
/// Remove the just-attempted ("new") supervisor private trust after a failed
/// publish. Returns the typed cleanup result so callers can record whether
/// the compensation itself failed — the activation rollback keys on that
/// structured verdict rather than parsing the formatted error message.
async fn cleanup_supervisor_private_trust_publish_attempt(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
authority: CommsTrustMutationAuthority,
removal_key: String,
context: &'static str,
) -> Result<(), MobError> {
if let Err(error) = self
.apply_private_trusted_peer_remove(comms.as_ref(), removal_key, authority)
.await
{
tracing::warn!(
%session_id,
%error,
context,
"failed to clean up supervisor private trust publish attempt"
);
return Err(MobError::from(error));
}
Ok(())
}
async fn cleanup_supervisor_private_trust_for_session(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
install: &SupervisorPrivateTrustInstall,
) {
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = self.runtime_adapter.as_ref() {
if let Err(error) = adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
{
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to stage local endpoint for supervisor private trust cleanup"
);
return;
}
let transition = match adapter
.stage_supervisor_revoke(session_id, install.peer_id.clone(), install.epoch)
.await
{
Ok(transition) => transition,
Err(error) => {
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to stage supervisor private trust cleanup"
);
return;
}
};
let revoke_freshness = match adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
{
Ok(authority) => authority,
Err(error) => {
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to build generated supervisor private trust cleanup freshness"
);
return;
}
};
let obligations =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(&transition, revoke_freshness);
let Some(obligation) = obligations.into_iter().find(|obligation| {
obligation.peer_id() == &install.peer_id && obligation.epoch() == install.epoch
}) else {
let reason =
"generated supervisor private trust cleanup effect was absent".to_string();
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
install.peer_id.clone(),
install.epoch,
reason.clone(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
reason,
"failed to stage supervisor private trust cleanup"
);
return;
};
if let Err(error) = self.apply_private_trusted_peer_remove(
comms.as_ref(),
install.removal_key.clone(),
match Self::supervisor_revoke_authority(&obligation) {
Ok(authority) => authority,
Err(error) => {
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
error.clone(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to build generated supervisor private trust cleanup authority"
);
return;
}
},
)
.await
{
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
error.to_string(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to clean up supervisor private trust"
);
return;
}
if let Err(error) = adapter
.stage_supervisor_trust_revoked(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
{
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to acknowledge supervisor private trust cleanup"
);
}
return;
}
let _ = comms;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
"skipping supervisor private trust cleanup because generated runtime adapter authority is unavailable"
);
}
#[cfg(feature = "runtime-adapter")]
async fn rollback_supervisor_private_trust_binding(
&self,
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
previous: &meerkat_runtime::meerkat_machine::SupervisorBinding,
current_peer_id: &str,
current_epoch: u64,
) -> Result<(), MobError> {
adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
match previous {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => {
let transition = adapter
.stage_supervisor_revoke(session_id, current_peer_id.to_string(), current_epoch)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
let revoke_freshness = adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
if let Some(obligation) =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
&transition,
revoke_freshness,
)
.into_iter()
.find(|obligation| {
obligation.peer_id().as_str() == current_peer_id
&& obligation.epoch() == current_epoch
})
{
adapter
.stage_supervisor_trust_revoked(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
}
Ok(())
}
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
name,
peer_id,
address,
signing_public_key,
epoch,
} => {
let current = adapter.supervisor_binding(session_id).await;
let transition = match current {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => adapter
.stage_supervisor_bind(
session_id,
name.clone(),
peer_id.clone(),
address.clone(),
signing_public_key.clone(),
*epoch,
)
.await,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { .. } => adapter
.stage_supervisor_authorize(
session_id,
name.clone(),
peer_id.clone(),
address.clone(),
signing_public_key.clone(),
*epoch,
)
.await,
other => {
return Err(MobError::WiringError(format!(
"supervisor private trust rollback for session '{session_id}' saw unsupported current binding {other:?}"
)));
}
}
.map_err(|error| MobError::WiringError(error.to_string()))?;
let publish_freshness = adapter
.supervisor_trust_publish_freshness_authority(session_id)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
let obligation =
meerkat_runtime::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
&transition,
publish_freshness,
)
.into_iter()
.find(|obligation| {
obligation.peer_id() == peer_id
&& obligation.epoch() == *epoch
&& obligation.signing_public_key().as_deref()
== Some(signing_public_key.as_str())
})
.ok_or_else(|| {
MobError::WiringError(format!(
"supervisor private trust rollback for session '{session_id}' produced no generated publish obligation"
))
})?;
let trusted_peer =
meerkat_runtime::comms_drain::trusted_peer_descriptor_from_supervisor_publish_obligation(
&obligation,
)
.map_err(MobError::WiringError)?;
self.apply_private_trusted_peer_add(
comms.as_ref(),
trusted_peer,
Self::supervisor_publish_authority(&obligation)
.map_err(MobError::WiringError)?,
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
adapter
.stage_supervisor_trust_published(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
Ok(())
}
_ => Err(MobError::WiringError(
"unknown supervisor binding variant during rollback".to_string(),
)),
}
}
async fn bridge_supervisor_payload(
&self,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let authority = self.supervisor_bridge.authority().await;
self.supervisor_payload_for_authority(&authority)
}
async fn bridge_supervisor_payload_for_recipient(
&self,
recipient: &TrustedPeerDescriptor,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let authority = self.supervisor_bridge.authority().await;
self.bridge_supervisor_payload_for_authority_and_recipient(&authority, recipient)
.await
}
async fn bridge_supervisor_payload_for_authority_and_recipient(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
recipient: &TrustedPeerDescriptor,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let spec = self
.supervisor_bridge
.supervisor_spec_for_authority_and_recipient(authority, recipient)
.await?;
Ok(super::bridge_protocol::BridgeSupervisorPayload {
supervisor: spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
})
}
fn supervisor_payload_for_authority(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let spec = Self::supervisor_spec_for_authority(&self.definition.id, authority)?;
Ok(super::bridge_protocol::BridgeSupervisorPayload {
supervisor: spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
})
}
fn bridge_bootstrap_token_from_binding(
binding: &crate::RuntimeBinding,
) -> Result<super::bridge_protocol::BridgeBootstrapToken, MobError> {
match binding {
crate::RuntimeBinding::External {
address,
bootstrap_token,
..
} => bootstrap_token
.as_ref()
.filter(|token| !token.is_empty())
.cloned()
.ok_or_else(|| {
MobError::WiringError(format!(
"external runtime binding for '{address}' is missing typed bootstrap_token field"
))
}),
crate::RuntimeBinding::Session => Err(MobError::Internal(
"bridge bootstrap token requested for session binding".to_string(),
)),
crate::RuntimeBinding::HostMaterialized { .. } => Err(MobError::Internal(
"placed members have no member bootstrap token; host trust was installed at the bind ceremony"
.to_string(),
)),
}
}
fn authorize_existing_peer_binding_for_rebind(
&mut self,
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
let crate::RuntimeBinding::External {
peer_id,
address,
pubkey,
..
} = binding
else {
return Err(MobError::Internal(format!(
"{context}: peer-only rebind authority requested for session binding"
)));
};
let canonical_address = super::bridge_protocol::canonicalize_bridge_address(address);
let observed_peer =
Self::peer_only_spec_from_parts(peer_id, &canonical_address, context, *pubkey)?;
let affected_identities: BTreeSet<_> = self
.dsl_authority
.state()
.member_peer_ids
.iter()
.filter(|(_, existing_peer_id)| existing_peer_id.0 == *peer_id)
.map(|(identity, _)| identity.clone())
.collect();
if affected_identities.is_empty() {
return Err(MobError::WiringError(format!(
"{context}: peer-only rebind for '{peer_id}' requires MobMachine member peer authority"
)));
}
let mut authorized_peer: Option<TrustedPeerDescriptor> = None;
for identity in &affected_identities {
let expected_peer_endpoint = self
.dsl_authority
.state()
.member_peer_endpoints
.get(identity)
.cloned()
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: peer-only rebind for '{}' lacks MobMachine endpoint authority",
identity.0
))
})?;
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerRebind {
agent_identity: identity.clone(),
expected_peer_endpoint,
},
context,
)?;
let endpoint =
Self::member_peer_rebind_endpoint_from_transition(&transition, identity, context)?;
let peer = Self::peer_only_spec_from_member_endpoint(&endpoint, context)?;
if peer.name != observed_peer.name
|| peer.peer_id != observed_peer.peer_id
|| peer.address != observed_peer.address
|| peer.pubkey != observed_peer.pubkey
{
return Err(MobError::WiringError(format!(
"{context}: observed peer-only rebind endpoint for '{peer_id}' is outside generated MobMachine authority"
)));
}
if let Some(existing) = &authorized_peer
&& (existing.name != peer.name
|| existing.peer_id != peer.peer_id
|| existing.address != peer.address
|| existing.pubkey != peer.pubkey)
{
return Err(MobError::WiringError(format!(
"{context}: generated MobMachine peer rebind authority disagrees across identities for '{peer_id}'"
)));
}
authorized_peer = Some(peer);
}
authorized_peer.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: peer-only rebind for '{peer_id}' produced no generated authority"
))
})
}
async fn bind_peer_only_member_for_binding(
&mut self,
peer: &TrustedPeerDescriptor,
binding: &crate::RuntimeBinding,
) -> Result<AuthorizedPeerOnlyBind, MobError> {
let payload = self.bridge_supervisor_payload_for_recipient(peer).await?;
self.bind_peer_only_member_for_binding_with_payload(peer, binding, &payload)
.await
}
async fn bind_peer_only_member_for_binding_with_payload(
&mut self,
peer: &TrustedPeerDescriptor,
binding: &crate::RuntimeBinding,
payload: &super::bridge_protocol::BridgeSupervisorPayload,
) -> Result<AuthorizedPeerOnlyBind, MobError> {
let crate::RuntimeBinding::External {
peer_id,
address: _,
bootstrap_token: _,
pubkey: _,
} = binding
else {
return Err(MobError::Internal(
"bind requested for non-external runtime binding".to_string(),
));
};
let authorized_peer =
self.authorize_existing_peer_binding_for_rebind(binding, "bind_peer_only_member")?;
if authorized_peer.name != peer.name
|| authorized_peer.peer_id != peer.peer_id
|| authorized_peer.address != peer.address
|| authorized_peer.pubkey != peer.pubkey
{
return Err(MobError::WiringError(format!(
"bind requested for peer '{peer_id}' without matching MobMachine member peer authority"
)));
}
let bootstrap_token = Self::bridge_bootstrap_token_from_binding(binding)?;
let command = super::bridge_protocol::BridgeCommand::BindMember(
super::bridge_protocol::BridgeBindPayload {
supervisor: payload.supervisor.clone(),
epoch: payload.epoch,
protocol_version: payload.protocol_version,
expected_peer_id: authorized_peer.peer_id.to_string(),
expected_address: authorized_peer.address.to_string(),
bootstrap_token,
},
);
let bind: super::bridge_protocol::BridgeBindResponse = self
.send_bridge_command_typed(
&authorized_peer,
&command,
// 60s (not 30s): tolerate async trust/peer-registration
// propagation lag before the bind reply lands (matches the
// responder's 60s wait). Bounds failure-lag only — under
// high-parallelism RBE loopback registration can lag tens of
// seconds; in real deployments it propagates in ms.
std::time::Duration::from_secs(60),
)
.await?;
let returned_address = super::bridge_protocol::canonicalize_bridge_address(&bind.address);
let authorized_peer_id = authorized_peer.peer_id.to_string();
let authorized_address = authorized_peer.address.to_string();
let expected_address =
super::bridge_protocol::canonicalize_bridge_address(&authorized_address);
if bind.peer_id != authorized_peer_id || returned_address != expected_address {
return Err(MobError::WiringError(format!(
"bind response changed authorized endpoint for peer '{peer_id}'"
)));
}
Ok(AuthorizedPeerOnlyBind {
peer: authorized_peer,
response: bind,
})
}
fn bridge_rejection_reply(
protocol_version: super::bridge_protocol::BridgeProtocolVersion,
value: &serde_json::Value,
) -> Option<super::bridge_protocol::BridgeRejectionReply> {
super::bridge_protocol::decode_bridge_rejection_reply(protocol_version, value)
}
fn bridge_rejection_error(rejection: super::bridge_protocol::BridgeRejectionReply) -> MobError {
MobError::from(rejection)
}
/// Whether an authenticated `BindHost` rejection proves that the host
/// could not have installed the offered binding. Only causes emitted
/// strictly before host persistence belong here. `AlreadyBound` and
/// `Internal` are deliberately excluded: either can arrive after an
/// earlier exact bind committed (including a lost ACK or post-persist
/// trust-install failure).
fn bind_rejection_certifies_no_remote_effect(
rejection: &super::bridge_protocol::BridgeRejectionReply,
) -> bool {
matches!(
rejection.typed_cause(),
Some(
super::bridge_protocol::BridgeRejectionCause::SenderMismatch
| super::bridge_protocol::BridgeRejectionCause::InvalidBootstrapToken
| super::bridge_protocol::BridgeRejectionCause::UnsupportedProtocolVersion
| super::bridge_protocol::BridgeRejectionCause::InvalidSupervisorSpec
| super::bridge_protocol::BridgeRejectionCause::InvalidPeerSpec
| super::bridge_protocol::BridgeRejectionCause::AddressMismatch
| super::bridge_protocol::BridgeRejectionCause::StaleFence
| super::bridge_protocol::BridgeRejectionCause::Unsupported
)
)
}
/// Authenticated RevokeHost rejections emitted before host persistence.
/// Internal/Unavailable are excluded because the host may already have
/// committed the durable revocation and failed while publishing its ACK.
fn revoke_rejection_certifies_no_remote_effect(
rejection: &super::bridge_protocol::BridgeRejectionReply,
) -> bool {
matches!(
rejection.typed_cause(),
Some(
super::bridge_protocol::BridgeRejectionCause::NotBound
| super::bridge_protocol::BridgeRejectionCause::StaleSupervisor
| super::bridge_protocol::BridgeRejectionCause::SenderMismatch
| super::bridge_protocol::BridgeRejectionCause::StaleFence
| super::bridge_protocol::BridgeRejectionCause::UnsupportedProtocolVersion
| super::bridge_protocol::BridgeRejectionCause::InvalidSupervisorSpec
| super::bridge_protocol::BridgeRejectionCause::InvalidPeerSpec
)
)
}
fn rebind_rejection_certifies_no_remote_effect(
rejection: &super::bridge_protocol::BridgeRejectionReply,
) -> bool {
matches!(
rejection.typed_cause(),
Some(
super::bridge_protocol::BridgeRejectionCause::NotBound
| super::bridge_protocol::BridgeRejectionCause::StaleSupervisor
| super::bridge_protocol::BridgeRejectionCause::SenderMismatch
| super::bridge_protocol::BridgeRejectionCause::StaleFence
| super::bridge_protocol::BridgeRejectionCause::UnsupportedProtocolVersion
| super::bridge_protocol::BridgeRejectionCause::InvalidSupervisorSpec
| super::bridge_protocol::BridgeRejectionCause::InvalidPeerSpec
)
)
}
fn bridge_rejection_error_with_reason(
rejection: &super::bridge_protocol::BridgeRejectionReply,
reason: String,
) -> MobError {
match rejection.typed_cause() {
Some(cause) => MobError::BridgeCommandRejected { cause, reason },
None => MobError::WiringError(reason),
}
}
async fn persist_rebound_binding(
&mut self,
prior_binding: &crate::RuntimeBinding,
authorized_peer: &TrustedPeerDescriptor,
bind_response: &super::bridge_protocol::BridgeBindResponse,
) -> Result<(), MobError> {
let crate::RuntimeBinding::External {
peer_id: prior_peer_id,
address: prior_address,
pubkey,
..
} = prior_binding
else {
return Ok(());
};
let bootstrap_token = Some(Self::bridge_bootstrap_token_from_binding(prior_binding)?);
let authorized_peer_id = authorized_peer.peer_id.to_string();
let authorized_address = authorized_peer.address.to_string();
let canonical_authorized_address =
super::bridge_protocol::canonicalize_bridge_address(&authorized_address);
let expected_address = super::bridge_protocol::canonicalize_bridge_address(prior_address);
if authorized_peer_id != *prior_peer_id
|| canonical_authorized_address != expected_address
|| authorized_peer.pubkey != *pubkey
{
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' lacks matching generated MobMachine endpoint authority"
)));
}
let returned_address =
super::bridge_protocol::canonicalize_bridge_address(&bind_response.address);
if bind_response.peer_id != authorized_peer_id
|| returned_address != canonical_authorized_address
{
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' attempted to change MobMachine-authorized endpoint"
)));
}
let affected_identities: BTreeSet<_> = self
.dsl_authority
.state()
.member_peer_ids
.iter()
.filter(|(_, peer_id)| peer_id.0 == authorized_peer_id)
.map(|(identity, _)| identity.clone())
.collect();
if affected_identities.is_empty() {
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' requires MobMachine member peer authority"
)));
}
let affected_domain_identities: BTreeSet<_> = affected_identities
.iter()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect();
let updated_entries = self
.roster
.write()
.await
.replace_backend_peer_binding_for_identities(
&affected_domain_identities,
&authorized_peer_id,
&canonical_authorized_address,
bootstrap_token.clone(),
);
for (identity, generation, pubkey) in updated_entries {
self.runtime_metadata
.upsert_external_binding_overlay(
&self.definition.id,
&crate::store::ExternalBindingOverlayRecord {
agent_identity: identity,
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: authorized_peer_id.clone(),
address: canonical_authorized_address.clone(),
pubkey,
bootstrap_token: None,
session_id: None,
}),
bootstrap_token: bootstrap_token.clone(),
status: crate::store::ExternalBindingOverlayStatus::Normalized,
updated_at: chrono::Utc::now(),
},
)
.await?;
}
Ok(())
}
async fn authorize_peer_only_member_ref_for_behavior(
&mut self,
member_ref: &MemberRef,
context: &'static str,
) -> Result<MemberRef, MobError> {
let Some(binding) = Self::runtime_binding_for_member_ref(member_ref) else {
return Ok(member_ref.clone());
};
let peer = Self::peer_only_spec_for_binding(&binding, context)?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(&binding))
.await?;
let bootstrap_token = Some(Self::bridge_bootstrap_token_from_binding(&binding)?);
Ok(MemberRef::BackendPeer {
peer_id: peer.peer_id.to_string(),
address: peer.address.to_string(),
pubkey: peer.pubkey,
bootstrap_token,
session_id: None,
})
}
async fn ensure_supervisor_authorized(
&mut self,
peer: &TrustedPeerDescriptor,
binding: Option<&crate::RuntimeBinding>,
) -> Result<TrustedPeerDescriptor, MobError> {
// A completed member rotation is fenced onto the pending authority.
// Never send the old authority back to a peer whose terminal receipt
// is already checkpointed: ordinary member behavior must wait for the
// exact durable rotation operation to finish local activation.
if let Some(current) = self.load_supervisor_authority().await?
&& let Some(pending) = current.pending_rotation.as_ref()
&& pending
.accepted_peer_ids
.iter()
.any(|peer_id| peer_id == &peer.peer_id.to_string())
{
return Err(Self::pending_supervisor_rotation_blocks_rebind(
¤t, pending, peer,
));
}
let payload = self.bridge_supervisor_payload_for_recipient(peer).await?;
let protocol_version = payload.protocol_version;
let command = super::bridge_protocol::BridgeCommand::AuthorizeSupervisor(payload);
// Transport requires the recipient be trusted before the request can be
// routed (see comms admission), so trust is installed before the send.
// The trust-install-to-terminality window is recorded as a MobMachine
// `pending_recipient_trust` obligation before the install, and the
// invariant is enforced by rolling trust newly installed by THIS
// attempt back below on every path where this `peer` is not a
// CONFIRMED, ACCEPTED supervisor (fail closed). Trust that pre-existed
// the call was established by an earlier confirmed terminality and is
// left in place.
self.record_pending_recipient_trust_obligation(peer, "ensure_supervisor_authorized")?;
let install = match self.supervisor_bridge.trust_recipient(peer).await {
Ok(install) => install,
Err(trust_error) => {
return Err(self.quarantine_uncertain_recipient_trust_install(
peer,
"ensure_supervisor_authorized",
trust_error,
));
}
};
// 60s (not 30s): the requester must tolerate the same async
// trust/peer-registration propagation lag the live peer waits out
// before replying (see `spawn_live_external_peer`, 60s). Bounds genuine
// failure-lag only — the happy path returns as soon as the reply lands.
// Under high-parallelism RBE loopback registration can lag tens of
// seconds; in real deployments it propagates in milliseconds.
let value = match self
.supervisor_bridge
.send_bridge_command(peer, &command, std::time::Duration::from_secs(60))
.await
{
Ok(value) => value,
Err(send_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, send_error)
.await);
}
};
if let Some(rejection) = Self::bridge_rejection_reply(protocol_version, &value) {
let should_rebind = match rejection.typed_cause() {
Some(cause) => match self.classify_bridge_rejection_recovery(cause) {
Ok(should_rebind) => should_rebind,
Err(classify_error) => {
// The recovery verdict could not be obtained, so the
// peer is not a confirmed supervisor: roll the
// just-installed trust back before propagating.
return Err(self
.rollback_supervisor_recipient_trust(peer, install, classify_error)
.await);
}
},
None => false,
};
if should_rebind && let Some(binding) = binding {
// A recoverable current-authority rejection normally permits a
// bootstrap re-bind. During a durable rotation operation that
// would reauthorize the old authority and can erase a member's
// unobserved terminal receipt. Refuse the fallback without
// mutating either the operation or the member authority.
let current_authority = match self.load_supervisor_authority().await {
Ok(current) => current,
Err(load_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, load_error)
.await);
}
};
if let Some(current) = current_authority
&& let Some(pending) = current.pending_rotation.as_ref()
{
let pending_error =
Self::pending_supervisor_rotation_blocks_rebind(¤t, pending, peer);
return Err(self
.rollback_supervisor_recipient_trust(peer, install, pending_error)
.await);
}
// The AuthorizeSupervisor was rejected with a recoverable cause,
// so we re-bind the same peer (the bind re-establishes its own
// recipient trust). If the bind itself fails, the recipient was
// never confirmed and its trust must not survive — roll it back
// fail-closed.
let authorized_bind =
match self.bind_peer_only_member_for_binding(peer, binding).await {
Ok(authorized_bind) => authorized_bind,
Err(bind_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, bind_error)
.await);
}
};
// The re-bind reached confirmed-accept terminality for this
// peer, closing the recipient-trust obligation window before
// the post-terminality persistence steps run.
self.resolve_pending_recipient_trust_obligation(
peer,
"ensure_supervisor_authorized rebind confirmed",
)?;
self.persist_rebound_binding(
binding,
&authorized_bind.peer,
&authorized_bind.response,
)
.await?;
return Ok(authorized_bind.peer);
}
return Err(self
.rollback_supervisor_recipient_trust(
peer,
install,
Self::bridge_rejection_error(rejection),
)
.await);
}
let _ack = match super::bridge_protocol::decode_bridge_ack(
&command,
value,
"authorize supervisor response",
) {
Ok(ack) => ack,
Err(decode_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, decode_error)
.await);
}
};
self.resolve_pending_recipient_trust_obligation(
peer,
"ensure_supervisor_authorized confirmed",
)?;
Ok(peer.clone())
}
/// Record the MobMachine `pending_recipient_trust` obligation for a peer
/// whose recipient trust is about to be installed ahead of authorization
/// terminality (dogma row R044). Set semantics make the input idempotent,
/// so nested authorize-then-bind windows for the same peer compose.
fn record_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.record_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn record_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RecordPendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
/// Close the `pending_recipient_trust` obligation after the peer reached
/// confirmed-accept authorization terminality.
fn resolve_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.resolve_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn resolve_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::ResolvePendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
/// Close the `pending_recipient_trust` obligation after a failure path
/// rolled the installed trust back (or established no trust at all).
fn rollback_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn rollback_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RollbackPendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
fn recipient_trust_was_newly_installed(
install: super::supervisor_bridge::RecipientTrustInstall,
) -> bool {
install == super::supervisor_bridge::RecipientTrustInstall::NewlyInstalled
}
/// `trust_recipient` applies the declarative peer endpoint before it
/// reconciles the live comms trust row. A returned error therefore does
/// not prove that no trust mutation occurred. Machine-owning callers
/// must keep the pending obligation open and stop this actor until cold
/// recovery can repair the exact DSL/live projection gap.
fn quarantine_uncertain_recipient_trust_install(
&mut self,
peer: &TrustedPeerDescriptor,
context: &'static str,
error: MobError,
) -> MobError {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
peer_id = %peer.peer_id,
context,
error = %error,
"recipient trust install failed after a possibly committed DSL mutation; retaining trust obligation and fail-stopping actor"
);
MobError::Internal(format!(
"recipient trust install for '{}' ({context}) failed after its declarative mutation may have committed: {error}; trust state and the pending recipient-trust obligation are retained for cold repair",
peer.peer_id
))
}
/// Fail-closed rollback for supervisor recipient trust installed ahead of an
/// `AuthorizeSupervisor` send. The recipient was trusted only so the bridge
/// request could be routed; if authorization did not terminate in a
/// confirmed accept, trust newly installed by this attempt must not
/// survive. Trust that pre-existed the attempt was established by an
/// earlier confirmed terminality and is left in place. The MobMachine
/// `pending_recipient_trust` obligation is rolled back once the trust
/// state is restored; if the untrust itself fails the obligation stays
/// pending because the window genuinely remains open. The authorization
/// `original_error` is preserved; an untrust or obligation-rollback
/// failure is folded into a typed `MobError` rather than silently dropped.
async fn rollback_supervisor_recipient_trust(
&mut self,
peer: &TrustedPeerDescriptor,
install: super::supervisor_bridge::RecipientTrustInstall,
original_error: MobError,
) -> MobError {
if Self::recipient_trust_was_newly_installed(install)
&& let Err(untrust_error) = self.supervisor_bridge.untrust_recipient(peer).await
{
self.durable_uncertainty_fail_stop = true;
return MobError::WiringError(format!(
"supervisor authorization failed ({original_error}); additionally failed to roll back installed recipient trust, so the pending obligation is retained and the actor is fail-stopping: {untrust_error}"
));
}
if let Err(obligation_error) = self.rollback_pending_recipient_trust_obligation(
peer,
"rollback_supervisor_recipient_trust",
) {
self.durable_uncertainty_fail_stop = true;
return MobError::WiringError(format!(
"supervisor authorization failed ({original_error}); additionally failed to roll back the pending recipient-trust obligation, so the actor is fail-stopping: {obligation_error}"
));
}
original_error
}
/// A fresh BindHost request crossed the transport send boundary, so the
/// host may already hold a durable binding even though this actor cannot
/// authenticate the terminal result. Keep recipient trust and the
/// machine-owned pending obligation open, then terminate the actor before
/// a Requested-only revoke or another command can falsely certify absence.
fn quarantine_uncertain_host_bind(
&mut self,
peer: &TrustedPeerDescriptor,
error: MobError,
) -> MobError {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
peer_id = %peer.peer_id,
error = %error,
"BindHost outcome is uncertain after request send; retaining trust and pending obligation for exact replay"
);
MobError::Internal(format!(
"BindHost outcome for host '{}' is uncertain after request send: {error}; recipient trust and pending obligation retained for exact replay",
peer.peer_id
))
}
fn quarantine_uncertain_host_revoke(
&mut self,
host_id: &mob_dsl::HostId,
error: MobError,
) -> MobError {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
host = %host_id.as_str(),
error = %error,
"RevokeHost outcome crossed the remote terminal boundary; retaining the Started anchor for cold exact replay"
);
MobError::Internal(format!(
"RevokeHost outcome for host '{}' is uncertain after remote delivery or terminal acceptance: {error}; Started anchor retained for cold exact replay",
host_id.as_str()
))
}
fn quarantine_uncertain_host_rebind(
&mut self,
host_id: &mob_dsl::HostId,
epoch: u64,
error: MobError,
) -> MobError {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
host = %host_id.as_str(),
epoch,
error = %error,
"RebindHost may have advanced the host authority; fail-stopping until cold exact convergence"
);
MobError::Internal(format!(
"RebindHost outcome for host '{}' epoch {epoch} is uncertain after request delivery: {error}; actor is fail-stopping for cold exact convergence",
host_id.as_str()
))
}
// -----------------------------------------------------------------------
// Cross-host route installs (multi-host §10.4 / §6.2).
//
// The MobMachine owns the INSTALL obligation ledger
// (`pending_route_installs`, the Record/Resolve/Rollback trio, and the
// RouteInstallRequested effect). Remove is a distinct synchronous
// pre-unwire authorization: every surviving remote lane must ACK while
// the exact edge/endpoints still exist, and it never enters the volatile
// ledger. Failed installs remain pending and observable; failed removals
// leave the durable edge wired and compensate any already-removed lanes.
// -----------------------------------------------------------------------
/// Derive host-scoped route work for an edge. Install callers record it
/// in the machine ledger; synchronous pre-unwire Remove callers use the
/// same exact carrier only as ephemeral authorization.
fn route_install_obligations_for_edge(
&self,
edge: &mob_dsl::WiringEdge,
kind: mob_dsl::RouteObligationKind,
) -> Vec<mob_dsl::RouteInstallObligation> {
let state = self.dsl_authority.state();
let mut hosts = BTreeSet::new();
for endpoint in [&edge.a, &edge.b] {
if let Some(host) = state.member_placement.get(endpoint) {
hosts.insert(host.clone());
}
}
hosts
.into_iter()
.map(|host| mob_dsl::RouteInstallObligation {
edge: edge.clone(),
host,
kind,
})
.collect()
}
/// Submit `RecordRouteInstall` and require the machine's
/// `RouteInstallRequested` effect for the same obligation (the
/// "accepted CommitHostBind but emitted no HostRegistered effect"
/// discipline).
fn record_route_install_obligation(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
context: &str,
) -> Result<(), MobError> {
if obligation.kind != mob_dsl::RouteObligationKind::Install {
return Err(MobError::Internal(
"pending route ledger accepts Install obligations only".to_string(),
));
}
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::RecordRouteInstall {
obligation: obligation.clone(),
},
context,
)?;
let requested = effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::RouteInstallRequested { obligation: requested }
if requested == obligation
)
});
if !requested {
return Err(MobError::Internal(format!(
"MobMachine accepted RecordRouteInstall but emitted no RouteInstallRequested effect for host '{}'",
obligation.host.as_str()
)));
}
Ok(())
}
/// Obtain generated authority for an exact synchronous Remove while the
/// edge is still wired. This opens no volatile outbox row: failure leaves
/// the durable graph unchanged, and only an ACKed removal may proceed to
/// the unwire commit.
fn authorize_route_removal_before_unwire(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
) -> Result<(), MobError> {
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::AuthorizeRouteRemovalBeforeUnwire {
obligation: obligation.clone(),
},
"authorize_route_removal_before_unwire",
)?;
if effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::RouteInstallRequested { obligation: requested }
if requested == obligation
)
}) {
return Ok(());
}
Err(MobError::Internal(format!(
"MobMachine authorized pre-unwire route removal but emitted no exact handoff for host '{}'",
obligation.host.as_str()
)))
}
/// Idempotent removal after every INSTALL lane of the obligation acked.
fn resolve_route_install_obligation(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
context: &str,
) -> Result<(), MobError> {
if obligation.kind != mob_dsl::RouteObligationKind::Install {
return Err(MobError::Internal(
"route-install resolve accepts pending Install obligations only".to_string(),
));
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::ResolveRouteInstall {
obligation: obligation.clone(),
},
context,
)
}
/// Idempotent supersession removal (DEC-P4C-10): the ONLY phase-4
/// consumer is an unwire superseding a pending Install (and the stale-
/// placement branch of realization) — never a failure path of the
/// install itself, because a failed install must LEAVE the obligation.
fn rollback_route_install_obligation(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
context: &str,
) -> Result<(), MobError> {
if obligation.kind != mob_dsl::RouteObligationKind::Install {
return Err(MobError::Internal(
"route-install rollback accepts pending Install obligations only".to_string(),
));
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::RollbackRouteInstall {
obligation: obligation.clone(),
},
context,
)
}
/// Roll back every pending Install obligation for an edge that is about
/// to be unwired: once the edge is gone a pending Install can never
/// re-validate (`edge_currently_wired` turns false) and would sit as
/// unserviceable residue. Sequencing: rollback stale Installs → commit
/// unwire. Synchronous Remove never enters the pending ledger.
fn rollback_superseded_install_obligations(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<(), MobError> {
let stale: Vec<mob_dsl::RouteInstallObligation> = self
.dsl_authority
.state()
.pending_route_installs
.iter()
.filter(|obligation| {
&obligation.edge == edge && obligation.kind == mob_dsl::RouteObligationKind::Install
})
.cloned()
.collect();
for obligation in stale {
self.rollback_route_install_obligation(
&obligation,
"unwire_supersedes_pending_install",
)?;
}
Ok(())
}
/// Canonical peer descriptor for a bound host from machine facts
/// (`host_endpoints` + `host_public_keys`).
fn bound_host_peer_descriptor(
&self,
host_id: &mob_dsl::HostId,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
let (endpoint, pubkey) = {
let state = self.dsl_authority.state();
(
state.host_endpoints.get(host_id).cloned(),
state.host_public_keys.get(host_id).copied(),
)
};
let (Some(endpoint), Some(pubkey)) = (endpoint, pubkey) else {
return Err(MobError::WiringError(format!(
"{context}: host '{}' has no recorded binding facts",
host_id.as_str()
)));
};
TrustedPeerDescriptor::unsigned_with_pubkey(
host_id.as_str(),
host_id.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::Internal(format!(
"{context}: bound host '{}' facts do not form a canonical peer descriptor: {error}",
host_id.as_str()
))
})
}
/// Cross-host trust descriptor for one member, built from MACHINE facts
/// (`member_peer_endpoints` — peer id, pubkey, canonical address). For a
/// LOCAL member with a composed controlling-side acceptor (ADJ-P4-2),
/// the transport address is overridden with the acceptor's advertised
/// address at realization — the machine fact keeps recording the
/// member's canonical endpoint (`member_peer_endpoints` is NOT
/// modified).
async fn cross_host_peer_spec_for_member(
&mut self,
identity: &AgentIdentity,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
let spec = self
.machine_member_peer_spec_for(identity, context)?
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: member '{identity}' has no machine-recorded peer endpoint"
))
})?;
#[cfg(not(target_arch = "wasm32"))]
let spec = {
let mut spec = spec;
let placed = {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity)
};
if !placed {
let advertised = self
.register_local_member_reverse_lane(identity)
.await?
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: local member '{identity}' has no process-scoped controlling acceptor; configure an explicit listen and dialable advertised address before installing a cross-host route"
))
})?;
spec.address = PeerAddress::parse(&advertised).map_err(|error| {
MobError::WiringError(format!(
"{context}: controlling acceptor advertises an invalid address '{advertised}': {error}"
))
})?;
}
spec
};
Ok(spec)
}
/// Realize one host-scoped route operation. Install resolves its pending
/// ledger row only when every lane ACKs. Remove is synchronous ephemeral
/// authority and returns success only after every selected lane ACKs.
///
/// Actor-loop discipline (ADJ-P4-12): these sends are awaited inline on
/// the actor task and are proven mailbox-independent — the reply
/// correlates inside the supervisor-bridge comms transport (its own
/// intake task), never through the actor mailbox, and the post-ack
/// `ResolveRouteInstall` is a direct `apply_dsl_input` by this method
/// already holding `&mut self`. The taint relay and the `HostStatus`
/// reconciliation sweep are the established precedents for this class.
async fn realize_route_install(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
) -> Result<(), MobError> {
self.realize_route_install_excluding_target(obligation, None)
.await
}
async fn realize_route_install_excluding_target(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
excluded_target: Option<&AgentIdentity>,
) -> Result<(), MobError> {
self.realize_route_install_excluding_target_with_peer_override(
obligation,
excluded_target,
None,
)
.await
}
/// Realize a host-scoped route operation while substituting an exact
/// historical counterpart endpoint. Retirement uses this only for
/// `Remove`: the ordinary route obligation binds the surviving host lane,
/// while the endpoint-migration handoff binds the stale peer id that must
/// be deleted before the edge can be unwired.
async fn realize_route_install_excluding_target_with_peer_override(
&mut self,
obligation: &mob_dsl::RouteInstallObligation,
excluded_target: Option<&AgentIdentity>,
counterpart_override: Option<(AgentIdentity, TrustedPeerDescriptor)>,
) -> Result<(), MobError> {
let host_peer =
self.bound_host_peer_descriptor(&obligation.host, "realize_route_install")?;
let (a_placed_here, b_placed_here) = {
let state = self.dsl_authority.state();
(
state.member_placement.get(&obligation.edge.a) == Some(&obligation.host),
state.member_placement.get(&obligation.edge.b) == Some(&obligation.host),
)
};
// (target member on the obligation's host, counterpart whose
// material it must trust).
let mut lanes: Vec<(AgentIdentity, AgentIdentity)> = Vec::new();
if a_placed_here {
let target = AgentIdentity::from(obligation.edge.a.0.as_str());
if excluded_target != Some(&target) {
lanes.push((target, AgentIdentity::from(obligation.edge.b.0.as_str())));
}
}
if b_placed_here {
let target = AgentIdentity::from(obligation.edge.b.0.as_str());
if excluded_target != Some(&target) {
lanes.push((target, AgentIdentity::from(obligation.edge.a.0.as_str())));
}
}
if lanes.is_empty() {
// The obligation's premise (an endpoint placed on this host) no
// longer holds — the placement moved or the member retired. The
// stale entry is superseded, not realizable; re-derivation from
// current facts owns any replacement obligation.
return match obligation.kind {
mob_dsl::RouteObligationKind::Install => {
self.rollback_route_install_obligation(
obligation,
"route_install_stale_placement",
)?;
Ok(())
}
mob_dsl::RouteObligationKind::Remove => Err(MobError::Internal(format!(
"authorized pre-unwire route removal for host '{}' resolved no target lane",
obligation.host.as_str()
))),
};
}
let authority = self.supervisor_bridge.authority().await;
let binding_generation = self.current_host_binding_generation(&obligation.host)?;
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&host_peer)
.await?;
for (target_identity, counterpart_identity) in lanes {
let peer_spec = match counterpart_override.as_ref() {
Some((override_identity, override_spec))
if override_identity == &counterpart_identity =>
{
override_spec.clone()
}
_ => {
self.cross_host_peer_spec_for_member(
&counterpart_identity,
"realize_route_install",
)
.await?
}
};
let payload = super::bridge_protocol::BridgePeerTrustPayload {
supervisor: supervisor_spec.clone().into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
mob_id: self.definition.id.to_string(),
agent_identity: target_identity.as_str().to_string(),
peer: peer_spec.into(),
};
let command = match obligation.kind {
mob_dsl::RouteObligationKind::Install => {
super::bridge_protocol::BridgeCommand::InstallPeerTrust(payload)
}
mob_dsl::RouteObligationKind::Remove => {
super::bridge_protocol::BridgeCommand::RemovePeerTrust(payload)
}
};
self.send_route_install_command(&host_peer, &command)
.await?;
}
if obligation.kind == mob_dsl::RouteObligationKind::Install {
self.resolve_route_install_obligation(obligation, "route_install_confirmed")?;
}
Ok(())
}
/// One route-install bridge send with the resend budget: exactly ONE
/// resend of the SAME payload, only on a bridge TIMEOUT (reply-loss
/// uncertainty — the host may have served the command and the ack was
/// lost). A delivered rejection — including `Unavailable` — is the
/// host's answer and fails immediately. A failed Install stays pending
/// for the retry lanes (`drive_route_installs`, authenticated periodic
/// `HostStatus`, host rebind, operator re-wire — ADJ-P4-1/ADJ-P4-7); a
/// synchronous pre-unwire Remove aborts the transaction so the still-wired
/// edge can be compensated.
async fn send_route_install_command(
&mut self,
host_peer: &TrustedPeerDescriptor,
command: &super::bridge_protocol::BridgeCommand,
) -> Result<(), MobError> {
match self
.send_bridge_command_typed::<super::bridge_protocol::BridgeAck>(
host_peer,
command,
ROUTE_INSTALL_BRIDGE_TIMEOUT,
)
.await
{
Ok(_ack) => Ok(()),
Err(error) if Self::route_install_send_is_resendable(&error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"route-install send failed on a retryable cause; resending the same payload once"
);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(host_peer, command, ROUTE_INSTALL_BRIDGE_TIMEOUT)
.await?;
Ok(())
}
Err(error) => Err(error),
}
}
/// Typed resend classification: ONLY a bridge timeout triggers the
/// single resend; every rejection (the host answered) fails immediately.
fn route_install_send_is_resendable(error: &MobError) -> bool {
matches!(error, MobError::BridgeRequestTimedOut { .. })
}
/// Post-commit obligation fold shared by every wire lane (single-edge
/// new/repair, batch, kickoff/restore riding `handle_wire`): record the
/// derived obligations, realize each, and NEVER fail the committed wire
/// operation — a partial install returns Ok with the obligation pending
/// and observable (ADJ-P4-9c: fail closed, never fail quiet; the
/// route-installs projection names what is outstanding).
async fn fold_route_install_obligations_after_wire(&mut self, edge: &mob_dsl::WiringEdge) {
let obligations =
self.route_install_obligations_for_edge(edge, mob_dsl::RouteObligationKind::Install);
self.record_and_realize_route_install_obligations(obligations, "wire_route_install")
.await;
}
fn route_removal_obligations_before_unwire(
&self,
edge: &mob_dsl::WiringEdge,
retiring_identity: Option<&AgentIdentity>,
) -> Vec<mob_dsl::RouteInstallObligation> {
let state = self.dsl_authority.state();
let mut hosts = BTreeSet::new();
for endpoint in [&edge.a, &edge.b] {
if retiring_identity.is_some_and(|retiring| endpoint.0 == retiring.as_str()) {
// The retiring placed runtime's trust store dies with the
// subsequent exact ReleaseMember. Requiring that unavailable
// runtime to ACK Remove would deadlock retirement; only
// surviving placed targets gate the unwire.
continue;
}
if let Some(host) = state.member_placement.get(endpoint) {
let identity = AgentIdentity::from(endpoint.0.as_str());
if self.confirmed_revoked_placed_host(&identity).as_ref() == Some(host) {
// An exact revoke tombstone proves this dormant runtime,
// including its trust store, is gone. There is no live
// endpoint to command and no Remove ACK to await. A later
// binding generation does not satisfy this predicate, so
// an active replacement survivor still receives Remove.
continue;
}
hosts.insert(host.clone());
}
}
hosts
.into_iter()
.map(|host| mob_dsl::RouteInstallObligation {
edge: edge.clone(),
host,
kind: mob_dsl::RouteObligationKind::Remove,
})
.collect()
}
/// Synchronously remove every remote survivor trust row before the
/// durable unwire. If any lane rejects/times out, keep the edge committed
/// as wired and repair Install best-effort; recovery sees the same wired
/// graph and repeats that repair rather than losing a volatile Remove.
async fn realize_route_removals_before_unwire(
&mut self,
edge: &mob_dsl::WiringEdge,
retiring_identity: Option<&AgentIdentity>,
) -> Result<(), MobError> {
let obligations = self.route_removal_obligations_before_unwire(edge, retiring_identity);
for obligation in &obligations {
let result = self.authorize_route_removal_before_unwire(obligation);
if let Err(error) = result {
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
if let Err(error) = self
.realize_route_install_excluding_target(obligation, retiring_identity)
.await
{
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
}
Ok(())
}
/// Remove one stale generation endpoint from every surviving placed
/// target before retirement unwires the edge. The route-removal authority
/// proves the exact host lane; the consumed endpoint-migration authority
/// proves `retained_spec.peer_id` is historical trust for the retiring
/// runtime.
async fn realize_historical_route_removals_before_unwire(
&mut self,
edge: &mob_dsl::WiringEdge,
retiring_identity: &AgentIdentity,
retained_spec: &TrustedPeerDescriptor,
_endpoint_migration_authority: CommsTrustMutationAuthority,
) -> Result<bool, MobError> {
let obligations =
self.route_removal_obligations_before_unwire(edge, Some(retiring_identity));
if obligations.is_empty() {
let retiring = mob_dsl::AgentIdentity::from_domain(retiring_identity);
let survivor = if edge.a == retiring {
&edge.b
} else if edge.b == retiring {
&edge.a
} else {
return Err(MobError::RetirementTopologyIncomplete(format!(
"retiring member '{retiring_identity}' is not an endpoint of the wired edge"
)));
};
let survivor = AgentIdentity::from(survivor.0.as_str());
// No remote Remove is required when the sole survivor is also an
// exact confirmed-revoked placed runtime. Its vanished trust
// store is the durable cleanup proof. Local survivors are handled
// by the caller's in-process comms lane instead.
return Ok(self.confirmed_revoked_placed_host(&survivor).is_some());
}
for obligation in &obligations {
if let Err(error) = self.authorize_route_removal_before_unwire(obligation) {
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
if let Err(error) = self
.realize_route_install_excluding_target_with_peer_override(
obligation,
Some(retiring_identity),
Some((retiring_identity.clone(), retained_spec.clone())),
)
.await
{
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
}
Ok(true)
}
async fn restore_route_installs_after_unwire_abort(&mut self, edge: &mob_dsl::WiringEdge) {
let installs =
self.route_install_obligations_for_edge(edge, mob_dsl::RouteObligationKind::Install);
self.record_and_realize_route_install_obligations(
installs,
"restore_route_installs_after_unwire_abort",
)
.await;
}
async fn record_and_realize_route_install_obligations(
&mut self,
obligations: Vec<mob_dsl::RouteInstallObligation>,
context: &'static str,
) {
for obligation in obligations {
match self.record_route_install_obligation(&obligation, context) {
Ok(()) => {
if let Err(error) = self.realize_route_install(&obligation).await {
tracing::warn!(
mob_id = %self.definition.id,
host = %obligation.host.as_str(),
kind = ?obligation.kind,
%error,
"cross-host route obligation pending; retained and drained by retry/rebind"
);
}
}
Err(error) => {
// A machine reject after a committed graph mutation can
// only mean the host lost `Bound` between resolution and
// record (revoke race): the edge stays committed and the
// obligation is re-derived at the next host bind (T2).
tracing::warn!(
mob_id = %self.definition.id,
host = %obligation.host.as_str(),
kind = ?obligation.kind,
%error,
"cross-host route obligation not admitted; re-derived at the next host bind"
);
}
}
}
}
/// Re-derive Install obligations from durable graph facts
/// (`wiring_edges` × `member_placement`, bound hosts only) and record
/// them. Set semantics dedupe; machine guards own admission (guard
/// rejects are debug-logged skips — ADJ-P4-1 re-derive posture).
fn record_derived_route_install_obligations(&mut self, host_filter: Option<&mob_dsl::HostId>) {
let derived: BTreeSet<mob_dsl::RouteInstallObligation> =
super::derive_install_obligations(self.dsl_authority.state(), host_filter);
for obligation in derived {
if let Err(error) =
self.record_route_install_obligation(&obligation, "route_install_re_derive")
{
tracing::debug!(
mob_id = %self.definition.id,
host = %obligation.host.as_str(),
%error,
"route-install re-derive skipped by machine admission"
);
}
}
}
/// Realize every pending Install (optionally scoped to one host).
/// Per-install transport failures leave the row pending and do not abort
/// the drain. An impossible Remove row is a machine invariant violation:
/// fail before sending anything rather than projecting or executing it.
async fn realize_pending_route_installs(
&mut self,
host_filter: Option<&mob_dsl::HostId>,
) -> Result<(), MobError> {
let pending: Vec<mob_dsl::RouteInstallObligation> = self
.dsl_authority
.state()
.pending_route_installs
.iter()
.filter(|obligation| match host_filter {
Some(filter) => &obligation.host == filter,
None => true,
})
.cloned()
.collect();
if let Some(invalid) = pending
.iter()
.find(|obligation| obligation.kind != mob_dsl::RouteObligationKind::Install)
{
return Err(MobError::Internal(format!(
"MobMachine invariant violation: pending route ledger contains non-Install obligation for host '{}'",
invalid.host.as_str()
)));
}
for obligation in pending {
if let Err(error) = self.realize_route_install(&obligation).await {
tracing::warn!(
mob_id = %self.definition.id,
host = %obligation.host.as_str(),
kind = ?obligation.kind,
%error,
"route-install drain left obligation pending; next trigger retries"
);
}
}
Ok(())
}
/// The explicit retry verb (ADJ-P4-9b): drain the PENDING obligation
/// ledger — `realize_pending_route_installs` is the one canonical drain
/// every trigger converges on. The ledger is install-only. The drive itself re-derives NOTHING: the
/// obligation set is the in-flight truth, so an idle drive sends
/// nothing (T-W3 pins the idempotency). Re-derivation from durable
/// graph facts belongs to the triggers that KNOW trust was invalidated:
/// host (re)bind (T2, `drain_route_installs_for_host`), controlling
/// recovery (T3, builder reseed), placed revival (T4,
/// `drive_route_installs_for_identity`), and an authenticated new host
/// runtime incarnation. Periodic `HostStatus` for the same incarnation
/// drains only this already-recorded ledger.
async fn handle_drive_route_installs(&mut self) -> Result<(), MobError> {
self.realize_pending_route_installs(None).await
}
/// T2 / runtime-incarnation drain: after a host (re)bind commit or an
/// authenticated host-process change, re-derive Install obligations for
/// every wired edge with an endpoint placed on THIS host (a restarted
/// host holds no trust rows) and realize every pending Install against
/// it. Per-obligation failures stay pending.
async fn drain_route_installs_for_host(
&mut self,
host_id: &mob_dsl::HostId,
) -> Result<(), MobError> {
self.record_derived_route_install_obligations(Some(host_id));
self.realize_pending_route_installs(Some(host_id)).await
}
/// T4 drain: after a placed member is re-materialized (revival), its
/// fresh runtime holds no peer trust rows — re-derive and realize the
/// obligations for every wired edge touching the identity.
async fn drive_route_installs_for_identity(&mut self, identity: &AgentIdentity) {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let edges: Vec<mob_dsl::WiringEdge> = self
.dsl_authority
.state()
.wiring_edges
.iter()
.filter(|edge| edge.a == dsl_identity || edge.b == dsl_identity)
.cloned()
.collect();
for edge in edges {
self.fold_route_install_obligations_after_wire(&edge).await;
}
}
// -----------------------------------------------------------------------
// Controlling-side reverse-lane acceptor (ADJ-P4-2).
// -----------------------------------------------------------------------
/// Register a local member's inbound identity on the controlling
/// acceptor so a remote member host can dial it (the reverse lane of a
/// cross-host edge). Returns the acceptor's advertised address when the
/// member is registered; `None` when no acceptor is composed or the
/// composer supplies no registration material for the session. The caller
/// turns `None` into a wiring error, so the route obligation remains
/// pending rather than publishing a process-local canonical endpoint.
///
/// Called ONLY while realizing a machine-emitted `RouteInstallRequested`
/// effect — the machine-effect-witnessed registry discipline
/// (ADJ-P4-2).
#[cfg(not(target_arch = "wasm32"))]
async fn register_local_member_reverse_lane(
&mut self,
identity: &AgentIdentity,
) -> Result<Option<String>, MobError> {
let Some(session_id) = self.machine_bridge_session_id_for_identity(identity)? else {
return Ok(None);
};
// Material first, bind second: a composer that supplies no
// registration material never pays for a bound listener.
let material = {
let Some(state) = self.controlling_acceptor.as_ref() else {
return Ok(None);
};
Arc::clone(&state.config.material)
};
let Some(registration) = material.registration_for(&session_id).await else {
tracing::debug!(
member = %identity,
session_id = %session_id,
"no controlling-acceptor registration material for local member; \
cross-host route installation remains pending"
);
return Ok(None);
};
let logical_owner = format!("{}\0{}\0{}", self.definition.id, identity, session_id);
let Some(state) = self.controlling_acceptor.as_mut() else {
return Ok(None);
};
// A session revival keeps its durable signing key but constructs a
// fresh comms runtime and inbox. Always refresh from current material;
// pubkey presence alone is not an attachment-incarnation witness.
let advertised = state
.refresh_registration(logical_owner, registration)
.await?;
Ok(Some(advertised))
}
/// Remove a local member's acceptor registration once it retains no
/// cross-host edges (unwire realization witness). Removal of an absent
/// entry is success.
#[cfg(not(target_arch = "wasm32"))]
async fn deregister_local_member_reverse_lane_if_unused(
&mut self,
identity: &AgentIdentity,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let has_cross_host_edges = {
let state = self.dsl_authority.state();
state.wiring_edges.iter().any(|edge| {
let other = if edge.a == dsl_identity {
Some(&edge.b)
} else if edge.b == dsl_identity {
Some(&edge.a)
} else {
None
};
other.is_some_and(|other| state.member_placement.contains_key(other))
})
};
if has_cross_host_edges {
return Ok(());
}
let Some(session_id) = self.machine_bridge_session_id_for_identity(identity)? else {
return Ok(());
};
let material = {
let Some(state) = self.controlling_acceptor.as_ref() else {
return Ok(());
};
Arc::clone(&state.config.material)
};
let Some(registration) = material.registration_for(&session_id).await else {
return Ok(());
};
let Some(state) = self.controlling_acceptor.as_mut() else {
return Ok(());
};
state.remove_registration(®istration.pubkey).await
}
async fn load_supervisor_authority_snapshot(
&self,
) -> Result<Option<SupervisorAuthorityLoad>, MobError> {
let Some(durable) = self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?
else {
return Ok(None);
};
Ok(Some(SupervisorAuthorityLoad { durable }))
}
async fn load_supervisor_authority(
&self,
) -> Result<Option<crate::store::SupervisorAuthorityRecord>, MobError> {
Ok(self
.load_supervisor_authority_snapshot()
.await?
.map(|loaded| loaded.durable))
}
fn pending_supervisor_rotation_blocks_rebind(
current: &crate::store::SupervisorAuthorityRecord,
pending: &crate::store::SupervisorPendingRotationRecord,
peer: &TrustedPeerDescriptor,
) -> MobError {
let operation = pending.operation_id.map_or_else(
|| "legacy operation awaiting migration".to_string(),
|operation_id| format!("operation {operation_id}"),
);
MobError::SupervisorRotationIncomplete {
previous_epoch: current.epoch,
attempted_epoch: pending.epoch,
attempted_public_peer_id: pending.public_peer_id.clone(),
rotated_peer_count: pending.accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation {operation} remains pending; refusing to reauthorize old authority for peer '{}'",
peer.peer_id
),
}
}
async fn send_bridge_command_typed<R: super::bridge_protocol::FromBridgeReply>(
&mut self,
peer: &TrustedPeerDescriptor,
command: &super::bridge_protocol::BridgeCommand,
timeout: std::time::Duration,
) -> Result<R, MobError> {
// Trust is installed ahead of the routed send (comms admission), with
// the window recorded as a MobMachine `pending_recipient_trust`
// obligation. Every non-confirmed outcome — send failure, terminal
// rejection, decode failure — rolls trust newly installed by THIS
// call back fail-closed; pre-existing trust is left in place.
self.record_pending_recipient_trust_obligation(peer, "send_bridge_command_typed")?;
let install = match self.supervisor_bridge.trust_recipient(peer).await {
Ok(install) => install,
Err(trust_error) => {
return Err(self.quarantine_uncertain_recipient_trust_install(
peer,
"send_bridge_command_typed",
trust_error,
));
}
};
let value = match self
.supervisor_bridge
.send_bridge_command(peer, command, timeout)
.await
{
Ok(value) => value,
Err(send_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, send_error)
.await);
}
};
if let Some(rejection) = Self::bridge_rejection_reply(command.protocol_version(), &value) {
return Err(self
.rollback_supervisor_recipient_trust(
peer,
install,
Self::bridge_rejection_error(rejection),
)
.await);
}
match super::bridge_protocol::decode_bridge_payload(command, value, "command") {
Ok(payload) => {
self.resolve_pending_recipient_trust_obligation(
peer,
"send_bridge_command_typed confirmed",
)?;
Ok(payload)
}
Err(decode_error) => Err(self
.rollback_supervisor_recipient_trust(peer, install, decode_error)
.await),
}
}
async fn observe_peer_only_binding(
&mut self,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<super::bridge_protocol::BridgeObservationResponse, MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "observe_peer_only_binding")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(binding))
.await?;
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::ObserveMember(payload);
self.send_bridge_command_typed(&peer, &command, timeout)
.await
}
async fn destroy_peer_only_binding(
&mut self,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<super::bridge_protocol::BridgeDestroyResponse, MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "destroy_peer_only_binding")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(binding))
.await?;
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::DestroyMember(payload);
self.send_bridge_command_typed(&peer, &command, timeout)
.await
}
async fn revoke_supervisor_for_retiring_entry(
&mut self,
entry: &RosterEntry,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "revoke_supervisor_for_binding")?;
if !self.remote_supervisor_revoked_for_entry(entry) {
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::RevokeSupervisor(payload);
match self
.send_bridge_command_typed::<super::bridge_protocol::BridgeAck>(
&peer, &command, timeout,
)
.await
{
Ok(_) => {}
Err(error) if Self::expected_revoke_cleanup_failure(&error).is_some() => {
tracing::debug!(
error = %error,
"supervisor revoke observed typed already-revoked terminality"
);
}
Err(error) => return Err(error),
}
// Persist remote terminal truth before touching the local route.
// If this append fails, retry contacts the remote again and its
// exact revoked-supervisor tombstone returns typed NotBound. Once
// durable, retries never depend on remote liveness again.
self.record_remote_member_supervisor_revoked(entry).await?;
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&entry.agent_identity)
{
target.take();
return Err(MobError::Internal(format!(
"fault-injected cancellation after remote supervisor revoke checkpoint for '{}'",
entry.agent_identity
)));
}
}
// The member is retiring, so the supervisor's own recipient trust is
// cleanup residue too. Revoke must close both the remote supervisor
// authority and this local transport trust before terminal publication.
self.supervisor_bridge.untrust_recipient(&peer).await?;
Ok(())
}
async fn wire_peer_only_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
recipient_binding: Option<&crate::RuntimeBinding>,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let recipient = self
.ensure_supervisor_authorized(recipient, recipient_binding)
.await?;
let mob_peer_overlay =
self.mob_peer_overlay_for_recipient(&recipient, "wire_peer_only_recipient")?;
if !mob_peer_overlay
.peers()
.iter()
.any(|peer| peer.peer_id == peer_spec.peer_id)
{
return Err(MobError::WiringError(format!(
"wire_peer_only_recipient: peer '{}' is absent from MobMachine overlay for recipient '{}'",
peer_spec.peer_id, recipient.peer_id
)));
}
let authority = self.supervisor_bridge.authority().await;
let sup_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&recipient)
.await?;
let command = super::bridge_protocol::BridgeCommand::WireMember(
super::bridge_protocol::BridgePeerWiringPayload {
supervisor: sup_spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
peer_spec: peer_spec.clone().into(),
mob_peer_overlay: Some(mob_peer_overlay.bridge_handoff()),
},
);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(&recipient, &command, timeout)
.await?;
Ok(())
}
async fn unwire_peer_only_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
recipient_binding: Option<&crate::RuntimeBinding>,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let recipient = self
.ensure_supervisor_authorized(recipient, recipient_binding)
.await?;
let mob_peer_overlay =
self.mob_peer_overlay_for_recipient(&recipient, "unwire_peer_only_recipient")?;
self.unwire_peer_only_recipient_with_overlay(
&recipient,
peer_spec,
mob_peer_overlay,
timeout,
)
.await
}
async fn unwire_peer_only_recipient_with_overlay(
&mut self,
recipient: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
mob_peer_overlay: super::provisioner::PeerOnlyTrustOverlay,
timeout: std::time::Duration,
) -> Result<(), MobError> {
if mob_peer_overlay
.peers()
.iter()
.any(|peer| peer.peer_id == peer_spec.peer_id)
{
return Err(MobError::WiringError(format!(
"unwire_peer_only_recipient: peer '{}' is still present in MobMachine overlay for recipient '{}'",
peer_spec.peer_id, recipient.peer_id
)));
}
let authority = self.supervisor_bridge.authority().await;
let sup_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(recipient)
.await?;
let command = super::bridge_protocol::BridgeCommand::UnwireMember(
super::bridge_protocol::BridgePeerWiringPayload {
supervisor: sup_spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
peer_spec: peer_spec.clone().into(),
mob_peer_overlay: Some(mob_peer_overlay.bridge_handoff()),
},
);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(recipient, &command, timeout)
.await?;
Ok(())
}
async fn unwire_retiring_peer_only_side(
&mut self,
retiring_entry: &RosterEntry,
retiring_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let Some(retiring_binding) = Self::runtime_binding_for_entry(retiring_entry) else {
return Ok(());
};
self.unwire_peer_only_recipient(retiring_spec, Some(&retiring_binding), peer_spec, timeout)
.await
}
async fn unwire_peer_only_survivor_for_respawn(
&mut self,
recipient: &TrustedPeerDescriptor,
recipient_binding: &crate::RuntimeBinding,
retiring_entry: &RosterEntry,
retiring_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let recipient = self
.ensure_supervisor_authorized(recipient, Some(recipient_binding))
.await?;
let overlay = self.mob_peer_overlay_for_retiring_cleanup(
&recipient,
retiring_entry,
"unwire_peer_only_survivor_for_respawn",
)?;
self.unwire_peer_only_recipient_with_overlay(&recipient, retiring_spec, overlay, timeout)
.await
}
fn mob_peer_overlay_for_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
context: &'static str,
) -> Result<super::provisioner::PeerOnlyTrustOverlay, MobError> {
let state = self.dsl_authority.state();
let expected_endpoint = mob_dsl::MemberPeerEndpoint::from(recipient);
let mut recipient_identity = None;
for (identity, endpoint) in &state.member_peer_endpoints {
if *endpoint == expected_endpoint {
if recipient_identity.replace(identity.clone()).is_some() {
return Err(MobError::WiringError(format!(
"{context}: recipient peer '{}' matches multiple MobMachine member endpoints",
recipient.peer_id
)));
}
}
}
let recipient_identity = recipient_identity.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: recipient peer '{}' is outside MobMachine member endpoint authority",
recipient.peer_id
))
})?;
let recipient_peer_id = recipient.peer_id.to_string();
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerOverlay {
agent_identity: recipient_identity.clone(),
expected_peer_endpoint: expected_endpoint,
},
context,
)?;
let obligation =
crate::generated::protocol_mob_member_peer_overlay::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_member_peer_overlay::MobTopologyFreshnessAuthority::from_live_topology_epoch(
self.dsl_topology_epoch.clone(),
Arc::clone(&self.dsl_authority_owner_token),
),
)
.into_iter()
.find(|obligation| {
obligation.agent_identity() == &recipient_identity
&& obligation.peer_id().0.as_str() == recipient_peer_id.as_str()
})
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: generated MobMachine peer overlay handoff missing for recipient '{}'",
recipient.peer_id
))
})?;
super::provisioner::PeerOnlyTrustOverlay::from_generated_mob_member_peer_overlay(
&obligation,
)
}
fn mob_peer_overlay_for_retiring_cleanup(
&mut self,
recipient: &TrustedPeerDescriptor,
retiring_entry: &RosterEntry,
context: &'static str,
) -> Result<super::provisioner::PeerOnlyTrustOverlay, MobError> {
let state = self.dsl_authority.state();
let expected_endpoint = mob_dsl::MemberPeerEndpoint::from(recipient);
let mut recipient_identity = None;
for (identity, endpoint) in &state.member_peer_endpoints {
if *endpoint == expected_endpoint {
if recipient_identity.replace(identity.clone()).is_some() {
return Err(MobError::WiringError(format!(
"{context}: recipient peer '{}' matches multiple MobMachine member endpoints",
recipient.peer_id
)));
}
}
}
let recipient_identity = recipient_identity.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: recipient peer '{}' is outside MobMachine member endpoint authority",
recipient.peer_id
))
})?;
let recipient_peer_id = recipient.peer_id.to_string();
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeRetiringMemberPeerOverlayCleanup {
recipient_identity: recipient_identity.clone(),
expected_recipient_endpoint: expected_endpoint,
retiring_identity: mob_dsl::AgentIdentity::from_domain(
&retiring_entry.agent_identity,
),
retiring_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&retiring_entry.agent_runtime_id,
),
fence_token: mob_dsl::FenceToken::from_domain(retiring_entry.fence_token),
generation: mob_dsl::Generation::from_domain(retiring_entry.generation),
},
context,
)?;
let obligation =
crate::generated::protocol_mob_member_peer_overlay::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_member_peer_overlay::MobTopologyFreshnessAuthority::from_live_topology_epoch(
self.dsl_topology_epoch.clone(),
Arc::clone(&self.dsl_authority_owner_token),
),
)
.into_iter()
.find(|obligation| {
obligation.agent_identity() == &recipient_identity
&& obligation.peer_id().0.as_str() == recipient_peer_id.as_str()
})
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: generated filtered peer overlay handoff missing for recipient '{}'",
recipient.peer_id
))
})?;
super::provisioner::PeerOnlyTrustOverlay::from_generated_mob_member_peer_overlay(
&obligation,
)
}
/// Mirror MobMachine's terminality verdict for an observed remote-member
/// runtime state.
///
/// The bridge consumer extracts the pure wire runtime-state observation;
/// MobMachine — not this shell — decides whether the observed state is
/// terminal. We feed the raw observation and mirror the emitted verdict,
/// failing closed (treating the observation as non-terminal, which forces a
/// conservative destroy) only if the machine emits no verdict.
fn observation_is_terminal(
&mut self,
observation: &super::bridge_protocol::BridgeObservationResponse,
) -> Result<bool, MobError> {
let observed_state = remote_member_runtime_observed_state(observation.state)?;
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::ClassifyRemoteMemberRuntimeObservation { observed_state },
"classify_remote_member_runtime_observation",
)?;
let (effect_observed_state, terminality) = effects
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RemoteMemberRuntimeTerminalityClassified {
observed_state,
terminality,
} => Some((observed_state, terminality)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted remote-member runtime observation but emitted no terminality verdict"
.into(),
)
})?;
if effect_observed_state != observed_state {
return Err(MobError::Internal(format!(
"MobMachine remote-member terminality drift: input={observed_state:?}, effect={effect_observed_state:?}"
)));
}
Ok(matches!(
terminality,
mob_dsl::MobRemoteMemberRuntimeTerminality::Terminal
))
}
/// Mirror MobMachine's bridge-rejection recovery verdict for a typed wire
/// rejection cause.
///
/// When an `AuthorizeSupervisor` command is rejected, the bridge consumer
/// extracts the pure wire rejection cause; MobMachine — not this shell —
/// owns whether that cause is recoverable by re-running `BindMember`
/// (`RebindRecover`) or must bubble up as fatal (`FatalBubbleUp`). We feed
/// the mapped cause and mirror the emitted verdict, returning `true` only
/// for `RebindRecover`. Fails closed (returns an error) if the machine emits
/// no verdict.
fn classify_bridge_rejection_recovery(
&mut self,
cause: super::bridge_protocol::BridgeRejectionCause,
) -> Result<bool, MobError> {
let rejection_cause = mob_bridge_rejection_cause(cause);
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::ClassifyBridgeRejectionRecovery { rejection_cause },
"classify_bridge_rejection_recovery",
)?;
let (effect_cause, recovery) = effects
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::BridgeRejectionRecoveryClassified {
rejection_cause,
recovery,
} => Some((rejection_cause, recovery)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted bridge rejection cause but emitted no recovery verdict"
.into(),
)
})?;
if effect_cause != rejection_cause {
return Err(MobError::Internal(format!(
"MobMachine bridge-rejection recovery drift: input={rejection_cause:?}, effect={effect_cause:?}"
)));
}
Ok(matches!(
recovery,
mob_dsl::MobBridgeRejectionRecovery::RebindRecover
))
}
fn fence_token_exhausted() -> MobError {
MobError::Internal(
"mob fence-token space is exhausted; refusing to wrap or reuse an incarnation fence"
.to_string(),
)
}
/// Issue a strictly increasing fence token. Fresh/recovered counters start
/// at 1, leaving 0 as the exhausted sentinel; this preserves `u64::MAX` as
/// the final allocatable token without permitting wrap or reuse.
fn allocate_fence_token(
next_fence_token: &std::sync::atomic::AtomicU64,
) -> Result<crate::ids::FenceToken, MobError> {
next_fence_token
.fetch_update(
std::sync::atomic::Ordering::Relaxed,
std::sync::atomic::Ordering::Relaxed,
|next| match next {
0 => None,
u64::MAX => Some(0),
_ => Some(next + 1),
},
)
.map(crate::ids::FenceToken::new)
.map_err(|_| Self::fence_token_exhausted())
}
fn issue_fence_token(&self) -> Result<crate::ids::FenceToken, MobError> {
Self::allocate_fence_token(&self.next_fence_token)
}
fn preview_fence_token(
next_fence_token: &std::sync::atomic::AtomicU64,
) -> Result<crate::ids::FenceToken, MobError> {
let next = next_fence_token.load(std::sync::atomic::Ordering::Relaxed);
if next == 0 {
return Err(Self::fence_token_exhausted());
}
Ok(crate::ids::FenceToken::new(next))
}
fn next_fence_token_preview(&self) -> Result<crate::ids::FenceToken, MobError> {
Self::preview_fence_token(&self.next_fence_token)
}
fn invalid_transition_to(&self, target: MobState) -> MobError {
MobError::InvalidTransition {
from: self.state(),
to: target,
}
}
async fn restore_failure_for(
&self,
agent_identity: &AgentIdentity,
) -> Option<super::handle::RestoreFailureDiagnostic> {
self.restore_diagnostics
.read()
.await
.get(agent_identity)
.cloned()
}
async fn ensure_member_not_broken(
&self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&crate::ids::AgentIdentity::from(
agent_identity.as_str(),
));
let lifecycle = self
.dsl_authority
.state()
.member_lifecycle_for_identity(&dsl_identity);
if lifecycle.status == mob_dsl::MobMemberLifecycleStatus::Broken {
let diag = self.restore_failure_for(agent_identity).await.unwrap_or(
super::handle::RestoreFailureDiagnostic {
bridge_session_id: None,
reason: lifecycle
.error
.unwrap_or_else(|| "member restore failed".to_string()),
},
);
return Err(MobError::MemberRestoreFailed {
member_id: agent_identity.clone(),
session_id: diag.bridge_session_id,
reason: diag.reason,
});
}
Ok(())
}
fn dsl_state(&self) -> MobState {
project_dsl_phase(self.dsl_authority.state().lifecycle_phase)
}
fn destroy_admitted(&self) -> bool {
self.dsl_authority.state().destroy_admitted
}
/// Project observable shell phase. A durable `MobDestroying` event closes
/// public live authority before all cleanup necessarily completes, so the
/// same-process projection must match restart projection and fail closed.
fn state(&self) -> MobState {
if self.destroy_admitted() {
MobState::Destroyed
} else {
self.dsl_state()
}
}
/// HARD INVARIANT: every applied machine input must publish through this
/// function — both the direct apply seam
/// ([`Self::apply_dsl_input_collect_transition`]) and every
/// prepared-commit seam (`commit_prepared_dsl_input[_after]`,
/// `commit_prepared_dsl_transition[_after]`) call it unconditionally
/// after committing to the DSL authority. The event-driven reconcilers
/// (placed kickoff, placed completion, remote turn) treat watch wakes as
/// a superset of machine-derived work arrival and idle on a slow safety
/// tick between wakes; a mutation path that bypasses this publish
/// degrades their convergence latency to that safety interval. Any new
/// commit path must call this function after mutating `dsl_authority`.
fn publish_machine_state_projection(&self) {
self.dsl_topology_epoch.store(
self.dsl_authority.state().topology_epoch,
std::sync::atomic::Ordering::Release,
);
let _ = self
.machine_state_watch_tx
.send(self.dsl_authority.state().clone());
}
fn apply_dsl_input(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input_collect_effects(input, context)
.map(|_| ())
}
fn apply_dsl_input_collect_effects(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<Vec<mob_dsl::MobMachineEffect>, MobError> {
Ok(self
.apply_dsl_input_collect_transition(input, context)?
.into_effects())
}
fn apply_dsl_input_collect_transition(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<mob_dsl::MobMachineTransition, MobError> {
let input_debug = format!("{input:?}");
let transition = mob_dsl::MobMachineMutator::apply(&mut self.dsl_authority, input)
.map_err(|e| {
MobError::Internal(format!(
"DSL authority ({context}) rejected {input_debug}: {e}"
))
})?;
self.queue_routed_effects_from(transition.effects());
if transition.from_phase != transition.to_phase {
// Publish the projected phase for external observers. This is
// the sole write seam for the dogma-#13 projection watch.
let _ = self.phase_watch_tx.send(self.state());
}
// Hard invariant (see publish_machine_state_projection): the direct
// apply seam publishes after EVERY applied input.
self.publish_machine_state_projection();
Ok(transition)
}
fn prepare_dsl_input(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<PreparedDslInput, MobError> {
self.prepare_dsl_inputs(std::slice::from_ref(&input), context)
}
fn prepare_dsl_input_transition(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<PreparedDslTransition, MobError> {
let input_debug = format!("{input:?}");
let mut authority = self.dsl_authority.prepare_authority();
let transition = mob_dsl::MobMachineMutator::apply(&mut authority, input).map_err(|e| {
MobError::Internal(format!(
"DSL authority prepare ({context}) rejected {input_debug}: {e}"
))
})?;
Ok(PreparedDslTransition {
authority,
transition,
})
}
fn prepare_dsl_signal_transition(
&self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<PreparedDslTransition, MobError> {
let signal_debug = format!("{signal:?}");
let mut authority = self.dsl_authority.prepare_authority();
let transition = authority.apply_signal(signal).map_err(|e| {
MobError::Internal(format!(
"DSL authority prepare ({context}): {e}; signal={signal_debug}; live_runtime_ids={:?}; runtime_fence_tokens={:?}",
self.dsl_authority.state().live_runtime_ids,
self.dsl_authority.state().runtime_fence_tokens,
))
})?;
Ok(PreparedDslTransition {
authority,
transition,
})
}
fn prepare_dsl_inputs(
&self,
inputs: &[mob_dsl::MobMachineInput],
context: &str,
) -> Result<PreparedDslInput, MobError> {
let mut authority = self.dsl_authority.prepare_authority();
let transitions = authority
.apply_batch(inputs.iter().cloned())
.map_err(|error| {
MobError::Internal(format!(
"DSL authority prepared batch ({context}) rejected: {error}"
))
})?;
let mut effects = Vec::new();
let mut phase_changed = false;
for transition in transitions {
if transition.from_phase != transition.to_phase {
phase_changed = true;
}
effects.extend(transition.into_effects());
}
Ok(PreparedDslInput {
authority,
effects,
phase_changed,
})
}
fn commit_prepared_dsl_input(&mut self, prepared: PreparedDslInput) -> Result<(), MobError> {
let effects = prepared.effects;
let phase_changed = prepared.phase_changed;
self.dsl_authority
.commit_prepared_authority(prepared.authority)
.map_err(|error| {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
// Hard invariant (see publish_machine_state_projection): every
// prepared-commit seam — this one and its _after/_transition
// siblings below — publishes after every committed input.
self.publish_machine_state_projection();
Ok(())
}
async fn commit_prepared_dsl_input_after<T, F, Fut>(
&mut self,
prepared: PreparedDslInput,
effect: F,
) -> Result<T, MobError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, MobError>>,
{
let effects = prepared.effects;
let phase_changed = prepared.phase_changed;
let result = self
.dsl_authority
.commit_prepared_authority_after(prepared.authority, effect)
.await
.map_err(|error| match error {
mob_dsl::MobMachinePreparedCommitEffectError::Commit(error) => {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
}
mob_dsl::MobMachinePreparedCommitEffectError::Effect(error) => error,
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(result)
}
fn commit_prepared_dsl_transition(
&mut self,
prepared: PreparedDslTransition,
) -> Result<(), MobError> {
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
self.dsl_authority
.commit_prepared_authority(prepared.authority)
.map_err(|error| {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(())
}
async fn commit_prepared_dsl_transition_after<T, F, Fut>(
&mut self,
prepared: PreparedDslTransition,
effect: F,
) -> Result<T, MobError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, MobError>>,
{
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
let result = self
.dsl_authority
.commit_prepared_authority_after(prepared.authority, effect)
.await
.map_err(|error| match error {
mob_dsl::MobMachinePreparedCommitEffectError::Commit(error) => {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
}
mob_dsl::MobMachinePreparedCommitEffectError::Effect(error) => error,
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(result)
}
fn prepare_supervisor_authority_persistence(
&self,
input: mob_dsl::MobMachineInput,
record: &crate::store::SupervisorAuthorityRecord,
context: &str,
) -> Result<PreparedSupervisorAuthorityPersistence, MobError> {
let prepared = self.prepare_dsl_input_transition(input, context)?;
let authority = crate::store::SupervisorAuthorityPersistenceAuthority::from_transition(
record,
&prepared.transition,
)?;
Ok(PreparedSupervisorAuthorityPersistence {
transition: prepared,
authority,
})
}
fn prepare_supervisor_authority_deletion(
&self,
record: &crate::store::SupervisorAuthorityRecord,
context: &str,
) -> Result<PreparedSupervisorAuthorityDeletion, MobError> {
let prepared = self.prepare_dsl_input_transition(
record.dsl_clear_authority_for_destroy_input(),
context,
)?;
let authority = crate::store::SupervisorAuthorityDeletionAuthority::from_transition(
record,
&prepared.transition,
)?;
Ok(PreparedSupervisorAuthorityDeletion {
transition: prepared,
authority,
})
}
fn supervisor_authority_record_is_machine_authorized(
&self,
record: &crate::store::SupervisorAuthorityRecord,
) -> bool {
let state = self.dsl_authority.state();
let peer_id = record.dsl_peer_id();
let signing_key = record.dsl_signing_key();
let protocol_version = record.dsl_protocol_version();
let current_matches = state.supervisor_authority_peer_id.as_ref() == Some(&peer_id)
&& state.supervisor_authority_signing_key == Some(signing_key)
&& state.supervisor_authority_epoch == Some(record.epoch)
&& state.supervisor_authority_protocol_version == Some(protocol_version.clone());
let pending_matches = state.supervisor_pending_authority_peer_id.as_ref() == Some(&peer_id)
&& state.supervisor_pending_authority_signing_key == Some(signing_key)
&& state.supervisor_pending_authority_epoch == Some(record.epoch)
&& state.supervisor_pending_authority_protocol_version == Some(protocol_version);
current_matches || pending_matches
}
fn supervisor_bridge_authority_for_record(
&self,
record: &crate::store::SupervisorAuthorityRecord,
) -> Result<crate::store::SupervisorAuthorityBridgeAuthority, MobError> {
Ok(
crate::store::SupervisorAuthorityBridgeAuthority::from_machine_state(
record,
self.dsl_authority.state(),
)?,
)
}
fn require_lifecycle_journal_effect(
transition: &mob_dsl::MobMachineTransition,
kind: mob_dsl::MobLifecycleJournalKind,
context: &str,
) -> Result<(), MobError> {
if transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::AppendLifecycleJournal {
kind: effect_kind,
agent_identity: None,
agent_runtime_id: None,
fence_token: None,
generation: None,
session_id: None,
}
if *effect_kind == kind
)
}) {
Ok(())
} else {
Err(MobError::Internal(format!(
"MobMachine {context} produced no generated {kind:?} lifecycle journal authority"
)))
}
}
fn require_placed_completion_lifecycle_intent_effect(
transition: &mob_dsl::MobMachineTransition,
intent: mob_dsl::PlacedCompletionLifecycleIntentKind,
active: bool,
context: &str,
) -> Result<(), MobError> {
if transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::PersistPlacedCompletionLifecycleIntent {
intent: effect_intent,
active: effect_active,
} if *effect_intent == intent && *effect_active == active
)
}) {
Ok(())
} else {
Err(MobError::Internal(format!(
"MobMachine {context} produced no exact generated lifecycle-intent persistence authority"
)))
}
}
#[allow(clippy::too_many_arguments)]
fn require_member_lifecycle_journal_effect(
transition: &mob_dsl::MobMachineTransition,
kind: mob_dsl::MobLifecycleJournalKind,
agent_identity: &AgentIdentity,
agent_runtime_id: &crate::ids::AgentRuntimeId,
fence_token: Option<crate::ids::FenceToken>,
generation: crate::ids::Generation,
session_id: Option<mob_dsl::SessionId>,
context: &str,
) -> Result<(), MobError> {
let expected_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let expected_runtime_id = mob_dsl::AgentRuntimeId::from_domain(agent_runtime_id);
let expected_fence = fence_token.map(mob_dsl::FenceToken::from_domain);
let expected_generation = mob_dsl::Generation::from_domain(generation);
let expected_session = session_id;
if transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::AppendLifecycleJournal {
kind: effect_kind,
agent_identity: Some(effect_identity),
agent_runtime_id: Some(effect_runtime_id),
fence_token: effect_fence,
generation: Some(effect_generation),
session_id: effect_session,
} if *effect_kind == kind
&& effect_identity == &expected_identity
&& effect_runtime_id == &expected_runtime_id
&& effect_fence == &expected_fence
&& effect_generation == &expected_generation
&& effect_session == &expected_session
)
}) {
Ok(())
} else {
Err(MobError::Internal(format!(
"MobMachine {context} produced no generated {kind:?} lifecycle journal authority for member '{agent_identity}'"
)))
}
}
fn operator_action_recorded_event_from_generated_effect(
transition: &mob_dsl::MobMachineTransition,
context: &str,
) -> Result<MobEventKind, MobError> {
let mut authorized = transition
.effects()
.iter()
.filter_map(|effect| match effect {
mob_dsl::MobMachineEffect::AppendOperatorActionProvenance {
tool_name,
principal_token,
caller_provenance,
audit_invocation_id,
} => Some(MobEventKind::OperatorActionRecorded {
tool_name: tool_name.clone(),
principal_token: principal_token.clone(),
caller_provenance: caller_provenance.clone(),
audit_invocation_id: audit_invocation_id.clone(),
}),
_ => None,
});
let Some(event) = authorized.next() else {
return Err(MobError::Internal(format!(
"MobMachine {context} produced no generated operator provenance journal authority"
)));
};
if authorized.next().is_some() {
return Err(MobError::Internal(format!(
"MobMachine {context} produced multiple generated operator provenance journal authorities"
)));
}
Ok(event)
}
fn prepare_command_admission(
&self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<PreparedDslInput, MobError> {
self.prepare_dsl_input(input, context).map_err(|error| {
tracing::debug!(
context,
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(target)
})
}
fn probe_command_admission(
&self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<(), MobError> {
self.prepare_command_admission(input, target, context)
.map(|_| ())
}
fn apply_command_admission(
&mut self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<Vec<mob_dsl::MobMachineEffect>, MobError> {
self.apply_dsl_input_collect_effects(input, context)
.map_err(|error| {
tracing::debug!(
context,
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(target)
})
}
fn machine_member_peer_spec_for(
&self,
identity: &AgentIdentity,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.member_peer_endpoints
.get(&dsl_identity)
.map(|endpoint| Self::peer_only_spec_from_member_endpoint(endpoint, context))
.transpose()
}
fn roster_member_peer_spec_for(
&self,
entry: &RosterEntry,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let Some(peer_id) = entry.peer_id else {
return Ok(None);
};
let Some(public_key) = entry.transport_public_key.as_deref() else {
return Ok(None);
};
let pubkey = meerkat_comms::PubKey::from_pubkey_string(public_key).map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid retained peer key for '{}': {error}",
entry.agent_identity
))
})?;
let comms_name = self.comms_name_for(entry)?;
TrustedPeerDescriptor::unsigned_with_pubkey(
comms_name.clone(),
peer_id.to_string(),
*pubkey.as_bytes(),
format!("inproc://{comms_name}"),
)
.map(Some)
.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid retained peer descriptor for '{}': {error}",
entry.agent_identity
))
})
}
async fn retained_member_peer_spec_from_wired_peer_trust(
&self,
entry: &RosterEntry,
peer_identities: &BTreeSet<AgentIdentity>,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let expected_name = self.comms_name_for(entry)?;
let mut retained = None;
for peer_identity in peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
if super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&peer_entry.agent_identity,
) {
// Retained trust on a placed peer lives on that peer's member
// host. Its projected session binding is a remote fence, not
// a controller-local comms lookup key.
continue;
}
let Some(comms) = self.provisioner_comms(&peer_entry.member_ref).await else {
continue;
};
let trusted_peers = comms
.trusted_peer_projection_snapshot_for_source(
meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind::MobMachineMemberTrustWiring,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"{context}: failed to read retained trusted peers for '{}': {error}",
peer_entry.agent_identity
))
})?;
for peer in trusted_peers {
if peer.name.as_str() != expected_name {
continue;
}
if let Some(previous) = retained.replace(peer.clone())
&& previous.peer_id != peer.peer_id
{
return Err(MobError::WiringError(format!(
"{context}: retained peer descriptor for '{}' disagrees across wired peers",
entry.agent_identity
)));
}
}
}
Ok(retained)
}
fn effects_include_wiring_graph_change(effects: &[mob_dsl::MobMachineEffect]) -> bool {
effects
.iter()
.any(|effect| matches!(effect, mob_dsl::MobMachineEffect::WiringGraphChanged { .. }))
}
fn wire_members_disposition_from_effects(
effects: &[mob_dsl::MobMachineEffect],
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<bool, MobError> {
let graph_changed = Self::effects_include_wiring_graph_change(effects);
let repair_requested = effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::WiringTrustRepairRequested {
edge: effect_edge
} if effect_edge == edge
)
});
match (graph_changed, repair_requested) {
(true, false) => Ok(true),
(false, true) => Ok(false),
(false, false) => Err(MobError::WiringError(format!(
"{context} produced no generated wiring graph or repair authority for edge {edge:?}"
))),
(true, true) => Err(MobError::WiringError(format!(
"{context} produced conflicting generated wiring graph and repair authority for edge {edge:?}"
))),
}
}
fn member_trust_wiring_handoff_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
self.member_trust_wiring_handoff_from_transition_with_freshness(
transition,
edge,
crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority::from_live_member_trust_authority(
&self.dsl_authority,
),
context,
operation,
)
}
fn member_trust_wiring_handoff_from_prepared_batch_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
freshness_authority: crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
self.member_trust_wiring_handoff_from_transition_with_freshness(
transition,
edge,
freshness_authority,
context,
operation,
)
}
fn member_trust_wiring_handoff_from_transition_with_freshness(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
mob_topology_freshness_authority: crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
let obligation =
crate::generated::protocol_mob_member_trust_wiring::extract_obligations_with_freshness(
transition,
mob_topology_freshness_authority,
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced no generated member wiring trust obligation"
))
})?;
let authority = match operation {
MemberTrustOperation::Wiring => MemberTrustAuthority::Wiring(obligation),
MemberTrustOperation::Repair => MemberTrustAuthority::Repair(obligation),
MemberTrustOperation::Unwiring => {
return Err(MobError::WiringError(
"member unwiring cannot use a wiring trust obligation".to_string(),
));
}
};
Ok(MemberTrustHandoff {
edge: edge.clone(),
authority,
operation,
})
}
fn wire_external_authority_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::ExternalPeerEdge,
context: &str,
) -> Result<WireTrustAuthority, MobError> {
let effects = transition.effects();
let graph_changed = Self::effects_include_wiring_graph_change(effects);
let wiring_obligation =
crate::generated::protocol_mob_external_peer_trust_wiring::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_external_peer_trust_wiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge);
let repair_obligation =
crate::generated::protocol_mob_external_peer_trust_repair::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_external_peer_trust_repair::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge);
let repair_requested = repair_obligation.is_some();
match (graph_changed, repair_requested) {
(true, false) => {
let obligation = wiring_obligation.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced external graph change without generated wiring trust obligation"
))
})?;
let expected_peer_id = edge.endpoint.peer_id.0.as_str();
Ok(WireTrustAuthority::ExternalGraphAdded(
crate::generated::protocol_mob_external_peer_trust_wiring::wiring_authority_for_peer(
&obligation,
expected_peer_id,
)
.map_err(MobError::WiringError)?,
))
}
(false, true) => {
let obligation = repair_obligation.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced external repair marker without generated repair trust obligation"
))
})?;
let expected_peer_id = edge.endpoint.peer_id.0.as_str();
Ok(WireTrustAuthority::ExternalRepairRequested(
crate::generated::protocol_mob_external_peer_trust_repair::repair_authority_for_peer(
&obligation,
expected_peer_id,
)
.map_err(MobError::WiringError)?,
))
}
(false, false) => Err(MobError::WiringError(format!(
"{context} produced no generated external-peer trust authority"
))),
(true, true) => Err(MobError::WiringError(format!(
"{context} produced conflicting generated external-peer trust authority"
))),
}
}
fn unwire_members_authority_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let obligation = crate::generated::protocol_mob_member_trust_unwiring::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_member_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced no generated member unwiring trust obligation"
))
})?;
Ok(MemberTrustHandoff {
edge: edge.clone(),
authority: MemberTrustAuthority::Unwiring(obligation),
operation: MemberTrustOperation::Unwiring,
})
}
fn authorize_member_trust_wiring(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustWiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.member_trust_wiring_handoff_from_transition(&transition, edge, context, operation)
}
fn authorize_member_trust_unwiring(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustUnwiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
/// Mint trust-removal handoffs for a dense set of live edges as one
/// prepared machine transaction. `apply_batch` preserves sequential guard
/// semantics while validating whole-state invariants once at the batch
/// boundary; applying one authorization input per edge made high-degree
/// retirement quadratic in debug/test builds.
fn authorize_member_trust_unwiring_batch(
&mut self,
edges: &[mob_dsl::WiringEdge],
context: &str,
) -> Result<BTreeMap<mob_dsl::WiringEdge, MemberTrustHandoff>, MobError> {
if edges.is_empty() {
return Ok(BTreeMap::new());
}
let mut authority = self.dsl_authority.prepare_authority();
let transitions = authority
.apply_batch(edges.iter().map(|edge| {
mob_dsl::MobMachineInput::AuthorizeMemberTrustUnwiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
}
}))
.map_err(|error| {
MobError::Internal(format!(
"DSL authority prepared batch ({context}) rejected: {error}"
))
})?;
let mut handoffs = BTreeMap::new();
let mut effects = Vec::new();
let mut phase_changed = false;
for (edge, transition) in edges.iter().zip(transitions) {
let handoff =
self.unwire_members_authority_from_transition(&transition, edge, context)?;
if handoffs.insert(edge.clone(), handoff).is_some() {
return Err(MobError::Internal(format!(
"DSL authority prepared batch ({context}) produced duplicate edge {edge:?}"
)));
}
if transition.from_phase != transition.to_phase {
phase_changed = true;
}
effects.extend(transition.into_effects());
}
self.commit_prepared_dsl_input(PreparedDslInput {
authority,
effects,
phase_changed,
})?;
Ok(handoffs)
}
fn authorize_member_endpoint_migration_trust_cleanup(
&mut self,
edge: &mob_dsl::WiringEdge,
entry: &RosterEntry,
retained_peer_endpoint: &mob_dsl::MemberPeerEndpoint,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberEndpointMigrationTrustCleanup {
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
retained_peer_endpoint: retained_peer_endpoint.clone(),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn authorize_member_trust_cleanup(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustCleanup {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn authorize_member_trust_cleanup_observed(
&mut self,
edge: &mob_dsl::WiringEdge,
a_identity: &AgentIdentity,
a_peer_id: &str,
b_identity: &AgentIdentity,
b_peer_id: &str,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustCleanupObserved {
edge: edge.clone(),
a_identity: mob_dsl::AgentIdentity::from_domain(a_identity),
a_peer_id: mob_dsl::PeerId(a_peer_id.to_string()),
b_identity: mob_dsl::AgentIdentity::from_domain(b_identity),
b_peer_id: mob_dsl::PeerId(b_peer_id.to_string()),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
#[allow(clippy::too_many_arguments)]
fn authorize_retiring_member_trust_cleanup_observed(
&mut self,
edge: &mob_dsl::WiringEdge,
retiring_entry: &RosterEntry,
a_identity: &AgentIdentity,
a_peer_id: &str,
b_identity: &AgentIdentity,
b_peer_id: &str,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeRetiringMemberTrustCleanupObserved {
edge: edge.clone(),
a_identity: mob_dsl::AgentIdentity::from_domain(a_identity),
a_peer_id: mob_dsl::PeerId(a_peer_id.to_string()),
b_identity: mob_dsl::AgentIdentity::from_domain(b_identity),
b_peer_id: mob_dsl::PeerId(b_peer_id.to_string()),
agent_identity: mob_dsl::AgentIdentity::from_domain(&retiring_entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&retiring_entry.agent_runtime_id,
),
fence_token: mob_dsl::FenceToken::from_domain(retiring_entry.fence_token),
generation: mob_dsl::Generation::from_domain(retiring_entry.generation),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn preview_dsl_input(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<mob_dsl::MobMachineState, MobError> {
let input_debug = format!("{input:?}");
let mut authority =
mob_dsl::MobMachineAuthority::recover_from_state(self.dsl_authority.state().clone())
.map_err(|error| {
MobError::Internal(format!(
"DSL authority preview ({context}) could not recover state: {error}"
))
})?;
let transition = mob_dsl::MobMachineMutator::apply(&mut authority, input).map_err(|e| {
MobError::Internal(format!(
"DSL authority preview ({context}) rejected {input_debug}: {e}"
))
})?;
let _ = transition;
Ok(authority.state().clone())
}
fn apply_dsl_signal(
&mut self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_signal_collect_transition(signal, context)
.map(|_| ())
}
fn apply_dsl_signal_collect_transition(
&mut self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<mob_dsl::MobMachineTransition, MobError> {
let signal_debug = format!("{signal:?}");
let transition = self
.dsl_authority
.apply_signal(signal)
.map_err(|e| {
MobError::Internal(format!(
"DSL authority ({context}): {e}; signal={signal_debug}; live_runtime_ids={:?}; runtime_fence_tokens={:?}",
self.dsl_authority.state().live_runtime_ids,
self.dsl_authority.state().runtime_fence_tokens,
))
})?;
self.queue_routed_effects_from(transition.effects());
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(transition)
}
/// Wave-c C-6p — harvest routed seam effects from a DSL transition's
/// effect list into the actor's pending-dispatch queue.
///
/// Non-routed variants (persist-kickoff, emit-lifecycle-notice,
/// topology-signal, etc.) stay on the in-process effect-drain path
/// reached from the individual command handlers and never enter the
/// composition dispatcher. Routed variants flow through
/// `flush_routed_effects` at the next async boundary.
fn queue_routed_effects_from(&mut self, effects: &[mob_dsl::MobMachineEffect]) {
for effect in effects {
if let Some(seam_effect) = super::composition::lift_routed_effect(effect) {
self.pending_routed_effects.push(seam_effect);
}
}
}
fn runtime_retire_blocked_on_pending_detach(
state: &mob_dsl::MobMachineState,
effect: &super::composition::MobSeamEffect,
) -> bool {
matches!(
effect.body(),
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_runtime_id,
..
} if state
.pending_session_ingress_detach_runtime_ids
.contains(agent_runtime_id)
)
}
fn has_exact_queued_runtime_retire(
&self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
) -> Result<bool, MobError> {
let mut exact_count = 0usize;
for effect in &self.pending_routed_effects {
let mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: queued_identity,
agent_runtime_id: queued_runtime_id,
session_id: queued_session_id,
} = effect.body()
else {
continue;
};
if queued_runtime_id != agent_runtime_id {
continue;
}
if queued_identity != agent_identity || queued_session_id != session_id {
return Err(MobError::Internal(format!(
"queued runtime-retire correlation for '{}' does not match machine retry authority",
agent_runtime_id.0
)));
}
exact_count += 1;
}
if exact_count > 1 {
return Err(MobError::Internal(format!(
"multiple queued runtime-retire effects exist for '{}'",
agent_runtime_id.0
)));
}
Ok(exact_count == 1)
}
fn discard_queued_runtime_retire_for(&mut self, agent_runtime_id: &mob_dsl::AgentRuntimeId) {
self.pending_routed_effects.retain(|effect| {
!matches!(
effect.body(),
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_runtime_id: queued_runtime_id,
..
} if queued_runtime_id == agent_runtime_id
)
});
}
fn ensure_runtime_retire_route_after_detach(
&mut self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
context: &str,
) -> Result<(), MobError> {
if self
.dsl_authority
.state()
.live_runtime_ids
.contains(agent_runtime_id)
{
if !self.has_exact_queued_runtime_retire(
agent_identity,
agent_runtime_id,
session_id,
)? {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RetryRuntimeRetire {
agent_identity: agent_identity.clone(),
agent_runtime_id: agent_runtime_id.clone(),
},
context,
)?;
}
} else {
// A routed RuntimeRetired observation is stronger than an
// unflushed local request. Do not dispatch a stale retire after
// the machine has already removed runtime liveness.
self.discard_queued_runtime_retire_for(agent_runtime_id);
}
Ok(())
}
async fn ensure_destroy_runtime_retire_route_after_detach(
&mut self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
domain_session_id: &SessionId,
context: &str,
) -> Result<(), MobError> {
// A cold destroy retry can reconstruct machine liveness from the
// durable MemberSpawned event after the runtime/session archive
// already committed but the terminal MemberRetired append failed.
// Archive authority is stronger than that replayed projection: do not
// route a fresh retire into an intentionally unregistered runtime.
if self
.retirement_archive_already_complete(domain_session_id)
.await?
{
self.discard_queued_runtime_retire_for(agent_runtime_id);
return Ok(());
}
self.ensure_runtime_retire_route_after_detach(
agent_identity,
agent_runtime_id,
session_id,
context,
)
}
/// Drain the pending-routed-effect queue, dispatching each payload
/// through the typed [`CompositionBinding`]. Consumer refusals are fed
/// through the generated composition closure before the rejected head is
/// removed; the producer machine owns the resulting failure/retry class.
/// Structural refusals retain the rejected head. In either failure case,
/// subsequent effects remain queued so the next flush sees them (FIFO).
///
/// In [`CompositionBinding::Standalone`] mode (single-machine / test
/// construction) this drains the queue without attempting any
/// dispatch — the producer has no consumer-side surface to target by
/// construction. In [`CompositionBinding::Wired`] mode,
/// [`DispatchRefusal::UnwiredConsumer`] surfaces as
/// [`MobError::WiringError`].
pub(super) async fn flush_routed_effects(&mut self) -> Result<(), MobError> {
use super::composition::{
dispatch_refusal_to_mob_error, dispatch_routed_effect, refusal_feedback_input,
};
use meerkat_runtime::composition::DispatchRefusal;
while let Some(effect) = self.pending_routed_effects.first().cloned() {
if Self::runtime_retire_blocked_on_pending_detach(self.dsl_authority.state(), &effect) {
tracing::debug!(
mob_id = %self.definition.id,
effect = ?effect.body(),
"deferring runtime retirement until session-ingress detach acknowledgement"
);
return Ok(());
}
match dispatch_routed_effect(&self.composition_binding, effect.clone()).await {
Ok(_outcome) => {
// Drop the head now that dispatch succeeded (or was a
// standalone no-op); keep subsequent queue entries
// intact so a later failure preserves FIFO ordering.
self.pending_routed_effects.remove(0);
}
Err(DispatchRefusal::ConsumerRefused { error, .. })
if !self.destroy_cleanup_active =>
{
// Generated failure closure: project the rejected effect +
// consumer's stable code back into the producer machine.
// MobMachine owns whether this is Broken, work-local, or a
// retirement retry anchor. Only the rejected head is
// removed after that feedback transition commits; every
// tail effect remains FIFO-visible.
let feedback = refusal_feedback_input(&effect, &error)?;
let transition = self.apply_dsl_input_collect_transition(
feedback,
"close_routed_effect_consumer_refusal",
)?;
let closed = closed_runtime_effect_refusal_from_transition(&transition)?;
self.pending_routed_effects.remove(0);
if let Some(agent_identity) = closed.broken_member.as_ref() {
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(closed.session_id.clone()),
reason: format!("{} [{}]", closed.reason, closed.refusal_code),
},
);
}
tracing::warn!(
mob_id = %self.definition.id,
kind = %closed.kind,
bridge_session_id = %closed.session_id,
refusal_code = %closed.refusal_code,
reason = %closed.reason,
queued_remaining = self.pending_routed_effects.len(),
"runtime consumer refusal closed through generated MobMachine feedback"
);
return Err(closed.into_mob_error());
}
Err(refusal) => {
// Structural routing failures retain the head. Destroy
// cleanup also retains consumer refusals so the existing
// incomplete-destroy retry anchor remains authoritative.
return Err(dispatch_refusal_to_mob_error(refusal));
}
}
}
Ok(())
}
fn classify_actor_boundary_flush(
result: Result<(), MobError>,
) -> ActorBoundaryFlushDisposition {
match result {
Ok(()) => ActorBoundaryFlushDisposition::Drained,
Err(error) if error.is_closed_runtime_effect_refusal() => {
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(error)
}
Err(error) => ActorBoundaryFlushDisposition::Fatal(error),
}
}
fn discard_pending_routed_effects_for_session(&mut self, session_id: &SessionId) {
let dsl_session_id = mob_dsl::SessionId::from_domain(session_id);
self.pending_routed_effects.retain(|effect| {
routed_effect_session_scope_dsl(effect.body()) != Some(&dsl_session_id)
});
}
/// Snapshot the DSL's current `member_state_markers` as a set of
/// runtime-id keys (in their stringified DSL form) currently marked
/// `Retiring`. The stringified form matches
/// `AgentRuntimeId::Display` (`"identity:generation"`), which is what
/// `mob_dsl::AgentRuntimeId::from_domain` produces.
fn retiring_runtime_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
self.dsl_authority
.state()
.member_state_markers
.iter()
.filter_map(|(runtime_id, member_state)| match member_state {
mob_dsl::MobMemberState::Retiring => Some(runtime_id.0.clone()),
mob_dsl::MobMemberState::Active => None,
})
.collect()
}
fn pending_kickoff_member_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
unresolved_kickoff_member_ids(self.dsl_authority.state())
}
fn ready_runtime_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
self.dsl_authority
.state()
.member_startup_runtime_ready
.iter()
.chain(self.dsl_authority.state().member_startup_ready.iter())
.map(|runtime_id| runtime_id.0.clone())
.collect()
}
fn machine_projection_for_identity(
&self,
agent_identity: &crate::ids::AgentIdentity,
) -> super::state::MobMemberMachineProjection {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl = self.dsl_authority.state();
let runtime_id = dsl.identity_to_runtime.get(&dsl_identity).cloned();
let state_marker = runtime_id
.as_ref()
.and_then(|runtime_id| dsl.member_state_markers.get(runtime_id).copied());
let live_runtime = runtime_id
.as_ref()
.is_some_and(|runtime_id| dsl.live_runtime_ids.contains(runtime_id));
let bound_session_id = dsl.member_session_bindings.get(&dsl_identity).cloned();
super::state::MobMemberMachineProjection {
runtime_id,
state_marker,
live_runtime,
bound_session_id,
}
}
fn machine_bridge_session_id_for_identity(
&self,
agent_identity: &crate::ids::AgentIdentity,
) -> Result<Option<SessionId>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
self.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.map(|session_id| {
SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"MobMachine has invalid current session binding '{}' for '{agent_identity}': {error}",
session_id.0
))
})
})
.transpose()
}
fn project_member_ref_session_binding(
member_ref: &MemberRef,
current_bridge_session_id: Option<SessionId>,
) -> Option<MemberRef> {
match member_ref {
MemberRef::Session { .. } => {
current_bridge_session_id.map(MemberRef::from_bridge_session_id)
}
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
..
} => Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: address.clone(),
pubkey: *pubkey,
bootstrap_token: bootstrap_token.clone(),
session_id: current_bridge_session_id,
}),
}
}
fn machine_member_ref_for_behavior(
&self,
entry: &RosterEntry,
context: &str,
) -> Result<MemberRef, MobError> {
self.ensure_placed_carrier_binding_active(&entry.agent_identity, context)?;
let bridge_session_id =
self.machine_bridge_session_id_for_identity(&entry.agent_identity)?;
Self::project_member_ref_session_binding(&entry.member_ref, bridge_session_id).ok_or_else(
|| {
MobError::Internal(format!(
"{context} requires MobMachine session binding for '{}'",
entry.agent_identity
))
},
)
}
async fn machine_member_material(
&mut self,
agent_identity: &AgentIdentity,
include_session_details: bool,
) -> Result<CanonicalMemberSnapshotMaterial, MobError> {
let roster_entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let domain_identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let include_local_session_details = include_session_details
&& !self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity);
let current_bridge_session_id =
self.machine_bridge_session_id_for_identity(&domain_identity)?;
let machine_runtime = self
.dsl_authority
.state()
.member_runtime_material_for_identity(&dsl_identity)
.map(|material| material.to_domain_for_identity(&domain_identity));
let member_present = roster_entry.is_some();
let (output_preview, tokens_used) = match current_bridge_session_id.as_ref() {
None => (None, 0),
Some(bridge_session_id) if include_local_session_details => {
match self.session_service.read(bridge_session_id).await {
Ok(view) => (
view.state.last_assistant_text.clone(),
view.billing.total_tokens,
),
Err(meerkat_core::service::SessionError::NotFound { .. }) => {
// An ordinary read hides archived documents, so its
// NotFound is NOT proof of absence. Only the typed
// resume seam's explicit Absent may record the
// permanent missing-session fact; an intact archived
// document (revivable or held) simply reports no
// local details.
let genuinely_absent = matches!(
self.session_service
.load_session_for_resume(bridge_session_id)
.await,
Ok(super::session_service::ResumeSessionLoad::Absent)
);
if genuinely_absent {
let _ = self
.record_missing_member_bridge_session(
agent_identity,
bridge_session_id,
"member_status",
)
.await;
} else {
tracing::debug!(
%agent_identity,
%bridge_session_id,
"member status read NotFound but the durable document is not \
absent (archived or transiently unreadable); not recording a \
missing bridge session"
);
}
(None, 0)
}
Err(_) => (None, 0),
}
}
Some(_) => (None, 0),
};
let machine_lifecycle = self
.dsl_authority
.state()
.member_lifecycle_for_identity(&dsl_identity);
let kickoff = kickoff_snapshot_from_machine_state(
agent_identity.as_str(),
self.dsl_authority.state(),
roster_entry
.as_ref()
.and_then(|entry| entry.kickoff.as_ref()),
);
let progress = if include_local_session_details {
match current_bridge_session_id.as_ref() {
Some(session_id) => match tokio::time::timeout(
MEMBER_PROGRESS_OBSERVATION_TIMEOUT,
self.session_service.execution_snapshot(session_id),
)
.await
{
Ok(Ok(Some(snapshot))) => {
let run_open = snapshot.active_run_id.is_some() && !snapshot.turn_terminal;
let pending_operations = snapshot
.pending_operation_ids
.as_ref()
.map_or(0_u64, |ids| ids.len() as u64);
let in_flight_work = pending_operations
.saturating_add(u64::from(snapshot.tool_calls_pending))
.saturating_add(u64::from(run_open));
let progress_token = format!(
"{:?}|{:?}|{}|{}|{}|{}",
snapshot.active_run_id,
snapshot.turn_phase,
snapshot.boundary_count,
snapshot.applied_cursor,
snapshot.tool_calls_pending,
pending_operations,
);
let observed_at_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.try_into()
.unwrap_or(u64::MAX);
if self.dsl_authority.state().lifecycle_phase == mob_dsl::MobPhase::Running
{
self.apply_dsl_input(
mob_dsl::MobMachineInput::ObserveMemberProgress {
agent_identity: dsl_identity.clone(),
run_open,
in_flight_work,
progress_token,
observed_at_ms,
},
"member_status_observe_progress",
)?;
}
let state = self.dsl_authority.state();
let run_state = if state
.member_run_open
.get(&dsl_identity)
.copied()
.unwrap_or(false)
{
super::handle::MemberRunState::RunOpen
} else {
super::handle::MemberRunState::Idle
};
let last_progress_event = match state
.member_last_progress_event
.get(&dsl_identity)
.copied()
.unwrap_or(mob_dsl::MemberProgressEventKind::Unchanged)
{
mob_dsl::MemberProgressEventKind::ExecutionAdvanced => {
super::handle::MemberProgressEvent::ExecutionAdvanced
}
mob_dsl::MemberProgressEventKind::BecameIdle => {
super::handle::MemberProgressEvent::BecameIdle
}
mob_dsl::MemberProgressEventKind::Unchanged => {
super::handle::MemberProgressEvent::Unchanged
}
};
let health = match state
.member_health_class
.get(&dsl_identity)
.copied()
.unwrap_or(mob_dsl::MemberHealthClass::Unknown)
{
mob_dsl::MemberHealthClass::Healthy => {
super::handle::MemberHealthClass::Healthy
}
mob_dsl::MemberHealthClass::Degraded => {
super::handle::MemberHealthClass::Degraded
}
mob_dsl::MemberHealthClass::Wedged => {
super::handle::MemberHealthClass::Wedged
}
mob_dsl::MemberHealthClass::Unknown => {
super::handle::MemberHealthClass::Unknown
}
};
Some(super::handle::MemberProgressSnapshot {
run_state,
in_flight_work: state
.member_in_flight_work
.get(&dsl_identity)
.copied()
.unwrap_or(0),
last_progress_at_ms: state
.member_last_progress_at_ms
.get(&dsl_identity)
.copied()
.unwrap_or(observed_at_ms),
last_progress_event,
health,
})
}
Ok(Ok(None) | Err(_)) | Err(_) => Some(super::handle::MemberProgressSnapshot {
run_state: super::handle::MemberRunState::Unknown,
in_flight_work: 0,
last_progress_at_ms: 0,
last_progress_event: super::handle::MemberProgressEvent::Unchanged,
health: super::handle::MemberHealthClass::Unknown,
}),
},
None => Some(super::handle::MemberProgressSnapshot {
run_state: super::handle::MemberRunState::Unknown,
in_flight_work: 0,
last_progress_at_ms: 0,
last_progress_event: super::handle::MemberProgressEvent::Unchanged,
health: super::handle::MemberHealthClass::Unknown,
}),
}
} else {
None
};
Ok(MobMemberLifecycleProjection::materialize(
MobMemberLifecycleInput {
member_present,
machine_lifecycle,
output_preview,
tokens_used,
agent_identity: domain_identity,
agent_runtime_id: machine_runtime
.as_ref()
.map(|(agent_runtime_id, _)| agent_runtime_id.clone()),
fence_token: machine_runtime.map(|(_, fence_token)| fence_token),
current_bridge_session_id,
peer_connectivity: None,
kickoff,
progress,
},
))
}
async fn record_missing_member_bridge_session(
&mut self,
agent_identity: &AgentIdentity,
bridge_session_id: &SessionId,
context: &'static str,
) -> Option<String> {
let domain_identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let current_binding_matches = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.is_some_and(|session_id| session_id.0 == bridge_session_id.to_string());
if !current_binding_matches {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
"observed missing bridge session for a stale/non-current member binding"
);
return None;
}
if let Some(reason) = self
.dsl_authority
.state()
.member_restore_failures
.get(&dsl_identity)
.cloned()
{
return Some(reason);
}
let reason = format!("missing bridge session snapshot for '{bridge_session_id}'");
if let Err(error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::RecoverMemberRestoreFailure {
agent_identity: dsl_identity,
reason: reason.clone(),
},
"record_missing_member_bridge_session",
) {
let fallback = format!("{reason}; failed to record restore failure: {error}");
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
error = %error,
"MobMachine rejected member restore-failure observation"
);
return Some(fallback);
}
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
reason = %reason,
"member bridge session is missing; marked member broken"
);
Some(reason)
}
/// Post-discard member-revival seam (#37).
///
/// Design decision: revival is a machine-classified, dispatch-triggered
/// rebuild — not a discard-time machine fact. The fail-closed live-session
/// discard happens inside the session layer (`PersistentSessionService`),
/// which has no mob knowledge; pushing a "NeedsRevival" fact from there
/// into MobMachine would mirror the session-registry's live cache into
/// machine state and create a shadow-truth synchronization seam. Instead,
/// the shell feeds the raw observation ("the member's CURRENT machine-owned
/// bridge session has no live materialization; the durable snapshot is
/// present/missing") at the dispatch admission boundary, and MobMachine —
/// the single owner of member lifecycle truth — classifies it:
///
/// - `ReviveAuthorized` (durable snapshot present): the machine records the
/// `member_revival_pending` obligation and authorizes exactly one shell
/// materialization attempt. The shell realizes it through the EXISTING
/// resume materialization path (`build_resumed_agent_config` →
/// `to_create_session_request` → `MobProvisioner::provision_member` with
/// a machine-minted self-owned provision owner) — the same path mob
/// resume-restore and spawn-with-resume use — then resolves the
/// obligation with `ResolveMemberRevivalSucceeded`/`Failed`.
/// - `BrokenRecorded` (durable snapshot missing): the existing terminal
/// Broken classification, surfaced as typed `MemberRestoreFailed`.
///
/// Fail-closed, no loops: a failed revival resolves into the machine-owned
/// Broken classification, whose `not_broken` guard refuses any further
/// revival authorization; subsequent dispatches reject typed via
/// `ensure_member_not_broken` before any classification. There is no shell
/// retry — one machine authorization, one materialization attempt, one
/// typed terminal outcome.
async fn revive_member_live_materialization(
&mut self,
entry: &RosterEntry,
member_ref: &MemberRef,
bridge_session_id: &SessionId,
recovered_binding_without_endpoint: bool,
restore_topology_immediately: bool,
) -> Result<(), MobError> {
let revival_lock = self.member_revival_lock_for(bridge_session_id).await;
let _revival_guard = revival_lock.lock().await;
match self
.session_service
.has_live_session(bridge_session_id)
.await
{
Ok(true) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %entry.agent_identity,
bridge_session_id = %bridge_session_id,
"member live materialization already present after acquiring revival lock"
);
return Ok(());
}
Ok(false) | Err(meerkat_core::service::SessionError::NotFound { .. }) => {}
Err(error) => return Err(MobError::SessionError(error)),
}
let agent_identity = entry.agent_identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
// Raw observation only: the live runtime is gone; is the durable
// snapshot still materializable? The verdict belongs to MobMachine.
// Presence is probed over the TYPED resume seam, not the
// archive-filtered metadata read: an intact Archived+Idle document
// would probe as absent there and be recorded as a terminal restore
// failure. Active, Revivable, AND ArchivedNotRevivable all mean the
// durable snapshot exists; only an explicit Absent means missing.
// The full document is loaded later, and only on the
// machine-authorized revival path.
let stored_session_present = if self.session_service.supports_persistent_sessions() {
!matches!(
self.session_service
.load_session_for_resume(bridge_session_id)
.await
.map_err(MobError::SessionError)?,
super::session_service::ResumeSessionLoad::Absent
)
} else {
false
};
let (observation, reason) = if stored_session_present {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotPresent,
format!("live session materialization missing for '{bridge_session_id}'"),
)
} else {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotMissing,
format!("missing bridge session snapshot for '{bridge_session_id}'"),
)
};
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::ClassifyMemberLiveMaterialization {
agent_identity: dsl_identity.clone(),
observation,
reason: reason.clone(),
},
"classify_member_live_materialization",
)?;
let (effect_observation, verdict) = transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::MemberLiveMaterializationClassified {
agent_identity: effect_identity,
observation,
verdict,
..
} if effect_identity == &dsl_identity => Some((*observation, *verdict)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted member live-materialization observation but emitted no revival verdict"
.into(),
)
})?;
if effect_observation != observation {
return Err(MobError::Internal(format!(
"MobMachine member-revival drift: input={observation:?}, effect={effect_observation:?}"
)));
}
match verdict {
mob_dsl::MemberRevivalVerdictKind::BrokenRecorded => {
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %reason,
"member live materialization is unrecoverable; MobMachine recorded terminal restore failure"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity,
session_id: Some(bridge_session_id.clone()),
reason,
})
}
mob_dsl::MemberRevivalVerdictKind::ReviveAuthorized => {
// Full-load only now that the machine authorized exactly one
// materialization attempt. A snapshot that vanished between
// the metadata presence probe and this load resolves through
// the SAME machine-owned failure path as any other
// materialization error — the revival obligation is never
// left dangling.
let materialization = match self
.session_service
.materialize_session_for_resume(bridge_session_id)
.await
{
Ok(
ResumeSessionLoad::Active(stored_session)
| ResumeSessionLoad::Revivable(stored_session),
) => {
self.materialize_revived_member_session(
entry,
member_ref,
bridge_session_id,
*stored_session,
recovered_binding_without_endpoint,
restore_topology_immediately,
)
.await
}
Ok(ResumeSessionLoad::Absent) => Err(MobError::Internal(format!(
"durable snapshot for bridge session '{bridge_session_id}' vanished \
between the metadata presence probe and revival materialization"
))),
Ok(ResumeSessionLoad::ArchivedNotRevivable { runtime_state }) => {
Err(MobError::SessionUnavailableForResume {
session_id: bridge_session_id.clone(),
reason:
crate::error::SessionResumeUnavailableReason::ArchivedNotRevivable,
runtime_state: runtime_state.map(|state| state.to_string()),
})
}
Err(error) => Err(MobError::SessionError(error)),
};
match materialization {
Ok(()) => {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalSucceeded {
agent_identity: dsl_identity.clone(),
},
"resolve_member_revival_succeeded",
)?;
self.restore_diagnostics
.write()
.await
.remove(&agent_identity);
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
"machine-authorized revival rebuilt the member's live session"
);
Ok(())
}
Err(error) => {
if revival_error_means_session_already_live(&error, bridge_session_id) {
match self
.session_service
.has_live_session(bridge_session_id)
.await
{
Ok(true) => {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalSucceeded {
agent_identity: dsl_identity.clone(),
},
"resolve_member_revival_already_live",
)?;
self.restore_diagnostics
.write()
.await
.remove(&agent_identity);
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
"machine-authorized revival found an already-live session; treating materialization as idempotent success"
);
return Ok(());
}
Ok(false)
| Err(meerkat_core::service::SessionError::NotFound { .. }) => {}
Err(error) => return Err(MobError::SessionError(error)),
}
}
let failure_reason = format!(
"machine-authorized revival of bridge session '{bridge_session_id}' failed: {error}"
);
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalFailed {
agent_identity: dsl_identity,
reason: failure_reason.clone(),
},
"resolve_member_revival_failed",
)?;
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: failure_reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %failure_reason,
"machine-authorized revival failed terminally; member is broken"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity,
session_id: Some(bridge_session_id.clone()),
reason: failure_reason,
})
}
}
}
}
}
async fn member_revival_lock_for(
&self,
bridge_session_id: &SessionId,
) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = self.member_revival_locks.lock().await;
locks
.entry(bridge_session_id.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
/// Controlling-side revival of a PLACED member (W-D.2 + ADJ-9/DEC-R6):
/// the SAME classify→resolve machine seam as the local revival, with the
/// realization swapped for a same-tuple `MaterializeMember { launch:
/// Resume }` re-issue against the member's bound host. The host's
/// replay admission (`same_generation`/`same_fence`/`same_digest`)
/// ensure-materializes from its durable row and acks the RECORDED
/// response verbatim; fence maps never advance. Fail-closed, one
/// machine authorization, one re-issue, one typed terminal outcome —
/// `not_broken` refuses any retry.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
async fn revive_placed_member(
&mut self,
agent_identity: &AgentIdentity,
reason: &str,
) -> Result<(), MobError> {
// Broken refuses retry TYPED before any classification (T-14).
self.ensure_member_not_broken(agent_identity).await?;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let (host, binding, host_durable) = {
let state = self.dsl_authority.state();
let host = state.member_placement.get(&dsl_identity).cloned();
let binding = state.member_session_bindings.get(&dsl_identity).cloned();
let host_durable = host
.as_ref()
.and_then(|host| state.host_durable_sessions.get(host).copied())
.unwrap_or(false);
(host, binding, host_durable)
};
let Some(host) = host else {
// Stale trigger: the member is no longer machine-placed (retired
// or re-committed locally between failure and trigger).
return Ok(());
};
let Some(binding) = binding else {
return Err(MobError::Internal(format!(
"placed member '{agent_identity}' has no machine session binding; the revival \
vocabulary requires one"
)));
};
let bridge_session_id = SessionId::parse(&binding.0).map_err(|error| {
MobError::Internal(format!(
"machine session binding for placed member '{agent_identity}' is invalid: {error}"
))
})?;
let revival_lock = self.member_revival_lock_for(&bridge_session_id).await;
let _revival_guard = revival_lock.lock().await;
// Raw observation (machine facts only; the shell never probes the
// remote session): the machine binding exists and the host declared
// durable sessions ⇒ the durable snapshot is recomposable host-side.
let (observation, classify_reason) = if host_durable {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotPresent,
format!(
"placed member live materialization lost on host '{}': {reason}",
host.as_str()
),
)
} else {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotMissing,
format!(
"host '{}' has no durable-session capability for placed member revival: {reason}",
host.as_str()
),
)
};
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::ClassifyMemberLiveMaterialization {
agent_identity: dsl_identity.clone(),
observation,
reason: classify_reason.clone(),
},
"classify_placed_member_live_materialization",
)?;
let (effect_observation, verdict) = transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::MemberLiveMaterializationClassified {
agent_identity: effect_identity,
observation,
verdict,
..
} if effect_identity == &dsl_identity => Some((*observation, *verdict)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted placed-member live-materialization observation but emitted no revival verdict"
.into(),
)
})?;
if effect_observation != observation {
return Err(MobError::Internal(format!(
"MobMachine placed-member-revival drift: input={observation:?}, effect={effect_observation:?}"
)));
}
match verdict {
mob_dsl::MemberRevivalVerdictKind::BrokenRecorded => {
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: classify_reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %classify_reason,
"placed member live materialization is unrecoverable; MobMachine recorded terminal restore failure"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity.clone(),
session_id: Some(bridge_session_id),
reason: classify_reason,
})
}
mob_dsl::MemberRevivalVerdictKind::ReviveAuthorized => {
match self
.materialize_revived_placed_member(agent_identity, &host, &binding)
.await
{
Ok(()) => {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalSucceeded {
agent_identity: dsl_identity,
},
"resolve_placed_member_revival_succeeded",
)?;
self.restore_diagnostics
.write()
.await
.remove(agent_identity);
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
host = %host.as_str(),
bridge_session_id = %bridge_session_id,
"machine-authorized placed revival re-materialized the member on its host"
);
Ok(())
}
Err(error) => {
let failure_reason = format!(
"machine-authorized placed revival of '{agent_identity}' on host '{}' failed: {error}",
host.as_str()
);
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalFailed {
agent_identity: dsl_identity,
reason: failure_reason.clone(),
},
"resolve_placed_member_revival_failed",
)?;
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: failure_reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %failure_reason,
"machine-authorized placed revival failed terminally; member is broken"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity.clone(),
session_id: Some(bridge_session_id),
reason: failure_reason,
})
}
}
}
}
}
/// Realize a machine-authorized placed revival: the same-tuple
/// `MaterializeMember { launch: Resume }` re-issue (DEC-R6) built from
/// the durable spec record and the machine-recorded
/// `(generation, fence, host)` facts, sent through the SAME
/// materialization sender the spawn lane uses. No
/// `CommitSpawnMembership` re-run — membership is already committed; the
/// shell verifies the ack echoes against the machine facts typed.
async fn promote_committed_placed_carrier_binding_generation(
&mut self,
expected: &crate::store::MobPlacedSpawnCarrierRecord,
host: &mob_dsl::HostId,
host_binding_generation: u64,
context: &str,
) -> Result<(), MobError> {
let identity = mob_dsl::AgentIdentity::from(expected.agent_identity.clone());
let machine_generation = self
.dsl_authority
.state()
.current_placed_spawn_host_binding_generations
.get(&identity)
.copied();
if expected.host_binding_generation == host_binding_generation {
if machine_generation == Some(host_binding_generation) {
return Ok(());
}
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' is durable at binding generation {} but machine authority is {:?}; actor is fail-stopping for cold recovery ({context})",
expected.agent_identity, host_binding_generation, machine_generation
)));
}
if expected.host_binding_generation == 0
|| host_binding_generation <= expected.host_binding_generation
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' cannot promote binding generation {} -> {}; actor is fail-stopping for cold recovery ({context})",
expected.agent_identity, expected.host_binding_generation, host_binding_generation
)));
}
let crate::store::PlacedSpawnCarrierPhase::Committed(committed) = &expected.phase else {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' binding promotion has no committed session facts; actor is fail-stopping for cold recovery ({context})",
expected.agent_identity
)));
};
let domain_identity = AgentIdentity::from(expected.agent_identity.as_str());
let member_session_id = mob_dsl::SessionId(committed.member_session_id.to_string());
// `RebindHost` cannot reach this path: both controller and host
// authorities require its binding generation to remain exact. A
// strict generation advance is possible only after `RevokeHost`
// removed the old Bound phase. For a formerly bound host, its
// authenticated revoke terminal already cleared every mob-scoped
// materialized/journal row, and the controller drained exact kickoff
// custody before publishing that local revoke. The old row therefore
// cannot reappear merely because TurnKey omits binding generation.
// Keep these exact disposals as idempotent crash-recovery guards before
// publishing the replacement carrier; the host is Bound again now, so
// classify any residual controller custody as rematerialization rather
// than inventing a second host-revocation claim.
if let Err(error) = self
.dispose_remote_turn_custody_for_exact_residency(
ExactRemoteTurnResidency {
agent_identity: &domain_identity,
host_id: host,
host_binding_generation: expected.host_binding_generation,
member_session_id: &member_session_id,
generation: expected.generation,
fence_token: expected.fence_token,
},
super::remote_flow_ticket::REMATERIALIZED_STEP_FAILURE_REASON,
"dispose_remote_turn_obligation_before_binding_promotion",
)
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' could not dispose exact generation-{} remote-turn custody before binding promotion; actor is fail-stopping ({context}): {error}",
expected.agent_identity, expected.host_binding_generation
)));
}
if let Err(error) = self
.dispose_placed_completion_custody_for_exact_residency(ExactRemoteTurnResidency {
agent_identity: &domain_identity,
host_id: host,
host_binding_generation: expected.host_binding_generation,
member_session_id: &member_session_id,
generation: expected.generation,
fence_token: expected.fence_token,
})
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' could not dispose exact generation-{} completion custody before binding promotion; actor is fail-stopping ({context}): {error}",
expected.agent_identity, expected.host_binding_generation
)));
}
if let Err(error) = self
.dispose_placed_kickoff_custody_for_exact_residency(ExactRemoteTurnResidency {
agent_identity: &domain_identity,
host_id: host,
host_binding_generation: expected.host_binding_generation,
member_session_id: &member_session_id,
generation: expected.generation,
fence_token: expected.fence_token,
})
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' could not dispose exact generation-{} kickoff custody before binding promotion; actor is fail-stopping ({context}): {error}",
expected.agent_identity, expected.host_binding_generation
)));
}
let input = mob_dsl::MobMachineInput::PromoteCommittedPlacedSpawnCarrierBinding {
spawn_id: mob_dsl::PlacedSpawnId(expected.spawn_id.to_string()),
agent_identity: identity,
generation: mob_dsl::Generation(expected.generation),
fence_token: mob_dsl::FenceToken(expected.fence_token),
host_id: host.clone(),
expected_host_binding_generation: expected.host_binding_generation,
host_binding_generation,
};
let prepared = match self.prepare_dsl_input_transition(input, context) {
Ok(prepared) => prepared,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' promotion could not prepare exact machine authority; actor is fail-stopping for cold recovery ({context}): {error}",
expected.agent_identity
)));
}
};
let mut promoted = expected.clone();
promoted.host_binding_generation = host_binding_generation;
let persistence =
match crate::store::MobPlacedSpawnBindingPromotionAuthority::from_transition(
expected,
&promoted,
&prepared.transition,
) {
Ok(persistence) => persistence,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' authenticated promotion proof could not mint its exact persistence witness; actor is fail-stopping for cold recovery ({context}): {error}",
expected.agent_identity
)));
}
};
let durable = match self
.runtime_metadata
.compare_and_promote_placed_spawn_binding(
&self.definition.id,
expected,
&promoted,
&persistence,
)
.await
{
Ok(
crate::store::PromotePlacedSpawnBindingResult::Promoted
| crate::store::PromotePlacedSpawnBindingResult::AlreadyPromotedExact,
) => true,
Ok(crate::store::PromotePlacedSpawnBindingResult::Conflict) => false,
Err(write_error) => match self
.runtime_metadata
.load_placed_spawn(&self.definition.id, &expected.agent_identity)
.await
{
Ok(Some(current)) if current == promoted => true,
Ok(current) => {
tracing::error!(
agent_identity = %expected.agent_identity,
error = %write_error,
current = ?current,
"placed carrier promotion write failed and exact reread did not prove the promoted row"
);
false
}
Err(read_error) => {
tracing::error!(
agent_identity = %expected.agent_identity,
error = %write_error,
read_error = %read_error,
"placed carrier promotion write and exact reread both failed"
);
false
}
},
};
if !durable {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' binding generation promotion {} -> {} conflicted or remained uncertain; actor is fail-stopping for cold recovery ({context})",
expected.agent_identity, expected.host_binding_generation, host_binding_generation
)));
}
let old_expected_member = super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: expected.agent_identity.clone(),
host_id: expected.host_id.to_string(),
binding_generation: expected.host_binding_generation,
member_session_id: committed.member_session_id.to_string(),
generation: expected.generation,
fence_token: expected.fence_token,
};
let promoted_expected_member = super::bridge_protocol::BridgeMemberIncarnation {
binding_generation: host_binding_generation,
..old_expected_member.clone()
};
if let Err(error) = self
.member_event_pumps
.stop_exact_pump_and_join(
&domain_identity,
&old_expected_member,
&promoted_expected_member,
)
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' durable binding generation {} was promoted before the old pump could be quiesced; actor is fail-stopping before machine publication ({context}): {error}",
expected.agent_identity, host_binding_generation
)));
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier '{}' durable binding generation {} was promoted before machine publication; actor is fail-stopping for cold recovery ({context}): {error}",
expected.agent_identity, host_binding_generation
)));
}
Ok(())
}
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
async fn materialize_revived_placed_member(
&mut self,
agent_identity: &AgentIdentity,
host: &mob_dsl::HostId,
binding: &mob_dsl::SessionId,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let (generation, fence_token, endpoint, pubkey) = {
let state = self.dsl_authority.state();
(
state
.identity_runtime_generations
.get(&dsl_identity)
.copied(),
state
.identity_runtime_fence_tokens
.get(&dsl_identity)
.copied(),
state.host_endpoints.get(host).cloned(),
state.host_public_keys.get(host).copied(),
)
};
let (Some(generation), Some(fence_token)) = (generation, fence_token) else {
return Err(MobError::Internal(format!(
"placed member '{agent_identity}' has no machine-recorded runtime tuple for revival"
)));
};
let (Some(endpoint), Some(pubkey)) = (endpoint, pubkey) else {
return Err(MobError::Internal(format!(
"placed member '{agent_identity}' is on host '{}' whose binding facts are gone (host revoked?)",
host.as_str()
)));
};
// Durable spec record (written under the materialization witness at
// spawn): the re-sent bytes MUST hash to the digest the host
// recorded — its replay admission guards `same_digest`.
let record = self
.runtime_metadata
.load_placed_spawn(&self.definition.id, agent_identity.as_str())
.await
.map_err(MobError::from)?
.filter(|record| {
record.generation == generation.0
&& record.fence_token == fence_token.0
&& record.host_id.to_string() == host.as_str()
&& matches!(
&record.phase,
crate::store::PlacedSpawnCarrierPhase::Committed(_)
)
})
.ok_or_else(|| {
MobError::Internal(format!(
"no exact committed placed-spawn carrier for '{agent_identity}' at generation {}; \
revival cannot re-issue the materialization",
generation.0
))
})?;
let recomputed = meerkat_contracts::wire::portable_member_spec_digest(&record.spec)
.map_err(|error| {
MobError::Internal(format!(
"stored placed-member spec for '{agent_identity}' failed canonical digest \
computation: {error}"
))
})?;
if recomputed != record.spec_digest {
return Err(MobError::Internal(format!(
"stored placed-member spec for '{agent_identity}' does not hash to its recorded \
digest; revival refuses a divergent re-issue"
)));
}
let host_peer = TrustedPeerDescriptor::unsigned_with_pubkey(
host.as_str(),
host.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::Internal(format!(
"bound host '{}' facts do not form a canonical peer descriptor: {error}",
host.as_str()
))
})?;
let authority = self.supervisor_bridge.authority().await;
let binding_generation = self.current_host_binding_generation(host)?;
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&host_peer)
.await?;
let payload = Box::new(super::bridge_protocol::BridgeMaterializePayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
generation: generation.0,
fence_token: fence_token.0,
spec: record.spec.clone(),
spec_digest: record.spec_digest.clone(),
launch: super::bridge_protocol::MaterializeLaunchMode::Resume {
session_id: binding.0.clone(),
},
});
// §19.L5: the ops owner for a placed member is the owner bridge
// session, re-authorized through the machine peer-endpoint facts.
let (owner_bridge_session_id, ops_registry) = self
.generated_peer_only_operation_owner_context(
agent_identity,
&crate::RuntimeBinding::HostMaterialized { host: host.clone() },
"revive_placed_member_ops_owner",
)
.await?;
let profile_name = {
let roster = self.roster.read().await;
roster
.get(agent_identity)
.map(|entry| entry.role.clone())
.ok_or_else(|| {
MobError::Internal(format!(
"placed member '{agent_identity}' has no roster entry for revival"
))
})?
};
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
// Revival owns an already-Committed carrier. Re-establish the SAME
// exact operation id when a true process restart dropped nonterminal
// registry rows, promote only exact Provisioning to Running, and
// never rediscover/mint an id from the returned endpoint.
if let Err(error) =
super::ops_adapter::MobOpsAdapter::ensure_committed_placed_provision_operation_exact(
ops_registry.as_ref(),
&owner_bridge_session_id,
&record.provision_operation_id,
&peer_name,
)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed revival for '{agent_identity}' could not validate its exact committed operation anchor; actor is fail-stopping for cold recovery: {error}"
)));
}
let receipt = self
.provisioner
.materialize_member(super::provisioner::MaterializeMemberRequest {
host_peer,
payload,
peer_name,
owner_bridge_session_id,
ops_registry,
provision_operation_id: record.provision_operation_id.clone(),
operation_anchor:
super::provisioner::PlacedProvisionOperationAnchor::ExistingCommitted,
timeout: super::provisioner::MATERIALIZE_BRIDGE_TIMEOUT,
})
.await?;
// Ack echoes vs machine facts (typed mismatch, never absorbed): the
// recorded response must name the SAME session and digest.
let Some(ack) = receipt.receipt.materialized_ack.as_deref() else {
return Err(MobError::Internal(format!(
"placed revival ack for '{agent_identity}' carried no materialized ack facts"
)));
};
if ack.session_id.to_string() != binding.0 {
return Err(MobError::Internal(format!(
"placed revival ack for '{agent_identity}' names session '{}' but the machine \
binding is '{}'",
ack.session_id, binding.0
)));
}
if ack.spec_digest_echo != record.spec_digest {
return Err(MobError::Internal(format!(
"placed revival ack digest echo for '{agent_identity}' diverged from the recorded \
spec digest"
)));
}
// The G2 ACK is authenticated by the exact bound request. Persist the
// carrier CAS before publishing G2 machine authority; only then may
// normal remote behavior resume for this logical member.
self.promote_committed_placed_carrier_binding_generation(
&record,
host,
binding_generation,
"promote placed carrier after revival ack",
)
.await?;
// T4 route-install drain (ADJ-P4-1): the re-materialized runtime
// holds no peer trust rows (trust is volatile on the member host) —
// re-derive + realize the obligations for every wired edge touching
// this identity. Failures leave obligations pending; the revival
// itself stands.
self.drive_route_installs_for_identity(agent_identity).await;
Ok(())
}
/// Realize a machine-authorized member revival through the existing resume
/// materialization path (the same one mob resume-restore and
/// spawn-with-resume use): rebuild the live session from the durable
/// snapshot under the member's unchanged machine-owned identity, runtime
/// incarnation, and session binding, then re-project comms drain ownership
/// and the machine-owned topology restore plan onto the fresh runtime.
async fn materialize_revived_member_session(
&mut self,
entry: &RosterEntry,
member_ref: &MemberRef,
bridge_session_id: &SessionId,
stored_session: meerkat_core::session::Session,
recovered_binding_without_endpoint: bool,
restore_topology_immediately: bool,
) -> Result<(), MobError> {
let agent_identity = entry.agent_identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
// Resolve and machine-authorize the profile material exactly like the
// resume-restore reconciliation.
let mut profile = if let Some(p) = entry.effective_profile_override.clone() {
p
} else {
self.definition
.resolve_profile(&entry.role, self.realm_profile_store.as_ref())
.await?
};
if let Some(model) = entry.effective_model_override.as_ref() {
profile.model.clone_from(model);
}
self.authorize_spawn_profile_material(
&agent_identity,
&entry.role,
&profile,
"revive_member_profile_authority",
)?;
// Revival inputs must equal spawn-time inputs: recompose the retained
// per-spawn overlay alongside profile bundles and mob-default tools.
let per_spawn_overlay = self
.per_spawn_external_tools
.read()
.await
.get(&agent_identity)
.cloned();
let external_tools = self.external_tools_for_profile(&profile, per_spawn_overlay)?;
let mut config = build::build_resumed_agent_config(build::BuildResumedAgentConfigParams {
base: build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &entry.role,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context: None,
labels: Some(entry.labels.clone()),
additional_instructions: None,
shell_env: None,
mob_tool_authority_context: None,
inherited_tool_filter: None,
// Revival resumes the persisted session; the effective policy
// is restored from durable session metadata by the factory.
tool_access_policy: None,
system_prompt_override: None,
},
expected_session_id: bridge_session_id,
resumed_session: stored_session,
})
.await?;
config.keep_alive = entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
// The kickoff prompt is Defer + Discard in `to_create_session_request`:
// no turn runs during materialization — the machine-admitted dispatch
// turn follows once the live session exists again.
let prompt = ContentInput::from(self.fallback_spawn_prompt(&entry.role, &agent_identity));
let req = build::to_create_session_request(&config, prompt);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
agent_identity.as_str(),
)?;
// Machine-minted self-owned provision owner for the unchanged binding
// (`RecoverMemberSessionBindingAlreadyCurrentRunning`): the same
// generated authority the resume-restore reconciliation uses.
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let is_replacing = replacing.is_some();
let dsl_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
let owner_transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::RecoverMemberSessionBinding {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
bridge_session_id: dsl_session_id.clone(),
replacing,
},
"revive_member_session_provision_owner",
)?;
let owner_authorized = owner_transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::SessionProvisionOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id,
} if effect_identity == &dsl_identity && session_id == &dsl_session_id
)
});
if !owner_authorized {
return Err(MobError::Internal(format!(
"MobMachine produced no session provision operation owner for revived member '{agent_identity}'"
)));
}
// The fail-closed discard dropped the live session task but leaves the
// runtime adapter's session entry holding the DEAD comms runtime as
// mob-owned peer ingress. Release it through the typed DetachIngress
// seam (keep_alive=false + no runtime aborts the stale drain task and
// clears ownership) so the fresh comms runtime can attach after the
// re-materialization. The runtime session entry itself stays
// registered — it is id-based and valid again once the live session
// exists.
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
match adapter
.update_peer_ingress_context(bridge_session_id, false, None)
.await
{
Ok(_) => {}
// Absent or already-terminal runtime state proves no stale
// drain/ownership remains — the post-condition this wants.
Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(error) => {
return Err(MobError::Internal(format!(
"failed to detach stale peer ingress for revived session '{bridge_session_id}': {error}"
)));
}
}
}
let receipt = self
.provisioner
.provision_member(ProvisionMemberRequest {
create_session: req,
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
binding: crate::RuntimeBinding::Session,
peer_name: peer_name.clone(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(bridge_session_id.clone()),
runtime_revival_intent:
super::provisioner::RuntimeRevivalIntent::MissingLiveMaterialization,
})
.await?;
let revived_session_id =
receipt
.member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"revival provisioned a non-session member for '{agent_identity}'"
))
})?;
if &revived_session_id != bridge_session_id {
return Err(MobError::Internal(format!(
"revival provisioned bridge session '{revived_session_id}' for machine binding '{bridge_session_id}'"
)));
}
// Snapshotless-head recovery first persists the exact replacement
// session binding so the old id cannot be classified Broken. When no
// live endpoint existed before takeover, enrich that same durable
// recovery fact with a follow-up endpoint carrier from the newly
// materialized comms incarnation before any topology is restored.
// An already-live replacement supplied its endpoint to the first
// carrier and therefore remains the ordinary one-event path.
if recovered_binding_without_endpoint {
let runtime = self
.provisioner
.comms_runtime(member_ref)
.await
.ok_or_else(|| {
MobError::Internal(format!(
"revived replacement session '{bridge_session_id}' has no comms runtime"
))
})?;
let endpoint = super::provisioner::SessionBackend::trusted_peer_spec_from_runtime(
&peer_name,
runtime.as_ref(),
)?
.ok_or_else(|| {
MobError::Internal(format!(
"revived replacement session '{bridge_session_id}' has no exact comms endpoint"
))
})?;
let event = super::builder::append_recovered_session_binding(
&mut self.dsl_authority,
&self.events,
&self.definition.id,
entry,
bridge_session_id,
Some(endpoint),
"explicit_resume_upgrade_recovered_member_peer_endpoint",
)
.await?;
self.roster.write().await.apply_event(&event);
let _ = self
.machine_state_watch_tx
.send(self.dsl_authority.state().clone());
}
// Re-project comms drain ownership onto the fresh comms runtime.
self.ensure_mob_comms_drain(&agent_identity, member_ref)
.await?;
if restore_topology_immediately {
// Dispatch-triggered one-member revival restores its topology
// immediately. Explicit mob Resume passes `false`: its shared
// all-member reconciliation must preflight the complete trust
// mutation set before any topology mutation occurs.
let plan = self.machine_restore_wiring_plan(&agent_identity)?;
let mut failed_restore_peer_ids: Vec<RespawnTopologyPeerId> = Vec::new();
for peer_identity in plan.local_peers {
if peer_identity == agent_identity {
continue;
}
let peer_agent_identity = crate::ids::AgentIdentity::from(peer_identity.as_str());
if let Err(error) = self
.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::Local(peer_agent_identity),
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_identity,
%error,
"revival: failed to restore machine-owned local peer edge"
);
failed_restore_peer_ids
.push(RespawnTopologyPeerId::from(peer_identity.as_str()));
}
}
for peer_spec in plan.external_peers {
let peer_id = RespawnTopologyPeerId::from(peer_spec.peer_id.as_str());
if let Err(error) = self
.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::External(peer_spec.clone()),
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_spec.name,
%error,
"revival: failed to restore machine-owned external peer edge"
);
failed_restore_peer_ids.push(peer_id);
}
}
let resolution = self.resolve_respawn_topology_restore_result(
&agent_identity,
failed_restore_peer_ids,
)?;
if resolution.result == mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed
{
tracing::warn!(
agent_identity = %agent_identity,
failed_peer_ids = ?resolution.failed_peer_ids,
"revival completed with degraded machine-owned topology edges"
);
}
}
Ok(())
}
fn active_machine_member_ids_for_profile(
&self,
profile_name: &ProfileName,
excluded_identity: &AgentIdentity,
) -> Vec<AgentIdentity> {
let dsl = self.dsl_authority.state();
dsl.member_profile_names
.iter()
.filter_map(|(identity, machine_profile_name)| {
if machine_profile_name.as_str() != profile_name.as_str()
|| identity.0.as_str() == excluded_identity.as_str()
|| !MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
{
return None;
}
Some(AgentIdentity::from(identity.0.as_str()))
})
.collect()
}
async fn project_member_list_from_machine(
&mut self,
include_retiring: bool,
) -> Vec<MobMemberListEntry> {
let entries_by_identity: BTreeMap<_, _> = {
let roster = self.roster.read().await;
roster
.list_all()
.cloned()
.map(|entry| (entry.agent_identity.clone(), entry))
.collect()
};
let machine_state = self.dsl_authority.state().clone();
let mut projected = Vec::with_capacity(machine_state.identity_to_runtime.len());
for identity in machine_state.identity_to_runtime.keys() {
let Some(entry) =
super::handle::MobHandle::project_member_list_entry_from_machine_identity(
identity,
entries_by_identity.get(&AgentIdentity::from(identity.0.as_str())),
&machine_state,
)
else {
continue;
};
if !include_retiring && entry.status == super::handle::MobMemberStatus::Retiring {
continue;
}
projected.push(entry);
}
projected
}
fn mob_handle_for_tools(&self) -> MobHandle {
MobHandle {
// §15.2 lane separation: the actor's internal handle-for-tools
// serves member-session operator tools — the AGENT authority
// lane. `MobOperatorToolDispatcher::new` preserves an existing
// agent-lane authority (including an upcall execution fence) and
// re-binds only non-agent handles defensively; this initial
// binding keeps direct in-crate consumers honest.
command_authority: CommandAuthority::agent_lane(),
command_tx: self.command_tx.clone(),
roster: self.roster.clone(),
definition: self.definition.clone(),
events: self.events.clone(),
run_store: self.run_store.clone(),
flow_streams: self.flow_streams.clone(),
session_service: self.session_service.clone(),
#[cfg(feature = "runtime-adapter")]
runtime_adapter: self.runtime_adapter.clone(),
restore_diagnostics: self.restore_diagnostics.clone(),
supervisor_bridge: self.supervisor_bridge.clone(),
machine_state_watch_rx: self.machine_state_watch_tx.subscribe(),
reachability_observations: Arc::clone(&self.reachability_observations),
phase_watch_rx: self.phase_watch_tx.subscribe(),
// W2-E: the actor's internal handle-for-tools does not carry the
// realtime factory — that seam lives on the caller-facing
// `MobHandle` returned from `MobBuilder`. Tools built from the
// actor do not dial realtime endpoints.
realtime_session_factory: None,
flow_target_provisioner: Arc::clone(&self.flow_target_provisioner),
}
}
fn kickoff_phase_from_dsl(
phase: mob_dsl::KickoffPhase,
) -> crate::roster::MobMemberKickoffPhase {
match phase {
mob_dsl::KickoffPhase::Pending => crate::roster::MobMemberKickoffPhase::Pending,
mob_dsl::KickoffPhase::Starting => crate::roster::MobMemberKickoffPhase::Starting,
mob_dsl::KickoffPhase::Started => crate::roster::MobMemberKickoffPhase::Started,
mob_dsl::KickoffPhase::CallbackPending => {
crate::roster::MobMemberKickoffPhase::CallbackPending
}
mob_dsl::KickoffPhase::Failed => crate::roster::MobMemberKickoffPhase::Failed,
mob_dsl::KickoffPhase::Cancelled => crate::roster::MobMemberKickoffPhase::Cancelled,
}
}
/// Map every DSL-emitted [`KickoffIntent`] to its stable lifecycle-notice
/// wire tag. This is a TOTAL projection: the MobMachine emits
/// `EmitKickoffLifecycleNotice` for every kickoff phase, so the shell
/// forwards every intent mechanically rather than narrowing the
/// machine-emitted set to `Failed`/`Cancelled`. There is no `None`-drop:
/// dropping a phase would silently discard a machine-owned lifecycle fact
/// that wired peers are entitled to observe.
pub(super) fn kickoff_notice_intent(
intent: crate::machines::mob_machine::KickoffIntent,
) -> &'static str {
use crate::machines::mob_machine::KickoffIntent;
match intent {
KickoffIntent::Pending => "mob.kickoff_pending",
KickoffIntent::Starting => "mob.kickoff_starting",
KickoffIntent::Started => "mob.kickoff_started",
KickoffIntent::CallbackPending => "mob.kickoff_callback_pending",
KickoffIntent::Failed => "mob.kickoff_failed",
KickoffIntent::Cancelled => "mob.kickoff_cancelled",
}
}
async fn clear_kickoff_state(&mut self, agent_identity: &AgentIdentity) {
match self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffClear {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"clear_kickoff_state",
)
.await
{
Ok(_) => {
self.roster.write().await.set_kickoff(agent_identity, None);
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
%error,
"kickoff clear failed"
);
}
}
}
async fn fail_startup_to_stopped(&mut self, failure_label: &'static str) {
if let Err(error) = self
.drive_placed_completion_lifecycle_cleanup(
None,
true,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
failure_label,
"startup failure Stop remains durably pending"
);
return;
}
let autonomous_stop_deadline = Instant::now() + STARTUP_FAILURE_AUTONOMOUS_STOP_DEADLINE;
loop {
match self.stop_all_autonomous_members().await {
Ok(()) => break,
Err(stop_error @ MobError::AutonomousStopInterruptsPending { .. }) => {
let now = Instant::now();
if now >= autonomous_stop_deadline {
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
failure_label,
"startup failure Stop did not prove exact autonomous cleanup before its deadline"
);
return;
}
tokio::time::sleep(
STARTUP_FAILURE_AUTONOMOUS_STOP_POLL_INTERVAL
.min(autonomous_stop_deadline - now),
)
.await;
}
Err(stop_error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
failure_label,
"startup failure Stop remains pending on autonomous cleanup"
);
return;
}
}
}
if let Err(error) = self.commit_stopped_lifecycle_after_cleanup().await {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
failure_label,
"authority rejected Stop after startup failure"
);
}
}
async fn apply_kickoff_input(
&mut self,
agent_identity: &AgentIdentity,
input: mob_dsl::MobMachineInput,
context: &'static str,
) -> Result<bool, MobError> {
// Prepare first and durably append every persistence effect before
// publishing the machine projection. A controller crash can therefore
// recover the accepted transition from the event log; no observer can
// see a kickoff phase that its durable carrier failed to record.
let prepared = self
.prepare_dsl_input_transition(input, context)
.map_err(|error| Self::kickoff_rejection_error(context, error))?;
let effects = prepared.transition.effects().to_vec();
let mut durable_events = Vec::new();
let mut projected_kickoff = None;
for effect in &effects {
match effect {
mob_dsl::MobMachineEffect::PersistKickoffUpdate {
member_id: _,
phase,
} => {
let kickoff = crate::roster::MobMemberKickoffSnapshot {
objective_id: Some(required_machine_kickoff_objective_id(
prepared.authority.state(),
agent_identity,
)?),
phase: Self::kickoff_phase_from_dsl(*phase),
error: None,
updated_at: SystemTime::now(),
};
durable_events.push(MobEventKind::MemberKickoffUpdated {
member: AgentIdentity::from(agent_identity.as_str()),
kickoff: kickoff.clone(),
});
projected_kickoff = Some(kickoff);
}
mob_dsl::MobMachineEffect::PersistKickoffFailureUpdate {
member_id: _,
phase,
error,
} => {
let kickoff = crate::roster::MobMemberKickoffSnapshot {
objective_id: Some(required_machine_kickoff_objective_id(
prepared.authority.state(),
agent_identity,
)?),
phase: Self::kickoff_phase_from_dsl(*phase),
error: Some(error.clone()),
updated_at: SystemTime::now(),
};
durable_events.push(MobEventKind::MemberKickoffUpdated {
member: AgentIdentity::from(agent_identity.as_str()),
kickoff: kickoff.clone(),
});
projected_kickoff = Some(kickoff);
}
mob_dsl::MobMachineEffect::PersistObjectiveOwnerBinding {
owner_id: _,
objective_id,
} => {
let objective_id = uuid::Uuid::parse_str(objective_id.as_str())
.map(meerkat_core::interaction::ObjectiveId)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine emitted invalid objective id '{objective_id}': {error}"
))
})?;
durable_events.push(MobEventKind::ObjectiveOwnerBound {
owner: agent_identity.clone(),
objective_id,
});
}
mob_dsl::MobMachineEffect::PersistObjectiveConclusion {
member_id: _,
objective_id,
outcome,
} => {
let objective_id = uuid::Uuid::parse_str(objective_id.as_str())
.map(meerkat_core::interaction::ObjectiveId)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine emitted invalid objective id '{objective_id}': {error}"
))
})?;
durable_events.push(MobEventKind::ObjectiveConcluded {
member: agent_identity.clone(),
objective_id,
outcome: outcome.clone(),
});
}
// Routed seam effects (`Request*`) were harvested above
// into `pending_routed_effects`; drain happens at the
// next async boundary via `flush_routed_effects`.
mob_dsl::MobMachineEffect::RequestRuntimeBinding { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeIngress { .. }
| mob_dsl::MobMachineEffect::RequestPeerRuntimeIngress { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeRetire { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeDestroy { .. } => {}
_ => {}
}
}
if !durable_events.is_empty() {
// Kickoff transitions are retryable record-before-publish
// boundaries. Reconcile an ambiguous batch response before the
// machine commit so cancellation cannot be durably present while
// the live authority remains Starting and originates the turn.
self.ensure_exact_structural_event_batch(durable_events)
.await?;
}
self.commit_prepared_dsl_transition(prepared)?;
if let Some(kickoff) = projected_kickoff {
self.roster
.write()
.await
.set_kickoff(agent_identity, Some(kickoff));
}
// Notices are effects of the now-durable, now-published transition.
// Failure is presentation-only and must not roll back authority.
for effect in effects {
if let mob_dsl::MobMachineEffect::EmitKickoffLifecycleNotice {
member_id: _,
intent,
} = effect
{
let notice_intent = Self::kickoff_notice_intent(intent);
if let Err(error) = self
.notify_kickoff_event(agent_identity, notice_intent)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
intent = %intent,
"failed to emit kickoff lifecycle notice"
);
}
}
}
Ok(true)
}
fn kickoff_rejection_error(context: &'static str, error: impl ToString) -> MobError {
MobError::MobMachineRejected {
context,
reason: error.to_string(),
}
}
/// Count of in-flight flow runs tracked by the mob actor's shell,
/// reported by test-only snapshots (`MobOrchestratorSnapshot`,
/// `MobLifecycleSnapshot`). Wave-c WAR-1: this is shell-state
/// introspection, not an authority read seam — `run_cancel_tokens`
/// is the actor's own `BTreeMap<RunId, CancellationToken>` and
/// carries no DSL semantics. Production callers on the
/// `apply_in_phase` / `can_accept_in_phase` path (historical, see
/// docs/architecture/machine-simplification-proposal.md) were
/// removed earlier, leaving only the cfg-test snapshot sites; gate
/// the method the same way its callers are gated so the
/// `NoDeadAuthorityWiring` rule can drop the silencing allow
/// without widening production reachability.
#[cfg(test)]
fn machine_active_run_count(&self) -> u32 {
self.run_cancel_tokens.len() as u32
}
/// Project an observable orchestrator snapshot from DSL state, if this mob
/// has an orchestrator. Returns `None` for plain mobs.
#[cfg(test)]
fn machine_orchestrator_snapshot(&self, phase: MobState) -> Option<MobOrchestratorSnapshot> {
if !self.has_orchestrator {
return None;
}
let coordinator_bound = self.dsl_authority.state().coordinator_bound;
Some(MobOrchestratorSnapshot {
phase,
coordinator_bound,
pending_spawn_count: self.dsl_authority.state().pending_spawn_count as u32,
active_flow_count: self.machine_active_run_count(),
// topology_revision and supervisor_active are shell diagnostics not
// tracked by the DSL; project supervisor_active from coordinator_bound
// to preserve existing test expectations.
topology_revision: 0,
supervisor_active: coordinator_bound,
})
}
/// Machine-routed eligibility gate for within-mob member operations
/// (spawn finalization, peer messaging, respawn finalization) that require
/// the mob to be live and running.
///
/// The lifecycle-phase eligibility verdict over the machine-owned phase
/// (plus the `destroy_admitted` projection marker) is a MobMachine fact: it
/// is exactly the machine's `ClassifyMemberOperationEligibility` verdict
/// (`Admitted` iff `Running` and destruction not admitted, else
/// `DeniedNotRunning`). The shell extracts no fact — it drives the machine
/// classifier (read-only; the classifier self-loops and never mutates
/// state) and mirrors the emitted eligibility to the same
/// `InvalidTransition { from: self.state(), to: Running }` rejection it
/// previously produced from a handwritten phase pre-check. Fails closed.
fn require_member_operation_eligible(&self) -> Result<(), MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ClassifyMemberOperationEligibility {},
"member_operation_eligibility",
)?;
let mut admission = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::MemberOperationEligibilityResolved {
admission: kind,
} = effect
{
if admission.replace(*kind).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple member-operation eligibility verdicts".into(),
));
}
}
}
match admission {
Some(mob_dsl::MobMemberOperationEligibilityKind::Admitted) => Ok(()),
Some(mob_dsl::MobMemberOperationEligibilityKind::DeniedNotRunning) => {
Err(self.invalid_transition_to(MobState::Running))
}
None => Err(MobError::Internal(
"MobMachine emitted no member-operation eligibility verdict".into(),
)),
}
}
/// Ask MobMachine whether a public stable-identity retire may cancel an
/// exact pending spawn incarnation. The generated classifier reads the
/// canonical committed-runtime, generation, and pending-session maps; the
/// actor only validates and mirrors its single structural verdict.
fn classify_retire_pending_spawn_disposition(
&self,
agent_identity: &AgentIdentity,
) -> Result<RetirePendingSpawnVerdict, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ClassifyRetirePendingSpawnDisposition {
agent_identity: dsl_identity.clone(),
},
"classify_retire_pending_spawn_disposition",
)?;
let mut verdict = None;
for effect in &prepared.effects {
let candidate = match effect {
mob_dsl::MobMachineEffect::RetirePendingSpawnCancellationAuthorized {
agent_identity: effect_identity,
agent_runtime_id,
generation,
pending_spawn_session_id,
} if effect_identity == &dsl_identity => Some(
RetirePendingSpawnVerdict::CancelCommittedIncarnation {
agent_runtime_id: agent_runtime_id.clone(),
generation: *generation,
pending_spawn_session_id: pending_spawn_session_id.clone(),
},
),
mob_dsl::MobMachineEffect::RetireCommittedIncarnationWithoutPendingSpawnResolved {
agent_identity: effect_identity,
agent_runtime_id,
generation,
} if effect_identity == &dsl_identity => Some(
RetirePendingSpawnVerdict::CommittedIncarnationWithoutPendingSpawn {
agent_runtime_id: agent_runtime_id.clone(),
generation: *generation,
},
),
mob_dsl::MobMachineEffect::RetireAbsentPendingSpawnPreservationResolved {
agent_identity: effect_identity,
} if effect_identity == &dsl_identity => {
Some(RetirePendingSpawnVerdict::PreservePendingSpawnForAbsentIdentity)
}
_ => None,
};
if let Some(candidate) = candidate
&& verdict.replace(candidate).is_some()
{
return Err(MobError::Internal(format!(
"MobMachine emitted multiple retire pending-spawn verdicts for '{agent_identity}'"
)));
}
}
verdict.ok_or_else(|| {
MobError::Internal(format!(
"MobMachine emitted no retire pending-spawn verdict for '{agent_identity}'"
))
})
}
fn drain_completed_peer_delivery_tasks(&mut self) {
while let Some(result) = self.peer_delivery_tasks.try_join_next() {
match result {
Ok(completion) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Err(error) => {
let _ = self.reconcile_peer_delivery_join_error(
error,
"peer delivery completion",
ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop,
);
}
}
}
}
fn reconcile_peer_delivery_join_error(
&mut self,
error: tokio::task::JoinError,
context: &'static str,
panic_disposition: ActorTaskJoinPanicDisposition,
) -> Option<MobError> {
let task_id = actor_task_join_error_task_id(&error);
#[cfg(not(target_arch = "wasm32"))]
let runtime_task_id = error.id();
#[cfg(not(target_arch = "wasm32"))]
let delivery = self
.peer_delivery_inflight
.iter()
.find(|(_, delivery)| delivery.task_id == runtime_task_id)
.map(|(delivery_id, delivery)| {
(*delivery_id, delivery.from.clone(), delivery.to.clone())
});
#[cfg(target_arch = "wasm32")]
let delivery: Option<(PeerDeliveryId, AgentIdentity, AgentIdentity)> = None;
let outcome = if error.is_cancelled() {
if panic_disposition.requires_fail_stop() {
self.durable_uncertainty_fail_stop = true;
let cancellation_error = MobError::Internal(format!(
"{context} task {task_id} was cancelled; \
disposition=cancelled_ambiguous_effect_fail_stop"
));
tracing::error!(
context,
task_id = %task_id,
delivery_id = ?delivery.as_ref().map(|(id, _, _)| *id),
from = ?delivery.as_ref().map(|(_, from, _)| from),
to = ?delivery.as_ref().map(|(_, _, to)| to),
disposition = "cancelled_ambiguous_effect_fail_stop",
error = %cancellation_error,
"peer delivery task was unexpectedly cancelled; effect outcome is unknown"
);
Some(cancellation_error)
} else {
tracing::debug!(
context,
task_id = %task_id,
delivery_id = ?delivery.as_ref().map(|(id, _, _)| *id),
from = ?delivery.as_ref().map(|(_, from, _)| from),
to = ?delivery.as_ref().map(|(_, _, to)| to),
disposition = "cancelled_teardown_terminal",
"peer delivery task was cancelled without a panic"
);
None
}
} else {
let panic_error = actor_task_join_panic_error(context, panic_disposition, error);
if panic_disposition.requires_fail_stop() {
self.durable_uncertainty_fail_stop = true;
}
tracing::error!(
context,
task_id = %task_id,
delivery_id = ?delivery.as_ref().map(|(id, _, _)| *id),
from = ?delivery.as_ref().map(|(_, from, _)| from),
to = ?delivery.as_ref().map(|(_, _, to)| to),
error = %panic_error,
"peer delivery effect may have escaped without a typed completion"
);
Some(panic_error)
};
if let Some((delivery_id, _, _)) = delivery {
self.peer_delivery_inflight.remove(&delivery_id);
} else if self.peer_delivery_tasks.is_empty() {
// A legacy/pre-registration task id should not strand volatile
// backpressure forever. Unknown identity plus a panic has already
// fail-stopped normal operation above.
self.peer_delivery_inflight.clear();
}
outcome
}
fn reconcile_actor_io_task_join(&mut self, result: Result<(), tokio::task::JoinError>) {
if let Err(error) = result {
if error.is_cancelled() {
tracing::error!(
task_id = %actor_task_join_error_task_id(&error),
disposition = "cancelled_ambiguous_effect_fail_stop",
"actor-owned I/O task was unexpectedly cancelled; effect outcome is unknown"
);
self.durable_uncertainty_fail_stop = true;
} else {
let disposition = ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop;
let _ = actor_task_join_panic_error("actor I/O completion", disposition, error);
self.durable_uncertainty_fail_stop = true;
}
}
}
fn drain_completed_actor_io_tasks(&mut self) {
while let Some(result) = self.actor_io_tasks.try_join_next() {
self.reconcile_actor_io_task_join(result);
}
}
async fn abort_and_join_actor_io_tasks(&mut self) {
self.actor_io_tasks.abort_all();
while let Some(result) = self.actor_io_tasks.join_next().await {
if let Err(error) = result
&& actor_task_join_error_is_panic(&error)
{
let _ = actor_task_join_panic_error(
"actor I/O teardown",
ActorTaskJoinPanicDisposition::TeardownTerminal,
error,
);
}
}
}
fn spawn_peer_message_delivery(
&mut self,
plan: PeerMessageDeliveryPlan,
reply_tx: tokio::sync::oneshot::Sender<Result<meerkat_core::comms::SendReceipt, MobError>>,
) {
let permit = match self.peer_delivery_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
let _ = reply_tx.send(Err(MobError::Internal(format!(
"peer message delivery lane saturated (limit={MAX_PENDING_PEER_DELIVERIES})"
))));
return;
}
};
let id = self.next_peer_delivery_ticket;
self.next_peer_delivery_ticket = self.next_peer_delivery_ticket.wrapping_add(1);
let cancel_token = tokio_util::sync::CancellationToken::new();
let from = plan.from.clone();
let to = plan.to.clone();
let task_cancel_token = cancel_token.clone();
let abort_handle = self.peer_delivery_tasks.spawn(async move {
let _permit = permit;
let result = tokio::select! {
result = plan.sender_comms.send(plan.command) => result.map_err(MobError::from),
() = task_cancel_token.cancelled() => Err(MobError::Internal(
"peer message delivery canceled because mob topology or lifecycle changed".to_string(),
)),
};
let _ = reply_tx.send(result);
PeerDeliveryCompletion { id }
});
self.peer_delivery_inflight.insert(
id,
PeerDeliveryInflight {
from,
to,
cancel_token,
#[cfg(not(target_arch = "wasm32"))]
task_id: abort_handle.id(),
},
);
#[cfg(target_arch = "wasm32")]
let _ = abort_handle;
}
async fn cancel_pending_peer_deliveries(
&mut self,
reason: &'static str,
) -> Result<(), MobError> {
self.cancel_peer_deliveries_matching(reason, |_| true).await
}
async fn cancel_peer_deliveries_for_member(
&mut self,
member: &AgentIdentity,
reason: &'static str,
) -> Result<(), MobError> {
self.cancel_peer_deliveries_matching(reason, |delivery| {
&delivery.from == member || &delivery.to == member
})
.await
}
async fn cancel_peer_deliveries_for_edge(
&mut self,
a: &AgentIdentity,
b: &AgentIdentity,
reason: &'static str,
) -> Result<(), MobError> {
self.cancel_peer_deliveries_matching(reason, |delivery| {
(&delivery.from == a && &delivery.to == b) || (&delivery.from == b && &delivery.to == a)
})
.await
}
async fn cancel_peer_deliveries_matching(
&mut self,
reason: &'static str,
mut predicate: impl FnMut(&PeerDeliveryInflight) -> bool,
) -> Result<(), MobError> {
let ids = self
.peer_delivery_inflight
.iter()
.filter_map(|(id, delivery)| predicate(delivery).then_some(*id))
.collect::<BTreeSet<_>>();
if ids.is_empty() {
return Ok(());
}
tracing::debug!(
mob_id = %self.definition.id,
pending = ids.len(),
reason,
"canceling pending peer delivery tasks"
);
for id in &ids {
if let Some(delivery) = self.peer_delivery_inflight.get(id) {
delivery.cancel_token.cancel();
}
}
let mut first_error = None;
while ids
.iter()
.any(|id| self.peer_delivery_inflight.contains_key(id))
{
match self.peer_delivery_tasks.join_next().await {
Some(Ok(completion)) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Some(Err(error)) => {
if let Some(error) = self.reconcile_peer_delivery_join_error(
error,
"peer delivery cancellation",
ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop,
) && first_error.is_none()
{
first_error = Some(error);
}
}
None => break,
}
}
for id in ids {
self.peer_delivery_inflight.remove(&id);
}
first_error.map_or(Ok(()), Err)
}
/// Deliver a lifecycle notification to every active member of the
/// definition-owned orchestrator profile.
///
/// This is *not* best-effort projection work: a genuine delivery fault
/// surfaces to the caller as a typed [`MobError`] instead of being
/// swallowed by a `tracing::warn`. Cases that legitimately have nothing to
/// deliver (no orchestrator declared or no active MobMachine member for its
/// profile) return `Ok(())`. Faults — an active machine member missing from
/// the mechanical roster, an autonomous host with no reachable injector, or
/// an injection/turn that fails — return `Err`.
///
/// The injector for an [`AutonomousHost`](crate::MobRuntimeMode::AutonomousHost)
/// orchestrator is resolved *synchronously* so its unavailability fails the
/// caller immediately. The actual delivery is dispatched on the bounded
/// `lifecycle_tasks` set (preserving backpressure); a delivery failure from
/// a previously dispatched task is surfaced when this method next drains the
/// set.
async fn notify_orchestrator_lifecycle(&mut self, message: String) -> Result<(), MobError> {
if let Some(error) = self.pending_lifecycle_delivery_error.take() {
return Err(error);
}
// Drain completed lifecycle tasks (non-blocking). A prior delivery that
// failed is surfaced here rather than dropped: lifecycle delivery owns a
// real fault, not a log line.
self.drain_completed_lifecycle_tasks()?;
let Some(orchestrator) = &self.definition.orchestrator else {
return Ok(());
};
let orchestrator_identities = self
.dsl_authority
.state()
.active_member_identities_for_profile(&orchestrator.profile);
if orchestrator_identities.is_empty() {
return Ok(());
}
let orchestrator_entries = {
let roster = self.roster.read().await;
orchestrator_identities
.into_iter()
.map(|orchestrator_identity| {
roster
.get(&orchestrator_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"active MobMachine orchestrator '{orchestrator_identity}' has no mechanical roster entry"
))
})
})
.collect::<Result<Vec<_>, MobError>>()?
};
for orchestrator_entry in orchestrator_entries {
while self.lifecycle_tasks.len() >= MAX_LIFECYCLE_NOTIFICATION_TASKS {
match self.lifecycle_tasks.join_next().await {
Some(join_result) => self.surface_lifecycle_task_outcome(join_result)?,
None => break,
}
}
let placed_incarnation = if super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&orchestrator_entry.agent_identity,
) {
Some(self.placed_member_incarnation(&orchestrator_entry)?)
} else {
None
};
let provisioner = self.provisioner.clone();
let runtime_mode = orchestrator_entry.runtime_mode;
let agent_identity = orchestrator_entry.agent_identity;
let bridge_session_id = self.machine_bridge_session_id_for_identity(&agent_identity)?;
let Some(member_ref) = Self::project_member_ref_session_binding(
&orchestrator_entry.member_ref,
bridge_session_id,
) else {
return Err(MobError::WiringError(format!(
"orchestrator lifecycle notification for '{agent_identity}' could not project an addressable member binding"
)));
};
let task_message = message.clone();
if let Some(expected_member) = placed_incarnation {
// Placement owns the delivery lane before runtime mode. An
// AutonomousHost session placed on a member host is just as remote
// as a TurnDriven one; its machine session binding must never be
// submitted to this controller's injector registry.
let task_member_ref = member_ref.clone();
self.lifecycle_tasks.spawn(async move {
provisioner
.start_turn_with_correlation(
&task_member_ref,
meerkat_core::service::StartTurnRequest {
injected_context: Vec::new(),
prompt: task_message.into(),
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::default(
),
},
Some(super::provisioner::PlacedTurnDeliveryContext {
input_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
transcript_interaction_id: None,
expected_member,
outcome_tracking: None,
}),
)
.await
.map(|_| ())
});
continue;
}
match runtime_mode {
crate::MobRuntimeMode::AutonomousHost => {
// Resolve the injector synchronously so an unreachable autonomous
// host fails the caller now instead of vanishing inside a
// detached task.
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Err(MobError::WiringError(format!(
"orchestrator lifecycle notification for '{agent_identity}' has no bridge session for autonomous-host injection"
)));
};
let Some(injector) = provisioner
.interaction_event_injector(bridge_session_id)
.await
else {
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "orchestrator lifecycle notification",
});
};
let task_identity = agent_identity.clone();
self.lifecycle_tasks.spawn(async move {
injector
.inject(
task_message.into(),
meerkat_core::PlainEventSource::Rpc,
meerkat_core::types::HandlingMode::Queue,
None,
)
.map_err(|error| {
MobError::Internal(format!(
"orchestrator lifecycle inject failed for '{task_identity}': {error}"
))
})
});
}
crate::MobRuntimeMode::TurnDriven => {
let task_member_ref = member_ref.clone();
self.lifecycle_tasks.spawn(async move {
provisioner
.start_turn_with_correlation(
&task_member_ref,
meerkat_core::service::StartTurnRequest {
injected_context: Vec::new(),
prompt: task_message.into(),
system_prompt: None,
event_tx: None,
runtime:
meerkat_core::service::StartTurnRuntimeSemantics::default(),
},
None,
)
.await
.map(|_| ())
});
}
}
}
Ok(())
}
fn retain_lifecycle_delivery_error(&mut self, error: MobError) {
if self.pending_lifecycle_delivery_error.is_none() {
self.pending_lifecycle_delivery_error = Some(error);
} else {
tracing::warn!(
error = %error,
"additional orchestrator lifecycle delivery failure observed while an earlier typed failure remains pending"
);
}
}
/// Drain all completed lifecycle notification tasks without blocking,
/// surfacing the first delivery fault as a typed [`MobError`].
fn drain_completed_lifecycle_tasks(&mut self) -> Result<(), MobError> {
let mut surfaced: Option<MobError> = None;
while let Some(join_result) = self.lifecycle_tasks.try_join_next() {
if let Err(error) = self.surface_lifecycle_task_outcome(join_result) {
// Drain the remainder before returning so the set does not grow
// unbounded, but report the first observed delivery fault.
if surfaced.is_none() {
surfaced = Some(error);
}
}
}
match surfaced {
Some(error) => Err(error),
None => Ok(()),
}
}
/// Interpret one steady-state lifecycle task result. Teardown uses
/// `abort_and_join_lifecycle_tasks`; cancellation at this boundary is
/// therefore an ambiguous delivery outcome, not expected teardown.
fn surface_lifecycle_task_outcome(
&mut self,
join_result: Result<Result<(), MobError>, tokio::task::JoinError>,
) -> Result<(), MobError> {
match join_result {
Ok(Ok(())) => Ok(()),
Ok(Err(delivery_error)) => {
tracing::warn!(
error = %delivery_error,
"orchestrator lifecycle delivery failed"
);
Err(delivery_error)
}
Err(join_error) if join_error.is_cancelled() => {
let task_id = actor_task_join_error_task_id(&join_error);
self.durable_uncertainty_fail_stop = true;
tracing::error!(
task_id = %task_id,
disposition = "cancelled_ambiguous_effect_fail_stop",
"orchestrator lifecycle delivery was cancelled; delivery outcome is unknown"
);
Err(MobError::Internal(format!(
"orchestrator lifecycle delivery task {task_id} was cancelled; \
disposition=cancelled_ambiguous_effect_fail_stop"
)))
}
Err(join_error) => {
let disposition = ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop;
let error = actor_task_join_panic_error(
"orchestrator lifecycle delivery",
disposition,
join_error,
);
self.durable_uncertainty_fail_stop = true;
Err(error)
}
}
}
fn retire_event_key(
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> String {
crate::ids::AgentRuntimeId::new(agent_identity.clone(), generation).to_string()
}
fn pending_spawn_maps_aligned(&self) -> bool {
self.pending_spawn_alignment_violation().is_none()
}
fn pending_spawn_alignment_violation(&self) -> Option<String> {
let expected = if self.has_orchestrator {
Some(self.dsl_authority.state().pending_spawn_count as usize)
} else {
None
};
if let Some(message) = self.pending_spawns.alignment_violation(expected) {
return Some(message);
}
if self.has_orchestrator {
let dsl_pending = self
.dsl_authority
.state()
.pending_spawn_sessions
.iter()
.map(|(identity, session_id)| (identity.0.clone(), session_id.0.clone()))
.collect::<BTreeMap<_, _>>();
let local_pending = self.pending_spawns.member_session_pairs();
if dsl_pending != local_pending {
return Some(format!(
"pending admission mismatch: dsl={dsl_pending:?}, local={local_pending:?}"
));
}
}
None
}
fn ensure_pending_spawn_alignment(&self, context: &str) -> Result<(), MobError> {
if let Some(message) = self.pending_spawn_alignment_violation() {
return Err(MobError::Internal(format!(
"{context}: pending spawn alignment violation: {message}"
)));
}
Ok(())
}
fn debug_assert_pending_spawn_alignment(&self) {
debug_assert!(
self.pending_spawn_maps_aligned(),
"pending spawn alignment must hold across pending maps and orchestrator count"
);
}
async fn insert_pending_spawn(
&mut self,
spawn_ticket: u64,
pending: PendingSpawn,
task: tokio::task::JoinHandle<()>,
started: AuthorizedMobSpawnStarted,
) -> Result<(), MobError> {
debug_assert_eq!(pending.agent_identity, started.agent_identity);
debug_assert_eq!(pending.admitted_bridge_session_id, started.session_id);
debug_assert_eq!(
started.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
debug_assert_eq!(
started.generated_started,
generated_mob_command_capabilities::CommandPlanKind::SpawnStarted
);
let impact = self.pending_spawns.insert(spawn_ticket, pending, task);
if let PendingSpawnInsertImpact::Collided { replaced } = impact {
let mut replaced = *replaced;
let replaced_identity = replaced.spawn.agent_identity.clone();
let respawn_origin = replaced.spawn.respawn_origin.clone();
let reason = format!("pending spawn slot collision for ticket {spawn_ticket}");
let mut cleanup_errors = Vec::new();
if let Some(task) = replaced.task.take() {
task.abort();
let _ = task.await;
}
// StageSpawn has already been accepted for the new slot in enqueue paths.
// If we replaced a prior slot at the same ticket, close that prior
// staged snapshot now so authority counters cannot drift silently.
if let Err(error) = self.complete_orchestrator_spawn(
Some(spawn_ticket),
&replaced_identity,
"pending spawn slot collision replaced existing entry",
) {
cleanup_errors.push(error.to_string());
}
// A ticket collision is mechanical cleanup, not a public Retire
// verdict. It therefore has no MobMachine-authorized committed
// incarnation witness to attach to the exact session/operation
// cleanup anchor.
if let Err(error) = self
.abort_pending_spawn_slot(&replaced, &reason, None)
.await
{
cleanup_errors.push(error.to_string());
}
if let Some(remote) = replaced.spawn.remote.as_ref() {
let pending_carrier = remote.pending_carrier.clone();
let cleanup_result = self
.fail_remote_spawn_exec(
&replaced_identity,
&pending_carrier,
true,
MobError::Internal(reason.clone()),
"materialize_canceled".to_string(),
"insert_pending_spawn_collision",
)
.await;
if self.durable_uncertainty_fail_stop {
cleanup_errors.push(cleanup_result.to_string());
}
}
if let Some(respawn_origin) = respawn_origin
&& let Err(error) = self
.durably_abandon_respawn_topology_if_terminal_exact(
&replaced_identity,
&respawn_origin,
)
.await
{
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
return Err(MobError::Internal(format!(
"{reason}; durable respawn-topology abandonment failed and the actor is fail-stopping for cold recovery: {error}"
)));
}
if self.durable_uncertainty_fail_stop {
replaced.fail(&reason);
return Err(MobError::Internal(format!(
"{reason}; collision cleanup remained durably uncertain and the actor is fail-stopping for cold recovery"
)));
}
replaced.fail(&reason);
tracing::warn!(
spawn_ticket,
"pending spawn slot collision replaced existing entry"
);
if !cleanup_errors.is_empty() {
return Err(Self::pending_spawn_cleanup_error(&reason, cleanup_errors));
}
}
self.debug_assert_pending_spawn_alignment();
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
spawn_ticket,
message = %message,
"pending spawn alignment violated after insert"
);
}
Ok(())
}
fn take_pending_spawn_slot(
&mut self,
spawn_ticket: u64,
) -> (Option<PendingSpawn>, Option<tokio::task::JoinHandle<()>>) {
self.pending_spawns
.take_slot(spawn_ticket)
.map_or((None, None), |slot| (Some(slot.spawn), slot.task))
}
fn complete_pending_spawn_slot(
&mut self,
spawn_ticket: u64,
context: &'static str,
) -> (Option<PendingSpawn>, Option<tokio::task::JoinHandle<()>>) {
let (pending, task) = self.take_pending_spawn_slot(spawn_ticket);
if pending.is_some() || task.is_some() {
if let Some(pending) = pending.as_ref() {
if let Ok(completed) = self.complete_orchestrator_spawn(
Some(spawn_ticket),
&pending.agent_identity,
context,
) {
debug_assert_eq!(completed.agent_identity, pending.agent_identity);
}
}
}
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
spawn_ticket,
context,
message = %message,
"pending spawn alignment violated after completion"
);
}
(pending, task)
}
fn stage_orchestrator_spawn(
&mut self,
agent_identity: &AgentIdentity,
session_id: &SessionId,
) -> Result<AuthorizedMobSpawnStart, MobError> {
let dsl_agent_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let dsl_session_id = mob_dsl::SessionId::from_domain(session_id);
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::StageSpawn {
agent_identity: dsl_agent_identity.clone(),
session_id: dsl_session_id.clone(),
},
"stage_spawn",
)?;
let authorized = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::PendingSpawnOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id: effect_session_id,
} if effect_identity == &dsl_agent_identity && effect_session_id == &dsl_session_id
)
});
if !authorized {
return Err(MobError::Internal(format!(
"MobMachine StageSpawn did not authorize pending operation owner for '{agent_identity}'"
)));
}
Ok(AuthorizedMobSpawnStart {
generated_can_start: generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn,
generated_owner:
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart,
agent_identity: agent_identity.clone(),
session_id: session_id.clone(),
})
}
fn apply_generated_self_owned_operation_owner(
provision_request: &mut ProvisionMemberRequest,
generated_owner: &AuthorizedMobSpawnStart,
) -> Result<(), MobError> {
if !matches!(provision_request.binding, crate::RuntimeBinding::Session)
|| provision_request.owner_bridge_session_id.is_some()
|| provision_request.ops_registry.is_some()
{
return Ok(());
}
provision_request.generated_self_owned_operation_owner =
Some(generated_owner.owner_session_id().clone());
Ok(())
}
fn preview_spawn_admission(
&self,
agent_identity: &AgentIdentity,
authorized_profile_material: &AuthorizedSpawnProfileMaterial,
bridge_session_id: Option<&SessionId>,
) -> Result<(), MobError> {
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let generation = self.mint_spawn_generation(agent_identity)?;
let domain_runtime_id =
crate::ids::AgentRuntimeId::new(domain_identity.clone(), generation);
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
// Spawn-ladder admission preview: `BeginSpawnExec` carries the spawn
// admission guards (the gate the old monolithic `Spawn` input owned),
// so a read-only peek of it is the canonical "would this spawn be
// admitted" probe.
self.preview_dsl_input(
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&domain_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(self.next_fence_token_preview()?),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: bridge_session_id.map(mob_dsl::SessionId::from_domain),
replacing,
// Local-arm probe: no phase-1 spawn path requests placement, so
// the multi-host denial guards are vacuous here. The two
// actor-level facts are real observations; the spec/profile
// facts are unobservable at this probe (it has no spawn spec).
placement: None,
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: self.default_external_tools_provider.is_some(),
default_llm_client_override_present: self.default_llm_client.is_some(),
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
},
"spawn_command_admission",
)
.map(|_| ())
.map_err(|_| self.invalid_transition_to(MobState::Running))
}
fn preview_spawn_command_admission(
&self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
self.require_member_operation_eligible()?;
self.probe_member_admission(agent_identity)
}
/// Read-only MobMachine probe for member-admission duplication. The machine
/// guard covers live runtime bindings, session bindings, and in-flight
/// pending spawns, so this fully replaces the former shell-side
/// `pending_spawns.contains_member` prechecks.
///
fn probe_member_admission(&self, agent_identity: &AgentIdentity) -> Result<(), MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ProbeMemberAdmission {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"probe_member_admission",
)?;
let mut verdict = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::MemberAdmissionProbed { verdict: kind, .. } = effect {
if verdict.replace(*kind).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple member-admission verdicts".into(),
));
}
}
}
match verdict {
Some(mob_dsl::MemberAdmissionVerdictKind::Admitted) => Ok(()),
Some(mob_dsl::MemberAdmissionVerdictKind::DuplicateRejected) => {
Err(MobError::MemberAlreadyExists(agent_identity.clone()))
}
None => Err(MobError::Internal(
"MobMachine emitted no member-admission verdict".into(),
)),
}
}
/// Spawn-lane generation mint (ADJ-24): a fresh identity starts at
/// `Generation::INITIAL`; an identity with retired machine history mints
/// the machine-owned monotone successor through
/// [`Self::compute_respawn_generation`] so the replacement tuple always
/// sorts above every recorded incarnation (host Superseding, never
/// StaleFence). The discriminating fact is the machine's own
/// `identity_runtime_generations` counter.
fn mint_spawn_generation(
&self,
agent_identity: &AgentIdentity,
) -> Result<crate::ids::Generation, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
if self
.dsl_authority
.state()
.identity_runtime_generations
.contains_key(&dsl_identity)
{
self.compute_respawn_generation(agent_identity)
} else {
Ok(crate::ids::Generation::INITIAL)
}
}
/// Read-only MobMachine probe for the next monotone respawn generation. The
/// machine owns the per-identity generation counter (seeded by Spawn/Reset),
/// so this fully replaces the former shell-side `snapshot.generation.next()`
/// increment.
fn compute_respawn_generation(
&self,
agent_identity: &AgentIdentity,
) -> Result<crate::ids::Generation, MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ComputeRespawnGeneration {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"compute_respawn_generation",
)?;
let mut next_generation = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::RespawnGenerationComputed {
next_generation: next,
..
} = effect
{
if next_generation.replace(next.0).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple respawn generations".into(),
));
}
}
}
match next_generation {
Some(next) => Ok(crate::ids::Generation::new(next)),
None => Err(MobError::Internal(
"MobMachine emitted no respawn generation".into(),
)),
}
}
fn authorize_spawn_profile_material(
&mut self,
agent_identity: &AgentIdentity,
profile_name: &ProfileName,
profile: &crate::profile::Profile,
context: &str,
) -> Result<AuthorizedSpawnProfileMaterial, MobError> {
self.authorize_spawn_profile_material_with_digest(
agent_identity,
profile_name,
profile,
None,
context,
)
}
/// Placed-spawn twin of [`Self::authorize_spawn_profile_material`]: the
/// compiled portable-spec digest is recorded by `AuthorizeSpawnProfile`
/// (single-shot; consumed by the remote commit's echo guard). Local
/// spawns pass `None` — the DSL denies a local spawn that carries a
/// recorded digest (`ResolvedSpecDigestAbsent` denial arm).
fn authorize_spawn_profile_material_with_digest(
&mut self,
agent_identity: &AgentIdentity,
profile_name: &ProfileName,
profile: &crate::profile::Profile,
resolved_spec_digest: Option<String>,
context: &str,
) -> Result<AuthorizedSpawnProfileMaterial, MobError> {
let (input, expected) = authorize_spawn_profile_input(
agent_identity,
profile_name,
profile,
resolved_spec_digest,
)?;
let transition = self.apply_dsl_input_collect_transition(input, context)?;
require_authorized_effect(&transition, &expected, context)?;
Ok(expected)
}
fn preview_run_flow_command_admission(&self, run_id: &RunId) -> Result<(), MobError> {
self.prepare_command_admission(
mob_dsl::MobMachineInput::RunFlow {
run_id: mob_dsl::RunId::from(run_id.to_string()),
step_ids: Default::default(),
ordered_steps: Vec::new(),
step_status: Default::default(),
output_recorded: Default::default(),
step_condition_results: Default::default(),
step_has_conditions: Default::default(),
step_dependencies: Default::default(),
step_dependency_modes: Default::default(),
step_branches: Default::default(),
step_collection_policies: Default::default(),
step_quorum_thresholds: Default::default(),
step_target_counts: Default::default(),
step_target_success_counts: Default::default(),
step_target_terminal_failure_counts: Default::default(),
escalation_threshold: 0,
max_step_retries: 0,
max_active_nodes: 0,
max_active_frames: 0,
max_frame_depth: 0,
},
MobState::Running,
"run_flow_command_admission",
)
.map(|_| ())
}
fn resolve_respawn_topology_restore_result(
&mut self,
agent_identity: &AgentIdentity,
failed_restore_peer_ids: Vec<RespawnTopologyPeerId>,
) -> Result<RespawnTopologyRestoreResolution, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from(agent_identity.as_str());
let dsl_failed_peer_ids = failed_restore_peer_ids
.iter()
.map(|peer_id| mob_dsl::RespawnTopologyPeerId::from(peer_id.as_str()))
.collect::<Vec<_>>();
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::ResolveRespawnTopologyRestore {
agent_identity: dsl_identity.clone(),
failed_peer_ids: dsl_failed_peer_ids.clone(),
},
"respawn_topology_restore_result",
)?;
let resolution = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RespawnTopologyRestoreResolved {
agent_identity: effect_identity,
result,
failed_peer_ids,
} if effect_identity == dsl_identity => Some((result, failed_peer_ids)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted respawn topology feedback but emitted no typed result"
.into(),
)
})?;
let (result, effect_failed_peer_ids) = resolution;
if effect_failed_peer_ids != dsl_failed_peer_ids {
return Err(MobError::Internal(
"MobMachine respawn topology feedback echoed inconsistent failed peer ids".into(),
));
}
let failed_peer_ids = effect_failed_peer_ids
.into_iter()
.map(|peer_id| RespawnTopologyPeerId::from(peer_id.0.as_str()))
.collect::<Vec<_>>();
match result {
mob_dsl::RespawnTopologyRestoreResultKind::Completed if failed_peer_ids.is_empty() => {
Ok(RespawnTopologyRestoreResolution {
result,
failed_peer_ids,
})
}
mob_dsl::RespawnTopologyRestoreResultKind::Completed => Err(MobError::Internal(
"MobMachine classified respawn topology restore as completed with failed peers"
.into(),
)),
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed
if !failed_peer_ids.is_empty() =>
{
Ok(RespawnTopologyRestoreResolution {
result,
failed_peer_ids,
})
}
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed => {
Err(MobError::Internal(
"MobMachine classified respawn topology restore as failed without failed peers"
.into(),
))
}
}
}
#[allow(clippy::too_many_arguments)]
fn resolve_submit_work_rejection_in_authority(
authority: &mut mob_dsl::MobMachineAuthority,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
origin: WorkOrigin,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let transition = match mob_dsl::MobMachineMutator::apply(
authority,
mob_dsl::MobMachineInput::ResolveSubmitWorkRejection {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
origin: dsl_origin,
},
) {
Ok(transition) => transition,
Err(err) => {
return MobError::Internal(format!(
"MobMachine rejected SubmitWork and failed to resolve typed rejection: {err}"
));
}
};
let reason = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::SubmitWorkRejected {
agent_runtime_id,
reason,
expected_fence_token,
actual_fence_token,
..
} if agent_runtime_id == *dsl_runtime_id => {
Some((reason, expected_fence_token, actual_fence_token))
}
_ => None,
});
match reason {
Some((mob_dsl::SubmitWorkRejectReasonKind::MobNotRunning, _, _)) => {
MobError::InvalidTransition {
from: current_state,
to: MobState::Running,
}
}
Some((mob_dsl::SubmitWorkRejectReasonKind::MemberNotFound, _, _)) => {
MobError::MemberNotFound(agent_identity.clone())
}
Some((
mob_dsl::SubmitWorkRejectReasonKind::StaleFenceToken,
Some(expected),
Some(actual),
)) => MobError::StaleFenceToken {
runtime_id: runtime_id.clone(),
expected: FenceToken::new(expected.0),
actual: FenceToken::new(actual.0),
},
Some((mob_dsl::SubmitWorkRejectReasonKind::StaleFenceToken, _, _)) => {
MobError::Internal(
"MobMachine rejected SubmitWork as stale without fence-token feedback".into(),
)
}
Some((mob_dsl::SubmitWorkRejectReasonKind::NotExternallyAddressable, _, _)) => {
MobError::NotExternallyAddressable(agent_identity.clone())
}
None => MobError::Internal(
"MobMachine rejected SubmitWork without typed rejection feedback".into(),
),
}
}
#[allow(clippy::too_many_arguments)]
fn resolve_submit_work_projection_missing_or_rejection(
authority: &mut mob_dsl::MobMachineAuthority,
declared_submit_work_admitted: bool,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
origin: WorkOrigin,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
if declared_submit_work_admitted {
return MobError::Internal(format!(
"MobMachine admitted SubmitWork for '{agent_identity}' but the roster projection has no member entry"
));
}
Self::resolve_submit_work_rejection_in_authority(
authority,
dsl_identity,
dsl_runtime_id,
dsl_fence_token,
runtime_id,
origin,
agent_identity,
current_state,
)
}
async fn resolve_spawn_policy_via_machine(
&mut self,
identity: &AgentIdentity,
) -> Result<Option<super::spawn_policy::SpawnSpec>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let (policy_enabled, revision) = {
let state = self.dsl_authority.state();
(state.spawn_policy_enabled, state.spawn_policy_revision)
};
if !policy_enabled {
return Ok(None);
}
let observed = self.spawn_policy.observe_resolution(identity).await;
let profile_name = observed
.as_ref()
.map(|spec| spec.profile.as_str().to_owned());
let runtime_mode = observed
.as_ref()
.and_then(|spec| spec.runtime_mode.map(mob_dsl::SpawnPolicyRuntimeMode::from));
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::ResolveSpawnPolicy {
agent_identity: dsl_identity.clone(),
revision,
profile_name,
runtime_mode,
},
"resolve_spawn_policy",
)?;
let recorded = transition.effects().iter().find_map(|effect| match effect {
mob_dsl::MobMachineEffect::SpawnPolicyResolutionRecorded {
agent_identity,
revision: effect_revision,
profile_name,
runtime_mode,
} if *agent_identity == dsl_identity && *effect_revision == revision => {
Some((profile_name.clone(), *runtime_mode))
}
_ => None,
});
match recorded {
Some((Some(profile), runtime_mode)) => Ok(Some(super::spawn_policy::SpawnSpec {
profile: crate::ids::ProfileName::from(profile),
runtime_mode: runtime_mode.map(crate::MobRuntimeMode::from),
})),
Some((None, None)) => Ok(None),
Some((None, Some(_))) => Err(MobError::Internal(
"MobMachine recorded spawn-policy runtime mode without profile".into(),
)),
None => Err(MobError::Internal(
"MobMachine accepted spawn-policy resolution but emitted no typed feedback".into(),
)),
}
}
fn resolve_cancel_all_work_rejection_in_authority(
authority: &mut mob_dsl::MobMachineAuthority,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
let transition = match mob_dsl::MobMachineMutator::apply(
authority,
mob_dsl::MobMachineInput::ResolveCancelAllWorkRejection {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
},
) {
Ok(transition) => transition,
Err(err) => {
return MobError::Internal(format!(
"MobMachine rejected CancelAllWork and failed to resolve typed rejection: {err}"
));
}
};
let reason = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::CancelAllWorkRejected {
agent_runtime_id,
reason,
expected_fence_token,
actual_fence_token,
} if agent_runtime_id == *dsl_runtime_id => {
Some((reason, expected_fence_token, actual_fence_token))
}
_ => None,
});
match reason {
Some((mob_dsl::CancelAllWorkRejectReasonKind::MobNotRunning, _, _)) => {
MobError::InvalidTransition {
from: current_state,
to: MobState::Running,
}
}
Some((mob_dsl::CancelAllWorkRejectReasonKind::MemberNotFound, _, _)) => {
MobError::MemberNotFound(agent_identity.clone())
}
Some((
mob_dsl::CancelAllWorkRejectReasonKind::StaleFenceToken,
Some(expected),
Some(actual),
)) => MobError::StaleFenceToken {
runtime_id: runtime_id.clone(),
expected: FenceToken::new(expected.0),
actual: FenceToken::new(actual.0),
},
Some((mob_dsl::CancelAllWorkRejectReasonKind::StaleFenceToken, _, _)) => {
MobError::Internal(
"MobMachine rejected CancelAllWork as stale without fence-token feedback"
.into(),
)
}
None => MobError::Internal(
"MobMachine rejected CancelAllWork without typed rejection feedback".into(),
),
}
}
fn preview_policy_spawn_submit_work_admission(
&self,
agent_identity: &AgentIdentity,
authorized_profile_material: &AuthorizedSpawnProfileMaterial,
work_ref: &WorkRef,
origin: WorkOrigin,
) -> Result<(), MobError> {
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let generation = self.mint_spawn_generation(agent_identity)?;
let domain_runtime_id = crate::ids::AgentRuntimeId::new(domain_identity, generation);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&domain_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(self.next_fence_token_preview()?);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let mut authority =
mob_dsl::MobMachineAuthority::recover_from_state(self.dsl_authority.state().clone())
.map_err(|error| {
MobError::Internal(format!(
"spawn preview could not recover DSL authority state: {error}"
))
})?;
// Spawn-ladder preview: drive the member through `BeginSpawnExec` →
// `CommitSpawnMembership` so it is committed into membership before the
// `SubmitWork` admission guard is evaluated against the recovered
// authority.
let preview_bridge_session_id = mob_dsl::SessionId::from_domain(&SessionId::new());
let begin_spawn = mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material.profile_material_digest.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(preview_bridge_session_id.clone()),
replacing: replacing.clone(),
// Local-arm probe: no phase-1 spawn path requests placement, so
// the multi-host denial guards are vacuous here. The two
// actor-level facts are real observations; the spec/profile
// facts are unobservable at this probe (it has no spawn spec).
placement: None,
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: self.default_external_tools_provider.is_some(),
default_llm_client_override_present: self.default_llm_client.is_some(),
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
};
mob_dsl::MobMachineMutator::apply(&mut authority, begin_spawn)
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
let commit_membership = mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material.profile_material_digest.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(preview_bridge_session_id),
replacing,
// Local arm: materialization-ack facts exist only for remote
// placement (phase 3).
member_peer_endpoint: None,
spec_digest_echo: None,
ack_engine_version: None,
placed_spawn_id: None,
provision_operation_id: None,
};
mob_dsl::MobMachineMutator::apply(&mut authority, commit_membership)
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: mob_dsl::WorkId::from_work_ref(work_ref),
origin: mob_dsl::WorkOrigin::from(origin),
},
)
.map(|_| ())
.map_err(|_| {
Self::resolve_submit_work_rejection_in_authority(
&mut authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&domain_runtime_id,
origin,
agent_identity,
self.state(),
)
})
}
fn complete_orchestrator_spawn(
&mut self,
spawn_ticket: Option<u64>,
agent_identity: &AgentIdentity,
context: &'static str,
) -> Result<AuthorizedMobSpawnCompleted, MobError> {
let dsl_agent_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let transition = match self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::CompleteSpawn {
agent_identity: dsl_agent_identity,
},
"complete_spawn",
) {
Ok(transition) => transition,
Err(error) => {
if let Some(spawn_ticket) = spawn_ticket {
tracing::warn!(
spawn_ticket,
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
} else {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
}
return Err(error);
}
};
let completed = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::EmitMemberLifecycleNotice {
kind: mob_dsl::MemberLifecycleKind::Spawned
}
)
});
if !completed {
let error = MobError::Internal(format!(
"MobMachine CompleteSpawn did not authorize spawn completion notice for '{agent_identity}'"
));
if let Some(spawn_ticket) = spawn_ticket {
tracing::warn!(
spawn_ticket,
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
} else {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
}
return Err(error);
}
let completed = AuthorizedMobSpawnCompleted {
generated_plan:
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart,
generated_effect: generated_mob_command_capabilities::CommandPlanKind::SpawnEffect,
agent_identity: agent_identity.clone(),
};
debug_assert_eq!(
completed.generated_effect,
generated_mob_command_capabilities::CommandPlanKind::SpawnEffect
);
debug_assert_eq!(
completed.generated_plan,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
Ok(completed)
}
async fn flow_tracker_alignment_violation(&self) -> Option<String> {
let snapshot = self.flow_tracker_snapshot().await;
let run_task_ids = snapshot.run_task_ids;
let run_token_ids = snapshot.cancel_token_ids;
if run_task_ids != run_token_ids {
return Some(format!(
"run task/token tracker mismatch: tasks={run_task_ids:?}, tokens={run_token_ids:?}"
));
}
let stream_ids = snapshot.stream_ids;
let unknown_streams = stream_ids
.iter()
.filter(|run_id| !run_task_ids.contains(*run_id))
.cloned()
.collect::<Vec<_>>();
if !unknown_streams.is_empty() {
return Some(format!(
"flow stream tracker contains unknown runs: {unknown_streams:?}"
));
}
None
}
async fn flow_tracker_snapshot(&self) -> super::MobFlowTrackerSnapshot {
super::MobFlowTrackerSnapshot {
run_task_ids: self
.run_tasks
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
cancel_token_ids: self
.run_cancel_tokens
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
stream_ids: self
.flow_streams
.lock()
.await
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
tracked_flows: self
.run_cancel_tokens
.iter()
.map(|(run_id, (_, flow_id))| (run_id.clone(), flow_id.clone()))
.collect(),
}
}
async fn ensure_flow_tracker_alignment(&self, context: &str) -> Result<(), MobError> {
if let Some(message) = self.flow_tracker_alignment_violation().await {
return Err(MobError::Internal(format!(
"{context}: flow tracker alignment violation: {message}"
)));
}
Ok(())
}
async fn cleanup_namespace(&self) -> Result<(), MobError> {
Ok(())
}
fn fallback_spawn_prompt(
&self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
) -> String {
format!(
"You have been spawned as '{}' (role: {}) in mob '{}'.",
agent_identity, profile_name, self.definition.id
)
}
/// Start the autonomous runtime for a member and optionally deliver its
/// initial prompt.
///
/// Sets up the keep-alive infrastructure (comms drain, dispatch capability)
/// then, when `prompt` is present, delivers it as a normal turn. Identity
/// reconciliation passes `None`: the resumed session already owns its
/// transcript and must not receive a manufactured kickoff.
///
/// Two paths:
/// - **Runtime-backed (adapter present):** Builds `Input::Prompt` and calls
/// `accept_input_with_completion` for a true admission ack. Spawns a
/// background task for completion wait + barrier signal.
/// - **No adapter (test/ephemeral):** Falls back to `provisioner.start_turn()`
/// in a spawned task with yield-check for immediate failure detection.
#[cfg(feature = "runtime-adapter")]
async fn start_autonomous_member(
&mut self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
prompt: Option<meerkat_core::types::ContentInput>,
) -> Result<(), MobError> {
self.ensure_autonomous_runtime_ready(agent_identity, member_ref)
.await?;
let startup_marker = {
let roster = self.roster.read().await;
roster
.get_by_identity(&AgentIdentity::from(agent_identity.as_str()))
.map(|entry| {
(
mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
)
})
}
.ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' missing roster entry for startup readiness"
))
})?;
if !self
.dsl_authority
.state()
.member_startup_ready
.contains(&startup_marker.0)
{
self.apply_dsl_input(
mob_dsl::MobMachineInput::StartupMarkReady {
agent_runtime_id: startup_marker.0,
fence_token: startup_marker.1,
},
"start_autonomous_member/startup_mark_ready",
)?;
}
let Some(prompt) = prompt else {
tracing::debug!(
agent_identity = %agent_identity,
"autonomous member runtime resumed without a fresh kickoff"
);
return Ok(());
};
let bridge_session_id = member_ref.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' must be session-backed"
))
})?;
let adapter = self.runtime_adapter.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' requires admission-capable substrate (runtime adapter)"
))
})?;
let kickoff_turn_metadata =
machine_kickoff_turn_metadata(self.dsl_authority.state(), agent_identity)?;
{
// Runtime-backed path: true admission ack via accept_input_with_completion.
use meerkat_runtime::{Input, InputHeader, PromptInput};
let input = Input::Prompt(PromptInput {
injected_context: Vec::new(),
header: InputHeader {
id: meerkat_core::lifecycle::InputId::new(),
timestamp: chrono::Utc::now(),
source: meerkat_runtime::InputOrigin::Operator,
durability: meerkat_runtime::InputDurability::Durable,
visibility: meerkat_runtime::InputVisibility::default(),
idempotency_key: None,
supersession_key: None,
correlation_id: None,
},
content: prompt,
typed_turn_appends: Vec::new(),
turn_metadata: kickoff_turn_metadata,
});
let (_outcome, completion_handle) = adapter
.accept_input_with_completion(bridge_session_id, input)
.await
.map_err(|e| {
MobError::Internal(format!(
"autonomous prompt admission failed for '{agent_identity}': {e}"
))
})?;
// Spawn background task for completion wait.
let log_id = agent_identity.clone();
let completion_command_tx = self.command_tx.clone();
let handle = tokio::spawn(async move {
if let Some(h) = completion_handle {
let outcome = h.wait().await;
let (ack_tx, ack_rx) = oneshot::channel();
if completion_command_tx
.send(RoutedMobCommand::internal(
MobCommand::KickoffOutcomeResolved {
agent_identity: log_id.clone(),
outcome,
ack_tx,
},
))
.await
.is_err()
{
tracing::warn!(
agent_identity = %log_id,
"mob actor dropped before kickoff outcome could be recorded"
);
} else {
let _ = ack_rx.await;
}
}
});
self.autonomous_initial_turns
.lock()
.await
.insert(agent_identity.clone(), InitialTurnHandle { handle });
}
tracing::debug!(agent_identity = %agent_identity, "autonomous member started");
Ok(())
}
async fn ensure_autonomous_runtime_ready(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
// Session registration + RuntimeLoop attachment is owned by the
// provisioner's lazy `runtime_session_state()` init (called during
// provision_member). stop_autonomous_member preserves registration
// (only aborts the drain), so resume just needs to re-spawn the drain.
self.ensure_mob_comms_drain(agent_identity, member_ref)
.await?;
self.ensure_autonomous_dispatch_capability(agent_identity, member_ref)
.await
}
async fn ensure_mob_comms_drain(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
if super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity) {
// The member host owns this session and its drain. The optional
// session carried by a projected BackendPeer is a remote fence,
// never a controller-local runtime key.
return Ok(());
}
#[cfg(all(not(target_arch = "wasm32"), feature = "runtime-adapter"))]
{
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Ok(());
};
let adapter =
self.runtime_adapter
.clone()
.ok_or_else(|| MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "local member comms-drain runtime adapter",
})?;
let comms_runtime = self
.provisioner
.comms_runtime(member_ref)
.await
.ok_or_else(|| MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "local member comms-drain startup",
})?;
let mob_id =
meerkat_runtime::meerkat_machine::dsl::MobId::from(self.definition.id.as_ref());
let spawned = adapter
.maybe_spawn_mob_comms_drain(bridge_session_id, comms_runtime, mob_id)
.await
.map_err(|err| {
MobError::Internal(format!(
"mob comms drain spawn failed for session {bridge_session_id}: {err}"
))
})?;
if spawned {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %bridge_session_id,
"updated peer ingress for mob member"
);
}
}
#[cfg(any(target_arch = "wasm32", not(feature = "runtime-adapter")))]
{
let _ = (agent_identity, member_ref);
}
Ok(())
}
async fn teardown_session_runtime_bindings_from_machine(&self) -> Result<(), MobError> {
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
let state = self.dsl_authority.state();
let session_ids = state
.member_session_bindings
.iter()
.filter(|(identity, _)| !state.member_placement.contains_key(*identity))
.map(|(_, session_id)| session_id)
.map(|session_id| {
SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"shutdown found invalid machine-owned session binding '{}': {error}",
session_id.0
))
})
})
.collect::<Result<Vec<_>, MobError>>()?;
let mut failures = Vec::new();
for session_id in session_ids {
// `unregister_session` now runs the two-phase drain internally
// (0.7.2 D1): it aborts the comms drain task *and* awaits its
// quiescence before committing teardown, so a separate
// pre-unregister `abort_comms_drain` here is redundant.
let unregister_result = match adapter.unregister_session(&session_id).await {
// Unregister is an independently-owned saga. Its caller
// grace can race the final teardown acknowledgement, so
// join (or deliberately retry) that exact registration
// once more before classifying the binding as unresolved.
// A second in-progress result remains a bounded,
// retryable shutdown failure.
Err(meerkat_runtime::RuntimeDriverError::UnregisterInProgress { .. }) => {
adapter.unregister_session(&session_id).await
}
result => result,
};
if let Err(error) = unregister_result {
failures.push(format!(
"failed to unregister runtime session {session_id} during mob teardown: {error}"
));
}
}
if !failures.is_empty() {
return Err(MobError::Internal(failures.join("; ")));
}
}
Ok(())
}
async fn ensure_autonomous_dispatch_capability_for_provisioner(
provisioner: &Arc<dyn MobProvisioner>,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
let bridge_session_id = member_ref.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' must be session-backed for injector dispatch"
))
})?;
if provisioner
.interaction_event_injector(bridge_session_id)
.await
.is_none()
{
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous member dispatch",
});
}
Ok(())
}
#[cfg(feature = "runtime-adapter")]
async fn resolve_kickoff_outcome(
&mut self,
agent_identity: &AgentIdentity,
outcome: Result<
meerkat_runtime::completion::CompletionOutcome,
meerkat_runtime::completion::CompletionWaitError,
>,
) -> Result<(), MobError> {
let outcome = match outcome {
Ok(outcome) => outcome,
Err(error) => {
return self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffResolveFailed {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
error: format!("runtime completion waiter failed: {error}"),
},
"resolve_kickoff_outcome_wait_error",
)
.await
.map(|_| ());
}
};
if let meerkat_runtime::completion::CompletionOutcome::CallbackPending {
tool_name,
args,
..
} = &outcome
{
tracing::debug!(
agent_identity = %agent_identity,
tool_name = %tool_name,
args = ?args,
"autonomous kickoff reached callback-pending boundary"
);
}
let _ = self
.apply_kickoff_input(
agent_identity,
kickoff_resolution_input(agent_identity, outcome),
"resolve_kickoff_outcome",
)
.await?;
Ok(())
}
async fn ensure_autonomous_dispatch_capability(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
if super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity) {
// Placed autonomous members receive input through the supervisor
// bridge; their injector capability is owned by the member host.
return Ok(());
}
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
agent_identity,
member_ref,
)
.await
}
/// Phase one of every autonomous stop: durably close kickoff origin, then
/// abort and join its volatile producer. This method performs no bridge
/// I/O, so every member in a mob-wide Stop reaches durable cancellation
/// even when another host is blackholed.
async fn request_autonomous_kickoff_stop(
&mut self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
let mut kickoff_handle = self
.autonomous_initial_turns
.lock()
.await
.remove(agent_identity);
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let kickoff_in_flight = self
.dsl_authority
.state()
.member_kickoff_pending
.contains(&dsl_identity)
|| self
.dsl_authority
.state()
.member_kickoff_starting
.contains(&dsl_identity)
|| self
.dsl_authority
.state()
.member_kickoff_callback_pending
.contains(&dsl_identity);
if kickoff_in_flight {
if let Err(error) = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffCancelRequested {
member_id: dsl_identity.clone(),
},
"request_autonomous_kickoff_stop",
)
.await
{
if self.durable_uncertainty_fail_stop {
// An unreadable write-then-error cancellation may already
// be durable. Join every volatile producer before the
// actor-loop fail-stop boundary so cold replay remains the
// only authority allowed to decide whether work can start.
if let Some(handle) = kickoff_handle.take() {
handle.abort_and_join().await;
}
self.quiesce_volatile_producers_after_fail_stop().await;
} else if let Some(handle) = kickoff_handle.take() {
// A proved-not-committed cancellation leaves the member
// running. Restore ownership of its waiter so a later
// Stop retry can still abort and join it.
self.autonomous_initial_turns
.lock()
.await
.insert(agent_identity.clone(), handle);
}
return Err(error);
}
}
if let Some(handle) = kickoff_handle {
handle.abort_and_join().await;
}
Ok(())
}
fn autonomous_stop_interrupt_incarnation(
&self,
entry: &RosterEntry,
) -> Result<AutonomousStopInterruptIncarnation, MobError> {
let expected_member =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity)
.then(|| self.placed_member_incarnation(entry))
.transpose()?;
Ok(AutonomousStopInterruptIncarnation {
member_ref: entry.member_ref.clone(),
expected_member,
})
}
async fn finish_autonomous_member_stop(
&self,
agent_identity: &AgentIdentity,
incarnation: &AutonomousStopInterruptIncarnation,
) -> Result<(), MobError> {
let member_ref = &incarnation.member_ref;
let expected_member = incarnation.expected_member.as_ref();
// Abort the comms drain but keep the session registered. A placed
// member's session and drain belong to the member host; its exact
// remote interrupt is the stop boundary.
#[cfg(feature = "runtime-adapter")]
if expected_member.is_none()
&& let (Some(adapter), Some(session_id)) =
(&self.runtime_adapter, member_ref.bridge_session_id())
{
adapter.abort_comms_drain(session_id).await.map_err(|err| {
MobError::Internal(format!(
"failed to abort comms drain for stopped member session {session_id}: {err}"
))
})?;
}
// Local sessions must be visibly idle before Stop can admit Resume.
// Placed sessions are fenced by the authenticated interrupt response.
let mut still_active = false;
if expected_member.is_none() {
for _ in 0..40 {
match self.provisioner.is_member_active(member_ref).await? {
Some(true) => tokio::time::sleep(std::time::Duration::from_millis(25)).await,
_ => {
still_active = false;
break;
}
}
still_active = true;
}
}
if still_active {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"autonomous member stop polling exhausted before member became idle"
);
}
Ok(())
}
/// Disposal owns an authenticated host Release/Revoke and then disposes
/// exact placed kickoff custody. It must not wait for the ordinary Stop
/// cancellation lane first: a blackholed cancel would otherwise prevent
/// the stronger release authority from ever running. Local members still
/// complete their in-process interrupt synchronously.
async fn stop_autonomous_member_for_disposal(
&mut self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
self.request_autonomous_kickoff_stop(agent_identity).await?;
if disposal_uses_host_release_authority(super::member_runtime_is_host_owned(
self.dsl_authority.state(),
agent_identity,
)) {
return Ok(());
}
let incarnation = AutonomousStopInterruptIncarnation {
member_ref: member_ref.clone(),
expected_member: None,
};
if let Err(error) = self.provisioner.interrupt_member(member_ref, None).await
&& !matches!(
error,
MobError::SessionError(meerkat_core::service::SessionError::NotFound { .. })
)
{
return Err(error);
}
self.finish_autonomous_member_stop(agent_identity, &incarnation)
.await
}
/// Poll or launch exact, timeout-bounded interrupts without awaiting their
/// bridge I/O on the actor loop. The durable Stop operation remains
/// retryable while receivers are pending; same-process successes are
/// retained so a slow host cannot make retries convoy healthy hosts.
fn drive_autonomous_stop_interrupts(
&mut self,
entries: &[RosterEntry],
) -> Result<(), MobError> {
let mut incarnations = BTreeMap::new();
for entry in entries {
incarnations.insert(
entry.agent_identity.clone(),
self.autonomous_stop_interrupt_incarnation(entry)?,
);
}
// Prune removed/replaced roster material before observing results.
// Dropping a receiver does not cancel its actor-owned task, but every
// remote command is timeout-bounded and exact-fenced; a stale result
// can no longer enter either success map.
self.autonomous_stop_interrupts.retain(|identity, task| {
incarnations
.get(identity)
.is_some_and(|current| current == &task.incarnation)
});
self.autonomous_stop_interrupted
.retain(|identity, completed| {
incarnations
.get(identity)
.is_some_and(|current| current == completed)
});
let mut first_error = None;
let mut failed_identities = BTreeSet::new();
let in_flight = self
.autonomous_stop_interrupts
.keys()
.cloned()
.collect::<Vec<_>>();
for agent_identity in in_flight {
let Some(task) = self.autonomous_stop_interrupts.get_mut(&agent_identity) else {
continue;
};
let result = task.result_rx.try_recv();
match result {
Ok(Ok(())) => {
if let Some(task) = self.autonomous_stop_interrupts.remove(&agent_identity) {
self.autonomous_stop_interrupted
.insert(agent_identity, task.incarnation);
}
}
Ok(Err(error)) => {
self.autonomous_stop_interrupts.remove(&agent_identity);
if matches!(&error, MobError::BridgeRequestTimedOut { .. }) {
tracing::debug!(
agent_identity = %agent_identity,
error = %error,
"autonomous stop interrupt timed out; exact retry remains pending"
);
} else {
failed_identities.insert(agent_identity);
if first_error.is_none() {
first_error = Some(error);
}
}
}
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {}
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
self.autonomous_stop_interrupts.remove(&agent_identity);
if first_error.is_none() {
first_error = Some(MobError::Internal(format!(
"autonomous stop interrupt task for '{agent_identity}' closed without a result"
)));
}
}
}
}
// Admit only the free portion of a fixed window. Rotation covers the
// full exact incarnation set, so a repeatedly failing low identity
// cannot monopolize every retry or starve the tail.
let capacity = MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS
.saturating_sub(self.autonomous_stop_interrupts.len());
let mut identities = incarnations.keys().cloned().collect::<Vec<_>>();
let mut rotation = 0usize;
if !identities.is_empty() {
rotation = self.autonomous_stop_interrupt_cursor % identities.len();
identities.rotate_left(rotation);
}
let mut launched = 0usize;
let mut last_selected_offset = None;
for (identity_offset, agent_identity) in identities.iter().enumerate() {
if launched == capacity {
break;
}
let Some(incarnation) = incarnations.get(agent_identity) else {
continue;
};
if self.autonomous_stop_interrupted.get(agent_identity) == Some(incarnation)
|| self.autonomous_stop_interrupts.contains_key(agent_identity)
|| failed_identities.contains(agent_identity)
{
continue;
}
let provisioner = Arc::clone(&self.provisioner);
let member_ref = incarnation.member_ref.clone();
let expected_member = incarnation.expected_member.clone();
let (result_tx, result_rx) = oneshot::channel();
self.actor_io_tasks.spawn(async move {
let result = provisioner
.interrupt_member(&member_ref, expected_member.as_ref())
.await;
let result = match result {
Err(MobError::SessionError(
meerkat_core::service::SessionError::NotFound { .. },
)) => Ok(()),
result => result,
};
let _ = result_tx.send(result);
});
self.autonomous_stop_interrupts.insert(
agent_identity.clone(),
AutonomousStopInterruptTask {
incarnation: incarnation.clone(),
result_rx,
},
);
launched += 1;
last_selected_offset = Some(identity_offset);
}
if !identities.is_empty() {
self.autonomous_stop_interrupt_cursor =
advance_rotating_cursor(identities.len(), rotation, last_selected_offset);
}
if let Some(error) = first_error {
return Err(error);
}
let pending = incarnations
.iter()
.filter(|(identity, incarnation)| {
self.autonomous_stop_interrupted.get(*identity) != Some(*incarnation)
})
.count();
if pending != 0 {
return Err(MobError::AutonomousStopInterruptsPending { pending });
}
Ok(())
}
async fn autonomous_shutdown_member_should_interrupt(
&mut self,
entry: &RosterEntry,
) -> Result<bool, MobError> {
let session_id = entry.member_ref.bridge_session_id().cloned();
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let placed_host = self
.dsl_authority
.state()
.member_placement
.get(&dsl_identity)
.cloned();
let (
session_has_live_actor,
session_projection_visible,
runtime_residue_present,
archive_authority_known,
) = match session_id.as_ref() {
Some(session_id) => {
let session_has_live_actor = self
.session_service
.live_session_actor_registered(session_id)
.await?;
let session_projection_visible = self
.session_service
.load_persisted_session(session_id)
.await?
.is_some();
#[cfg(feature = "runtime-adapter")]
let runtime_residue_present = match self.runtime_adapter.as_ref() {
Some(adapter) => adapter
.archive_runtime_residue_present(session_id)
.await
.map_err(|error| {
MobError::Internal(format!(
"autonomous shutdown runtime-residue observation failed for '{session_id}': {error}"
))
})?,
None => false,
};
#[cfg(not(feature = "runtime-adapter"))]
let runtime_residue_present = false;
let archive_authority_known = self
.session_service
.session_known_to_archive_authority(session_id)
.await?;
(
session_has_live_actor,
session_projection_visible,
runtime_residue_present,
archive_authority_known,
)
}
None => (false, false, false, false),
};
// A placed host release is terminal only for the exact current
// identity/host/session/generation/fence tuple. Keep the durable event
// as the shell observation and let MobMachine combine it with the
// retained placement/carrier state; a historical release from before
// MobReset or another incarnation must never suppress interruption.
let placed_release_confirmed = match (placed_host.as_ref(), session_id.as_ref()) {
(Some(host), Some(session_id)) => {
let all_events = self.events.replay_all().await?;
let mob_events = all_events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
let member_session_id = session_id.to_string();
mob_events[epoch_start..].iter().any(|event| {
matches!(
&event.kind,
MobEventKind::RemoteMemberReleaseConfirmed {
agent_identity,
host_id,
member_session_id: confirmed_session_id,
generation,
fence_token,
} if agent_identity == &entry.agent_identity
&& host_id == host.as_str()
&& confirmed_session_id == &member_session_id
&& generation == &entry.generation
&& fence_token == &entry.fence_token
)
})
}
_ => false,
};
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(entry.fence_token);
let dsl_generation = mob_dsl::Generation::from_domain(entry.generation);
let dsl_session_id = session_id.as_ref().map(mob_dsl::SessionId::from_domain);
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::ResolveAutonomousShutdownMemberAction {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
generation: dsl_generation,
session_id: dsl_session_id.clone(),
session_has_live_actor,
session_projection_visible,
runtime_residue_present,
archive_authority_known,
placed_release_confirmed,
},
"resolve_autonomous_shutdown_member_action",
)?;
let mut resolved = transition
.effects()
.iter()
.filter_map(|effect| match effect {
mob_dsl::MobMachineEffect::AutonomousShutdownMemberActionResolved {
action,
agent_identity,
agent_runtime_id,
fence_token,
generation,
session_id,
} if agent_identity == &dsl_identity
&& agent_runtime_id == &dsl_runtime_id
&& fence_token == &dsl_fence_token
&& generation == &dsl_generation
&& session_id == &dsl_session_id =>
{
Some(*action)
}
_ => None,
});
let action = resolved.next().ok_or_else(|| {
MobError::Internal(format!(
"MobMachine returned no autonomous shutdown action for '{}'",
entry.agent_identity
))
})?;
if resolved.next().is_some() {
return Err(MobError::Internal(format!(
"MobMachine returned multiple autonomous shutdown actions for '{}'",
entry.agent_identity
)));
}
Ok(matches!(
action,
mob_dsl::AutonomousShutdownMemberActionKind::Interrupt
))
}
async fn stop_all_autonomous_members(&mut self) -> Result<(), MobError> {
let entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost)
.cloned()
.collect::<Vec<_>>()
};
// Physical roster retention outlives archive when the terminal
// MemberRetired journal append fails. Feed raw carrier observations to
// MobMachine and mirror its exact-incarnation action; the actor does
// not infer whether a row is live or a publication-only retry anchor.
let mut interruptible_entries = Vec::with_capacity(entries.len());
for entry in entries {
if self
.autonomous_shutdown_member_should_interrupt(&entry)
.await?
{
interruptible_entries.push(entry);
} else {
self.autonomous_stop_interrupts
.remove(&entry.agent_identity);
self.autonomous_stop_interrupted
.remove(&entry.agent_identity);
}
}
let entries = interruptible_entries;
if entries.is_empty() {
self.autonomous_stop_interrupts.clear();
self.autonomous_stop_interrupted.clear();
return Ok(());
}
// Phase 1 is deliberately complete-before-error: one member's local
// persistence failure must not prevent later members from recording
// their own cancellation intent.
let mut first_error = None;
for entry in &entries {
if let Err(error) = self
.request_autonomous_kickoff_stop(&entry.agent_identity)
.await
{
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed recording autonomous kickoff stop intent"
);
if first_error.is_none() {
first_error = Some(error);
}
if self.durable_uncertainty_fail_stop {
break;
}
}
}
if let Some(error) = first_error {
return Err(error);
}
// Durable Cancelled authority permanently closes fresh kickoff
// origin for every member before any interrupt begins. Exact host
// cancellation/terminal folding is owned by the reconciler.
let state = self.dsl_authority.state();
let placed_kickoff_pending = state.pending_placed_kickoff_outcomes.len();
let placed_kickoff_resolved = state.resolved_placed_kickoff_outcomes.len();
if autonomous_stop_phase(placed_kickoff_pending, placed_kickoff_resolved)
== AutonomousStopPhase::WaitForPlacedKickoffCleanup
{
return Err(MobError::PlacedKickoffCleanupPending {
pending: placed_kickoff_pending,
resolved: placed_kickoff_resolved,
});
}
// Phase 2 launches all exact interrupts together and returns without
// waiting for their bridge I/O. A later retry observes completion.
self.drive_autonomous_stop_interrupts(&entries)?;
let mut first_error = None;
for entry in &entries {
let incarnation = self
.autonomous_stop_interrupted
.get(&entry.agent_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"autonomous stop lost interrupt success for '{}'",
entry.agent_identity
))
})?;
if let Err(error) = self
.finish_autonomous_member_stop(&entry.agent_identity, &incarnation)
.await
{
if first_error.is_none() {
first_error = Some(error);
}
} else {
self.autonomous_stop_interrupted
.remove(&entry.agent_identity);
}
}
if let Some(error) = first_error {
return Err(error);
}
Ok(())
}
/// Fence every local session at the explicit stopped -> running resume
/// seam. A same-handle resume keeps the exact active sidecar already
/// owned by this provisioner. A reconstructed handle retires the foreign
/// exact attachment (or discards an unattached actor) and returns the
/// members that must be rebuilt after the durable Resume transition.
///
/// Only `MemberRef::Session` is local session ownership. An unplaced
/// `BackendPeer` remains external peer-only material even if a legacy
/// machine journal still carries a stale session binding for it.
async fn prepare_explicit_resume_member_sessions(
&self,
) -> Result<Vec<ExplicitResumeMemberRebuild>, MobError> {
let entries = {
let roster = self.roster.read().await;
roster.list().cloned().collect::<Vec<_>>()
};
let candidates = {
let machine = self.dsl_authority.state();
entries
.into_iter()
.filter_map(|entry| {
if !matches!(&entry.member_ref, MemberRef::Session { .. }) {
return None;
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(
&crate::ids::AgentIdentity::from(entry.agent_identity.as_str()),
);
if machine.member_placement.contains_key(&dsl_identity) {
return None;
}
let session_id = match machine.member_session_bindings.get(&dsl_identity) {
Some(session_id) => match SessionId::parse(&session_id.0) {
Ok(session_id) => session_id,
Err(error) => {
return Some(Err(MobError::Internal(format!(
"MobMachine has invalid explicit-resume session binding '{}' for '{}': {error}",
session_id.0, entry.agent_identity
))));
}
},
None => return None,
};
let member_ref = Self::project_member_ref_session_binding(
&entry.member_ref,
Some(session_id.clone()),
)
.ok_or_else(|| {
MobError::Internal(format!(
"explicit resume cannot project the current session binding for '{}'",
entry.agent_identity
))
});
Some(member_ref.map(|member_ref| (entry, member_ref, session_id)))
})
.collect::<Result<Vec<_>, MobError>>()?
};
let mut rebuild = Vec::new();
let mut listed_sessions = None;
for (entry, member_ref, session_id) in candidates {
let requires_materialization = self
.provisioner
.prepare_member_session_for_explicit_resume(&session_id)
.await?;
if !requires_materialization {
continue;
}
let current_snapshot_present = self.session_service.supports_persistent_sessions()
&& self
.session_service
.load_persisted_session_metadata(&session_id)
.await
.map_err(MobError::SessionError)?
.is_some();
if current_snapshot_present || !self.session_service.supports_persistent_sessions() {
rebuild.push(ExplicitResumeMemberRebuild {
entry,
member_ref,
bridge_session_id: session_id,
requires_materialization,
repoints_session_binding: false,
recovered_peer_endpoint: None,
});
continue;
}
if listed_sessions.is_none() {
listed_sessions = Some(
self.session_service
.list(meerkat_core::service::SessionQuery::default())
.await
.map_err(MobError::SessionError)?,
);
}
let replacement = super::builder::latest_persisted_session_for_member(
self.session_service.as_ref(),
listed_sessions.as_deref().unwrap_or_default(),
&session_id,
&self.definition.id,
&entry.role,
&entry.agent_identity,
)
.await?;
let Some((replacement_session_id, _replacement_session)) = replacement else {
// No canonical persisted successor exists. Let the ordinary
// machine-owned classification record this exact binding as
// terminal Broken after Resume commits.
rebuild.push(ExplicitResumeMemberRebuild {
entry,
member_ref,
bridge_session_id: session_id,
requires_materialization,
repoints_session_binding: false,
recovered_peer_endpoint: None,
});
continue;
};
let replacement_member_ref = Self::project_member_ref_session_binding(
&entry.member_ref,
Some(replacement_session_id.clone()),
)
.ok_or_else(|| {
MobError::Internal(format!(
"explicit resume cannot project replacement session binding for '{}'",
entry.agent_identity
))
})?;
let replacement_requires_materialization = self
.provisioner
.prepare_member_session_for_explicit_resume(&replacement_session_id)
.await?;
// Publish an endpoint from a live successor only when this exact
// provisioner is allowed to reuse that attachment. If preparation
// retires foreign attachment A, its descriptor is stale by
// definition and must never be journaled over replacement B. The
// existing post-materialization upgrade below records B instead.
let recovered_peer_endpoint = if replacement_requires_materialization {
None
} else {
let fallback_name = render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
entry.agent_identity.as_str(),
)?;
match self
.provisioner
.comms_runtime(&replacement_member_ref)
.await
{
Some(runtime) => {
super::provisioner::SessionBackend::trusted_peer_spec_from_runtime(
&fallback_name,
runtime.as_ref(),
)?
}
None => None,
}
};
rebuild.push(ExplicitResumeMemberRebuild {
entry,
member_ref: replacement_member_ref,
bridge_session_id: replacement_session_id,
requires_materialization: replacement_requires_materialization,
repoints_session_binding: true,
recovered_peer_endpoint,
});
}
Ok(rebuild)
}
/// Rebuild only the sessions whose prior exact attachment was retired by
/// the explicit resume preparation. The lifecycle is already durably
/// Running, so the existing machine-owned classify -> materialize ->
/// resolve seam is authoritative. Terminal member restore failures remain
/// a partial-resume success; infrastructure failures are surfaced after
/// every independent candidate has had a chance to converge.
async fn rebuild_explicit_resume_member_sessions(
&mut self,
rebuild: Vec<ExplicitResumeMemberRebuild>,
) -> Result<(), MobError> {
let mut first_infrastructure_error = None;
for rebuild in rebuild {
let ExplicitResumeMemberRebuild {
mut entry,
member_ref,
bridge_session_id,
requires_materialization,
repoints_session_binding,
recovered_peer_endpoint,
} = rebuild;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(
&crate::ids::AgentIdentity::from(entry.agent_identity.as_str()),
);
if self
.dsl_authority
.state()
.member_restore_failures
.contains_key(&dsl_identity)
{
continue;
}
if repoints_session_binding {
// `RecoverMemberSessionBinding` is a Running-only machine
// transition. The helper stages binding + endpoint on one
// detached authority, appends the replay-authority event, and
// publishes the machine only after that write succeeds. Apply
// the same event to the live roster projection afterward.
let event = super::builder::append_recovered_session_binding(
&mut self.dsl_authority,
&self.events,
&self.definition.id,
&entry,
&bridge_session_id,
recovered_peer_endpoint.clone(),
"explicit_resume_repoint_missing_member_session_binding",
)
.await?;
self.roster.write().await.apply_event(&event);
let _ = self
.machine_state_watch_tx
.send(self.dsl_authority.state().clone());
}
if !requires_materialization {
continue;
}
// A cold actor has no persisted copy of per-spawn external tools.
// The public SpawnMemberCustomizer contract explicitly re-supplies
// those process-local mechanics at SpawnSource::Resume. Startup
// reconciliation already seeds this map for a Running mob; the
// explicit Stopped -> Running seam must do the same before its
// machine-authorized revival build.
let mut restore_spec = super::handle::SpawnMemberSpec::new(
entry.role.clone(),
entry.agent_identity.clone(),
);
restore_spec.runtime_mode = Some(entry.runtime_mode);
restore_spec.labels = Some(entry.labels.clone());
restore_spec.override_profile = entry.effective_profile_override.clone();
restore_spec.model_override = entry.effective_model_override.clone();
self.customize_spawn_spec(super::handle::SpawnSource::Resume, None, &mut restore_spec)?;
if restore_spec.identity != entry.agent_identity {
return Err(MobError::Internal(format!(
"spawn customizer cannot change explicit-resume identity from '{}' to '{}'",
entry.agent_identity, restore_spec.identity
)));
}
if restore_spec.role_name != entry.role {
return Err(MobError::Internal(format!(
"spawn customizer cannot change explicit-resume profile for '{}' from '{}' to '{}'",
entry.agent_identity, entry.role, restore_spec.role_name
)));
}
entry.effective_profile_override = restore_spec.override_profile.clone();
entry.effective_model_override = restore_spec.model_override.clone();
entry.labels = restore_spec
.labels
.clone()
.unwrap_or_else(|| entry.labels.clone());
{
let mut retained = self.per_spawn_external_tools.write().await;
if let Some(tools) = restore_spec.external_tools {
retained.insert(entry.agent_identity.clone(), tools);
} else {
retained.remove(&entry.agent_identity);
}
}
{
let mut roster = self.roster.write().await;
let mut snapshot = roster.snapshot();
if let Some(roster_entry) = snapshot.get_by_identity_mut(&entry.agent_identity) {
roster_entry.effective_profile_override =
entry.effective_profile_override.clone();
roster_entry.effective_model_override = entry.effective_model_override.clone();
}
*roster = super::roster_authority::RosterAuthority::from_roster(snapshot);
}
match self
.revive_member_live_materialization(
&entry,
&member_ref,
&bridge_session_id,
repoints_session_binding && recovered_peer_endpoint.is_none(),
false,
)
.await
{
Ok(()) => {}
Err(MobError::MemberRestoreFailed { .. }) => {
// The machine durably classified this one member Broken.
// Other members and the resumed mob remain usable.
}
Err(error) => {
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %entry.agent_identity,
bridge_session_id = %bridge_session_id,
error = %error,
"explicit resume could not converge one member runtime"
);
if first_infrastructure_error.is_none() {
first_infrastructure_error = Some(error);
}
}
}
}
if let Some(error) = first_infrastructure_error {
return Err(error);
}
Ok(())
}
/// Ensure all autonomous roster members have their runtime ready.
///
/// Called on mob startup and resume. Does NOT fire synthetic kickoff turns —
/// the keep-alive runtime infrastructure is sufficient. On resume, the
/// member's session already has its conversation history. Once a recovered
/// local autonomous member's drain and injector have both been observed,
/// publish that exact runtime/fence through MobMachine's existing startup
/// readiness transition. Durable membership alone is not readiness.
async fn ensure_autonomous_runtimes_from_roster(
&mut self,
allow_stopped_resume_reopen: bool,
) -> Result<(), MobError> {
let lifecycle = self.dsl_authority.state();
if lifecycle_origin_fenced(lifecycle) {
if !allow_stopped_resume_reopen {
tracing::debug!(
mob_id = %self.definition.id,
intent = ?lifecycle.placed_completion_lifecycle_intent,
"autonomous runtime startup remains fenced by durable lifecycle intent"
);
return Ok(());
}
if self.state() != MobState::Stopped
|| lifecycle.placed_completion_lifecycle_intent
!= Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop)
{
return Err(MobError::LifecycleOperationPending {
intent: format!("{:?}", lifecycle.placed_completion_lifecycle_intent),
});
}
}
let broken_members = self
.dsl_authority
.state()
.member_restore_failures
.keys()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<HashSet<_>>();
let placed_members = self
.dsl_authority
.state()
.member_placement
.keys()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<HashSet<_>>();
let entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| {
entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& !broken_members.contains(&entry.agent_identity)
&& !placed_members.contains(&entry.agent_identity)
})
.cloned()
.collect::<Vec<_>>()
};
if entries.is_empty() {
// Turn-driven resumed members still need their mob-owned comms
// drain rebound even though they do not need autonomous dispatch.
}
let mut first_error: Option<MobError> = None;
let all_entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| {
entry.runtime_mode != crate::MobRuntimeMode::AutonomousHost
&& !broken_members.contains(&entry.agent_identity)
&& !placed_members.contains(&entry.agent_identity)
})
.cloned()
.collect::<Vec<_>>()
};
for entry in &all_entries {
let ensure_result = tokio::time::timeout(std::time::Duration::from_secs(5), async {
self.provisioner
.ensure_runtime_session_state(&entry.member_ref)
.await?;
self.ensure_mob_comms_drain(&entry.agent_identity, &entry.member_ref)
.await
})
.await;
let result = match ensure_result {
Ok(result) => result,
Err(_elapsed) => {
tracing::warn!(
agent_identity = %entry.agent_identity,
"timed out ensuring mob comms drain ready"
);
// A readiness timeout is a confirmed-never fault, not a
// resumable no-op: surface it so Resume cannot report
// success when comms-drain readiness was never observed.
if first_error.is_none() {
first_error = Some(MobError::ReadyWaitTimedOut {
pending_member_ids: vec![AgentIdentity::from(
entry.agent_identity.as_str(),
)],
});
}
continue;
}
};
if let Err(error) = result {
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed ensuring mob comms drain ready"
);
if first_error.is_none() {
first_error = Some(error);
}
}
}
for entry in &entries {
let ensure_result = tokio::time::timeout(
std::time::Duration::from_secs(5),
self.ensure_autonomous_runtime_ready(&entry.agent_identity, &entry.member_ref),
)
.await;
let result = match ensure_result {
Ok(result) => result,
Err(_elapsed) => {
tracing::warn!(
agent_identity = %entry.agent_identity,
"timed out ensuring autonomous runtime ready"
);
// A readiness timeout is a confirmed-never fault, not a
// resumable no-op: surface it so Resume cannot report
// success when autonomous-runtime readiness was never
// observed.
if first_error.is_none() {
first_error = Some(MobError::ReadyWaitTimedOut {
pending_member_ids: vec![AgentIdentity::from(
entry.agent_identity.as_str(),
)],
});
}
continue;
}
};
if let Err(error) = result {
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed ensuring autonomous runtime ready"
);
if first_error.is_none() {
first_error = Some(error);
}
continue;
}
// Cold Running recovery rebuilds session runtime mechanics before
// the actor starts, then reaches this shared check without the
// volatile startup marker created by the original process. Feed
// the successful observation through the same exact-runtime/fence
// machine transition used by fresh spawn. Never infer this from
// the durable roster, and never apply it while the mob is Stopped:
// same-handle resume performs these checks before its durable
// Resume commit, while a rebuilt attachment checks again after it.
if self.state() == MobState::Running {
let runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
if !self
.dsl_authority
.state()
.member_startup_ready
.contains(&runtime_id)
&& let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::StartupMarkReady {
agent_runtime_id: runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
},
"ensure_autonomous_runtimes_from_roster/startup_mark_ready",
)
{
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed publishing autonomous runtime startup readiness"
);
if first_error.is_none() {
first_error = Some(error);
}
}
}
}
if let Some(error) = first_error {
return Err(error);
}
Ok(())
}
/// Chokepoint (a) — MobCommand admission (DEC-P5E-4, ADJ-P5-12/15).
///
/// Runs against the actor's OWN machine state (fully serialized — no
/// TOCTOU) and fires BEFORE any handler logic, so a denied principal
/// learns nothing about lifecycle/roster state.
fn admit_command_scope(&self, authority: &CommandAuthority, cmd: MobCommand) -> ScopeAdmission {
// A remote member-operator request may await durable Pending IO before
// routing its effect. Revalidate its exact residency first, even for
// commands classified as internal/query admissions: a fenced handle
// must never gain an unfenced early-return path. Deferred commands
// re-enter this gate and therefore revalidate again.
if let Some(fence) = authority.member_operator_execution_fence()
&& let Err(error) = self.validate_member_operator_execution_fence(fence)
{
cmd.reject_with_error(error);
return ScopeAdmission::Denied;
}
let Some(required) = cmd.required_control_scope() else {
// Internal/machine lane: no principal semantics.
return ScopeAdmission::Admitted(cmd);
};
match authority.kind() {
// §15.2: the agent authority lane's own gates (machine-composed
// MobToolAuthorityContext admission) already ran; principal
// grants neither gate nor are satisfied here.
CommandAuthorityKind::AgentLane => ScopeAdmission::Admitted(cmd),
// An operator-class command minted without authority is a wiring
// fault: fail closed at runtime, loud in test lanes (ADJ-P5-15).
// Any legitimate case found later must be explicitly
// reclassified or carry the originating principal — silent
// admission is forbidden (FLAG-P5E-6).
CommandAuthorityKind::Internal => {
debug_assert!(
false,
"internal authority presented operator-class command {} \
(required {required:?})",
cmd.kind()
);
tracing::error!(
command_kind = cmd.kind(),
?required,
"internal authority presented an operator-class command; denied"
);
cmd.reject_scope_denied(ScopeDenial {
required,
presented: BTreeSet::new(),
});
ScopeAdmission::Denied
}
CommandAuthorityKind::Principal => {
let Some(principal) = authority.control_principal() else {
// Principal-lane authority without a principal cannot
// exist by construction; fail closed if it ever does.
cmd.reject_scope_denied(ScopeDenial {
required,
presented: BTreeSet::new(),
});
return ScopeAdmission::Denied;
};
let policy = self.resolve_control_policy_for(principal);
for required in std::iter::once(required) {
if let Err(denial) = policy.require(required) {
cmd.reject_scope_denied(denial);
return ScopeAdmission::Denied;
}
}
ScopeAdmission::Admitted(cmd)
}
}
}
fn validate_member_operator_execution_fence(
&self,
fence: &MemberOperatorExecutionFence,
) -> Result<(), MobError> {
if self.member_operator_residency_is_current(
&fence.agent_identity,
fence.generation,
fence.fence_token,
&fence.host_id,
fence.host_binding_generation,
&fence.requester_member_session_id,
) {
return Ok(());
}
Err(MobError::StaleMemberOperatorAuthority {
member_id: AgentIdentity::from(fence.agent_identity.as_str()),
reason: format!(
"expected host='{}' host_generation={} session='{}' member_generation={} fence={}",
fence.host_id,
fence.host_binding_generation,
fence.requester_member_session_id,
fence.generation,
fence.fence_token,
),
})
}
fn member_operator_residency_is_current(
&self,
agent_identity: &str,
generation: u64,
fence_token: u64,
host_id: &str,
host_binding_generation: u64,
member_session_id: &str,
) -> bool {
let identity = mob_dsl::AgentIdentity::from(agent_identity);
let host = mob_dsl::HostId::from(host_id);
let state = self.dsl_authority.state();
state.identity_to_runtime.contains_key(&identity)
&& state
.identity_runtime_generations
.get(&identity)
.map(|value| value.0)
== Some(generation)
&& state
.identity_runtime_fence_tokens
.get(&identity)
.map(|value| value.0)
== Some(fence_token)
&& state
.member_session_bindings
.get(&identity)
.map(|value| value.0.as_str())
== Some(member_session_id)
&& state.member_placement.get(&identity) == Some(&host)
&& state
.current_placed_spawn_host_binding_generations
.get(&identity)
.copied()
== Some(host_binding_generation)
&& state.host_bind_phase.get(&host) == Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_binding_generations.get(&host).copied() == Some(host_binding_generation)
}
async fn prune_stale_member_operator_requests(&self) -> Result<u64, MobError> {
let state = self.dsl_authority.state();
let mut current = BTreeSet::new();
for (identity, host) in &state.member_placement {
let Some(generation) = state.identity_runtime_generations.get(identity) else {
continue;
};
let Some(fence_token) = state.identity_runtime_fence_tokens.get(identity) else {
continue;
};
let Some(member_session_id) = state.member_session_bindings.get(identity) else {
continue;
};
let Some(host_binding_generation) = state
.current_placed_spawn_host_binding_generations
.get(identity)
.copied()
else {
continue;
};
if !self.member_operator_residency_is_current(
identity.0.as_str(),
generation.0,
fence_token.0,
host.as_str(),
host_binding_generation,
member_session_id.0.as_str(),
) {
continue;
}
current.insert(crate::store::MobMemberOperatorResidency {
agent_identity: identity.0.clone(),
generation: generation.0,
fence_token: fence_token.0,
host_id: host.as_str().to_string(),
host_binding_generation,
member_session_id: member_session_id.0.clone(),
});
}
let authority =
crate::store::MobMemberOperatorPruneAuthority::from_actor_current_residencies(current);
self.runtime_metadata
.prune_stale_member_operator_requests(&self.definition.id, &authority)
.await
.map_err(MobError::from)
}
/// Revalidate an asynchronously submitted remote-outcome command against
/// the actor's current full placed residency. A pump can enqueue this
/// command just before a host-binding promotion aborts it; mailbox order
/// then delivers the command only after G2 publication. Exact actor-time
/// validation prevents that queued G1 page from mutating custody or ACK
/// state after the promotion barrier.
fn validate_remote_turn_outcome_residency(
&self,
obligation: &mob_dsl::RemoteTurnObligation,
) -> Result<(), MobError> {
let state = self.dsl_authority.state();
let identity = &obligation.agent_identity;
let host = &obligation.host_id;
let exact = state.member_placement.get(identity) == Some(host)
&& state
.current_placed_spawn_host_binding_generations
.get(identity)
.copied()
== Some(obligation.host_binding_generation)
&& state.member_session_bindings.get(identity) == Some(&obligation.member_session_id)
&& state.identity_runtime_generations.get(identity) == Some(&obligation.generation)
&& state.identity_runtime_fence_tokens.get(identity) == Some(&obligation.fence_token)
&& state.host_bind_phase.get(host) == Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_binding_generations.get(host).copied()
== Some(obligation.host_binding_generation);
if exact {
return Ok(());
}
Err(MobError::Internal(format!(
"remote-turn outcome residency for '{}' is no longer current (host='{}', host_generation={}, session='{}', member_generation={}, fence={})",
obligation.agent_identity.0,
obligation.host_id.0,
obligation.host_binding_generation,
obligation.member_session_id.0,
obligation.generation.0,
obligation.fence_token.0,
)))
}
fn validate_placed_kickoff_outcome_residency(
&self,
obligation: &mob_dsl::PlacedKickoffObligation,
) -> Result<(), MobError> {
let state = self.dsl_authority.state();
let identity = &obligation.agent_identity;
let host = &obligation.host_id;
let exact = state.member_placement.get(identity) == Some(host)
&& state
.current_placed_spawn_host_binding_generations
.get(identity)
.copied()
== Some(obligation.host_binding_generation)
&& state.member_session_bindings.get(identity) == Some(&obligation.member_session_id)
&& state.identity_runtime_generations.get(identity) == Some(&obligation.generation)
&& state.identity_runtime_fence_tokens.get(identity) == Some(&obligation.fence_token)
&& state.host_bind_phase.get(host) == Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_binding_generations.get(host).copied()
== Some(obligation.host_binding_generation)
&& state.member_kickoff_objective_ids.get(identity) == Some(&obligation.objective_id)
&& state.member_kickoff_input_ids.get(identity) == Some(&obligation.input_id);
if exact {
return Ok(());
}
Err(MobError::Internal(format!(
"placed-kickoff outcome residency for '{}' is no longer current (host='{}', host_generation={}, session='{}', member_generation={}, fence={}, input='{}')",
obligation.agent_identity.0,
obligation.host_id.0,
obligation.host_binding_generation,
obligation.member_session_id.0,
obligation.generation.0,
obligation.fence_token.0,
obligation.input_id.0,
)))
}
async fn prepared_kickoff_snapshot(
&self,
prepared: &PreparedDslTransition,
agent_identity: &AgentIdentity,
) -> Result<crate::roster::MobMemberKickoffSnapshot, MobError> {
let timestamp_hint = self
.roster
.read()
.await
.get(agent_identity)
.and_then(|entry| entry.kickoff.clone());
placed_kickoff_snapshot_for_structural_event(
prepared.authority.state(),
agent_identity,
timestamp_hint.as_ref(),
SystemTime::now(),
)
}
async fn emit_prepared_kickoff_notices(
&self,
agent_identity: &AgentIdentity,
effects: &[mob_dsl::MobMachineEffect],
) {
for effect in effects {
if let mob_dsl::MobMachineEffect::EmitKickoffLifecycleNotice { intent, .. } = effect {
let notice_intent = Self::kickoff_notice_intent(*intent);
if let Err(error) = self
.notify_kickoff_event(agent_identity, notice_intent)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
intent = %intent,
"failed to emit durable placed-kickoff lifecycle notice"
);
}
}
}
}
async fn apply_placed_kickoff_host_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
input: mob_dsl::MobMachineInput,
outcome: crate::event::PlacedKickoffHostOutcomeEvent,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
self.validate_placed_kickoff_outcome_residency(&obligation)?;
let structural_replay = !self
.dsl_authority
.state()
.pending_placed_kickoff_outcomes
.contains(&obligation);
let prepared =
self.prepare_dsl_input_transition(input, "resolve_placed_kickoff_outcome")?;
let effects = prepared.transition.effects().to_vec();
if structural_replay {
if !effects.is_empty() {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(
"placed-kickoff outcome replay unexpectedly emitted fresh effects".to_string(),
));
}
self.require_existing_placed_kickoff_structural_carrier(
PlacedKickoffStructuralReplay::Resolved {
obligation: &obligation_event,
outcome: &outcome,
},
)
.await?;
return self.commit_prepared_dsl_transition(prepared);
}
let identity = obligation_event.agent_identity.clone();
let existing = self
.existing_placed_kickoff_structural_carrier(PlacedKickoffStructuralReplay::Resolved {
obligation: &obligation_event,
outcome: &outcome,
})
.await?;
let kickoff = match existing {
Some(kickoff) => kickoff,
None => {
let kickoff = self.prepared_kickoff_snapshot(&prepared, &identity).await?;
self.ensure_exact_structural_event_batch(vec![
MobEventKind::PlacedKickoffOutcomeResolved {
obligation: obligation_event,
outcome,
kickoff: kickoff.clone(),
},
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: kickoff.clone(),
},
])
.await?;
kickoff
}
};
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed-kickoff Resolve is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.roster
.write()
.await
.set_kickoff(&identity, Some(kickoff));
self.emit_prepared_kickoff_notices(&identity, &effects)
.await;
Ok(())
}
async fn resolve_placed_kickoff_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
record: super::bridge_protocol::BridgeTurnOutcomeRecord,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
self.validate_placed_kickoff_outcome_residency(&obligation)?;
if record.input_id != obligation.input_id.0
|| record.generation != obligation.generation.0
|| record.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"placed-kickoff outcome record does not match exact custody".to_string(),
));
}
let (input, outcome) = match record.outcome {
super::bridge_protocol::WireFlowTurnOutcome::InteractionComplete => (
mob_dsl::MobMachineInput::ResolvePlacedKickoffStarted {
obligation: obligation.clone(),
},
crate::event::PlacedKickoffHostOutcomeEvent::InteractionComplete,
),
super::bridge_protocol::WireFlowTurnOutcome::InteractionCallbackPending => (
mob_dsl::MobMachineInput::ResolvePlacedKickoffCallbackPending {
obligation: obligation.clone(),
},
crate::event::PlacedKickoffHostOutcomeEvent::InteractionCallbackPending,
),
super::bridge_protocol::WireFlowTurnOutcome::InteractionFailed { detail } => {
let error = if detail.truncated {
format!(
"{} [host retained {} of {} UTF-8 bytes]",
detail.text,
detail.text.len(),
detail.original_utf8_bytes
)
} else {
detail.text
};
(
mob_dsl::MobMachineInput::ResolvePlacedKickoffFailed {
obligation: obligation.clone(),
error: error.clone(),
},
crate::event::PlacedKickoffHostOutcomeEvent::InteractionFailed { error },
)
}
other => {
return Err(MobError::Internal(format!(
"placed-kickoff host journal carried non-interaction terminal {other:?}"
)));
}
};
self.apply_placed_kickoff_host_outcome_in_actor(obligation_event, input, outcome)
.await
}
async fn resolve_placed_kickoff_cancelled_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
self.apply_placed_kickoff_host_outcome_in_actor(
obligation_event,
mob_dsl::MobMachineInput::ResolvePlacedKickoffCancelled { obligation },
crate::event::PlacedKickoffHostOutcomeEvent::InteractionCancelled,
)
.await
}
async fn apply_placed_kickoff_cancel_receipt_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
response: super::bridge_protocol::BridgeTrackedInputCancelResponse,
) -> Result<(), MobError> {
let expected_member = super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: obligation_event.agent_identity.to_string(),
host_id: obligation_event.host_id.clone(),
binding_generation: obligation_event.host_binding_generation,
member_session_id: obligation_event.member_session_id.clone(),
generation: obligation_event.generation.get(),
fence_token: obligation_event.fence_token.get(),
};
if response.expected_member != expected_member
|| response.input_id != obligation_event.input_id
{
return Err(MobError::Internal(
"tracked kickoff cancellation receipt does not match exact custody".to_string(),
));
}
match response.outcome {
super::bridge_protocol::BridgeTrackedInputCancelOutcome::NoEffect => {
self.reject_placed_kickoff_before_admission_in_actor(
obligation_event,
super::placed_kickoff_reconciler::TRACKED_INPUT_CANCEL_NO_EFFECT.to_string(),
)
.await
}
super::bridge_protocol::BridgeTrackedInputCancelOutcome::Cancelled => {
let ack_event = obligation_event.clone();
self.resolve_placed_kickoff_cancelled_in_actor(obligation_event)
.await?;
self.acknowledge_placed_kickoff_outcome_in_actor(
ack_event.clone(),
super::bridge_protocol::BridgeTurnOutcomeAck {
generation: ack_event.generation.get(),
fence_token: ack_event.fence_token.get(),
input_id: ack_event.input_id.clone(),
},
)
.await
}
super::bridge_protocol::BridgeTrackedInputCancelOutcome::Terminal { record } => {
self.resolve_placed_kickoff_outcome_in_actor(obligation_event, record)
.await
}
_ => Err(MobError::Internal(
"member host returned an unsupported tracked-input cancellation outcome"
.to_string(),
)),
}
}
async fn start_placed_kickoff_obligation_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
// Start itself binds the input id, so validate the full placement and
// objective tuple here without requiring `member_kickoff_input_ids`
// to be populated yet. The generated transition repeats these guards.
let state = self.dsl_authority.state();
let identity = &obligation.agent_identity;
let host = &obligation.host_id;
let structural_replay =
state.retained_placed_kickoff_obligations.get(identity) == Some(&obligation);
let exact = state.member_placement.get(identity) == Some(host)
&& state
.current_placed_spawn_host_binding_generations
.get(identity)
.copied()
== Some(obligation.host_binding_generation)
&& state.member_session_bindings.get(identity) == Some(&obligation.member_session_id)
&& state.identity_runtime_generations.get(identity) == Some(&obligation.generation)
&& state.identity_runtime_fence_tokens.get(identity) == Some(&obligation.fence_token)
&& state.member_kickoff_objective_ids.get(identity) == Some(&obligation.objective_id);
if !exact {
return Err(MobError::Internal(format!(
"placed-kickoff start tuple is stale for '{}'",
obligation.agent_identity.0
)));
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::StartPlacedKickoff { obligation },
"start_placed_kickoff",
)?;
if !structural_replay {
let tracked_key = PlacedTrackedInputKey::kickoff(&obligation_event);
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.reserve_fresh_tracked_input(
tracked_key,
PlacedTrackedInputOwner::Kickoff(obligation_event.clone()),
)?;
}
let effects = prepared.transition.effects().to_vec();
if structural_replay {
if !effects.is_empty() {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(
"placed-kickoff Start replay unexpectedly emitted fresh effects".to_string(),
));
}
self.require_existing_placed_kickoff_structural_carrier(
PlacedKickoffStructuralReplay::Record {
obligation: &obligation_event,
},
)
.await?;
return self.commit_prepared_dsl_transition(prepared);
}
let domain_identity = obligation_event.agent_identity.clone();
let existing = self
.existing_placed_kickoff_structural_carrier(PlacedKickoffStructuralReplay::Record {
obligation: &obligation_event,
})
.await?;
let kickoff = match existing {
Some(kickoff) => kickoff,
None => {
let kickoff = self
.prepared_kickoff_snapshot(&prepared, &domain_identity)
.await?;
self.ensure_exact_structural_event_batch(vec![
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation_event,
},
MobEventKind::MemberKickoffUpdated {
member: domain_identity.clone(),
kickoff: kickoff.clone(),
},
])
.await?;
kickoff
}
};
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed-kickoff Record is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.roster
.write()
.await
.set_kickoff(&domain_identity, Some(kickoff));
self.emit_prepared_kickoff_notices(&domain_identity, &effects)
.await;
Ok(())
}
async fn acknowledge_placed_kickoff_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
ack: super::bridge_protocol::BridgeTurnOutcomeAck,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
self.validate_placed_kickoff_outcome_residency(&obligation)?;
if ack.input_id != obligation.input_id.0
|| ack.generation != obligation.generation.0
|| ack.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"placed-kickoff outcome ACK does not match exact custody".to_string(),
));
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::AcknowledgePlacedKickoffOutcome { obligation },
"acknowledge_placed_kickoff_outcome",
)?;
self.ensure_exact_structural_event(MobEventKind::PlacedKickoffOutcomeAcknowledged {
obligation: obligation_event,
})
.await?;
self.commit_prepared_dsl_transition(prepared)
}
async fn reject_placed_kickoff_before_admission_in_actor(
&mut self,
obligation_event: crate::event::PlacedKickoffObligationEvent,
error: String,
) -> Result<(), MobError> {
let obligation =
super::remote_flow_ticket::placed_kickoff_obligation_from_event(&obligation_event);
self.validate_placed_kickoff_outcome_residency(&obligation)?;
let structural_replay = !self
.dsl_authority
.state()
.pending_placed_kickoff_outcomes
.contains(&obligation);
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RejectPlacedKickoffBeforeAdmission {
obligation,
error: error.clone(),
},
"reject_placed_kickoff_before_admission",
)?;
let effects = prepared.transition.effects().to_vec();
if structural_replay {
if !effects.is_empty() {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(
"placed-kickoff rejection replay unexpectedly emitted fresh effects"
.to_string(),
));
}
self.require_existing_placed_kickoff_structural_carrier(
PlacedKickoffStructuralReplay::RejectedNoEffect {
obligation: &obligation_event,
error: &error,
},
)
.await?;
return self.commit_prepared_dsl_transition(prepared);
}
let identity = obligation_event.agent_identity.clone();
let existing = self
.existing_placed_kickoff_structural_carrier(
PlacedKickoffStructuralReplay::RejectedNoEffect {
obligation: &obligation_event,
error: &error,
},
)
.await?;
let kickoff = match existing {
Some(kickoff) => kickoff,
None => {
let kickoff = self.prepared_kickoff_snapshot(&prepared, &identity).await?;
self.ensure_exact_structural_event_batch(vec![
MobEventKind::PlacedKickoffRejectedNoEffect {
obligation: obligation_event,
error,
kickoff: kickoff.clone(),
},
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: kickoff.clone(),
},
])
.await?;
kickoff
}
};
self.commit_prepared_dsl_transition(prepared)?;
self.roster
.write()
.await
.set_kickoff(&identity, Some(kickoff));
self.emit_prepared_kickoff_notices(&identity, &effects)
.await;
Ok(())
}
/// Resolve the sealed policy for `principal` against the actor's own
/// machine grant facts. The owner short-circuits to full scope WITHOUT a
/// wall-clock read (A16); named principals read the clock exactly once
/// per decision (DEC-P5P-6).
fn resolve_control_policy_for(&self, principal: &MobControlPrincipal) -> ResolvedControlPolicy {
let now_ms = if matches!(principal, MobControlPrincipal::Owner) {
0
} else {
super::scope_gate::control_now_ms()
};
ResolvedControlPolicy::resolve(principal, self.dsl_authority.state(), now_ms)
}
/// One canonical path, no silent sibling (DEC-P5P-8): the raw
/// machine-input lanes must not mint or revoke grants — a raw grant
/// input would be ungated (no AdminGrants check) AND unpersisted (no
/// durable record): an in-memory-only scope escalation that silently
/// vanishes on restart. Pre-1.0: no compatibility carve-out.
fn reject_raw_grant_machine_input(input: &mob_dsl::MobMachineInput) -> Result<(), MobError> {
match input {
mob_dsl::MobMachineInput::GrantOperatorScopes { .. }
| mob_dsl::MobMachineInput::RevokeOperatorScopes { .. } => Err(MobError::Internal(
"grant inputs must route through the typed grant verbs \
(grant_scopes / revoke_scopes); the raw machine-input lane \
cannot mint or revoke control-scope grants"
.into(),
)),
mob_dsl::MobMachineInput::RecordRemoteTurnObligation { .. }
| mob_dsl::MobMachineInput::AbortRemoteTurnObligation { .. }
| mob_dsl::MobMachineInput::CommitRemoteTurnOutcome { .. }
| mob_dsl::MobMachineInput::ResolveRemoteTurnObligation { .. }
| mob_dsl::MobMachineInput::AcknowledgeRemoteTurnOutcome { .. }
| mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { .. }
| mob_dsl::MobMachineInput::RecordPlacedCompletionObligation { .. }
| mob_dsl::MobMachineInput::RequestPlacedCompletionCancellation { .. }
| mob_dsl::MobMachineInput::ResolvePlacedCompletionOutcome { .. }
| mob_dsl::MobMachineInput::ClosePlacedCompletionOutcome { .. }
| mob_dsl::MobMachineInput::AcknowledgePlacedCompletionOutcome { .. }
| mob_dsl::MobMachineInput::DisposePlacedCompletionOutcome { .. }
| mob_dsl::MobMachineInput::BeginPlacedCompletionLifecycleQuiesce { .. }
| mob_dsl::MobMachineInput::EndPlacedCompletionLifecycleQuiesce { .. }
| mob_dsl::MobMachineInput::StartPlacedKickoff { .. }
| mob_dsl::MobMachineInput::ResolvePlacedKickoffStarted { .. }
| mob_dsl::MobMachineInput::ResolvePlacedKickoffCallbackPending { .. }
| mob_dsl::MobMachineInput::ResolvePlacedKickoffFailed { .. }
| mob_dsl::MobMachineInput::ResolvePlacedKickoffCancelled { .. }
| mob_dsl::MobMachineInput::RejectPlacedKickoffBeforeAdmission { .. }
| mob_dsl::MobMachineInput::AcknowledgePlacedKickoffOutcome { .. }
| mob_dsl::MobMachineInput::DisposePlacedKickoffObligation { .. } => {
Err(MobError::Internal(
"remote interaction custody inputs must route through sealed actor verbs with their durable intent, host observation, ACK, or release proof"
.into(),
))
}
_ => Ok(()),
}
}
fn remote_turn_carrier_present(
events: &[crate::event::MobEvent],
mob_id: &MobId,
desired: &MobEventKind,
) -> Result<bool, MobError> {
fn phase_and_obligation(
kind: &MobEventKind,
) -> Option<(u8, &crate::event::RemoteTurnObligationEvent)> {
match kind {
MobEventKind::RemoteTurnObligationRecorded { obligation } => Some((0, obligation)),
MobEventKind::StepTargetCompleted {
remote_turn_obligation: Some(obligation),
..
}
| MobEventKind::StepTargetFailed {
remote_turn_obligation: Some(obligation),
..
} => Some((1, obligation)),
MobEventKind::RemoteTurnOutcomeResolved { obligation } => Some((2, obligation)),
MobEventKind::RemoteTurnOutcomeAcknowledged { obligation } => Some((3, obligation)),
MobEventKind::RemoteTurnOutcomeDisposed { obligation } => Some((4, obligation)),
_ => None,
}
}
let (desired_phase, desired_obligation) =
phase_and_obligation(desired).ok_or_else(|| {
MobError::Internal("attempted to ensure a non remote-turn carrier".to_string())
})?;
let mob_events = events
.iter()
.filter(|event| &event.mob_id == mob_id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
for event in &mob_events[epoch_start..] {
if event.kind == *desired {
return Ok(true);
}
let Some((phase, obligation)) = phase_and_obligation(&event.kind) else {
continue;
};
if phase == desired_phase
&& obligation.dispatch_sequence == desired_obligation.dispatch_sequence
{
return Err(MobError::Internal(format!(
"remote-turn carrier conflict at phase {desired_phase} sequence {}",
desired_obligation.dispatch_sequence
)));
}
}
Ok(false)
}
async fn ensure_remote_turn_carrier(&self, desired: MobEventKind) -> Result<(), MobError> {
let events = self.events.replay_all().await?;
if Self::remote_turn_carrier_present(&events, &self.definition.id, &desired)? {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&desired)?;
return Ok(());
}
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await;
match append {
Ok(_) => {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&desired)?;
Ok(())
}
Err(error) => {
// Wrote-then-error is resolved by an actor-serialized read.
// Only a proven exact carrier turns the ambiguous response
// into success; absence preserves the original error.
let events = self.events.replay_all().await?;
if Self::remote_turn_carrier_present(&events, &self.definition.id, &desired)? {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rebuild_from_events(&events, &self.definition.id)?;
Ok(())
} else {
Err(MobError::from(error))
}
}
}
}
async fn ensure_placed_completion_carrier(
&mut self,
desired: MobEventKind,
) -> Result<(), MobError> {
let indexed = self
.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.desired_present(&desired);
match indexed {
Ok(true) => return Ok(()),
Ok(false) => {}
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
}
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await;
match append {
Ok(_) => {
if let Err(error) = self
.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&desired)
{
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
Ok(())
}
Err(error) => {
let replay = match self.events.replay_all().await {
Ok(replay) => replay,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed-completion append failed ({error}) and replay was unreadable; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
};
let present = {
let mut index = self
.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match index.rebuild_from_events(&replay, &self.definition.id) {
Ok(()) => index.desired_present(&desired),
Err(conflict) => Err(conflict),
}
};
match present {
Ok(true) => Ok(()),
Ok(false) => Err(MobError::from(error)),
Err(conflict) => {
self.durable_uncertainty_fail_stop = true;
Err(conflict)
}
}
}
}
}
fn validate_placed_completion_residency(
&self,
obligation: &mob_dsl::PlacedCompletionObligation,
) -> Result<(), MobError> {
let state = self.dsl_authority.state();
let identity = &obligation.agent_identity;
let host = &obligation.host_id;
let exact = state.member_placement.get(identity) == Some(host)
&& state
.current_placed_spawn_host_binding_generations
.get(identity)
== Some(&obligation.host_binding_generation)
&& state.host_binding_generations.get(host)
== Some(&obligation.host_binding_generation)
&& state.member_session_bindings.get(identity) == Some(&obligation.member_session_id)
&& state.identity_runtime_generations.get(identity) == Some(&obligation.generation)
&& state.identity_runtime_fence_tokens.get(identity) == Some(&obligation.fence_token);
if exact {
Ok(())
} else {
Err(MobError::Internal(format!(
"placed completion sequence {} no longer matches current residency",
obligation.dispatch_sequence
)))
}
}
async fn request_placed_completion_cancellation_in_actor(
&mut self,
obligation_event: crate::event::PlacedCompletionObligationEvent,
) -> Result<(), MobError> {
let obligation =
super::placed_completion_reconciler::obligation_from_event(&obligation_event)?;
let pending = self
.dsl_authority
.state()
.pending_placed_completion_outcomes
.contains(&obligation);
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RequestPlacedCompletionCancellation { obligation },
"request_placed_completion_cancellation",
)?;
if pending {
self.ensure_placed_completion_carrier(
MobEventKind::PlacedCompletionCancellationRequested {
obligation: obligation_event,
},
)
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion cancellation carrier is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
return Ok(());
}
self.commit_prepared_dsl_transition(prepared)
}
async fn resolve_placed_completion_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedCompletionObligationEvent,
record: super::bridge_protocol::BridgeTurnOutcomeRecord,
) -> Result<(), MobError> {
let obligation =
super::placed_completion_reconciler::obligation_from_event(&obligation_event)?;
let was_pending = self
.dsl_authority
.state()
.pending_placed_completion_outcomes
.contains(&obligation);
let was_resolved = self
.dsl_authority
.state()
.resolved_placed_completion_outcomes
.contains(&obligation);
if was_pending || was_resolved {
self.validate_placed_completion_residency(&obligation)?;
}
if record.input_id != obligation.input_id.0
|| record.generation != obligation.generation.0
|| record.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"placed completion terminal does not match exact custody".to_string(),
));
}
let waiter_record = record.clone();
let outcome = match record.outcome {
super::bridge_protocol::WireFlowTurnOutcome::InteractionComplete => {
crate::event::PlacedCompletionHostOutcomeEvent::InteractionComplete
}
super::bridge_protocol::WireFlowTurnOutcome::InteractionCallbackPending => {
crate::event::PlacedCompletionHostOutcomeEvent::InteractionCallbackPending
}
super::bridge_protocol::WireFlowTurnOutcome::InteractionFailed { detail } => {
let error = if detail.truncated {
format!(
"{} [host retained {} of {} UTF-8 bytes]",
detail.text,
detail.text.len(),
detail.original_utf8_bytes
)
} else {
detail.text
};
crate::event::PlacedCompletionHostOutcomeEvent::InteractionFailed { error }
}
other => {
return Err(MobError::Internal(format!(
"placed completion host journal carried non-interaction terminal {other:?}"
)));
}
};
let desired = MobEventKind::PlacedCompletionOutcomeResolved {
obligation: obligation_event.clone(),
outcome,
};
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::ResolvePlacedCompletionOutcome { obligation },
"resolve_placed_completion_outcome",
)?;
if was_pending {
self.ensure_placed_completion_carrier(desired).await?;
} else {
let exact = self
.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.require_exact(&desired);
if let Err(error) = exact {
if was_resolved {
self.durable_uncertainty_fail_stop = true;
}
return Err(error);
}
}
if !was_pending && !was_resolved {
return self.commit_prepared_dsl_transition(prepared);
}
// Commit semantic custody first, resolve the exact volatile observer
// second, and only then publish the resolved/ACK-pending watch state.
// This ordering is shared by ordinary pump fold and the authenticated
// CancelTrackedInput::Terminal alternate authority.
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
if let Err(error) = self
.dsl_authority
.commit_prepared_authority(prepared.authority)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Resolve carrier is durable but prepared commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.queue_routed_effects_from(&effects);
let expected_member = super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: obligation_event.agent_identity.to_string(),
host_id: obligation_event.host_id,
binding_generation: obligation_event.host_binding_generation,
member_session_id: obligation_event.member_session_id,
generation: obligation_event.generation.get(),
fence_token: obligation_event.fence_token.get(),
};
if let Err(error) = self
.member_event_pumps
.resolve_placed_completion_waiter(&expected_member, &waiter_record)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Resolve committed but waiter handoff failed before ACK publication; actor is fail-stopping for cold recovery: {error}"
)));
}
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(())
}
async fn close_placed_completion_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedCompletionObligationEvent,
closure: crate::event::PlacedCompletionClosureEvent,
) -> Result<(), MobError> {
let obligation =
super::placed_completion_reconciler::obligation_from_event(&obligation_event)?;
let was_pending = self
.dsl_authority
.state()
.pending_placed_completion_outcomes
.contains(&obligation);
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::ClosePlacedCompletionOutcome { obligation },
"close_placed_completion_outcome",
)?;
let desired = MobEventKind::PlacedCompletionOutcomeClosed {
obligation: obligation_event.clone(),
closure,
};
if was_pending {
self.ensure_placed_completion_carrier(desired).await?;
} else {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.require_exact(&desired)?;
return self.commit_prepared_dsl_transition(prepared);
}
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
if let Err(error) = self
.dsl_authority
.commit_prepared_authority(prepared.authority)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Close carrier is durable but prepared commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.queue_routed_effects_from(&effects);
let expected_member = super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: obligation_event.agent_identity.to_string(),
host_id: obligation_event.host_id,
binding_generation: obligation_event.host_binding_generation,
member_session_id: obligation_event.member_session_id,
generation: obligation_event.generation.get(),
fence_token: obligation_event.fence_token.get(),
};
if let Err(error) = self
.member_event_pumps
.resolve_placed_completion_waiter_closed(
&expected_member,
&obligation_event.input_id,
closure,
)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Close committed but waiter handoff failed; actor is fail-stopping for cold recovery: {error}"
)));
}
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(())
}
async fn acknowledge_placed_completion_outcome_in_actor(
&mut self,
obligation_event: crate::event::PlacedCompletionObligationEvent,
ack: super::bridge_protocol::BridgeTurnOutcomeAck,
) -> Result<(), MobError> {
let obligation =
super::placed_completion_reconciler::obligation_from_event(&obligation_event)?;
let was_resolved = self
.dsl_authority
.state()
.resolved_placed_completion_outcomes
.contains(&obligation);
if was_resolved {
self.validate_placed_completion_residency(&obligation)?;
}
if ack.input_id != obligation.input_id.0
|| ack.generation != obligation.generation.0
|| ack.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"placed completion ACK does not match exact custody".to_string(),
));
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::AcknowledgePlacedCompletionOutcome { obligation },
"acknowledge_placed_completion_outcome",
)?;
let desired = MobEventKind::PlacedCompletionOutcomeAcknowledged {
obligation: obligation_event,
};
if was_resolved {
self.ensure_placed_completion_carrier(desired).await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion ACK carrier is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
return Ok(());
}
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.require_exact(&desired)?;
self.commit_prepared_dsl_transition(prepared)
}
async fn ensure_placed_completion_quiesce_marker(
&mut self,
quiescing: bool,
intent: mob_dsl::PlacedCompletionLifecycleIntentKind,
) -> Result<(), MobError> {
let desired = if quiescing {
MobEventKind::PlacedCompletionLifecycleQuiesceStarted { intent }
} else {
MobEventKind::PlacedCompletionLifecycleQuiesceEnded { intent }
};
let latest_matches = |events: &[crate::event::MobEvent]| {
events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.filter_map(|event| match event.kind {
MobEventKind::PlacedCompletionLifecycleQuiesceStarted { intent: recorded } => {
Some(Some(recorded))
}
MobEventKind::PlacedCompletionLifecycleQuiesceEnded { .. }
| MobEventKind::MobReset => Some(None),
_ => None,
})
.next_back()
== Some(if quiescing { Some(intent) } else { None })
};
let replay = self.events.replay_all().await?;
if latest_matches(&replay) {
return Ok(());
}
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired,
})
.await;
match append {
Ok(_) => Ok(()),
Err(error) => {
let replay = match self.events.replay_all().await {
Ok(replay) => replay,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"completion quiesce marker append failed ({error}) and replay was unreadable; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
};
if latest_matches(&replay) {
Ok(())
} else {
Err(MobError::from(error))
}
}
}
}
async fn begin_placed_completion_lifecycle_quiesce(
&mut self,
intent: mob_dsl::PlacedCompletionLifecycleIntentKind,
) -> Result<(), MobError> {
let target = match intent {
mob_dsl::PlacedCompletionLifecycleIntentKind::Stop => MobState::Stopped,
mob_dsl::PlacedCompletionLifecycleIntentKind::Complete => MobState::Completed,
mob_dsl::PlacedCompletionLifecycleIntentKind::Destroy => MobState::Destroyed,
mob_dsl::PlacedCompletionLifecycleIntentKind::Reset
| mob_dsl::PlacedCompletionLifecycleIntentKind::RetireAll => MobState::Running,
};
let prepared = self
.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::BeginPlacedCompletionLifecycleQuiesce { intent },
"begin_placed_completion_lifecycle_quiesce",
)
.map_err(|_| self.invalid_transition_to(target))?;
Self::require_placed_completion_lifecycle_intent_effect(
&prepared.transition,
intent,
true,
"begin_placed_completion_lifecycle_quiesce",
)?;
// The actor has already accepted the lifecycle command and processes no
// later public command until this handler returns. Drain every mutating
// live effect admitted before it, then prove/close the one active
// channel (if any) for every retained member. Only after this barrier is
// it safe to publish the durable work-origin fence: queued Open/Control
// commands will subsequently fail machine eligibility, and no direct WS
// input plane survives into Stopped/Completed/Destroyed cleanup.
self.drain_member_live_mutations_for_lifecycle().await?;
self.close_all_member_live_channels_for_lifecycle(
"begin placed completion lifecycle quiesce",
)
.await?;
self.ensure_placed_completion_quiesce_marker(true, intent)
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"completion quiesce Start is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(())
}
async fn end_placed_completion_lifecycle_quiesce(
&mut self,
intent: mob_dsl::PlacedCompletionLifecycleIntentKind,
) -> Result<(), MobError> {
if !self
.dsl_authority
.state()
.placed_completion_lifecycle_quiescing
{
return Ok(());
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::EndPlacedCompletionLifecycleQuiesce { intent },
"end_placed_completion_lifecycle_quiesce",
)?;
let (persisted_intent, remains_active) = match self.dsl_authority.state().lifecycle_phase {
mob_dsl::MobPhase::Running => (intent, false),
mob_dsl::MobPhase::Stopped => {
(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop, true)
}
mob_dsl::MobPhase::Completed => {
(mob_dsl::PlacedCompletionLifecycleIntentKind::Complete, true)
}
mob_dsl::MobPhase::Destroyed => {
return Err(MobError::Internal(
"cannot end placed completion lifecycle intent after Destroyed".to_string(),
));
}
};
Self::require_placed_completion_lifecycle_intent_effect(
&prepared.transition,
persisted_intent,
remains_active,
"end_placed_completion_lifecycle_quiesce",
)?;
self.ensure_placed_completion_quiesce_marker(remains_active, persisted_intent)
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"completion quiesce End is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(())
}
async fn resume_lifecycle_after_quiesce(&mut self) -> Result<(), MobError> {
let intent = self
.dsl_authority
.state()
.placed_completion_lifecycle_intent
.ok_or_else(|| {
MobError::Internal(
"resume found completion quiesce without typed lifecycle intent".to_string(),
)
})?;
let prepared = self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Resume, "resume_input")
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
Self::require_placed_completion_lifecycle_intent_effect(
&prepared.transition,
intent,
false,
"resume_input",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Resumed,
"resume_input",
)?;
// Store-first: a crash after this marker but before machine commit is
// still Stopped and cannot originate work; retry reuses the latest End
// marker and commits the prepared Resume.
self.ensure_placed_completion_quiesce_marker(false, intent)
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"completion quiesce End is durable but Resume commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(())
}
async fn commit_stopped_lifecycle_after_cleanup(&mut self) -> Result<(), MobError> {
let prepared = self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Stop, "stop_input")
.map_err(|_| self.invalid_transition_to(MobState::Stopped))?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Stopped,
"stop_input",
)?;
if let Err(error) = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobStopped,
})
.await
{
// The event store may have written before returning an error. A
// cold replay can decide from the exact MobStopped carrier; the
// live actor must not continue under an ambiguous public phase.
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"Stop completion marker append was ambiguous; actor is fail-stopping for cold recovery: {error}"
)));
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"MobStopped is durable but Stop machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(())
}
/// Close work origin, durably request exact cancellation, and make one
/// bounded convergence attempt per matching Pending row. Resolved rows
/// deliberately return retryable: the actor must unblock so the sole pump
/// can obtain ACK confirmation through its normal sequential cursor lane.
async fn drive_placed_completion_lifecycle_cleanup(
&mut self,
only_identity: Option<&AgentIdentity>,
require_drained: bool,
intent: Option<mob_dsl::PlacedCompletionLifecycleIntentKind>,
) -> Result<(), MobError> {
if only_identity.is_none() {
let intent = intent.ok_or_else(|| {
MobError::Internal(
"global placed completion cleanup requires typed lifecycle intent".to_string(),
)
})?;
self.begin_placed_completion_lifecycle_quiesce(intent)
.await?;
} else if intent.is_some() {
return Err(MobError::Internal(
"member-scoped placed completion cleanup cannot carry global lifecycle intent"
.to_string(),
));
}
let matches_identity = |obligation: &mob_dsl::PlacedCompletionObligation| {
only_identity.is_none_or(|identity| obligation.agent_identity.0 == identity.as_str())
};
let pending = self
.dsl_authority
.state()
.pending_placed_completion_outcomes
.iter()
.filter(|obligation| matches_identity(obligation))
.cloned()
.collect::<Vec<_>>();
for obligation in pending {
if !self
.dsl_authority
.state()
.cancel_requested_placed_completion_outcomes
.contains(&obligation)
{
self.request_placed_completion_cancellation_in_actor(
super::placed_completion_reconciler::obligation_event(&obligation)?,
)
.await?;
}
}
let state = self.dsl_authority.state();
let pending = state
.pending_placed_completion_outcomes
.iter()
.filter(|obligation| matches_identity(obligation))
.count();
let resolved = state
.resolved_placed_completion_outcomes
.iter()
.filter(|obligation| matches_identity(obligation))
.count();
if !require_drained || (pending == 0 && resolved == 0) {
Ok(())
} else {
Err(MobError::PlacedCompletionCleanupPending { pending, resolved })
}
}
async fn existing_placed_kickoff_structural_carrier(
&mut self,
expected: PlacedKickoffStructuralReplay<'_>,
) -> Result<Option<crate::roster::MobMemberKickoffSnapshot>, MobError> {
let replay = match self.events.replay_all().await {
Ok(events) => events,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"{} replay for input '{}' could not read its durable carrier; actor is fail-stopping for cold recovery: {error}",
expected.carrier_name(),
expected.input_id()
)));
}
};
match existing_placed_kickoff_structural_carrier(&replay, &self.definition.id, expected) {
Ok(existing) => Ok(existing),
Err(error) => {
self.durable_uncertainty_fail_stop = true;
Err(error)
}
}
}
async fn require_existing_placed_kickoff_structural_carrier(
&mut self,
expected: PlacedKickoffStructuralReplay<'_>,
) -> Result<(), MobError> {
let carrier_name = expected.carrier_name();
let input_id = expected.input_id().to_string();
if self
.existing_placed_kickoff_structural_carrier(expected)
.await?
.is_some()
{
return Ok(());
}
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"{carrier_name} replay for input '{input_id}' has no exact durable carrier"
)))
}
async fn ensure_exact_structural_event(
&mut self,
desired: MobEventKind,
) -> Result<(), MobError> {
let present = |events: &[crate::event::MobEvent]| {
let mob_events = events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
mob_events[epoch_start..]
.iter()
.any(|event| event.kind == desired)
};
if present(&self.events.replay_all().await?) {
if matches!(
&desired,
MobEventKind::RemoteTurnObligationRecorded { .. }
| MobEventKind::PlacedKickoffObligationRecorded { .. }
) {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&desired)?;
}
return Ok(());
}
match self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
{
Ok(_) => {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&desired)?;
Ok(())
}
Err(error) => {
let replay = match self.events.replay_all().await {
Ok(events) => events,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"structural event append failed ({error}) and durable reconciliation remained unreadable; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
};
if present(&replay) {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rebuild_from_events(&replay, &self.definition.id)?;
Ok(())
} else {
Err(MobError::from(error))
}
}
}
}
/// Append an exact structural batch idempotently. Event-store batches are
/// all-or-nothing; after an ambiguous write response, an exact replay of
/// every requested event is the only proof that permits machine commit.
async fn ensure_exact_structural_event_batch(
&mut self,
desired: Vec<MobEventKind>,
) -> Result<(), MobError> {
if desired.is_empty() {
return Ok(());
}
let mob_id = self.definition.id.clone();
let replay = self.events.replay_all().await?;
match exact_structural_batch_present(&replay, &mob_id, &desired) {
Ok(true) => {
for kind in &desired {
if matches!(
kind,
MobEventKind::RemoteTurnObligationRecorded { .. }
| MobEventKind::PlacedKickoffObligationRecorded { .. }
) {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(kind)?;
}
}
return Ok(());
}
Ok(false) => {}
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
}
let events = desired
.iter()
.cloned()
.map(|kind| NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind,
})
.collect();
match self.events.append_batch(events).await {
Ok(_) => {
let mut index = self
.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for kind in &desired {
index.observe(kind)?;
}
Ok(())
}
Err(error) => {
let replay = match self.events.replay_all().await {
Ok(events) => events,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"structural batch append failed ({error}) and durable reconciliation remained unreadable; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
};
match exact_structural_batch_present(&replay, &mob_id, &desired) {
Ok(true) => {
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rebuild_from_events(&replay, &self.definition.id)?;
Ok(())
}
Ok(false) => Err(MobError::from(error)),
Err(conflict) => {
self.durable_uncertainty_fail_stop = true;
Err(conflict)
}
}
}
}
}
/// `GrantScopes` arm: AdminGrants self-gate → machine transition →
/// durable grant record under the transition witness → commit → reply
/// (ADJ-P5-2 records realization; the CommitHostBind prepare→persist→
/// commit discipline). A durable-write failure leaves the machine
/// UNADVANCED; retry is idempotent (grant = full replace).
async fn handle_grant_scopes_command(
&mut self,
caller: MobControlPrincipal,
principal: meerkat_core::auth::PrincipalId,
scopes: BTreeSet<mob_dsl::ControlScope>,
expires_at_ms: Option<u64>,
) -> Result<crate::control_policy::OperatorGrant, MobError> {
self.resolve_control_policy_for(&caller)
.require(mob_dsl::ControlScope::AdminGrants)?;
// No clock in the write path: a past expiry is accepted verbatim
// (ADJ-P5-7) — the enforcement seam makes it inert.
let machine_key = crate::control_policy::machine_principal(&principal);
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::GrantOperatorScopes {
principal: machine_key.clone(),
scopes: scopes.clone(),
expires_at_ms,
},
"grant_scopes commit",
)?;
let recorded = prepared
.transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::GrantRecorded {
principal,
scopes,
expires_at_ms,
} => Some((principal.clone(), scopes.clone(), *expires_at_ms)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted GrantOperatorScopes but emitted no GrantRecorded effect"
.into(),
)
})?;
let record = crate::store::MobOperatorGrantRecord {
principal: recorded.0.0.clone(),
scopes: recorded.1.iter().map(|scope| scope.to_wire()).collect(),
expires_at_ms: recorded.2,
};
let permit = crate::store::MobOperatorGrantPersistenceAuthority::from_transition(
&record,
&prepared.transition,
)?;
self.runtime_metadata
.put_mob_operator_grant(&self.definition.id, &record, &permit)
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(crate::control_policy::OperatorGrant {
principal: recorded.0.0,
scopes: recorded.1,
expires_at_ms: recorded.2,
})
}
/// `RevokeScopes` arm: AdminGrants self-gate → shell computes the
/// `(revoked, remaining)` partition from the actor's own state (the
/// machine revalidates it, dsl :7641-7687) → machine transition →
/// durable record update/delete under the witness → commit → reply
/// `removed`. Idempotent per ADJ-P5-3: never-granted scopes intersect
/// away; an absent grant is a no-op `removed: false`.
async fn handle_revoke_scopes_command(
&mut self,
caller: MobControlPrincipal,
principal: meerkat_core::auth::PrincipalId,
scopes: Option<BTreeSet<mob_dsl::ControlScope>>,
) -> Result<bool, MobError> {
self.resolve_control_policy_for(&caller)
.require(mob_dsl::ControlScope::AdminGrants)?;
let machine_key = crate::control_policy::machine_principal(&principal);
let state = self.dsl_authority.state();
let Some(recorded) = state.operator_grant_scopes.get(&machine_key).cloned() else {
// Absent grant: drive the machine's idempotent Absent arm so the
// machine stays the one judge; no event, no record touch.
self.apply_dsl_input(
mob_dsl::MobMachineInput::RevokeOperatorScopes {
principal: machine_key,
revoked: BTreeSet::new(),
remaining: BTreeSet::new(),
},
"revoke_scopes absent no-op",
)?;
return Ok(false);
};
let recorded_expiry = state
.operator_grant_expiries
.get(&machine_key)
.copied()
.flatten();
let revoked: BTreeSet<mob_dsl::ControlScope> = match &scopes {
None => recorded.clone(),
Some(requested) => requested.intersection(&recorded).copied().collect(),
};
if revoked.is_empty() {
// Revoking never-granted scopes is a no-op, not an error
// (ADJ-P5-3); the grant record is untouched.
return Ok(false);
}
let remaining: BTreeSet<mob_dsl::ControlScope> =
recorded.difference(&revoked).copied().collect();
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RevokeOperatorScopes {
principal: machine_key.clone(),
revoked: revoked.clone(),
remaining: remaining.clone(),
},
"revoke_scopes commit",
)?;
if !prepared
.transition
.effects()
.iter()
.any(|effect| matches!(effect, mob_dsl::MobMachineEffect::GrantRevoked { .. }))
{
return Err(MobError::Internal(
"MobMachine accepted RevokeOperatorScopes but emitted no GrantRevoked effect"
.into(),
));
}
let removed = remaining.is_empty();
if removed {
let permit = crate::store::MobOperatorGrantDeletionAuthority::from_transition(
&machine_key.0,
&prepared.transition,
)?;
self.runtime_metadata
.delete_mob_operator_grant(&self.definition.id, &machine_key.0, &permit)
.await?;
} else {
// Partial revoke retains the recorded expiry (dsl :7671); the
// record mirrors the post-transition remaining set verbatim, and
// the permit validates the expiry against the PREPARED
// post-state (the effect does not carry it).
let record = crate::store::MobOperatorGrantRecord {
principal: machine_key.0.clone(),
scopes: remaining.iter().map(|scope| scope.to_wire()).collect(),
expires_at_ms: recorded_expiry,
};
let permit =
crate::store::MobOperatorGrantPersistenceAuthority::from_revoke_transition(
&record,
&prepared.transition,
prepared.authority.state(),
)?;
self.runtime_metadata
.put_mob_operator_grant(&self.definition.id, &record, &permit)
.await?;
}
self.commit_prepared_dsl_transition(prepared)?;
Ok(removed)
}
/// `Grants` arm: AdminGrants self-gate → RAW projection of the machine
/// grant maps (ADJ-P5-4: expired rows appear verbatim, NO expired flag —
/// one clock, one evaluator, at the enforcement seam only). A scopes key
/// with a desynced expiry row projects `expires_at_ms: None` while the
/// resolve seam fail-closes it (ADJ-P5-8): the projection shows raw
/// truth, the policy denies.
fn handle_grants_command(
&self,
caller: MobControlPrincipal,
) -> Result<Vec<crate::control_policy::OperatorGrant>, MobError> {
self.resolve_control_policy_for(&caller)
.require(mob_dsl::ControlScope::AdminGrants)?;
let state = self.dsl_authority.state();
Ok(state
.operator_grant_scopes
.iter()
.map(|(principal, scopes)| crate::control_policy::OperatorGrant {
principal: principal.0.clone(),
scopes: scopes.clone(),
expires_at_ms: state
.operator_grant_expiries
.get(principal)
.copied()
.flatten(),
})
.collect())
}
/// Cancel and join derived lifecycle-notification delivery.
/// Safe to call repeatedly; completed and cancelled tasks are drained.
async fn abort_and_join_lifecycle_tasks(&mut self) {
self.lifecycle_tasks.abort_all();
while let Some(result) = self.lifecycle_tasks.join_next().await {
if let Err(error) = result
&& actor_task_join_error_is_panic(&error)
{
let _ = actor_task_join_panic_error(
"orchestrator lifecycle delivery teardown",
ActorTaskJoinPanicDisposition::TeardownTerminal,
error,
);
}
}
}
/// Join barrier for every background task/listener owned directly by the
/// actor (as opposed to keyed pending-spawn/flow/autonomous tables).
/// Safe to call repeatedly; all task sets and optional handles are drained.
async fn shutdown_actor_owned_background_work(&mut self) {
self.member_live_mutation_tasks.abort_all();
while let Some(result) = self.member_live_mutation_tasks.join_next().await {
if let Err(error) = result
&& actor_task_join_error_is_panic(&error)
{
let _ = actor_task_join_panic_error(
"mutating member-live teardown",
ActorTaskJoinPanicDisposition::TeardownTerminal,
error,
);
}
}
self.abort_and_join_actor_io_tasks().await;
self.peer_delivery_tasks.abort_all();
while let Some(result) = self.peer_delivery_tasks.join_next().await {
match result {
Ok(completion) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Err(error) => {
let _ = self.reconcile_peer_delivery_join_error(
error,
"peer delivery teardown",
ActorTaskJoinPanicDisposition::TeardownTerminal,
);
}
}
}
self.peer_delivery_inflight.clear();
self.abort_and_join_lifecycle_tasks().await;
self.member_event_pumps.stop_all_and_join().await;
// These listener tasks retain live command/bridge ownership. Stop and
// join them before the supervisor bridge and before command-channel
// closure, so a cold replacement cannot race the previous process.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
if let Some(responder) = self.upcall_responder.take() {
responder.shutdown().await;
}
#[cfg(not(target_arch = "wasm32"))]
if let Some(acceptor) = self.controlling_acceptor.take() {
acceptor.shutdown().await;
}
self.supervisor_bridge.shutdown().await;
}
/// Crash-semantics quiescence for an actor whose durable commit outcome is
/// uncertain. This deliberately applies no Stop/Shutdown input and writes
/// no compensating durable fact: cold replay is the sole authority that
/// may decide whether the uncertain append committed.
async fn quiesce_volatile_producers_after_fail_stop(&mut self) {
let run_tasks = std::mem::take(&mut self.run_tasks)
.into_values()
.collect::<Vec<_>>();
for task in &run_tasks {
task.abort();
}
for task in run_tasks {
let _ = task.await;
}
self.run_cancel_tokens.clear();
self.flow_streams.lock().await.clear();
// Pending provisioners are volatile crash-owned producers. Abort the
// tasks without invoking their semantic failure/cleanup ladders.
let pending_spawns = self.pending_spawns.drain_all();
for mut slot in pending_spawns {
if let Some(task) = slot.task.take() {
task.abort();
let _ = task.await;
}
}
let autonomous_turns = {
let mut turns = self.autonomous_initial_turns.lock().await;
std::mem::take(&mut *turns)
.into_values()
.collect::<Vec<_>>()
};
for turn in autonomous_turns {
turn.abort_and_join().await;
}
self.shutdown_actor_owned_background_work().await;
}
fn enqueue_identity_reconcile(&mut self, identity: AgentIdentity) {
if self.identity_reconcile_enqueued.insert(identity.clone()) {
self.identity_reconcile_queue.push_back(identity);
}
}
fn identity_reconcile_now_ms() -> Result<u64, MobError> {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
.map_err(|error| {
MobError::Internal(format!("identity reconcile clock failed: {error}"))
})
}
fn identity_evidence_digest(bytes: &[u8]) -> String {
format!("sha256:{:x}", Sha256::digest(bytes))
}
async fn replace_identity_reconcile_status(
&mut self,
identity: &AgentIdentity,
intent_revision: Option<u64>,
lease_epoch: Option<u64>,
decision: crate::identity::IdentityReconcileDecision,
detail: Option<String>,
) {
let observed_at_ms = Self::identity_reconcile_now_ms().unwrap_or_default();
let status = crate::identity::IdentityConvergenceStatus {
identity: identity.clone(),
intent_revision,
lease_epoch,
decision: Some(decision),
observed_at_ms,
detail,
};
let identity_status = Arc::clone(&self.identity_status);
let mob_id = self.definition.id.clone();
let projected_identity = identity.clone();
// Status is deliberately output-only. Queue its best-effort projection
// on actor-owned I/O custody so a slow, unavailable, or corrupt status
// store cannot delay the actuator or the causal re-observation that
// follows it. Shutdown may abort this replaceable diagnostic write.
#[cfg(not(target_arch = "wasm32"))]
self.actor_io_tasks.spawn(async move {
if let Err(error) = identity_status
.replace_identity_convergence_status(&mob_id, &status)
.await
{
tracing::warn!(
mob_id = %mob_id,
agent_identity = %projected_identity,
error = %error,
"identity convergence status projection failed"
);
}
});
// wasm has no Send task executor in this crate. Keep the same
// output-only error semantics there without changing the native actor
// convergence boundary.
#[cfg(target_arch = "wasm32")]
if let Err(error) = identity_status
.replace_identity_convergence_status(&mob_id, &status)
.await
{
tracing::warn!(
mob_id = %mob_id,
agent_identity = %projected_identity,
error = %error,
"identity convergence status projection failed"
);
}
}
async fn record_identity_reconcile_disposition(
&mut self,
identity: &AgentIdentity,
authority: &IdentityReconcileCompletionAuthority,
disposition: IdentityMemberActuationDisposition,
) {
let current_authority = match self
.identity
.observe_identity_intent(&self.definition.id, identity)
.await
{
Ok(observation) => IdentityReconcileAuthorityKey::from_observation(&observation),
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %identity,
error = %error,
"could not verify identity authority for terminal actuation disposition"
);
self.schedule_identity_reconcile_pass_error(identity, &error.to_string());
return;
}
};
self.clear_stale_identity_reconcile_schedule(identity, ¤t_authority);
// An unavailable intent read uses `Unavailable` only as an ephemeral
// scheduling key. If the verification read immediately recovers, bind
// the disposition to those exact bytes instead of treating recovery
// as a superseding declaration and hot-requeueing the same failure.
let effective_authority = if authority.intent == IdentityReconcileAuthorityKey::Unavailable
{
IdentityReconcileCompletionAuthority {
intent: current_authority.clone(),
lease_epoch: authority.lease_epoch,
}
} else {
authority.clone()
};
if current_authority != effective_authority.intent {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %identity,
completed_intent_revision = ?effective_authority.intent.intent_revision(),
current_intent_revision = ?current_authority.intent_revision(),
"discarding stale identity actuation completion from superseded intent authority"
);
self.enqueue_identity_reconcile(identity.clone());
return;
}
let intent_revision = effective_authority.intent.intent_revision();
let lease_epoch = effective_authority.lease_epoch;
let requeue_now = match disposition {
IdentityMemberActuationDisposition::Applied => {
self.identity_reconcile_failures
.write()
.await
.remove(identity);
self.identity_reconcile_backoff.remove(identity);
self.identity_reconcile_parked.remove(identity);
true
}
IdentityMemberActuationDisposition::Conflict { detail } => {
self.identity_reconcile_failures
.write()
.await
.remove(identity);
// A CAS conflict is progress evidence from another actuator,
// not a repeated failure; re-observe without backoff debt.
self.identity_reconcile_backoff.remove(identity);
tracing::debug!(
agent_identity = %identity,
detail = %detail,
"identity member CAS conflicted; re-observing the target"
);
true
}
IdentityMemberActuationDisposition::RepairBlocked { detail } => {
self.identity_reconcile_failures.write().await.insert(
identity.clone(),
IdentityReconcileFailureState {
authority: effective_authority.intent.clone(),
detail: detail.clone(),
},
);
self.identity_reconcile_backoff.remove(identity);
self.identity_reconcile_parked.insert(
identity.clone(),
IdentityReconcileParkState {
authority: effective_authority.intent.clone(),
},
);
self.replace_identity_reconcile_status(
identity,
intent_revision,
lease_epoch,
crate::identity::IdentityReconcileDecision::RepairBlocked,
Some(detail),
)
.await;
false
}
IdentityMemberActuationDisposition::Backoff { detail } => {
self.identity_reconcile_failures.write().await.insert(
identity.clone(),
IdentityReconcileFailureState {
authority: effective_authority.intent.clone(),
detail: detail.clone(),
},
);
let now = Instant::now();
let entry = self
.identity_reconcile_backoff
.entry(identity.clone())
.or_insert(IdentityReconcileBackoffState {
authority: effective_authority.intent.clone(),
consecutive_failures: 0,
next_attempt_at: now,
retry_enqueued: false,
});
if entry.authority != effective_authority.intent {
*entry = IdentityReconcileBackoffState {
authority: effective_authority.intent.clone(),
consecutive_failures: 0,
next_attempt_at: now,
retry_enqueued: false,
};
}
entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
let delay = identity_reconcile_backoff_delay(entry.consecutive_failures);
entry.next_attempt_at = now.checked_add(delay).unwrap_or(now);
entry.retry_enqueued = false;
if identity_reconcile_failure_should_log(entry.consecutive_failures) {
tracing::warn!(
agent_identity = %identity,
consecutive_failures = entry.consecutive_failures,
retry_delay_ms = delay.as_millis() as u64,
detail = %detail,
"identity member actuation failed; backing off before the next attempt"
);
}
self.replace_identity_reconcile_status(
identity,
intent_revision,
lease_epoch,
crate::identity::IdentityReconcileDecision::Backoff,
Some(detail),
)
.await;
// Deliberately no immediate requeue. The actor sleeps until
// the exact earliest backoff deadline and then enqueues only
// the identities due at that instant. An immediate requeue
// here plus the old 25ms tick was the 2026-07-29 hot-retry
// loop.
false
}
IdentityMemberActuationDisposition::ParkedTransportClosed { detail } => {
self.identity_reconcile_failures.write().await.insert(
identity.clone(),
IdentityReconcileFailureState {
authority: effective_authority.intent.clone(),
detail: detail.clone(),
},
);
self.identity_reconcile_backoff.remove(identity);
self.identity_reconcile_parked.insert(
identity.clone(),
IdentityReconcileParkState {
authority: effective_authority.intent.clone(),
},
);
tracing::error!(
agent_identity = %identity,
detail = %detail,
"callback transport is closed; parking identity reconciliation until a new declaration or process restart"
);
self.replace_identity_reconcile_status(
identity,
intent_revision,
lease_epoch,
crate::identity::IdentityReconcileDecision::RepairBlocked,
Some(format!(
"parked: {detail}; the callback transport cannot recover in-process"
)),
)
.await;
false
}
};
if requeue_now {
// Terminal completion is the causal wake. It is serialized by this
// actor before best-effort reply delivery, so a dropped reply
// cannot strand convergence until the slow safety sweep.
self.enqueue_identity_reconcile(identity.clone());
#[cfg(test)]
IDENTITY_RECONCILE_COMPLETION_REQUEUES
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
}
fn clear_stale_identity_reconcile_schedule(
&mut self,
identity: &AgentIdentity,
current_authority: &IdentityReconcileAuthorityKey,
) {
let stale_backoff = self
.identity_reconcile_backoff
.get(identity)
.is_some_and(|state| &state.authority != current_authority);
if stale_backoff {
self.identity_reconcile_backoff.remove(identity);
}
let stale_park = self
.identity_reconcile_parked
.get(identity)
.is_some_and(|state| &state.authority != current_authority);
if stale_park {
self.identity_reconcile_parked.remove(identity);
}
}
/// Whether reconcile scheduling may admit this identity right now. Parked
/// identities never re-enter; identities under an actuation-backoff
/// deadline re-enter only once the deadline has passed.
fn identity_reconcile_admittable(
&mut self,
identity: &AgentIdentity,
current_authority: Option<&IdentityReconcileAuthorityKey>,
) -> bool {
if let Some(current_authority) = current_authority {
self.clear_stale_identity_reconcile_schedule(identity, current_authority);
}
if self.identity_reconcile_parked.contains_key(identity) {
return false;
}
match self.identity_reconcile_backoff.get(identity) {
Some(backoff) => Instant::now() >= backoff.next_attempt_at,
None => true,
}
}
/// Duration until the exact earliest retry not already under actor
/// custody. `None` means the actor needs no reconciliation timer at all.
fn identity_reconcile_next_backoff_wait(&self) -> Option<Duration> {
let now = Instant::now();
self.identity_reconcile_backoff
.values()
.filter(|state| !state.retry_enqueued)
.map(|state| state.next_attempt_at.saturating_duration_since(now))
.min()
}
/// Move every retry whose deadline has elapsed into the ordinary actor
/// queue. This is O(number of failing identities) in volatile memory and
/// performs no store connection, row scan, decode, or session read.
fn enqueue_due_identity_reconcile_backoffs(&mut self) {
let now = Instant::now();
let due = self
.identity_reconcile_backoff
.iter()
.filter(|(_, state)| !state.retry_enqueued && now >= state.next_attempt_at)
.map(|(identity, _)| identity.clone())
.collect::<Vec<_>>();
for identity in due {
if let Some(state) = self.identity_reconcile_backoff.get_mut(&identity) {
state.retry_enqueued = true;
}
self.enqueue_identity_reconcile(identity);
}
}
/// A pass-level error that escaped the typed classifier must still get an
/// exact deadline. Without this fallback, removing the 1Hz full-table
/// sweep would strand it; immediately requeueing it would recreate the
/// hot loop.
fn schedule_identity_reconcile_pass_error(&mut self, identity: &AgentIdentity, detail: &str) {
let now = Instant::now();
let entry = self
.identity_reconcile_backoff
.entry(identity.clone())
.or_insert(IdentityReconcileBackoffState {
authority: IdentityReconcileAuthorityKey::Unavailable,
consecutive_failures: 0,
next_attempt_at: now,
retry_enqueued: false,
});
entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
let delay = identity_reconcile_backoff_delay(entry.consecutive_failures);
entry.next_attempt_at = now.checked_add(delay).unwrap_or(now);
entry.retry_enqueued = false;
if identity_reconcile_failure_should_log(entry.consecutive_failures) {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %identity,
consecutive_failures = entry.consecutive_failures,
retry_delay_ms = delay.as_millis() as u64,
detail,
"identity reconciliation pass failed; retry scheduled at exact backoff deadline"
);
}
}
fn identity_unobserved_facts(
intent: crate::identity::IdentityAuthorityCondition,
lease: crate::identity::IdentityLeaseCondition,
) -> crate::identity::IdentityReconcileFacts {
crate::identity::IdentityReconcileFacts {
intent,
lease,
external_binding_required: false,
initial_delivery_required: false,
session_creation_receipt: crate::identity::IdentityReceiptCondition::Unavailable,
retirement_receipt: crate::identity::IdentityReceiptCondition::Unavailable,
session: crate::identity::IdentitySessionCondition::Unavailable,
runtime: crate::identity::IdentityResourceCondition::Unavailable,
member: crate::identity::IdentityResourceCondition::Unavailable,
external_binding_receipt: crate::identity::IdentityReceiptCondition::NotRequired,
external_trust: crate::identity::IdentityExternalTrustCondition::NotRequired,
external_ceremony: crate::identity::IdentityExternalCeremonyCondition::NotRequired,
initial_delivery_receipt: crate::identity::IdentityReceiptCondition::NotRequired,
initial_delivery: crate::identity::IdentityInitialDeliveryCondition::NotRequired,
wiring: crate::identity::IdentityResourceCondition::Unavailable,
}
}
fn identity_authority_condition(
observation: &crate::identity::IdentityStoredObservation<
crate::identity::IdentityIntentRecord,
>,
) -> crate::identity::IdentityAuthorityCondition {
use crate::identity::{
DesiredSessionAuthorityPolicy, IdentityAuthorityCondition, IdentityIntent,
IdentityStoredObservation,
};
match observation {
IdentityStoredObservation::Missing => IdentityAuthorityCondition::Missing,
IdentityStoredObservation::Unsupported { .. }
| IdentityStoredObservation::Malformed { .. } => IdentityAuthorityCondition::Malformed,
IdentityStoredObservation::Valid(record) => match &record.intent {
IdentityIntent::Present { session, .. } => match session.authority_policy {
DesiredSessionAuthorityPolicy::CreateIfAbsent => {
IdentityAuthorityCondition::PresentCreateIfAbsent
}
DesiredSessionAuthorityPolicy::RequireExisting => {
IdentityAuthorityCondition::PresentRequireExisting
}
},
IdentityIntent::Absent { .. } => IdentityAuthorityCondition::Absent,
},
}
}
fn identity_lease_condition(
&self,
observation: &crate::identity::IdentityStoredObservation<
crate::identity::IdentityLeaseRecord,
>,
now_ms: u64,
) -> crate::identity::IdentityLeaseCondition {
use crate::identity::{IdentityLeaseCondition, IdentityStoredObservation};
match observation {
IdentityStoredObservation::Missing => IdentityLeaseCondition::Missing,
IdentityStoredObservation::Unsupported { .. }
| IdentityStoredObservation::Malformed { .. } => IdentityLeaseCondition::Malformed,
IdentityStoredObservation::Valid(record) => match &record.active {
None => IdentityLeaseCondition::Missing,
Some(active)
if active.holder_id == self.identity_reconcile_holder_id
&& active.incarnation_id == self.identity_reconcile_incarnation_id
&& now_ms < active.expires_at_ms =>
{
IdentityLeaseCondition::HeldByCurrentIncarnation
}
Some(active) if now_ms >= active.expires_at_ms => {
IdentityLeaseCondition::HeldByExpiredIncarnation
}
Some(_) => IdentityLeaseCondition::HeldByOtherLiveIncarnation,
},
}
}
async fn observe_identity_session(
&self,
desired: &crate::identity::DesiredSessionTarget,
) -> Result<crate::identity::IdentitySessionObservation, MobError> {
#[cfg(test)]
IDENTITY_RECONCILE_SESSION_DOCUMENT_LOADS
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let authority_before = self
.observe_bounded_persisted_session_authority(desired)
.await?;
// Typed resume-seam read: `load_persisted_session` deliberately
// hides archived documents, and mapping that hidden state to
// authoritative Missing let identity reconciliation mark an intact
// Archived+Idle member Broken before explicit resume could revive
// it. Archived-but-not-revivable observes as UNAVAILABLE (a holding
// state), never as absence.
let loaded = match self
.session_service
.load_session_for_resume(&desired.session_id)
.await
{
Ok(
super::session_service::ResumeSessionLoad::Active(session)
| super::session_service::ResumeSessionLoad::Revivable(session),
) => Ok(Some(*session)),
Ok(super::session_service::ResumeSessionLoad::Absent) => Ok(None),
Ok(super::session_service::ResumeSessionLoad::ArchivedNotRevivable {
runtime_state,
}) => {
let state = runtime_state.map_or_else(
|| "<no runtime record>".to_string(),
|state| state.to_string(),
);
return Ok(crate::identity::IdentitySessionObservation::unavailable(
format!(
"session '{}' is archived and not revivable from runtime state {state}; \
the transcript is intact and preserved — refusing to observe it as missing",
desired.session_id
),
));
}
Err(error) => Err(error),
};
let authority_after = self
.observe_bounded_persisted_session_authority(desired)
.await?;
if authority_after != authority_before {
return Err(MobError::Internal(format!(
"persisted authority changed while observing session {}",
desired.session_id
)));
}
match identity_session_observation_from_persisted_load(desired, loaded, authority_before) {
Ok(observation) => Ok(observation),
Err(error) => Ok(crate::identity::IdentitySessionObservation::unavailable(
format!("persisted session observation construction failed: {error}"),
)),
}
}
async fn observe_identity_member(
&self,
identity: &AgentIdentity,
intent: &crate::identity::IdentityIntentRecord,
) -> crate::identity::IdentityResourceObservation {
if self.pending_spawns.contains_member(identity) {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: "member materialization is in flight".to_string(),
};
}
let Some(store) = self.identity_member.as_ref() else {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: "atomic identity/member store capability is unavailable".to_string(),
};
};
match store
.observe_identity_member_target(&self.definition.id, identity)
.await
{
Ok(observation) => observation.resource_observation_against(intent),
Err(error) => crate::identity::IdentityResourceObservation::Unavailable {
detail: error.to_string(),
},
}
}
/// The retained sealed-intent slice proves only an empty local/local topology.
/// Direct desired wiring remains sealed in the intent substrate, but any
/// non-empty desired or observed incident topology is a typed refusal; no
/// structural, generated-graph, or comms-trust mutation is attempted.
async fn observe_identity_wiring_empty(
&self,
identity: &AgentIdentity,
desired_incident_wiring: &BTreeSet<crate::identity::DesiredIdentityEdge>,
) -> crate::identity::IdentityResourceObservation {
use crate::store::IdentityWiringTargetObservation;
use meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind;
if !desired_incident_wiring.is_empty() {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "non-empty desired wiring requires a target-atomic graph/trust actuator outside the retained sealed-intent reconciliation slice"
.to_string(),
};
}
let Some(member_store) = self.identity_member.as_ref() else {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: "atomic identity/wiring observation capability is unavailable".to_string(),
};
};
let structural = match member_store
.observe_identity_wiring_target(&self.definition.id, identity)
.await
{
Ok(observation) => observation,
Err(error) => {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: error.to_string(),
};
}
};
match &structural {
IdentityWiringTargetObservation::Malformed {
observed_version,
detail,
} => {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: observed_version.clone(),
detail: detail.clone(),
};
}
IdentityWiringTargetObservation::Present { incident_edges, .. }
if !incident_edges.is_empty() =>
{
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: structural.target_precondition().map(|target| match target {
crate::identity::IdentityTargetObservationVersion::Version { version } => {
version
}
crate::identity::IdentityTargetObservationVersion::Absent {
absence_version,
} => absence_version,
crate::identity::IdentityTargetObservationVersion::InsertIfAbsent => {
"insert-if-absent".to_string()
}
}),
detail:
"non-empty structural wiring is RepairBlocked in the retained sealed-intent reconciliation slice"
.to_string(),
};
}
IdentityWiringTargetObservation::Absent { .. }
| IdentityWiringTargetObservation::Present { .. } => {}
}
for edge in &self.dsl_authority.state().wiring_edges {
if edge.a.0 == identity.as_str() || edge.b.0 == identity.as_str() {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "non-empty generated wiring graph is RepairBlocked in the retained sealed-intent reconciliation slice"
.to_string(),
};
}
}
let roster_entries: Vec<RosterEntry> = self.roster.read().await.list().cloned().collect();
let current_entry = roster_entries
.iter()
.find(|entry| &entry.agent_identity == identity);
let machine_spec = match self
.machine_member_peer_spec_for(identity, "identity empty-wiring observation")
{
Ok(spec) => spec,
Err(error) => {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: error.to_string(),
};
}
};
let mut current_live = false;
if let Some(entry) = current_entry {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
if self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity)
{
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail:
"identity recovery wiring supports only controlling-session local endpoints"
.to_string(),
};
}
let member_ref = match self
.machine_member_ref_for_behavior(entry, "identity empty-wiring observation")
{
Ok(member_ref) => member_ref,
Err(error) => {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: error.to_string(),
};
}
};
match member_ref {
MemberRef::BackendPeer { .. } => {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "identity recovery wiring supports only controlling-session local endpoints"
.to_string(),
};
}
MemberRef::Session { .. } => {
if let Some(comms) = self.provisioner_comms(&member_ref).await {
let rows = match comms
.trusted_peer_projection_snapshot_for_source(
GeneratedCommsTrustAuthoritySourceKind::MobMachineMemberTrustWiring,
)
.await
{
Ok(rows) => rows,
Err(error) => {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: error.to_string(),
};
}
};
if !rows.is_empty() {
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "non-empty source-scoped outgoing trust is RepairBlocked in the retained sealed-intent reconciliation slice"
.to_string(),
};
}
current_live = true;
}
// A normal keep_alive=false member may have no current
// process-local comms actor. There is then no live trust
// store to inspect; durable structural/generated wiring
// remains the resource authority checked above.
}
}
}
if let Some(expected) = machine_spec.as_ref() {
let current_peer_id = Self::trusted_peer_removal_key(expected);
for entry in roster_entries {
if &entry.agent_identity == identity {
continue;
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
if self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity)
{
continue;
}
let member_ref = match self
.machine_member_ref_for_behavior(&entry, "identity empty-wiring inbound proof")
{
Ok(member_ref @ MemberRef::Session { .. }) => member_ref,
Ok(MemberRef::BackendPeer { .. }) | Err(_) => continue,
};
let Some(comms) = self.provisioner_comms(&member_ref).await else {
continue;
};
let rows = match comms
.trusted_peer_projection_snapshot_for_source(
GeneratedCommsTrustAuthoritySourceKind::MobMachineMemberTrustWiring,
)
.await
{
Ok(rows) => rows,
Err(error) => {
return crate::identity::IdentityResourceObservation::Unavailable {
detail: error.to_string(),
};
}
};
if rows
.iter()
.any(|row| Self::trusted_peer_removal_key(row) == current_peer_id)
{
return crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "non-empty source-scoped inbound trust is RepairBlocked in the retained sealed-intent reconciliation slice"
.to_string(),
};
}
}
}
let version = Self::identity_evidence_digest(
format!(
"identity={identity};structural={structural:?};graph=empty;trust=empty;live={current_live}"
)
.as_bytes(),
);
// Empty wiring is a content fact, not a liveness requirement. A
// dormant local endpoint has no process-local trust store to mutate;
// member realization is classified independently.
crate::identity::IdentityResourceObservation::Matching { version }
}
async fn observe_identity_receipt_condition(
&self,
identity: &AgentIdentity,
slot: &crate::identity::IdentityOperationSlot,
) -> crate::identity::IdentityReceiptCondition {
use crate::identity::{
IdentityOperationSubject, IdentityReceiptCondition, IdentityStoredObservation,
};
match self
.identity
.observe_identity_operation_receipt(
&self.definition.id,
&IdentityOperationSubject::Identity {
identity: identity.clone(),
},
slot,
)
.await
{
Err(_) => IdentityReceiptCondition::Unavailable,
Ok(IdentityStoredObservation::Missing) => IdentityReceiptCondition::Missing,
Ok(
IdentityStoredObservation::Unsupported { .. }
| IdentityStoredObservation::Malformed { .. },
) => IdentityReceiptCondition::Malformed,
Ok(IdentityStoredObservation::Valid(_)) => IdentityReceiptCondition::Matching,
}
}
#[cfg(feature = "runtime-adapter")]
fn identity_runtime_process_phase_is_fresh_idle(
record: &meerkat_runtime::store::DecodedMachineLifecycleObservation,
) -> bool {
let binding = record.binding();
// Supervisor custody is recovered by its owning runtime protocol. It
// is not process-phase authority, so identity convergence must not
// mirror or classify that record.
record.runtime_state() == Some(meerkat_runtime::RuntimeState::Idle)
&& binding.agent_runtime_id().is_none()
&& binding.fence_token().is_none()
&& binding.runtime_generation().is_none()
&& binding.runtime_epoch_id().is_none()
&& record.run().current_run_id().is_none()
&& record.run().pre_run_phase().is_none()
}
#[cfg(feature = "runtime-adapter")]
async fn observe_identity_runtime_target(
&self,
desired: &crate::identity::DesiredSessionTarget,
) -> (
crate::identity::IdentityResourceObservation,
Option<meerkat_runtime::RuntimeSessionLifecycleObservation>,
) {
use meerkat_runtime::store::MachineLifecycleObservation;
let Some(runtime) = self.runtime_adapter.as_ref() else {
return (
crate::identity::IdentityResourceObservation::Unavailable {
detail: "identity runtime adapter is unavailable".to_string(),
},
None,
);
};
let observed = match runtime
.observe_cold_runtime_lifecycle(&desired.session_id)
.await
{
Ok(observed) => observed,
Err(meerkat_runtime::store::RuntimeStoreError::Unsupported(detail)) => {
return (
crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: format!(
"runtime lifecycle observation capability is unsupported: {detail}"
),
},
None,
);
}
Err(error) => {
return (
crate::identity::IdentityResourceObservation::Unavailable {
detail: error.to_string(),
},
None,
);
}
};
let live = runtime.contains_session(&desired.session_id).await;
let resource = match observed.lifecycle() {
MachineLifecycleObservation::Missing if !live => {
crate::identity::IdentityResourceObservation::Missing {
absence_version: format!("runtime-lifecycle-absent:{}", observed.runtime_id()),
}
}
MachineLifecycleObservation::Missing => {
crate::identity::IdentityResourceObservation::Malformed {
observed_version: None,
detail: "a live runtime exists without its durable lifecycle row".to_string(),
}
}
MachineLifecycleObservation::Decoded { record, version }
if Self::identity_runtime_process_phase_is_fresh_idle(record)
&& record.unregister_progress().is_none() =>
{
// Runtime lifecycle content is durable realization; the live
// map is process-local mechanics. Normal mob members use
// keep_alive=false, so their executor may quiesce after
// materialization while the exact unbound Idle row remains
// converged. Member liveness is observed independently below
// and must not force an endless runtime re-registration loop.
crate::identity::IdentityResourceObservation::Matching {
version: version.as_str().to_string(),
}
}
MachineLifecycleObservation::Decoded { record, .. }
if live
&& Self::identity_runtime_process_phase_is_fresh_idle(record)
&& record.unregister_progress().is_some() =>
{
// Valid same-resource teardown authority is recovered by the
// runtime owner. It is transient, not malformed evidence and
// not authority for identity repair.
crate::identity::IdentityResourceObservation::Unavailable {
detail: "runtime unregister recovery is still in progress".to_string(),
}
}
MachineLifecycleObservation::Decoded { version, .. } if !live => {
// Every persisted phase is observation of a dead process. The
// runtime actuator replaces it with a fresh unbound Idle shell
// over the existing session under this exact row version.
crate::identity::IdentityResourceObservation::Divergent {
version: version.as_str().to_string(),
detail: "durable runtime lifecycle has no live process shell".to_string(),
}
}
MachineLifecycleObservation::Decoded { version, .. } => {
crate::identity::IdentityResourceObservation::Malformed {
observed_version: Some(version.as_str().to_string()),
detail: "live runtime lifecycle is not a canonical stable Idle shell"
.to_string(),
}
}
MachineLifecycleObservation::Unsupported {
record_version,
evidence_digest,
version,
} => crate::identity::IdentityResourceObservation::Malformed {
observed_version: Some(version.as_str().to_string()),
detail: format!(
"unsupported runtime lifecycle record version {record_version} ({evidence_digest})"
),
},
MachineLifecycleObservation::Malformed {
evidence_digest,
version,
detail,
..
} => crate::identity::IdentityResourceObservation::Malformed {
observed_version: Some(version.as_str().to_string()),
detail: format!("malformed runtime lifecycle {evidence_digest}: {detail}"),
},
};
(resource, Some(observed))
}
#[cfg(not(feature = "runtime-adapter"))]
async fn observe_identity_runtime(
&self,
desired: &crate::identity::DesiredSessionTarget,
) -> crate::identity::IdentityResourceObservation {
let _ = desired;
crate::identity::IdentityResourceObservation::Unavailable {
detail: "identity runtime reconciliation requires runtime-adapter".to_string(),
}
}
fn identity_actuation_permit(
&self,
identity: &AgentIdentity,
intent: &crate::identity::IdentityIntentRecord,
lease: &crate::identity::IdentityLeaseClaim,
target: crate::identity::IdentityActuatorTarget,
target_observation: crate::identity::IdentityTargetObservationVersion,
) -> crate::identity::IdentityActuationPermit {
crate::identity::IdentityActuationPermit {
mob_id: self.definition.id.clone(),
identity: identity.clone(),
target,
intent_revision: intent.intent_revision,
intent_digest: intent.intent_digest.clone(),
intent_authority_digest: intent.authority_digest.clone(),
lease_epoch: lease.epoch,
lease_holder_id: lease.holder_id.clone(),
lease_incarnation_id: lease.incarnation_id.clone(),
lease_expires_at_ms: lease.expires_at_ms,
target_observation,
}
}
/// Refresh the bounded lease immediately before a target write.
///
/// Observation and provisioning may legitimately take longer than one
/// lease window (the HomeCore snapshot is 82 MB). The resource witness and
/// intent authority remain unchanged; only the store-owned lease claim is
/// renewed/reclaimed here, then atomically revalidated by the actuator.
async fn renew_identity_actuation_lease(
&self,
identity: &AgentIdentity,
) -> Result<Option<crate::identity::IdentityLeaseClaim>, MobError> {
use crate::identity::IdentityLeaseClaimOutcome;
match self
.identity
.claim_or_renew_identity_lease(
&self.definition.id,
identity,
&self.identity_reconcile_holder_id,
&self.identity_reconcile_incarnation_id,
IDENTITY_RECONCILE_LEASE_TTL_MS,
)
.await
.map_err(MobError::from)?
{
IdentityLeaseClaimOutcome::Acquired(claim)
| IdentityLeaseClaimOutcome::Renewed(claim) => Ok(Some(claim)),
IdentityLeaseClaimOutcome::HeldByOther(_) => Ok(None),
}
}
fn refresh_identity_permit_lease(
mut permit: crate::identity::IdentityActuationPermit,
lease: &crate::identity::IdentityLeaseClaim,
) -> crate::identity::IdentityActuationPermit {
permit.lease_epoch = lease.epoch;
permit.lease_holder_id.clone_from(&lease.holder_id);
permit
.lease_incarnation_id
.clone_from(&lease.incarnation_id);
permit.lease_expires_at_ms = lease.expires_at_ms;
permit
}
async fn enqueue_identity_intent_observations(
&mut self,
observations: BTreeMap<
AgentIdentity,
crate::identity::IdentityStoredObservation<crate::identity::IdentityIntentRecord>,
>,
) {
for (identity, observation) in observations {
match observation {
crate::identity::IdentityStoredObservation::Valid(record) => {
// Startup discovery and the slow cross-process safety scan
// both respect the same park/backoff gate as the causal
// queue.
let authority = IdentityReconcileAuthorityKey::from_intent(&record);
if self.identity_reconcile_admittable(&identity, Some(&authority)) {
self.enqueue_identity_reconcile(identity);
}
}
crate::identity::IdentityStoredObservation::Unsupported {
evidence_digest,
detail,
} => {
self.replace_identity_reconcile_status(
&identity,
None,
None,
crate::identity::IdentityReconcileDecision::RepairBlocked,
Some(format!(
"unsupported identity intent row {evidence_digest}: {detail}"
)),
)
.await;
}
crate::identity::IdentityStoredObservation::Malformed {
evidence_digest,
detail,
} => {
self.replace_identity_reconcile_status(
&identity,
None,
None,
crate::identity::IdentityReconcileDecision::RepairBlocked,
Some(format!(
"malformed identity intent row {evidence_digest}: {detail}"
)),
)
.await;
}
crate::identity::IdentityStoredObservation::Missing => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %identity,
"identity intent scan returned an impossible missing row"
);
}
}
}
}
/// Cold discovery is exhaustive but bounded per store read. It may walk
/// every page once because a new actor has no causal in-memory queue.
async fn enqueue_all_identity_intents(&mut self) {
let mut cursor = None;
loop {
let page = match self
.identity
.scan_identity_intents_page(
&self.definition.id,
cursor.as_ref(),
crate::store::IDENTITY_INTENT_SCAN_PAGE_MAX,
)
.await
{
Ok(page) => page,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"paged identity intent startup discovery failed"
);
return;
}
};
let next = page.next;
self.enqueue_identity_intent_observations(page.observations)
.await;
let Some(next) = next else {
return;
};
if cursor.as_ref().is_some_and(|current| &next <= current) {
tracing::error!(
mob_id = %self.definition.id,
"identity intent startup discovery returned a non-advancing cursor"
);
return;
}
cursor = Some(next);
}
}
/// One timer wake reads and admits at most one bounded keyset page.
async fn enqueue_next_identity_intent_safety_page(&mut self) {
let after = self.identity_reconcile_safety_cursor.clone();
match self
.identity
.scan_identity_intents_page(
&self.definition.id,
after.as_ref(),
IDENTITY_RECONCILE_SAFETY_PAGE_LIMIT,
)
.await
{
Ok(page) => {
if let (Some(current), Some(next)) = (after.as_ref(), page.next.as_ref())
&& next <= current
{
tracing::error!(
mob_id = %self.definition.id,
"identity intent safety scan returned a non-advancing cursor; restarting the scan"
);
self.identity_reconcile_safety_cursor = None;
return;
}
self.identity_reconcile_safety_cursor = page.next;
self.enqueue_identity_intent_observations(page.observations)
.await;
}
Err(error) => tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"bounded identity intent safety scan failed; the same page will retry"
),
}
}
async fn reconcile_next_identity(
&mut self,
session_witnesses: &mut IdentityConvergedSessionWitnesses,
) {
let Some(identity) = self.identity_reconcile_queue.pop_front() else {
return;
};
self.identity_reconcile_enqueued.remove(&identity);
// The park/backoff gate covers every admission path (startup/safety
// discovery, terminal-completion requeue, and exact-deadline wake).
if !self.identity_reconcile_admittable(&identity, None) {
return;
}
match self
.reconcile_identity_once(&identity, session_witnesses)
.await
{
Ok(true) => self.enqueue_identity_reconcile(identity),
Ok(false) => {
// A due retry stays marked under actor custody while an async
// member materialization is pending; its serialized
// completion will either clear or reschedule the backoff. If
// no async completion owns it and no typed disposition
// replaced the deadline, release it back to the timer without
// resetting the accumulated failure debt.
if !self.pending_spawns.contains_member(&identity)
&& let Some(backoff) = self.identity_reconcile_backoff.get_mut(&identity)
&& backoff.retry_enqueued
{
let delay = identity_reconcile_backoff_delay(backoff.consecutive_failures);
let now = Instant::now();
backoff.next_attempt_at = now.checked_add(delay).unwrap_or(now);
backoff.retry_enqueued = false;
}
}
Err(error) => {
self.schedule_identity_reconcile_pass_error(&identity, &error.to_string());
}
}
}
/// Return the cached converged session observation only while both the
/// sealed intent authority and the runtime store's exact bounded physical
/// boundary authority still match. Authority read failure propagates:
/// integrity observation never falls back to a Session body.
async fn identity_converged_session_observation(
&self,
session_witnesses: &mut IdentityConvergedSessionWitnesses,
identity: &AgentIdentity,
intent: &crate::identity::IdentityIntentRecord,
desired: &crate::identity::DesiredSessionTarget,
) -> Result<Option<crate::identity::IdentitySessionObservation>, MobError> {
let Some(witness) = session_witnesses.get(identity).cloned() else {
return Ok(None);
};
let current = witness.intent_revision == intent.intent_revision
&& witness.intent_digest == intent.intent_digest
&& witness.authority_digest == intent.authority_digest
&& witness.persisted_authority.session_id() == &desired.session_id;
if !current {
session_witnesses.remove(identity);
return Ok(None);
}
match self.session_service.persisted_session_authority_read_cost() {
super::session_service::PersistedSessionAuthorityReadCost::Unsupported => {
return Err(MobError::from(
meerkat_core::service::SessionError::Unsupported(format!(
"session service cannot cheaply revalidate exact persisted authority for {}",
desired.session_id
)),
));
}
super::session_service::PersistedSessionAuthorityReadCost::Bounded => {}
}
let observed = self
.session_service
.observe_persisted_session_authority(&desired.session_id)
.await
.map_err(MobError::from)?;
let Some(observed) = observed else {
return Err(MobError::Internal(format!(
"persisted authority disappeared for converged session {}",
desired.session_id
)));
};
if observed != witness.persisted_authority {
session_witnesses.remove(identity);
return Ok(None);
}
Ok(Some(witness.observation))
}
/// Observe the small store-issued identity of the current persisted
/// boundary. There is deliberately no Session-body fallback.
async fn observe_bounded_persisted_session_authority(
&self,
desired: &crate::identity::DesiredSessionTarget,
) -> Result<Option<crate::identity::IdentitySessionStoreAuthority>, MobError> {
match self.session_service.persisted_session_authority_read_cost() {
super::session_service::PersistedSessionAuthorityReadCost::Unsupported => {
return Err(MobError::from(
meerkat_core::service::SessionError::Unsupported(format!(
"session service cannot observe exact bounded persisted authority for {}",
desired.session_id
)),
));
}
super::session_service::PersistedSessionAuthorityReadCost::Bounded => {}
}
let authority = self
.session_service
.observe_persisted_session_authority(&desired.session_id)
.await
.map_err(MobError::from)?;
if let Some(authority) = authority.as_ref()
&& authority.session_id() != &desired.session_id
{
return Err(MobError::Internal(format!(
"persisted authority is bound to {}, not observed session {}",
authority.session_id(),
desired.session_id
)));
}
Ok(authority)
}
async fn reconcile_identity_once(
&mut self,
identity: &AgentIdentity,
session_witnesses: &mut IdentityConvergedSessionWitnesses,
) -> Result<bool, MobError> {
match self
.reconcile_identity_pass(identity, session_witnesses, true)
.await?
{
IdentityReconcilePassDisposition::Outcome(requeue) => Ok(requeue),
IdentityReconcilePassDisposition::ReverifySession => {
match self
.reconcile_identity_pass(identity, session_witnesses, false)
.await?
{
IdentityReconcilePassDisposition::Outcome(requeue) => Ok(requeue),
IdentityReconcilePassDisposition::ReverifySession => {
Err(MobError::Internal(
"identity reconcile requested session reverification from a fresh document observation"
.to_string(),
))
}
}
}
}
}
async fn reconcile_identity_pass(
&mut self,
identity: &AgentIdentity,
session_witnesses: &mut IdentityConvergedSessionWitnesses,
allow_session_witness: bool,
) -> Result<IdentityReconcilePassDisposition, MobError> {
use crate::identity::{
IDENTITY_OPERATION_RECEIPT_SCHEMA_VERSION, IdentityActuatorTarget,
IdentityAuthorityCondition, IdentityExternalCeremonyCondition,
IdentityExternalTrustCondition, IdentityInitialDeliveryCondition, IdentityIntent,
IdentityLeaseClaimOutcome, IdentityLeaseCondition, IdentityOperationKind,
IdentityOperationReceipt, IdentityOperationReceiptInsertOutcome,
IdentityOperationReceiptPayload, IdentityOperationSlot, IdentityOperationSubject,
IdentityReceiptCondition, IdentityReconcileDecision, IdentityStoredObservation,
IdentityTargetObservationVersion,
};
let intent_observation = match self
.identity
.observe_identity_intent(&self.definition.id, identity)
.await
{
Ok(observation) => observation,
Err(error) => {
let decision = crate::identity::classify_identity_reconciliation(
Self::identity_unobserved_facts(
IdentityAuthorityCondition::Unavailable,
IdentityLeaseCondition::Unavailable,
),
);
let authority = IdentityReconcileCompletionAuthority {
intent: IdentityReconcileAuthorityKey::Unavailable,
lease_epoch: None,
};
self.record_identity_reconcile_disposition(
identity,
&authority,
IdentityMemberActuationDisposition::Backoff {
detail: error.to_string(),
},
)
.await;
debug_assert_eq!(decision, IdentityReconcileDecision::Backoff);
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
};
let intent_condition = Self::identity_authority_condition(&intent_observation);
let intent_record = match &intent_observation {
IdentityStoredObservation::Valid(record) => Some(record.clone()),
IdentityStoredObservation::Missing
| IdentityStoredObservation::Unsupported { .. }
| IdentityStoredObservation::Malformed { .. } => None,
};
if let Some(record) = intent_record.as_ref()
&& let Some(reason) = identity_reconciliation_slice_intent_rejection(record)
{
let authority = IdentityReconcileCompletionAuthority {
intent: IdentityReconcileAuthorityKey::from_intent(record),
lease_epoch: None,
};
self.record_identity_reconcile_disposition(
identity,
&authority,
IdentityMemberActuationDisposition::RepairBlocked { detail: reason },
)
.await;
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
let now_ms = Self::identity_reconcile_now_ms()?;
let lease_observation = match self
.identity
.observe_identity_lease(&self.definition.id, identity)
.await
{
Ok(observation) => observation,
Err(error) => {
let decision = crate::identity::classify_identity_reconciliation(
Self::identity_unobserved_facts(
intent_condition,
IdentityLeaseCondition::Unavailable,
),
);
let authority = IdentityReconcileCompletionAuthority {
intent: intent_record
.as_ref()
.map_or(IdentityReconcileAuthorityKey::Unavailable, |record| {
IdentityReconcileAuthorityKey::from_intent(record)
}),
lease_epoch: None,
};
self.record_identity_reconcile_disposition(
identity,
&authority,
IdentityMemberActuationDisposition::Backoff {
detail: error.to_string(),
},
)
.await;
debug_assert_eq!(decision, IdentityReconcileDecision::Backoff);
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
};
let lease_condition = self.identity_lease_condition(&lease_observation, now_ms);
let preliminary = crate::identity::classify_identity_reconciliation(
Self::identity_unobserved_facts(intent_condition, lease_condition),
);
if preliminary == IdentityReconcileDecision::AcquireLease {
self.replace_identity_reconcile_status(
identity,
intent_record.as_ref().map(|record| record.intent_revision),
None,
preliminary,
None,
)
.await;
let outcome = self
.identity
.claim_or_renew_identity_lease(
&self.definition.id,
identity,
&self.identity_reconcile_holder_id,
&self.identity_reconcile_incarnation_id,
IDENTITY_RECONCILE_LEASE_TTL_MS,
)
.await
.map_err(MobError::from)?;
#[cfg(any(test, feature = "test-support"))]
if matches!(&outcome, IdentityLeaseClaimOutcome::Acquired(_)) {
super::trigger_identity_recovery_fail_stop(
super::IdentityRecoveryFailStopPoint::LeaseAcquired,
);
}
return Ok(IdentityReconcilePassDisposition::Outcome(matches!(
outcome,
IdentityLeaseClaimOutcome::Acquired(_) | IdentityLeaseClaimOutcome::Renewed(_)
)));
}
if lease_condition != IdentityLeaseCondition::HeldByCurrentIncarnation {
if preliminary == IdentityReconcileDecision::Backoff {
let authority = IdentityReconcileCompletionAuthority {
intent: intent_record
.as_ref()
.map_or(IdentityReconcileAuthorityKey::Unavailable, |record| {
IdentityReconcileAuthorityKey::from_intent(record)
}),
lease_epoch: None,
};
self.record_identity_reconcile_disposition(
identity,
&authority,
IdentityMemberActuationDisposition::Backoff {
detail: format!(
"identity lease observation classified as {lease_condition:?}"
),
},
)
.await;
} else {
self.replace_identity_reconcile_status(
identity,
intent_record.as_ref().map(|record| record.intent_revision),
None,
preliminary,
None,
)
.await;
}
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
let Some(intent_record) = intent_record else {
self.replace_identity_reconcile_status(identity, None, None, preliminary, None)
.await;
return Ok(IdentityReconcilePassDisposition::Outcome(false));
};
let active_lease = match lease_observation {
IdentityStoredObservation::Valid(record) => record.active.ok_or_else(|| {
MobError::Internal(
"current-incarnation lease observation had no active claim".to_string(),
)
})?,
IdentityStoredObservation::Missing
| IdentityStoredObservation::Unsupported { .. }
| IdentityStoredObservation::Malformed { .. } => {
return Err(MobError::Internal(
"current-incarnation lease condition had no valid record".to_string(),
));
}
};
let IdentityIntent::Present {
session, member, ..
} = &intent_record.intent
else {
unreachable!("absent intent was refused at the recovery-slice boundary")
};
let desired_incident_wiring = match &intent_record.retirement_plan {
crate::identity::IdentityRetirementPlan::Targets {
incident_wiring, ..
} => incident_wiring,
crate::identity::IdentityRetirementPlan::NoKnownRealization => {
return Err(MobError::Internal(
"present identity intent is missing its store-sealed incident wiring plan"
.to_string(),
));
}
};
// Converged steady state must not reload the session document: the
// cached witness substitutes the expensive read while the sealed
// intent authority is unchanged. Every other per-pass observation
// below stays fresh, so realization drift that those observations can
// see keeps its scan-cadence repair latency.
let witness_session_observation = if allow_session_witness {
self.identity_converged_session_observation(
session_witnesses,
identity,
&intent_record,
session,
)
.await?
} else {
session_witnesses.remove(identity);
None
};
let session_observed_from_witness = witness_session_observation.is_some();
let session_observation = match witness_session_observation {
Some(observation) => observation,
None => self.observe_identity_session(session).await?,
};
#[cfg(feature = "runtime-adapter")]
let (runtime_observation, runtime_target_observation) =
self.observe_identity_runtime_target(session).await;
#[cfg(not(feature = "runtime-adapter"))]
let runtime_observation = self.observe_identity_runtime(session).await;
let member_observation = self.observe_identity_member(identity, &intent_record).await;
let wiring_observation = self
.observe_identity_wiring_empty(identity, desired_incident_wiring)
.await;
let normalized_tombstone = intent_record.tombstone_generation.unwrap_or(0);
let session_creation_slot = IdentityOperationSlot::SessionCreationConsumed {
tombstone_generation: normalized_tombstone,
session_id: session.session_id.clone(),
lineage_id: session.lineage_id.clone(),
lineage_generation: session.lineage_generation,
};
let session_creation_receipt = match session.authority_policy {
crate::identity::DesiredSessionAuthorityPolicy::CreateIfAbsent => {
self.observe_identity_receipt_condition(identity, &session_creation_slot)
.await
}
crate::identity::DesiredSessionAuthorityPolicy::RequireExisting => {
IdentityReceiptCondition::NotRequired
}
};
let (initial_delivery_required, initial_delivery_receipt, initial_delivery) =
if let Some(delivery) = &member.initial_delivery {
let slot = IdentityOperationSlot::InitialDelivery {
tombstone_generation: normalized_tombstone,
session_id: session.session_id.clone(),
lineage_id: session.lineage_id.clone(),
lineage_generation: session.lineage_generation,
delivery_generation: delivery.delivery_generation,
};
let receipt = self
.observe_identity_receipt_condition(identity, &slot)
.await;
let evidence = match receipt {
IdentityReceiptCondition::Missing => {
IdentityInitialDeliveryCondition::ProvenAbsent
}
IdentityReceiptCondition::Matching => {
// Until the runtime exposes an exact InputId readback,
// ambiguous legacy content must never authorize replay.
IdentityInitialDeliveryCondition::Indeterminate
}
IdentityReceiptCondition::Unavailable => {
IdentityInitialDeliveryCondition::Unavailable
}
IdentityReceiptCondition::Conflicting
| IdentityReceiptCondition::Malformed
| IdentityReceiptCondition::NotRequired => {
IdentityInitialDeliveryCondition::Malformed
}
};
(true, receipt, evidence)
} else {
(
false,
IdentityReceiptCondition::NotRequired,
IdentityInitialDeliveryCondition::NotRequired,
)
};
let external_binding_required = matches!(
member.execution(),
crate::identity::DesiredExecution::External { .. }
);
let mut facts = crate::identity::IdentityReconcileFacts {
intent: intent_condition,
lease: lease_condition,
external_binding_required,
initial_delivery_required,
session_creation_receipt,
retirement_receipt: IdentityReceiptCondition::NotRequired,
session: session_observation.condition(),
runtime: runtime_observation.condition(),
member: member_observation.condition(),
external_binding_receipt: IdentityReceiptCondition::NotRequired,
external_trust: IdentityExternalTrustCondition::NotRequired,
external_ceremony: IdentityExternalCeremonyCondition::NotRequired,
initial_delivery_receipt,
initial_delivery,
wiring: wiring_observation.condition(),
};
if external_binding_required {
// External ceremony is fresh process-local input. This narrow
// release slice deliberately waits rather than persisting it or
// fabricating trust from the desired address.
facts.external_binding_receipt = IdentityReceiptCondition::Unavailable;
facts.external_trust = IdentityExternalTrustCondition::Unavailable;
facts.external_ceremony = IdentityExternalCeremonyCondition::TemporarilyUnavailable;
}
let decision = crate::identity::classify_identity_reconciliation(facts);
if session_observed_from_witness && decision != IdentityReconcileDecision::Converged {
// The fresh cheap observations stopped corroborating the cached
// session witness. Discard it and re-run this pass over a fresh
// document read before any status projection or actuation, so a
// permit is never minted from a cached session witness.
session_witnesses.remove(identity);
return Ok(IdentityReconcilePassDisposition::ReverifySession);
}
if !session_observed_from_witness {
if decision == IdentityReconcileDecision::Converged {
let persisted_authority = session_observation
.store_authority()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"converged session observation had no store-issued authority for {}",
session.session_id
))
})?;
session_witnesses.insert(
identity.clone(),
IdentityConvergedSessionWitness {
intent_revision: intent_record.intent_revision,
intent_digest: intent_record.intent_digest.clone(),
authority_digest: intent_record.authority_digest.clone(),
observation: session_observation.clone(),
persisted_authority,
},
);
} else {
session_witnesses.remove(identity);
}
}
let unsupported_obligation_detail =
identity_reconciliation_slice_unsupported_obligation(decision).then(|| {
format!(
"generated obligation {decision:?} has no actuator in the retained sealed-intent reconciliation slice"
)
});
let reported_decision = if unsupported_obligation_detail.is_some() {
IdentityReconcileDecision::RepairBlocked
} else {
decision
};
let current_intent_authority = IdentityReconcileAuthorityKey::from_intent(&intent_record);
let prior_actuator_failure = self
.identity_reconcile_failures
.read()
.await
.get(identity)
.filter(|failure| failure.authority == current_intent_authority)
.map(|failure| failure.detail.clone());
let classification_detail = matches!(
decision,
IdentityReconcileDecision::Backoff
| IdentityReconcileDecision::RepairBlocked
| IdentityReconcileDecision::Quarantined
)
.then(|| {
format!(
"fresh identity observations classified as {facts:?}; runtime={runtime_observation:?}; member={member_observation:?}; wiring={wiring_observation:?}"
)
});
let disposition_detail = unsupported_obligation_detail
.clone()
.or(prior_actuator_failure.clone())
.or(classification_detail.clone());
let completion_authority = IdentityReconcileCompletionAuthority {
intent: current_intent_authority,
lease_epoch: Some(active_lease.epoch),
};
if decision == IdentityReconcileDecision::Converged {
self.identity_reconcile_backoff.remove(identity);
self.identity_reconcile_parked.remove(identity);
self.identity_reconcile_failures
.write()
.await
.remove(identity);
}
if unsupported_obligation_detail.is_some() {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::RepairBlocked {
detail: disposition_detail.unwrap_or_else(|| {
format!("unsupported identity recovery obligation {decision:?}")
}),
},
)
.await;
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
if decision == IdentityReconcileDecision::Backoff {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::Backoff {
detail: disposition_detail.unwrap_or_else(|| {
"fresh identity observations requested retry backoff".to_string()
}),
},
)
.await;
return Ok(IdentityReconcilePassDisposition::Outcome(false));
}
self.replace_identity_reconcile_status(
identity,
Some(intent_record.intent_revision),
Some(active_lease.epoch),
reported_decision,
disposition_detail,
)
.await;
let action: Result<bool, MobError> = async {
match decision {
IdentityReconcileDecision::SealSessionCreationConsumed => {
let Some(write_lease) = self.renew_identity_actuation_lease(identity).await?
else {
return Ok(true);
};
let authority = session_observation
.store_authority()
.cloned()
.ok_or_else(|| {
MobError::Internal(
"session-creation receipt decision had no store-issued authority"
.to_string(),
)
})?;
let permit = self.identity_actuation_permit(
identity,
&intent_record,
&write_lease,
IdentityActuatorTarget::SessionCreationReceipt,
IdentityTargetObservationVersion::InsertIfAbsent,
);
let mut receipt = IdentityOperationReceipt {
schema_version: IDENTITY_OPERATION_RECEIPT_SCHEMA_VERSION,
mob_id: self.definition.id.clone(),
subject: IdentityOperationSubject::Identity {
identity: identity.clone(),
},
effect_kind: IdentityOperationKind::SessionCreationConsumed,
slot: session_creation_slot,
receipt_id: meerkat_core::ops::OperationId::new(),
intent_revision: Some(intent_record.intent_revision),
intent_digest: Some(intent_record.intent_digest.clone()),
intent_authority_digest: Some(intent_record.authority_digest.clone()),
tombstone_generation: intent_record.tombstone_generation,
audit_lease_epoch: Some(write_lease.epoch),
request_digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000".to_string(),
payload: IdentityOperationReceiptPayload::SessionCreationConsumed {
authority,
},
};
receipt.request_digest = receipt
.canonical_request_digest()
.map_err(|error| MobError::Internal(error.to_string()))?;
match self
.identity
.insert_identity_operation_receipt_if_absent(&receipt, &permit)
.await
.map_err(MobError::from)?
{
IdentityOperationReceiptInsertOutcome::Inserted(_)
| IdentityOperationReceiptInsertOutcome::ExistingExact(_)
| IdentityOperationReceiptInsertOutcome::Conflict(_) => Ok(true),
}
}
IdentityReconcileDecision::EnsureRuntimeRegistration => {
let Some(write_lease) = self.renew_identity_actuation_lease(identity).await?
else {
return Ok(true);
};
let target = runtime_observation
.target_precondition()
.map_err(|error| MobError::Internal(error.to_string()))?
.ok_or_else(|| {
MobError::Internal(
"runtime registration decision had no target-local witness"
.to_string(),
)
})?;
let permit = self.identity_actuation_permit(
identity,
&intent_record,
&write_lease,
IdentityActuatorTarget::Runtime,
target,
);
#[cfg(feature = "runtime-adapter")]
let completion_authority =
IdentityReconcileCompletionAuthority::from_permit(&permit);
#[cfg(feature = "runtime-adapter")]
{
let runtime = self.runtime_adapter.as_ref().ok_or_else(|| {
MobError::Internal("identity runtime adapter is unavailable".to_string())
})?;
let observed = runtime_target_observation.ok_or_else(|| {
MobError::Internal(
"runtime registration decision had no exact lifecycle observation"
.to_string(),
)
})?;
let fence = self.identity.prepare_runtime_write_fence(
permit,
session.clone(),
&observed,
)?;
match runtime
.register_session_if_runtime_lifecycle_current(observed, fence)
.await
{
meerkat_runtime::RuntimeSessionRegistrationOutcome::Applied { .. } => {
#[cfg(any(test, feature = "test-support"))]
super::trigger_identity_recovery_fail_stop(
super::IdentityRecoveryFailStopPoint::RuntimeCasApplied,
);
Ok(true)
}
meerkat_runtime::RuntimeSessionRegistrationOutcome::AlreadyExact {
..
}
| meerkat_runtime::RuntimeSessionRegistrationOutcome::Conflict {
..
} => Ok(true),
meerkat_runtime::RuntimeSessionRegistrationOutcome::RepairBlocked {
reason,
..
} => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::RepairBlocked {
detail: reason,
},
)
.await;
Ok(false)
}
meerkat_runtime::RuntimeSessionRegistrationOutcome::Backoff {
reason,
} => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::Backoff { detail: reason },
)
.await;
Ok(false)
}
}
}
#[cfg(not(feature = "runtime-adapter"))]
{
Err(MobError::Internal(
"identity runtime reconciliation requires runtime-adapter".to_string(),
))
}
}
IdentityReconcileDecision::EnsureMemberMaterialization => {
let Some(write_lease) = self.renew_identity_actuation_lease(identity).await?
else {
return Ok(true);
};
let target = member_observation
.target_precondition()
.map_err(|error| MobError::Internal(error.to_string()))?
.ok_or_else(|| {
MobError::Internal(
"member materialization decision had no target-local witness"
.to_string(),
)
})?;
let permit = self.identity_actuation_permit(
identity,
&intent_record,
&write_lease,
IdentityActuatorTarget::Member,
target,
);
self.identity
.validate_identity_actuation_permit(&permit)
.await
.map_err(MobError::from)?;
let completion_authority =
IdentityReconcileCompletionAuthority::from_permit(&permit);
let mut spec = super::spec_compiler::spawn_spec_from_desired_member(
identity,
session,
member,
)?;
let key = super::IdentityLocalMaterializationKey::new(
self.definition.id.clone(),
identity.clone(),
intent_record.intent_revision,
intent_record.intent_digest.clone(),
intent_record.authority_digest.clone(),
);
match super::identity_local_services::resolve_identity_local_external_tools(
self.identity_local_external_tools_provider.as_deref(),
&key,
&member.material.required_local_callback_tools,
) {
Ok(external_tools) => spec.external_tools = external_tools,
Err(super::IdentityLocalExternalToolsError::Missing) => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::Backoff {
detail: format!(
"process-local external-tool services are not registered for sealed materialization {key}"
),
},
)
.await;
return Ok(false);
}
Err(super::IdentityLocalExternalToolsError::Unavailable { reason }) => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::Backoff {
detail: format!(
"process-local external-tool services are unavailable for sealed materialization {key}: {reason}"
),
},
)
.await;
return Ok(false);
}
Err(super::IdentityLocalExternalToolsError::AuthorityMismatch {
registered,
}) => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::RepairBlocked {
detail: format!(
"process-local external-tool authority mismatch: expected {key}, registered {registered}"
),
},
)
.await;
return Ok(false);
}
Err(super::IdentityLocalExternalToolsError::DefinitionMismatch {
detail,
}) => {
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
IdentityMemberActuationDisposition::RepairBlocked {
detail: format!(
"process-local external-tool definition mismatch for sealed materialization {key}: {detail}"
),
},
)
.await;
return Ok(false);
}
}
let (reply_tx, mut reply_rx) = oneshot::channel();
self.enqueue_spawn(
spec,
super::handle::SpawnSource::IdentityReconcile,
Some(permit),
None,
None,
reply_tx,
)
.await;
match reply_rx.try_recv() {
Ok(reply) => {
let disposition = identity_member_actuation_disposition(&reply);
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
disposition,
)
.await;
Ok(false)
}
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
let reply = Err(MobError::Internal(
"identity member materialization reply closed before custody transfer"
.to_string(),
));
let disposition = identity_member_actuation_disposition(&reply);
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
disposition,
)
.await;
Ok(false)
}
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {
// The serialized spawn-completion command records
// the typed disposition and requeues this identity
// before attempting reply delivery. Drop this
// internal observer deliberately: convergence is
// causally driven by actor custody, never by a
// detached reply waiter or the safety sweep.
drop(reply_rx);
Ok(false)
}
}
}
IdentityReconcileDecision::Backoff
| IdentityReconcileDecision::RepairBlocked
| IdentityReconcileDecision::AwaitLease
| IdentityReconcileDecision::Converged
| IdentityReconcileDecision::Quarantined => Ok(false),
IdentityReconcileDecision::AcquireLease
| IdentityReconcileDecision::SealRetirementProven
| IdentityReconcileDecision::EnsureSessionAuthority
| IdentityReconcileDecision::AwaitExternalBindingCeremony
| IdentityReconcileDecision::EnsureExternalBindingReceipt
| IdentityReconcileDecision::EnsureExternalBinding
| IdentityReconcileDecision::EnsureInitialDeliveryReceipt
| IdentityReconcileDecision::EnsureInitialDelivery
| IdentityReconcileDecision::AwaitInitialDelivery
| IdentityReconcileDecision::ReconcileWiring
| IdentityReconcileDecision::RetireMemberMaterialization
| IdentityReconcileDecision::RetireRuntimeRegistration
| IdentityReconcileDecision::ReleaseSessionAuthority
| IdentityReconcileDecision::Tombstoned => Err(MobError::Internal(format!(
"unsupported identity recovery obligation {decision:?} escaped its RepairBlocked admission boundary"
))),
}
}
.await;
match action {
Ok(requeue) => Ok(IdentityReconcilePassDisposition::Outcome(requeue)),
Err(error) => {
let disposition = identity_actuation_error_disposition(&error);
self.record_identity_reconcile_disposition(
identity,
&completion_authority,
disposition,
)
.await;
Ok(IdentityReconcilePassDisposition::Outcome(false))
}
}
}
#[inline(never)]
fn dispatch_command_boxed<'a>(
&'a mut self,
authority: &'a CommandAuthority,
cmd: MobCommand,
command_rx: &'a mut mpsc::Receiver<RoutedMobCommand>,
deferred_commands: &'a mut VecDeque<RoutedMobCommand>,
host_status_polls_in_flight: &'a mut BTreeSet<mob_dsl::HostId>,
) -> ActorCommandFuture<'a, ActorLoopControl> {
match cmd {
// Keep this catalog-classified fail-closed gate as ordinary AST so
// the machine-authority drift audit can verify the exact
// MobCommand -> MobMachineInput relationship.
MobCommand::SetSpawnPolicy { policy, reply_tx } => Box::pin(async move {
let enabled = policy.is_some();
let result = self
.apply_dsl_input(
mob_dsl::MobMachineInput::SetSpawnPolicy { enabled },
"set_spawn_policy",
)
.map_err(|error| {
MobError::Internal(format!(
"SetSpawnPolicy rejected by MobMachine guards before shell policy write: {error}"
))
});
if result.is_ok() {
self.spawn_policy.set(policy).await;
}
let _ = reply_tx.send(result);
ActorLoopControl::ProceedBoundary
}),
cmd => boxed_actor_dispatch!(cmd, {
MobCommand::Spawn {
spec,
spawn_source,
owner_bridge_session_id,
ops_registry,
reply_tx,
} => {
tracing::debug!(
member_id = %spec.identity,
profile = %spec.role_name,
owner_bound = owner_bridge_session_id.is_some(),
"MobActor received Spawn command"
);
Box::pin(self.enqueue_spawn(
*spec,
spawn_source,
None,
owner_bridge_session_id,
ops_registry,
reply_tx,
))
.await;
}
MobCommand::SpawnProvisioned {
spawn_ticket,
result,
} => {
let mut completions = vec![(spawn_ticket, result)];
loop {
match command_rx.try_recv() {
Ok(RoutedMobCommand {
cmd:
MobCommand::SpawnProvisioned {
spawn_ticket,
result,
},
..
}) => completions.push((spawn_ticket, result)),
// Deferred commands keep their routing authority
// and re-enter the scope gate when popped.
Ok(other) => {
deferred_commands.push_back(other);
break;
}
Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break,
Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => break,
}
}
// Box::pin: the `handle_spawn_provisioned_batch`
// future tips past clippy's `large_futures` 16 KiB
// threshold after Track-B R5 added
// `peer_projection_*` state to `MeerkatMachine` —
// the enum size cascades through the actor's
// transitive borrows. Heap-allocating the future
// keeps the polling stack frame small without
// reshaping the handler.
Box::pin(self.handle_spawn_provisioned_batch(completions)).await;
}
MobCommand::RevivePlacedMember {
agent_identity,
reason,
} => {
// Internal fire-and-forget trigger: the outcome is
// machine-recorded (revival pending → resolved, or
// Broken) and mirrored into restore diagnostics; typed
// failures are logged for the operator.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
if let Err(error) =
Box::pin(self.revive_placed_member(&agent_identity, &reason)).await
{
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
error = %error,
"placed member revival failed"
);
}
// Builds without the remote materialization lane cannot
// realize the re-issue; the trigger is honestly logged
// (placed spawns cannot be created on these builds).
#[cfg(not(all(feature = "runtime-adapter", not(target_arch = "wasm32"))))]
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
reason = %reason,
"placed member revival trigger on a build without the remote \
materialization lane"
);
}
MobCommand::HostStatusPollCompleted {
host_id,
binding_epoch,
binding_generation,
binding_incarnation,
result,
} => {
let host_id = mob_dsl::HostId::from(host_id);
host_status_polls_in_flight.remove(&host_id);
let binding_is_current = {
let state = self.dsl_authority.state();
state.host_bind_phase.get(&host_id) == Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_authority_epochs.get(&host_id).copied()
== Some(binding_epoch)
&& state.host_binding_generations.get(&host_id).copied()
== Some(binding_generation)
&& self.host_binding_incarnations.get(&host_id).copied()
== Some(binding_incarnation)
};
if !binding_is_current {
tracing::debug!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
binding_epoch,
binding_generation,
binding_incarnation,
"discarding stale periodic host-status response"
);
return ActorLoopControl::SkipBoundary;
}
match result {
Ok(status) => {
match self.reconcile_host_status_response(&host_id, status).await {
Ok(runtime_incarnation_changed) => {
// Member trust is process-local. A fresh
// authenticated host incarnation therefore
// re-derives this host's obligations from
// the MobMachine wiring graph; an unchanged
// incarnation only retries already-pending
// rows. Both convergence actions precede
// the reachability success projection so
// readiness never races route recovery.
let route_result = self
.converge_routes_after_host_runtime_observation(
&host_id,
runtime_incarnation_changed,
)
.await;
if let Err(error) = route_result {
self.reachability_observations
.mark_host_failure(host_id.as_str(), &error);
tracing::error!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %error,
"authenticated host status found an invalid pending route-install ledger"
);
} else {
self.reachability_observations
.mark_host_success(host_id.as_str());
}
}
Err(error) => {
self.reachability_observations
.mark_host_failure(host_id.as_str(), &error);
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %error,
"periodic host status response failed reconciliation"
);
}
}
}
Err(error) => {
if Self::host_status_rejection_requires_fail_stop(&error) {
self.durable_uncertainty_fail_stop = true;
}
self.reachability_observations
.mark_host_failure(host_id.as_str(), &error);
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %error,
"periodic host status/reconciliation poll failed"
);
}
}
}
MobCommand::HostRuntimeIncarnationObserved {
expected_member,
runtime_incarnation,
reply_tx,
} => {
let result = match self
.validate_member_events_runtime_observation(&expected_member)
{
Err(error) => Err(error),
Ok(host_id) => {
let changed = self.record_host_runtime_incarnation(
&host_id,
runtime_incarnation,
"member event page",
);
match self
.converge_routes_after_host_runtime_observation(&host_id, changed)
.await
{
Ok(()) => {
self.reachability_observations
.mark_host_member_events_success(host_id.as_str());
Ok(())
}
Err(error) => {
self.reachability_observations
.mark_host_failure(host_id.as_str(), &error);
Err(error)
}
}
}
};
let _ = reply_tx.send(result);
}
MobCommand::HostOrphanReleaseCompleted { key, result } => {
let binding_is_current = {
let state = self.dsl_authority.state();
state.host_bind_phase.get(&key.host_id)
== Some(&mob_dsl::HostBindPhase::Bound)
&& self.host_binding_incarnations.get(&key.host_id).copied()
== Some(key.binding_incarnation)
};
if !binding_is_current {
self.orphan_release_reservations.remove(&key);
tracing::debug!(
mob_id = %self.definition.id,
host = %key.host_id.as_str(),
binding_incarnation = key.binding_incarnation,
agent_identity = %key.agent_identity.0,
generation = key.generation.0,
fence_token = key.fence_token.0,
"discarding orphan release completion from a superseded host binding"
);
return ActorLoopControl::SkipBoundary;
}
let succeeded = result.is_ok();
let completion_was_owned = Self::absorb_host_orphan_release_completion(
&mut self.orphan_release_reservations,
&key,
succeeded,
);
if !completion_was_owned {
tracing::warn!(
mob_id = %self.definition.id,
host = %key.host_id.as_str(),
binding_incarnation = key.binding_incarnation,
agent_identity = %key.agent_identity.0,
generation = key.generation.0,
fence_token = key.fence_token.0,
"orphan release completion had no exact actor-owned in-flight key"
);
}
match result {
Ok(disposal) => {
tracing::info!(
target: "mob.host.orphan_released",
mob_id = %self.definition.id,
host = %key.host_id.as_str(),
binding_incarnation = key.binding_incarnation,
agent_identity = %key.agent_identity.0,
generation = key.generation.0,
fence_token = key.fence_token.0,
disposal = ?disposal,
"orphan host row released at the stale tuple"
);
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
host = %key.host_id.as_str(),
binding_incarnation = key.binding_incarnation,
agent_identity = %key.agent_identity.0,
generation = key.generation.0,
fence_token = key.fence_token.0,
error = %error,
"orphan release failed; the next sweep retries"
);
}
}
}
MobCommand::PlacedBehaviorCompleted {
agent_identity,
expected_member,
completion,
} => {
let current = {
let entry = self.roster.read().await.get(&agent_identity).cloned();
entry
.as_ref()
.map(|entry| self.placed_member_incarnation(entry))
.transpose()
};
let completion_gate = current.and_then(|current| {
if current.as_ref() == Some(&expected_member) {
Ok(())
} else {
Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleFence,
reason: format!(
"placed behavior completed for superseded member incarnation {expected_member:?}; current is {current:?}"
),
})
}
});
match completion {
super::state::PlacedBehaviorCompletion::MemberHistory {
result,
reply_tx,
} => {
let _ = reply_tx.send(completion_gate.and(result));
}
super::state::PlacedBehaviorCompletion::LiveClose { result, reply_tx } => {
let _ = reply_tx.send(completion_gate.and(result));
}
super::state::PlacedBehaviorCompletion::LiveStatus { result, reply_tx } => {
let _ = reply_tx.send(completion_gate.and(result));
}
}
}
MobCommand::Retire {
agent_identity,
reply_tx,
} => {
// MobMachine owns the cancel-vs-preserve decision from its
// canonical committed incarnation and pending-session
// maps. The actor realizes cancellation only when the
// verdict names the exact pending session; an absent
// retire mechanically preserves a later incarnation.
// A previous exact-incarnation cancellation may have
// committed `CancelPendingSpawn` before its mechanical
// abort/archive failed. Drain only those retained anchors
// for this stable identity before asking MobMachine about
// the current incarnation. If cleanup is still refused we
// return without classifying, so a distinct later pending
// session remains untouched. Once cleanup succeeds the
// current machine state receives the normal typed verdict.
let result = if let Err(error) = self
.drain_pending_spawn_cleanup_anchors_for_member(
&agent_identity,
"retire command retained pending-spawn cleanup",
)
.await
{
Err(error)
} else {
match self.classify_retire_pending_spawn_disposition(&agent_identity) {
Ok(RetirePendingSpawnVerdict::CancelCommittedIncarnation {
agent_runtime_id,
generation,
pending_spawn_session_id,
}) => match self
.cancel_pending_spawn_for_retire_incarnation(
&agent_identity,
RetirePendingSpawnCleanupIncarnation {
agent_runtime_id: agent_runtime_id.clone(),
generation,
pending_spawn_session_id: pending_spawn_session_id.clone(),
},
"retire command received",
)
.await
{
Ok(canceled) => {
tracing::info!(
agent_identity = %agent_identity,
agent_runtime_id = %agent_runtime_id.0,
generation = generation.0,
pending_spawn_session_id = %pending_spawn_session_id.0,
canceled,
"MobMachine-authorized retire canceled exact pending spawn incarnation"
);
self.handle_retire(agent_identity).await
}
Err(error) => Err(error),
},
Ok(
RetirePendingSpawnVerdict::CommittedIncarnationWithoutPendingSpawn {
agent_runtime_id,
generation,
},
) => {
tracing::debug!(
agent_identity = %agent_identity,
agent_runtime_id = %agent_runtime_id.0,
generation = generation.0,
"MobMachine resolved retire against committed incarnation without pending spawn"
);
self.handle_retire(agent_identity).await
}
Ok(
RetirePendingSpawnVerdict::PreservePendingSpawnForAbsentIdentity,
) => {
tracing::debug!(
agent_identity = %agent_identity,
"MobMachine resolved absent retire and preserved any pending later incarnation"
);
self.handle_retire(agent_identity).await
}
Err(error) => Err(error),
}
};
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
}
MobCommand::Respawn {
agent_identity,
initial_message,
reply_tx,
} => {
let respawn_identity = agent_identity.clone();
match Box::pin(self.handle_respawn(agent_identity, initial_message)).await {
Ok(RespawnProgress::Completed(receipt)) => {
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(Ok(receipt));
}
}
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
Ok(RespawnProgress::DeferredPlaced {
identity,
old_fence_token,
spawn_reply_rx,
}) => {
// Completed off the actor loop: the remote lane
// finalizes through this mailbox, which must stay
// free to process it.
if !self.respawn_topology_reply_withheld {
let roster = Arc::clone(&self.roster);
self.actor_io_tasks.spawn(async move {
if let Some(result) = Self::complete_placed_respawn(
roster,
identity,
old_fence_token,
spawn_reply_rx,
)
.await
{
let _ = reply_tx.send(result);
}
});
}
}
Err(error) => {
match self
.durably_abandon_respawn_topology_if_terminal(&respawn_identity)
.await
{
Ok(()) if !self.respawn_topology_reply_withheld => {
let _ = reply_tx.send(Err(error));
}
Ok(()) => {
tracing::error!(
agent_identity = %respawn_identity,
"withholding respawn failure reply because the actor is fail-stopping for cold recovery"
);
}
Err(abandon_error) => {
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
tracing::error!(
agent_identity = %respawn_identity,
error = %error,
abandonment_error = %abandon_error,
"durable topology abandonment failed; withholding respawn failure reply and fail-stopping for cold recovery"
);
}
}
}
}
}
MobCommand::RetireAll { reply_tx } => {
let result = async {
self.probe_command_admission(
mob_dsl::MobMachineInput::RetireAll,
MobState::Running,
"retire_all_preflight",
)?;
self.ensure_pending_spawn_alignment("retire_all preflight")?;
// First mint every member's durable retirement-start
// anchor. A failure before that boundary must not
// publish a global lifecycle fence that cancels an
// unrelated in-flight spawn.
self.retire_all_members("retire_all").await?;
self.drive_placed_completion_lifecycle_cleanup(
None,
false,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::RetireAll),
)
.await?;
self.end_placed_completion_lifecycle_quiesce(
mob_dsl::PlacedCompletionLifecycleIntentKind::RetireAll,
)
.await
}
.await;
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
}
MobCommand::SubmitWork { payload, reply_tx } => {
tracing::debug!(
agent_identity = %payload.runtime_id.identity,
runtime_id = %payload.runtime_id,
work_ref = %payload.work_ref,
origin = ?payload.origin,
handling_mode = ?payload.handling_mode,
ack_mode = ?payload.ack_mode,
"MobActor handling SubmitWork command"
);
match Box::pin(self.handle_submit_work(payload)).await {
Ok(SubmitWorkDispatchCompletion::Completed) => {
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(Ok(()));
}
}
Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id: _,
member_ref,
req,
completion_tx,
llm_identity_applied_tx,
placed_identity,
placed_incarnation,
placed_input_id,
}) => {
if !self.respawn_topology_reply_withheld {
self.spawn_turn_admission_reply(
self.provisioner.clone(),
member_ref,
req,
completion_tx,
llm_identity_applied_tx,
reply_tx,
placed_identity
.map(|identity| (self.command_tx.clone(), identity)),
placed_incarnation,
placed_input_id,
);
}
}
Ok(SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref,
req,
placed_identity,
placed_incarnation,
placed_input_id,
placed_completion_obligation,
placed_completion_context,
}) => {
// Register the exact waiter on the actor before
// spawning the sender. Every post-Record failure
// durably requests cancellation; none can strand
// machine Pending custody.
let remote = match (
placed_identity.as_ref(),
placed_incarnation,
placed_input_id,
placed_completion_obligation,
placed_completion_context,
) {
(
Some(identity),
Some(expected_member),
Some(input_id),
Some(obligation),
Some(context),
) => {
match self.prepare_placed_completion_wait(
identity,
context,
expected_member,
input_id,
obligation.clone(),
) {
Ok(prepared) => Some(prepared),
Err(error) => {
let aftercare_handle = self.mob_handle_for_tools();
let aftercare_identity =
obligation.agent_identity.clone();
let aftercare_obligation = obligation.clone();
let error = self
.unwind_placed_completion_prepare_failure(
obligation, error,
)
.await;
self.try_fire_placed_revival_trigger(
placed_identity.clone(),
&error,
);
let _ = reply_tx.send(Err(error));
self.actor_io_tasks.spawn(async move {
Self::persist_placed_completion_cancellation_until_durable(
&aftercare_handle,
&aftercare_identity,
&aftercare_obligation,
"pre_send_waiter_registration_failed",
)
.await;
});
return ActorLoopControl::SkipBoundary;
}
}
}
(None, None, None, None, None) => None,
(_, _, _, obligation, _) => {
let error = MobError::Internal(
"turn completion placement/custody fields drifted"
.to_string(),
);
let aftercare = obligation.as_ref().map(|obligation| {
(
self.mob_handle_for_tools(),
obligation.agent_identity.clone(),
obligation.clone(),
)
});
let error = if let Some(obligation) = obligation {
self.unwind_placed_completion_prepare_failure(
obligation, error,
)
.await
} else {
error
};
let _ = reply_tx.send(Err(error));
if let Some((handle, identity, obligation)) = aftercare {
self.actor_io_tasks.spawn(async move {
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"pre_send_custody_shape_drift",
)
.await;
});
}
return ActorLoopControl::SkipBoundary;
}
};
self.spawn_turn_completed_reply(
self.provisioner.clone(),
member_ref,
req,
reply_tx,
placed_identity.map(|identity| (self.command_tx.clone(), identity)),
remote,
);
}
Err(error) => {
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(Err(error));
}
}
}
}
MobCommand::SendPeerMessage {
from,
to,
content,
handling_mode,
reply_tx,
} => {
match Box::pin(self.prepare_send_peer_message(from, to, content, handling_mode))
.await
{
Ok(plan) => {
self.spawn_peer_message_delivery(plan, reply_tx);
}
Err(error) => {
let _ = reply_tx.send(Err(error));
}
}
}
MobCommand::DeclareMemberOutboundTaint {
identity,
taint,
reply_tx,
} => {
let result =
Box::pin(self.declare_member_outbound_taint(identity, taint)).await;
let _ = reply_tx.send(result);
}
#[cfg(feature = "runtime-adapter")]
MobCommand::KickoffOutcomeResolved {
agent_identity,
outcome,
ack_tx,
} => {
if let Err(error) = self.resolve_kickoff_outcome(&agent_identity, outcome).await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
"failed to persist kickoff outcome"
);
}
let _ = ack_tx.send(());
}
MobCommand::RunFlow {
run_id,
flow_id,
activation_params,
scoped_event_tx,
reply_tx,
} => {
let result = self
.handle_run_flow(run_id, flow_id, activation_params, scoped_event_tx)
.await;
let _ = reply_tx.send(result);
}
MobCommand::PreviewRunFlowAdmission { reply_tx } => {
let result = self.preview_run_flow_command_admission(&RunId::new());
let _ = reply_tx.send(result);
}
MobCommand::CancelFlow { run_id, reply_tx } => {
let result = self.handle_cancel_flow(run_id).await;
let _ = reply_tx.send(result);
}
MobCommand::FlowStatus { run_id, reply_tx } => {
let result = self
.run_store
.get_run(&run_id)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowRunCommand {
run_id,
command,
context,
reply_tx,
} => {
let result = self
.commit_flow_run_command_in_actor(&run_id, *command, context)
.await;
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowTerminalization {
run_id,
flow_id,
target,
command,
context,
reply_tx,
} => {
let result = self
.commit_flow_terminalization_in_actor(
run_id, flow_id, target, *command, context,
)
.await;
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowFrameStorePlan {
run_id,
plan,
reply_tx,
} => {
let result = self
.commit_flow_frame_store_plan_in_actor(&run_id, *plan)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ProjectMachineInput { input, reply_tx } => {
if let Err(error) = Self::reject_raw_grant_machine_input(&input) {
let _ = reply_tx.send(Err(error));
return ActorLoopControl::SkipBoundary;
}
let result = self
.apply_dsl_input(*input, "project_machine_input")
.map(|()| self.dsl_authority.state().clone());
let _ = reply_tx.send(result);
}
MobCommand::ApplyMachineInputEffects { input, reply_tx } => {
if let Err(error) = Self::reject_raw_grant_machine_input(&input) {
let _ = reply_tx.send(Err(error));
return ActorLoopControl::SkipBoundary;
}
let result =
self.apply_dsl_input_collect_effects(*input, "apply_machine_input_effects");
let _ = reply_tx.send(result);
}
MobCommand::ValidateCommandAuthority { reply_tx } => {
// Admission already ran at the actor mailbox boundary.
// Reaching the handler is the serialized validation ACK.
let _ = reply_tx.send(Ok(()));
}
MobCommand::AdmitControlScope { reply_tx, .. } => {
// The requested scope was enforced at the serialized
// mailbox gate. No projection or mutation occurs here.
let _ = reply_tx.send(Ok(()));
}
MobCommand::PruneStaleMemberOperatorRequests { reply_tx } => {
let result = self.prune_stale_member_operator_requests().await;
let _ = reply_tx.send(result);
}
MobCommand::ReserveRemoteTurnObligation {
mut intent,
reply_tx,
} => {
let result = async {
let dispatch_sequence = next_remote_turn_dispatch_sequence(
&self.run_store,
&self.definition.id,
self.dsl_authority
.state()
.remote_turn_dispatch_sequence,
)
.await?;
let target_identity = intent.obligation.agent_identity.clone();
let dsl_identity = mob_dsl::AgentIdentity(target_identity.to_string());
let (
host_id,
generation,
fence_token,
host_binding_generation,
member_session_id,
) = {
let state = self.dsl_authority.state();
let missing = |fact: &str| {
MobError::Internal(format!(
"remote-turn target '{target_identity}' has no current {fact}"
))
};
(
state
.member_placement
.get(&dsl_identity)
.cloned()
.ok_or_else(|| missing("host placement"))?,
state
.identity_runtime_generations
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("generation"))?,
state
.identity_runtime_fence_tokens
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("fence token"))?,
state
.current_placed_spawn_host_binding_generations
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("host binding generation"))?,
state
.member_session_bindings
.get(&dsl_identity)
.cloned()
.ok_or_else(|| missing("member session binding"))?,
)
};
let entry = self
.roster
.read()
.await
.get(&target_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(target_identity.clone()))?;
if entry.generation.get() != generation.0
|| entry.fence_token.get() != fence_token.0
{
return Err(MobError::Internal(format!(
"remote-turn roster incarnation for '{target_identity}' is stale against machine authority"
)));
}
let crate::event::MemberRef::BackendPeer {
session_id: route_session_id,
..
} = &entry.member_ref
else {
return Err(MobError::Internal(format!(
"remote-turn target '{target_identity}' has no peer transport route"
)));
};
if !optional_route_session_matches_machine(
route_session_id.as_ref(),
&member_session_id.0,
) {
return Err(MobError::Internal(format!(
"remote-turn route session for '{target_identity}' is stale against machine authority"
)));
}
intent.obligation.host_id = host_id.0.clone();
intent.obligation.host_binding_generation = host_binding_generation;
intent.obligation.member_session_id = member_session_id.0.clone();
intent.obligation.generation = crate::ids::Generation::new(generation.0);
intent.obligation.fence_token =
crate::ids::FenceToken::new(fence_token.0);
intent.obligation.dispatch_sequence = dispatch_sequence;
intent.expected_member =
meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: target_identity.to_string(),
host_id: host_id.0.clone(),
binding_generation: host_binding_generation,
member_session_id: member_session_id.0,
generation: generation.0,
fence_token: fence_token.0,
};
let encoded_len = serde_json::to_vec(&intent)
.map_err(|error| {
MobError::Internal(format!(
"remote-turn intent preflight encoding failed: {error}"
))
})?
.len();
if encoded_len > crate::run::REMOTE_TURN_INTENT_MAX_ENCODED_BYTES {
return Err(MobError::Internal(format!(
"remote-turn intent is {encoded_len} bytes; conservative signed-wire maximum is {} bytes",
crate::run::REMOTE_TURN_INTENT_MAX_ENCODED_BYTES
)));
}
let obligation =
super::remote_flow_ticket::obligation_from_event(&intent.obligation);
let carrier = super::remote_flow_ticket::obligation_event(&obligation)?;
let prepared = self
.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordRemoteTurnObligation {
obligation: obligation.clone(),
},
"reserve_remote_turn_obligation",
)
.map_err(|error| {
let state = self.dsl_authority.state();
let runtime_id = state.identity_to_runtime.get(&dsl_identity);
MobError::Internal(format!(
"{error}; remote-turn admission facts: placement={:?}, host_phase={:?}, durable={:?}, tracked_cancel={:?}, protocol={:?}..={:?}, session={:?}, runtime={runtime_id:?}, live={}, startup_ready={}, retiring={:?}, revival_pending={}, spawn_phase={:?}, materialization_failure={}, generation={:?}, fence={:?}, dispatch_highwater={}",
state.member_placement.get(&dsl_identity),
state.host_bind_phase.get(&host_id),
state.host_durable_sessions.get(&host_id),
state.host_tracked_input_cancel.get(&host_id),
state.host_protocol_min.get(&host_id),
state.host_protocol_max.get(&host_id),
state.member_session_bindings.get(&dsl_identity),
runtime_id.is_some_and(|runtime_id| state.live_runtime_ids.contains(runtime_id)),
runtime_id.is_some_and(|runtime_id| state.member_startup_ready.contains(runtime_id)),
runtime_id.and_then(|runtime_id| state.member_state_markers.get(runtime_id)),
state.member_revival_pending.contains(&dsl_identity),
state.spawn_exec_phase.get(&dsl_identity),
state.member_materialization_failures.contains_key(&dsl_identity),
state.identity_runtime_generations.get(&dsl_identity),
state.identity_runtime_fence_tokens.get(&dsl_identity),
state.remote_turn_dispatch_sequence,
))
})?;
let intent = *intent;
let reserved_intent = intent.clone();
let member_ref = entry.member_ref.clone();
let run_id = intent.obligation.run_id.clone();
let tracked_key = PlacedTrackedInputKey::remote_turn(&carrier);
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.reserve_fresh_tracked_input(
tracked_key,
PlacedTrackedInputOwner::RemoteTurn(carrier.clone()),
)?;
self.run_store
.put_remote_turn_intent(&run_id, &intent)
.await?;
// The private intent is already the replay authority,
// so reserve its host TurnKey in the derived index
// even if the public Record projection append returns
// ambiguously and is repaired later.
let public_record =
MobEventKind::RemoteTurnObligationRecorded {
obligation: carrier,
};
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.observe(&public_record)?;
// The private intent is the replay-complete authority
// carrier. The public Record is an ordered projection:
// attempt it before machine commit, but an ambiguous
// append result cannot roll back or reuse the sequence.
// Recovery reprojects from the intent when needed.
let carrier_append = self
.ensure_remote_turn_carrier(public_record)
.await;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"remote-turn private/public Record is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
if let Err(error) = carrier_append {
tracing::warn!(
run_id = %run_id,
dispatch_sequence = intent.obligation.dispatch_sequence,
error = %error,
"remote-turn public Record append was ambiguous; durable intent remains authoritative for reprojection"
);
}
Ok(super::remote_flow_ticket::ReservedRemoteTurnIntent {
intent: reserved_intent,
member_ref,
})
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::CommitRemoteTurnReceipt { receipt, reply_tx } => {
let result = self.commit_remote_turn_receipt_in_actor(*receipt).await;
let _ = reply_tx.send(result);
}
MobCommand::CloseRemoteTurnAfterTrackedCancel { receipt, reply_tx } => {
let result = self
.close_remote_turn_after_tracked_cancel_in_actor(*receipt)
.await;
let _ = reply_tx.send(result);
}
MobCommand::EnsureRemoteTurnRecord {
obligation,
reply_tx,
} => {
let result = async {
let obligation = *obligation;
let dsl_obligation =
super::remote_flow_ticket::obligation_from_event(&obligation);
let intent = self
.run_store
.list_remote_turn_intents(&obligation.run_id)
.await?
.into_iter()
.find(|intent| {
intent.obligation.dispatch_sequence
== obligation.dispatch_sequence
})
.ok_or_else(|| {
MobError::Internal(format!(
"remote-turn Record repair has no private intent for run '{}' sequence {}",
obligation.run_id, obligation.dispatch_sequence
))
})?;
intent
.validate_for(&obligation.run_id, &self.definition.id)
.map_err(|error| {
MobError::Internal(format!(
"remote-turn Record repair private intent is invalid: {error}"
))
})?;
if intent.obligation != obligation {
return Err(MobError::Internal(format!(
"remote-turn Record repair tuple conflicts with private intent at sequence {}",
obligation.dispatch_sequence
)));
}
let state = self.dsl_authority.state();
let current = state
.pending_remote_turn_outcomes
.contains(&dsl_obligation)
|| state
.committed_remote_turn_outcomes
.contains(&dsl_obligation)
|| state
.resolved_remote_turn_outcomes
.contains(&dsl_obligation);
if !current {
return Err(MobError::Internal(format!(
"remote-turn Record repair has no current custody at sequence {}",
obligation.dispatch_sequence
)));
}
self.ensure_remote_turn_carrier(
MobEventKind::RemoteTurnObligationRecorded { obligation },
)
.await
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::FinalizeRemoteTurnPrivacyCleanup { cleanup, reply_tx } => {
let result = self
.finalize_remote_turn_privacy_cleanup_in_actor(*cleanup)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ConvergeRecoveredFlowRun { run_id, reply_tx } => {
let result = self.converge_recovered_flow_run_in_actor(run_id).await;
let _ = reply_tx.send(result);
}
MobCommand::ResolveRemoteTurnOutcome {
obligation,
record,
reply_tx,
} => {
let result = async {
let obligation = *obligation;
self.validate_remote_turn_outcome_residency(&obligation)?;
if record.input_id != obligation.input_id.0
|| record.generation != obligation.generation.0
|| record.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"remote-turn outcome record does not match exact custody"
.to_string(),
));
}
let carrier = super::remote_flow_ticket::obligation_event(&obligation)?;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::ResolveRemoteTurnObligation { obligation },
"resolve_remote_turn_outcome",
)?;
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnOutcomeResolved {
obligation: carrier,
})
.await?;
self.commit_prepared_dsl_transition(prepared)
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::AcknowledgeRemoteTurnOutcome {
obligation,
ack,
reply_tx,
} => {
let result = async {
let obligation = *obligation;
self.validate_remote_turn_outcome_residency(&obligation)?;
if ack.input_id != obligation.input_id.0
|| ack.generation != obligation.generation.0
|| ack.fence_token != obligation.fence_token.0
{
return Err(MobError::Internal(
"remote-turn outcome ACK does not match exact custody".to_string(),
));
}
let carrier = super::remote_flow_ticket::obligation_event(&obligation)?;
let run_id = carrier.run_id.clone();
let dispatch_sequence = carrier.dispatch_sequence;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::AcknowledgeRemoteTurnOutcome { obligation },
"acknowledge_remote_turn_outcome",
)?;
self.ensure_remote_turn_carrier(
MobEventKind::RemoteTurnOutcomeAcknowledged {
obligation: carrier,
},
)
.await?;
// Delete while Resolved custody is still present. If
// cleanup fails, an ACK retry re-enters this exact
// actor path; publishing machine absence first would
// strand the private rows until process restart.
self.run_store
.delete_remote_turn_receipt(&run_id, dispatch_sequence)
.await?;
self.run_store
.delete_remote_turn_intent(&run_id, dispatch_sequence)
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::RequestPlacedCompletionCancellation {
obligation,
reply_tx,
} => {
let result = self
.request_placed_completion_cancellation_in_actor(*obligation)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ResolvePlacedCompletionOutcome {
obligation,
record,
reply_tx,
} => {
let result = self
.resolve_placed_completion_outcome_in_actor(*obligation, record)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ClosePlacedCompletionOutcome {
obligation,
closure,
reply_tx,
} => {
let result = self
.close_placed_completion_outcome_in_actor(*obligation, closure)
.await;
let _ = reply_tx.send(result);
}
MobCommand::AcknowledgePlacedCompletionOutcome {
obligation,
ack,
reply_tx,
} => {
let result = self
.acknowledge_placed_completion_outcome_in_actor(*obligation, ack)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ResolvePlacedKickoffOutcome {
obligation,
record,
reply_tx,
} => {
let result = self
.resolve_placed_kickoff_outcome_in_actor(*obligation, record)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ResolvePlacedKickoffCancelled {
obligation,
reply_tx,
} => {
let result = self
.resolve_placed_kickoff_cancelled_in_actor(*obligation)
.await;
let _ = reply_tx.send(result);
}
MobCommand::AcknowledgePlacedKickoffOutcome {
obligation,
ack,
reply_tx,
} => {
let result = self
.acknowledge_placed_kickoff_outcome_in_actor(*obligation, ack)
.await;
let _ = reply_tx.send(result);
}
MobCommand::RejectPlacedKickoffBeforeAdmission {
obligation,
error,
reply_tx,
} => {
let result = self
.reject_placed_kickoff_before_admission_in_actor(*obligation, error)
.await;
let _ = reply_tx.send(result);
}
MobCommand::PreviewMachineInput { input, reply_tx } => {
let result = self.preview_dsl_input(*input, "preview_machine_input");
let _ = reply_tx.send(result);
}
MobCommand::QueryMachineState { reply_tx } => {
let _ = reply_tx.send(self.dsl_authority.state().clone());
}
MobCommand::GrantScopes {
caller,
principal,
scopes,
expires_at_ms,
reply_tx,
} => {
// One acting principal (zealot M2): the explicit caller
// must be the routed transport principal.
debug_assert!(
authority.carried_principal() == Some(&caller),
"grant caller must equal the routed command authority principal"
);
let result = self
.handle_grant_scopes_command(caller, principal, scopes, expires_at_ms)
.await;
let _ = reply_tx.send(result);
}
MobCommand::RevokeScopes {
caller,
principal,
scopes,
reply_tx,
} => {
debug_assert!(
authority.carried_principal() == Some(&caller),
"revoke caller must equal the routed command authority principal"
);
let result = self
.handle_revoke_scopes_command(caller, principal, scopes)
.await;
let _ = reply_tx.send(result);
}
MobCommand::Grants { caller, reply_tx } => {
debug_assert!(
authority.carried_principal() == Some(&caller),
"grants caller must equal the routed command authority principal"
);
let _ = reply_tx.send(self.handle_grants_command(caller));
}
#[cfg(test)]
MobCommand::AuthorizeMemberTrustCleanupForTest { edge, reply_tx } => {
let result = self
.authorize_member_trust_cleanup(
&edge,
"authorize_member_trust_cleanup_for_test",
)
.and_then(|handoff| match handoff.authority {
MemberTrustAuthority::Unwiring(obligation) => Ok(obligation),
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Repair(_) => {
Err(MobError::WiringError(
"member trust cleanup returned non-unwiring authority"
.to_string(),
))
}
});
let _ = reply_tx.send(result);
}
#[cfg(test)]
MobCommand::StagePendingSpawnForRetireTest {
agent_identity,
pending_spawn_session_id,
operation_id,
reply_tx,
} => {
let result = self
.stage_pending_spawn_for_retire_test(
agent_identity,
pending_spawn_session_id,
operation_id,
)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ApplyExternalPeerReciprocalTrust {
key,
target_comms,
peer,
reply_tx,
} => {
let result = self
.apply_external_peer_reciprocal_trust(key, target_comms, peer)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ProjectMachineSignal { signal } => {
let foreign_runtime_id =
foreign_runtime_observation(self.dsl_authority.state(), &signal).cloned();
if let Err(error) = self.apply_dsl_signal(signal, "project_machine_signal") {
if let Some(agent_runtime_id) = foreign_runtime_id {
tracing::debug!(
?agent_runtime_id,
error = %error,
"ignored foreign runtime lifecycle observation"
);
} else {
tracing::error!(
error = %error,
"typed composition signal projection failed"
);
}
}
}
MobCommand::RecordMissingMemberBridgeSession {
agent_identity,
bridge_session_id,
} => {
// Ready-wait saw a missing bridge-session snapshot. Re-check the
// live member binding under actor ownership before marking the
// member broken; a rebind that landed during the handle's read
// makes this a stale observation the guard drops.
let _ = self
.record_missing_member_bridge_session(
&agent_identity,
&bridge_session_id,
"ready_wait",
)
.await;
}
MobCommand::FlowFinished { run_id } => {
if let Err(error) = self
.handle_flow_cleanup(run_id, "flow finished cleanup")
.await
{
tracing::error!(error = %error, "flow finished cleanup failed");
// The FlowEngine future is already gone. If its
// durable terminalization cannot be proved, no live
// owner remains for Pending remote-turn custody.
// Fail-stop into cold recovery instead of leaving a
// nonterminal run that the reconciler must mistake
// for actor-owned work.
self.durable_uncertainty_fail_stop = true;
}
}
MobCommand::FlowCanceledCleanup {
run_id,
terminalized,
} => {
if !terminalized {
tracing::error!(
%run_id,
"flow canceled cleanup refused: spawned cancellation path did not prove persisted terminalization"
);
self.durable_uncertainty_fail_stop = true;
} else if let Err(error) = self
.handle_flow_cleanup(run_id, "flow canceled cleanup")
.await
{
tracing::error!(error = %error, "flow canceled cleanup failed");
self.durable_uncertainty_fail_stop = true;
}
}
#[cfg(test)]
MobCommand::FlowTrackerCounts { reply_tx } => {
let snapshot = self.flow_tracker_snapshot().await;
let tasks = snapshot.run_task_ids.len();
let tokens = snapshot.cancel_token_ids.len();
let _ = reply_tx.send((tasks, tokens));
}
#[cfg(test)]
MobCommand::OrchestratorSnapshot { reply_tx } => {
let phase = self.state();
let _ = reply_tx.send(
self.machine_orchestrator_snapshot(phase)
.unwrap_or_default(),
);
}
#[cfg(test)]
MobCommand::LifecycleSnapshot { reply_tx } => {
let _ = reply_tx.send(super::state::MobLifecycleSnapshot {
phase: self.state(),
active_run_count: self.machine_active_run_count(),
cleanup_pending: false,
});
}
#[cfg(test)]
MobCommand::LifecycleNotificationBurst {
count,
message,
reply_tx,
} => {
let mut burst_result = Ok(());
for index in 0..count {
if let Err(error) = self
.notify_orchestrator_lifecycle(format!("{message} #{index}"))
.await
{
burst_result = Err(error);
break;
}
}
let _ = reply_tx.send(burst_result);
}
#[cfg(test)]
MobCommand::DslT2Snapshot { reply_tx } => {
let dsl = self.dsl_authority.state();
let _ = reply_tx.send(super::state::MobDslT2Snapshot {
destroy_admitted: dsl.destroy_admitted,
flow_authority_schema_version: dsl.flow_authority_schema_version,
owner_bridge_session_id: dsl.owner_bridge_session_id.clone(),
owner_bridge_destroy_on_archive: dsl.owner_bridge_destroy_on_archive,
implicit_delegation_mob: dsl.implicit_delegation_mob,
supervisor_authority_peer_id: dsl.supervisor_authority_peer_id.clone(),
supervisor_authority_signing_key: dsl.supervisor_authority_signing_key,
supervisor_authority_epoch: dsl.supervisor_authority_epoch,
supervisor_authority_protocol_version: dsl
.supervisor_authority_protocol_version
.clone(),
supervisor_pending_authority_peer_id: dsl
.supervisor_pending_authority_peer_id
.clone(),
supervisor_pending_authority_signing_key: dsl
.supervisor_pending_authority_signing_key,
supervisor_pending_authority_epoch: dsl.supervisor_pending_authority_epoch,
supervisor_pending_authority_protocol_version: dsl
.supervisor_pending_authority_protocol_version
.clone(),
supervisor_pending_authority_operation_id: dsl
.supervisor_pending_authority_operation_id
.clone(),
supervisor_pending_authority_member_target_names: dsl
.supervisor_pending_authority_member_target_names
.clone(),
supervisor_pending_authority_member_target_addresses: dsl
.supervisor_pending_authority_member_target_addresses
.clone(),
supervisor_pending_authority_accepted_peer_ids: dsl
.supervisor_pending_authority_accepted_peer_ids
.clone(),
pending_recipient_trust: dsl.pending_recipient_trust.clone(),
host_binding_generations: dsl.host_binding_generations.clone(),
host_binding_generation_highwater: dsl
.host_binding_generation_highwater
.clone(),
confirmed_host_binding_revocations: dsl
.confirmed_host_binding_revocations
.clone(),
replacement_host_bind_endpoints: dsl
.replacement_host_bind_endpoints
.clone(),
replacement_host_binding_generations: dsl
.replacement_host_binding_generations
.clone(),
member_state_markers: dsl.member_state_markers.clone(),
wiring_edges: dsl.wiring_edges.clone(),
external_peer_edges: dsl.external_peer_edges.clone(),
external_peer_edges_by_key: dsl.external_peer_edges_by_key.clone(),
pending_respawn_topology: dsl.pending_respawn_topology.clone(),
abandoned_respawn_topology: dsl.abandoned_respawn_topology.clone(),
identity_to_runtime: dsl.identity_to_runtime.clone(),
identity_runtime_generations: dsl.identity_runtime_generations.clone(),
identity_runtime_fence_tokens: dsl.identity_runtime_fence_tokens.clone(),
member_profile_names: dsl.member_profile_names.clone(),
member_runtime_modes: dsl.member_runtime_modes.clone(),
member_peer_ids: dsl.member_peer_ids.clone(),
member_peer_endpoints: dsl.member_peer_endpoints.clone(),
member_prior_peer_endpoints: dsl.member_prior_peer_endpoints.clone(),
member_restore_failures: dsl.member_restore_failures.clone(),
member_restore_failure_codes: dsl.member_restore_failure_codes.clone(),
runtime_retire_refusal_codes: dsl.runtime_retire_refusal_codes.clone(),
runtime_retire_refusal_reasons: dsl.runtime_retire_refusal_reasons.clone(),
runtime_retire_pending_sessions: dsl
.runtime_retire_pending_sessions
.clone(),
remote_runtime_retired_ids: dsl.remote_runtime_retired_ids.clone(),
remote_supervisor_revoked_ids: dsl.remote_supervisor_revoked_ids.clone(),
member_revival_pending: dsl.member_revival_pending.clone(),
member_kickoff_objective_ids: dsl.member_kickoff_objective_ids.clone(),
objective_owner_ids: dsl.objective_owner_ids.clone(),
objective_outcomes: dsl.objective_outcomes.clone(),
concluded_objective_ids: dsl.concluded_objective_ids.clone(),
member_run_open: dsl.member_run_open.clone(),
member_in_flight_work: dsl.member_in_flight_work.clone(),
member_progress_tokens: dsl.member_progress_tokens.clone(),
member_last_observed_at_ms: dsl.member_last_observed_at_ms.clone(),
member_last_progress_at_ms: dsl.member_last_progress_at_ms.clone(),
member_last_progress_event: dsl.member_last_progress_event.clone(),
member_health_class: dsl.member_health_class.clone(),
member_session_bindings: dsl.member_session_bindings.clone(),
spawn_exec_phase: dsl.spawn_exec_phase.clone(),
pending_spawn_sessions: dsl.pending_spawn_sessions.clone(),
pending_session_ingress_detach_runtime_ids: dsl
.pending_session_ingress_detach_runtime_ids
.clone(),
topology_epoch: dsl.topology_epoch,
spawn_policy_enabled: dsl.spawn_policy_enabled,
spawn_policy_revision: dsl.spawn_policy_revision,
spawn_policy_resolution_revision: dsl
.spawn_policy_resolution_revision
.clone(),
spawn_policy_resolution_profiles: dsl
.spawn_policy_resolution_profiles
.clone(),
spawn_policy_resolution_runtime_modes: dsl
.spawn_policy_resolution_runtime_modes
.clone(),
spawn_policy_resolution_absent: dsl.spawn_policy_resolution_absent.clone(),
spawn_profile_authority_profile_names: dsl
.spawn_profile_authority_profile_names
.clone(),
spawn_profile_authority_models: dsl.spawn_profile_authority_models.clone(),
spawn_profile_authority_material_digests: dsl
.spawn_profile_authority_material_digests
.clone(),
spawn_profile_authority_tool_config_digests: dsl
.spawn_profile_authority_tool_config_digests
.clone(),
spawn_profile_authority_skills_digests: dsl
.spawn_profile_authority_skills_digests
.clone(),
spawn_profile_authority_provider_params_digests: dsl
.spawn_profile_authority_provider_params_digests
.clone(),
spawn_profile_authority_output_schema_digests: dsl
.spawn_profile_authority_output_schema_digests
.clone(),
spawn_profile_authority_external_addressable: dsl
.spawn_profile_authority_external_addressable
.clone(),
orphan_budget: dsl.orphan_budget,
topology_default_policy: dsl.topology_default_policy,
external_member_rebind_capability: dsl
.external_member_rebind_capability
.clone(),
desired_members: dsl.desired_members.clone(),
members_to_spawn: dsl.members_to_spawn.clone(),
members_to_retire: dsl.members_to_retire.clone(),
pending_placed_spawn_ids: dsl.pending_placed_spawn_ids.clone(),
pending_placed_spawn_generations: dsl
.pending_placed_spawn_generations
.clone(),
pending_placed_spawn_fence_tokens: dsl
.pending_placed_spawn_fence_tokens
.clone(),
pending_placed_spawn_hosts: dsl.pending_placed_spawn_hosts.clone(),
pending_autonomous_placed_spawns: dsl
.pending_autonomous_placed_spawns
.clone(),
pending_placed_spawn_host_binding_generations: dsl
.pending_placed_spawn_host_binding_generations
.clone(),
pending_placed_spawn_spec_digests: dsl
.pending_placed_spawn_spec_digests
.clone(),
pending_placed_spawn_provision_operation_ids: dsl
.pending_placed_spawn_provision_operation_ids
.clone(),
pending_placed_spawn_operation_owner_session_ids: dsl
.pending_placed_spawn_operation_owner_session_ids
.clone(),
current_placed_spawn_ids: dsl.current_placed_spawn_ids.clone(),
current_placed_spawn_host_binding_generations: dsl
.current_placed_spawn_host_binding_generations
.clone(),
current_placed_spawn_provision_operation_ids: dsl
.current_placed_spawn_provision_operation_ids
.clone(),
current_placed_spawn_operation_owner_session_ids: dsl
.current_placed_spawn_operation_owner_session_ids
.clone(),
pending_placed_carrier_cleanup: dsl.pending_placed_carrier_cleanup.clone(),
remote_turn_dispatch_sequence: dsl.remote_turn_dispatch_sequence,
committed_remote_turn_outcomes: dsl.committed_remote_turn_outcomes.clone(),
resolved_remote_turn_outcomes: dsl.resolved_remote_turn_outcomes.clone(),
pending_placed_kickoff_outcomes: dsl
.pending_placed_kickoff_outcomes
.clone(),
resolved_placed_kickoff_outcomes: dsl
.resolved_placed_kickoff_outcomes
.clone(),
});
}
MobCommand::StartupKickoffSnapshot { reply_tx } => {
let _ = reply_tx.send(super::state::MobStartupKickoffSnapshot {
pending_kickoff_member_ids: self.pending_kickoff_member_ids_from_dsl(),
ready_runtime_ids: self.ready_runtime_ids_from_dsl(),
});
}
MobCommand::ProjectMemberList {
include_retiring,
reply_tx,
} => {
let _ = reply_tx.send(Ok(self
.project_member_list_from_machine(include_retiring)
.await));
}
MobCommand::ProjectMemberStatus {
agent_identity,
reply_tx,
} => {
let _ = reply_tx.send(
self.machine_member_material(&agent_identity, true)
.await
.map(|material| material.to_snapshot()),
);
}
MobCommand::GetIdentityIntent {
agent_identity,
reply_tx,
} => {
let result = self
.identity
.observe_identity_intent(&self.definition.id, &agent_identity)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::GetIdentityConvergenceStatus {
agent_identity,
reply_tx,
} => {
let result = self
.identity_status
.load_identity_convergence_status(
&self.definition.id,
&agent_identity,
)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::ConcludeObjective {
agent_identity,
objective_id,
outcome,
reply_tx,
} => {
let result = self
.apply_kickoff_input(
&agent_identity,
mob_dsl::MobMachineInput::ConcludeObjective {
member_id: mob_dsl::AgentIdentity::from_domain(&agent_identity),
objective_id: objective_id.to_string(),
outcome,
},
"conclude_objective",
)
.await
.map(|_| ());
let _ = reply_tx.send(result);
}
MobCommand::BindObjectiveOwner {
owner_identity,
objective_id,
reply_tx,
} => {
let result = self
.apply_kickoff_input(
&owner_identity,
mob_dsl::MobMachineInput::BindObjectiveOwner {
owner_id: mob_dsl::AgentIdentity::from_domain(&owner_identity),
objective_id: objective_id.to_string(),
},
"bind_objective_owner",
)
.await
.map(|_| ());
let _ = reply_tx.send(result);
}
MobCommand::MemberMachineProjection {
agent_identity,
reply_tx,
} => {
let _ =
reply_tx.send(Ok(self.machine_projection_for_identity(&agent_identity)));
}
MobCommand::Stop { reply_tx } => {
let stop_intent_preexisting = self
.dsl_authority
.state()
.placed_completion_lifecycle_intent
== Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop);
let result = if self.state() == MobState::Destroyed {
Err(self.invalid_transition_to(MobState::Stopped))
} else if let Err(error) = self
.drive_placed_completion_lifecycle_cleanup(
None,
true,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop),
)
.await
{
Err(error)
} else {
match self.probe_command_admission(
mob_dsl::MobMachineInput::Stop,
MobState::Stopped,
"stop_command_admission",
) {
Ok(()) => {
let mut stop_result =
self.fail_all_pending_spawns("mob is stopping").await;
if stop_result.is_ok() {
if let Err(error) =
self.cancel_pending_peer_deliveries("mob is stopping").await
{
stop_result = Err(error);
}
// Lifecycle delivery is a real fault, not
// best-effort: fold a failure into the stop
// result rather than swallowing it. Cleanup
// still proceeds so the mob can stop.
if stop_result.is_ok()
&& !stop_intent_preexisting
&& let Err(error) = self
.notify_orchestrator_lifecycle(format!(
"Mob '{}' is stopping.",
self.definition.id
))
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"stop encountered orchestrator lifecycle delivery error"
);
stop_result = Err(error);
}
// Cancel checkpointer gates before stopping host loops so
// in-flight saves that complete after the loop stops don't
// race with subsequent external cleanup (e.g. DML deletes).
if stop_result.is_ok() {
self.provisioner.cancel_all_checkpointers().await;
}
}
if stop_result.is_ok() {
let loop_result = self.stop_all_autonomous_members().await;
if let Err(error) = loop_result {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"stop encountered autonomous loop cleanup error"
);
if stop_result.is_ok() {
stop_result = Err(error);
}
}
}
if stop_result.is_ok() {
if self.has_orchestrator
&& let Err(error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::StopOrchestrator,
"stop_orchestrator",
)
{
stop_result = Err(MobError::Internal(format!(
"orchestrator StopOrchestrator transition failed during stop: {error}"
)));
}
if stop_result.is_ok()
&& let Err(error) =
self.commit_stopped_lifecycle_after_cleanup().await
{
stop_result = Err(error);
}
}
if stop_result.is_err() {
self.provisioner.rearm_all_checkpointers().await;
}
stop_result
}
Err(error) => Err(error),
}
};
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
}
MobCommand::ResumeLifecycle { reply_tx } => {
let result = match self.probe_command_admission(
mob_dsl::MobMachineInput::Resume,
MobState::Running,
"resume_command_admission",
) {
Ok(()) => async {
// Re-enable checkpointers cancelled during stop.
self.provisioner.rearm_all_checkpointers().await;
let rebuild = match self
.prepare_explicit_resume_member_sessions()
.await
{
Ok(rebuild) => rebuild,
Err(error) => {
self.provisioner.cancel_all_checkpointers().await;
return Err(error);
}
};
let rebuilt_attachment = !rebuild.is_empty();
// Same-handle resume preserves the prior
// pre-commit readiness contract. A reconstructed
// handle cannot become ready until its foreign
// attachments have been retired and the durable
// Resume transition authorizes the existing
// machine-owned revival seam.
if !rebuilt_attachment
&& let Err(error) =
self.ensure_autonomous_runtimes_from_roster(true).await
{
if let Err(stop_error) = self.stop_all_autonomous_members().await {
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
"resume cleanup failed while stopping autonomous loops"
);
}
self.provisioner.cancel_all_checkpointers().await;
return Err(error);
}
// Resume's durable End{Stop}/Resumed carrier is
// the commit point. Do not enqueue the external
// coordinator notification before it: an
// absent/failed carrier must leave both the mob
// and coordinator stopped. Same-handle pre-commit
// failures still roll back freshly restarted
// loops. A reconstructed handle may already have
// retired foreign attachments; failure remains a
// stopped, discoverable session set that an
// explicit retry can rebuild.
if let Err(error) = self.resume_lifecycle_after_quiesce().await {
if !rebuilt_attachment
&& let Err(stop_error) =
self.stop_all_autonomous_members().await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
"resume transition rollback failed while stopping autonomous loops"
);
}
self.provisioner.cancel_all_checkpointers().await;
return Err(error);
}
let mut post_commit_error = None;
if rebuilt_attachment {
let rebuild_result = self
.rebuild_explicit_resume_member_sessions(rebuild)
.await;
let readiness_result =
self.ensure_autonomous_runtimes_from_roster(false).await;
if let Err(error) = rebuild_result {
post_commit_error = Some(error);
}
if let Err(error) = readiness_result
&& post_commit_error.is_none()
{
post_commit_error = Some(error);
}
}
#[cfg(feature = "runtime-adapter")]
{
// All exact session attachments are settled before
// topology repair. The shared reconciler consumes its
// generated trust handoffs directly; routing the same
// effects again here would duplicate live mutations.
let mut topology_roster = self.roster.read().await.snapshot();
let topology_result = super::builder::reconcile_resume_topology(
&self.definition,
&mut topology_roster,
self.provisioner.as_ref(),
&self.supervisor_bridge,
&self.runtime_metadata,
&mut self.dsl_authority,
&self.dsl_topology_epoch,
)
.await;
*self.roster.write().await =
RosterAuthority::from_roster(topology_roster);
self.publish_machine_state_projection();
if let Err(error) = topology_result
&& post_commit_error.is_none()
{
post_commit_error = Some(error);
}
}
// A cold actor reconstructed while Stopped skips
// the startup-time operation-binding restore. Once
// explicit Resume has settled every attachment and
// the shared topology seam has recovered the exact
// current peer endpoint, rebuild those generated
// owner bindings through the same seam used by a
// cold Running actor. Peer-only members have no
// local bridge session of their own, so this is
// their only durable route back to the owner
// bridge's operation registry before
// respawn/retire. Binding after topology also
// avoids anchoring a legacy pre-rebind address.
if let Err(error) =
self.restore_generated_member_operation_bindings().await
&& post_commit_error.is_none()
{
post_commit_error = Some(error);
}
if self.has_orchestrator {
let orchestrator_transition_succeeded = match self
.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResumeOrchestrator,
"resume_orchestrator_after_durable_resume",
) {
Ok(()) => true,
Err(error) => {
if post_commit_error.is_none() {
// The mob is durably Running;
// surface the local transition
// failure without pretending
// Resume rolled back.
post_commit_error = Some(MobError::Internal(format!(
"mob resumed durably but orchestrator ResumeOrchestrator transition failed: {error}"
)));
}
false
}
};
if orchestrator_transition_succeeded
&& self.notify_orchestrator_on_resume
{
let orchestrator_entries = if let Some(orchestrator) =
self.definition.orchestrator.as_ref()
{
let orchestrator_identities = self
.dsl_authority
.state()
.active_member_identities_for_profile(
&orchestrator.profile,
);
let roster = self.roster.read().await;
orchestrator_identities
.into_iter()
.map(|orchestrator_identity| {
roster
.get(&orchestrator_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"active MobMachine orchestrator '{orchestrator_identity}' has no mechanical roster entry during explicit resume"
))
})
})
.collect::<Result<Vec<_>, MobError>>()
} else {
Ok(Vec::new())
};
match orchestrator_entries {
Ok(orchestrator_entries) => {
for orchestrator_entry in orchestrator_entries {
if let Err(error) = super::builder::realize_orchestrator_resume_notification(
self.definition.as_ref(),
&orchestrator_entry,
self.session_service.as_ref(),
self.provisioner.as_ref(),
&self.dsl_authority,
)
.await
&& post_commit_error.is_none()
{
post_commit_error = Some(error);
}
}
}
Err(error) if post_commit_error.is_none() => {
post_commit_error = Some(error);
}
Err(_) => {}
}
}
}
post_commit_error.map_or(Ok(()), Err)
}
.await,
Err(error) => Err(error),
};
let _ = reply_tx.send(result);
}
MobCommand::Complete { reply_tx } => {
let result = async {
// Proved-nonmutating shell checks run before the
// durable typed Complete intent. Any later failure is
// a recoverable Complete retry, never an untyped latch.
self.require_host_authority_anchors_clear_for_action("complete mob")
.await?;
self.ensure_pending_spawn_alignment("complete preflight")?;
self.ensure_flow_tracker_alignment("complete preflight")
.await?;
self.drive_placed_completion_lifecycle_cleanup(
None,
false,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Complete),
)
.await?;
// Final Complete admission intentionally occurs in
// handle_complete only after Retire/Release has had a
// chance to Dispose every exact host-row custody key.
self.fail_all_pending_spawns("mob is completing").await?;
self.cancel_pending_peer_deliveries("mob is completing")
.await?;
self.handle_complete().await
}
.await;
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
}
MobCommand::Destroy { reply_tx } => {
let result = self.handle_destroy().await;
let destroy_succeeded = result.is_ok();
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
if destroy_succeeded && !self.durable_uncertainty_fail_stop {
// Destroy is terminal for the current ownership model:
// once a mob is destroyed, the actor task exits and no
// further commands are accepted.
self.abort_and_join_lifecycle_tasks().await;
return ActorLoopControl::BreakActor;
}
}
MobCommand::Reset { reply_tx } => {
let prior_state = self.state();
let result = async {
self.require_host_authority_anchors_clear_for_action("reset mob")
.await?;
self.ensure_pending_spawn_alignment("reset preflight")?;
self.ensure_flow_tracker_alignment("reset preflight")
.await?;
self.drive_placed_completion_lifecycle_cleanup(
None,
true,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Reset),
)
.await?;
self.probe_command_admission(
mob_dsl::MobMachineInput::Reset,
MobState::Running,
"reset_command_admission",
)?;
self.fail_all_pending_spawns("mob is resetting").await?;
self.cancel_pending_peer_deliveries("mob is resetting")
.await?;
self.handle_reset(prior_state).await
}
.await;
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
}
MobCommand::RotateSupervisor { reply_tx } => {
let result = self.handle_rotate_supervisor().await;
let _ = reply_tx.send(result);
}
MobCommand::BindHost { request, reply_tx } => {
let result = self.handle_bind_host(*request).await;
let _ = reply_tx.send(result);
}
MobCommand::RevokeHost { host_id, reply_tx } => {
let result = self.handle_revoke_host(&host_id).await;
let _ = reply_tx.send(result);
}
MobCommand::PollEvents {
after_cursor,
limit,
reply_tx,
} => {
let result = self
.events
.poll(after_cursor, limit)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::ReplayAllEvents { reply_tx } => {
let result = self.events.replay_all().await.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::RecordOperatorActionProvenance {
tool_name,
authority_context,
reply_tx,
} => {
let result = async {
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordOperatorActionProvenance {
tool_name,
principal_token: authority_context.principal_token().clone(),
caller_provenance: authority_context.caller_provenance().cloned(),
audit_invocation_id: authority_context
.audit_invocation_id()
.map(ToOwned::to_owned),
},
"record_operator_action_provenance",
)?;
let kind = Self::operator_action_recorded_event_from_generated_effect(
&prepared.transition,
"record_operator_action_provenance",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
Ok(())
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::ForceCancel {
agent_identity,
reply_tx,
} => {
let result = self.handle_force_cancel(agent_identity).await;
let _ = reply_tx.send(result);
}
MobCommand::HardCancelMember {
agent_identity,
reason,
reply_tx,
} => {
self.handle_hard_cancel_member(agent_identity, reason, reply_tx)
.await;
}
MobCommand::MemberHistory {
agent_identity,
from_index,
limit,
reply_tx,
} => {
self.handle_member_history(agent_identity, from_index, limit, reply_tx)
.await;
}
MobCommand::MemberLiveOpen {
agent_identity,
turning_mode,
transport,
reply_tx,
} => {
self.handle_member_live_open(agent_identity, turning_mode, transport, reply_tx)
.await;
}
MobCommand::MemberLiveClose {
agent_identity,
channel_id,
reply_tx,
} => {
self.handle_member_live_close(agent_identity, channel_id, reply_tx)
.await;
}
MobCommand::MemberLiveStatus {
agent_identity,
channel_id,
reply_tx,
} => {
self.handle_member_live_status(agent_identity, channel_id, reply_tx)
.await;
}
MobCommand::MemberLiveControl {
agent_identity,
channel_id,
verb,
reply_tx,
} => {
self.handle_member_live_control(agent_identity, channel_id, verb, reply_tx)
.await;
}
MobCommand::EnsureMemberEventPump {
agent_identity,
reply_tx,
} => {
let result = self.ensure_member_event_pump(&agent_identity).await;
if result.is_ok() {
// Obligation-driven liveness (A17): the pump must
// outlive taps while the remote turn is outstanding.
self.member_event_pumps
.mark_obligation_keepalive(&agent_identity);
}
let _ = reply_tx.send(result);
}
MobCommand::EnsureMemberEventTap {
agent_identity,
reply_tx,
} => {
// Atomic ensure+tap: a tap opened AFTER a separate
// ensure can miss the fresh pump's first pages.
let result = match self.member_pump_tap_material(&agent_identity).await {
Ok(material) => {
Ok(self.member_event_pumps.ensure_pump_with_tap(material).await)
}
Err(error) => Err(error),
};
let _ = reply_tx.send(result);
}
MobCommand::Wire {
local,
target,
reply_tx,
} => {
let result = self.handle_wire(local, target).await;
let _ = reply_tx.send(result);
}
MobCommand::WireMembersBatch { edges, reply_tx } => {
let result = self.handle_wire_members_batch(edges).await;
let _ = reply_tx.send(result);
}
MobCommand::Unwire {
local,
target,
reply_tx,
} => {
let result = self.handle_unwire(local, target).await;
let _ = reply_tx.send(result);
}
MobCommand::DriveRouteInstalls { reply_tx } => {
let result = self.handle_drive_route_installs().await;
let _ = reply_tx.send(result);
}
MobCommand::CancelAllWork {
runtime_id,
fence_token,
reply_tx,
} => {
let result = self.handle_cancel_all_work(runtime_id, fence_token).await;
let _ = reply_tx.send(result);
}
MobCommand::QueryPhase { reply_tx } => {
let _ = reply_tx.send(Ok(self.state()));
}
@control
MobCommand::SetSpawnPolicy { reply_tx, .. } => {
// The outer AST-visible catalog gate owns this variant.
// Keep the erased inner match safely exhaustive if that
// ownership boundary is ever changed.
let _ = reply_tx.send(Err(MobError::Internal(
"SetSpawnPolicy escaped its catalog-gated dispatch arm".into(),
)));
ActorLoopControl::SkipBoundary
}
#[cfg(any(test, feature = "test-support"))]
MobCommand::CrashStopPreservingDurableWorkForTest { reply_tx } => {
// Process-crash simulation for durable recovery tests:
// abort volatile producers without applying Shutdown,
// canceling flow state, or deleting private custody.
// The exact stores remain the only restart authority.
self.quiesce_volatile_producers_after_fail_stop().await;
let _ = reply_tx.send(Ok(()));
ActorLoopControl::BreakActor
}
MobCommand::Shutdown { reply_tx } => {
if let Err(error) = self.probe_command_admission(
mob_dsl::MobMachineInput::Shutdown,
MobState::Stopped,
"shutdown_command_admission",
) {
let _ = reply_tx.send(Err(error));
return ActorLoopControl::SkipBoundary;
}
if let Err(error) = self.drain_member_live_mutations_for_lifecycle().await {
let _ = reply_tx.send(Err(error));
return ActorLoopControl::SkipBoundary;
}
if let Err(error) = self
.close_controller_local_member_live_channels_for_shutdown(
"shutdown mob runtime",
)
.await
{
let _ = reply_tx.send(Err(error));
return ActorLoopControl::SkipBoundary;
}
let mut result = self
.fail_all_pending_spawns("mob runtime is shutting down")
.await;
if result.is_err() {
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
if !self.durable_uncertainty_fail_stop {
return ActorLoopControl::SkipBoundary;
}
} else {
if let Err(error) = self
.cancel_pending_peer_deliveries("mob runtime is shutting down")
.await
{
result = Err(error);
}
if let Err(error) = self.cancel_all_flow_tasks().await {
tracing::warn!(error = %error, "shutdown flow cancellation encountered errors");
if result.is_ok() {
result = Err(error);
}
}
if let Err(error) = self.stop_all_autonomous_members().await {
tracing::warn!(error = %error, "shutdown loop stop encountered errors");
if result.is_ok() {
result = Err(error);
}
}
// Lifecycle notifications are actor-owned mechanical
// delivery, not teardown retry anchors. A notification
// can be blocked in the session service while machine
// unregister waits for the same session to quiesce, so
// cancel and join notifications before opening runtime
// teardown. The full background-work barrier below is
// idempotent and still owns every other task/listener.
if result.is_ok() {
self.abort_and_join_lifecycle_tasks().await;
}
// The member-stop phase owns the exact executor
// attachments. Preserve them as retry anchors when an
// interrupt is still pending; tearing them down here
// would make the handle-level lifecycle retry target
// an authority this failed attempt already removed.
if result.is_ok() {
if let Err(error) =
self.teardown_session_runtime_bindings_from_machine().await
{
tracing::warn!(error = %error, "shutdown session binding teardown failed");
result = Err(error);
}
}
if result.is_ok() {
self.shutdown_actor_owned_background_work().await;
if let Err(error) = self.apply_command_admission(
mob_dsl::MobMachineInput::Shutdown,
MobState::Stopped,
"shutdown_input",
) {
tracing::warn!(error = %error, "shutdown admission apply failed");
result = Err(error);
}
}
let succeeded = result.is_ok();
if !self.respawn_topology_reply_withheld {
let _ = reply_tx.send(result);
}
if !self.durable_uncertainty_fail_stop {
if succeeded {
return ActorLoopControl::BreakActor;
}
// Required teardown retains this actor as the
// retry owner. Never publish Stopped or exit while
// a runtime binding is unresolved.
return ActorLoopControl::SkipBoundary;
}
}
ActorLoopControl::ProceedBoundary
}
}),
}
}
/// Main actor loop: process commands sequentially until Shutdown.
pub(super) async fn run(mut self, mut command_rx: mpsc::Receiver<RoutedMobCommand>) {
if matches!(self.dsl_state(), MobState::Running) {
if let Err(error) = self.restore_generated_member_operation_bindings().await {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"failed to restore generated mob member operation bindings during actor startup; entering Stopped"
);
self.fail_startup_to_stopped("mob member operation binding restore failure")
.await;
}
}
if let Err(error) = self.recover_durable_member_live_open_cleanups().await {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"durable member-live Open cleanup recovery failed; fail-stopping before command admission"
);
self.quiesce_volatile_producers_after_fail_stop().await;
return;
}
// Prune only after generated member bindings have recovered: before
// that point the machine snapshot may not yet contain every current
// session/residency atom, and treating absence as stale would evict a
// valid replay row. A failed binding restore transitions to Stopped
// and deliberately skips pruning this incomplete snapshot.
if matches!(self.state(), MobState::Running)
&& let Err(error) = self.prune_stale_member_operator_requests().await
{
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"failed to prune stale member-operator ledger rows during actor recovery"
);
self.fail_startup_to_stopped("member-operator ledger recovery pruning failure")
.await;
}
if matches!(self.state(), MobState::Running) {
if let Err(error) = self.ensure_autonomous_runtimes_from_roster(false).await {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"failed to start autonomous host loops during actor startup; entering Stopped"
);
// Per-session recovery failures are actor-owned lifecycle
// facts. Persist Stop and retain the handle so an explicit
// MobHandle::resume retry can return the original typed
// capability failure; startup must not make an Ok builder
// result look Running or ready.
self.fail_startup_to_stopped("autonomous runtime startup failure")
.await;
}
}
// Desired-state recovery is independent from the legacy roster
// projection. Every cold actor incarnation starts by reading the sole
// durable intent authority and scheduling one level-triggered pass per
// identity.
self.enqueue_all_identity_intents().await;
let mut deferred_commands: VecDeque<RoutedMobCommand> = VecDeque::new();
let mut host_status_polls_in_flight = BTreeSet::new();
// Run-scoped converged session witnesses: volatile by construction, so
// a cold incarnation always re-verifies every session document.
let mut identity_session_witnesses = IdentityConvergedSessionWitnesses::new();
let mut host_status_poll = tokio::time::interval(super::handle::HOST_STATUS_POLL_INTERVAL);
let mut identity_reconcile_safety_scan =
tokio::time::interval(IDENTITY_RECONCILE_SAFETY_PAGE_INTERVAL);
#[cfg(not(target_arch = "wasm32"))]
{
host_status_poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
identity_reconcile_safety_scan
.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
// Tokio intervals tick immediately. Bind/rebind already perform an
// eager observation, so consume that first tick and start the
// periodic driver one cadence later.
host_status_poll.tick().await;
identity_reconcile_safety_scan.tick().await;
}
// `tokio_with_wasm` intervals already wait one full period before
// their first tick. Awaiting a startup tick on wasm would therefore
// park the actor for the five-minute identity safety cadence.
enum RegularActorWake {
Routed(Option<RoutedMobCommand>),
HostStatusPoll,
IdentityReconcile,
IdentityBackoffDeadline,
IdentitySafetyScan,
}
'actor: loop {
// Detached completion paths can re-enter through `continue` or
// `SkipBoundary`, bypassing the ordinary post-command boundary.
// Honor fail-stop before polling any new work so a confirmed
// durable panic quarantine transfers ownership to cold recovery.
if self.durable_uncertainty_fail_stop {
tracing::error!(
mob_id = %self.definition.id,
"durable mutation outcome remained uncertain; crash-quiescing and terminating mob actor for cold replay"
);
self.quiesce_volatile_producers_after_fail_stop().await;
command_rx.close();
break;
}
self.drain_completed_actor_io_tasks();
self.drain_completed_peer_delivery_tasks();
if let Err(error) = self.drain_completed_lifecycle_tasks() {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"orchestrator lifecycle completion reconciliation failed"
);
self.retain_lifecycle_delivery_error(error);
}
self.drain_completed_member_live_mutations().await;
if self.durable_uncertainty_fail_stop {
// A drain may have just classified an actor-owned task as an
// ambiguous effect. Re-enter the loop-top fail-stop before
// selecting or dispatching even one more command.
continue;
}
let actor_io_pending = !self.actor_io_tasks.is_empty();
let peer_delivery_pending = !self.peer_delivery_tasks.is_empty();
let lifecycle_pending = !self.lifecycle_tasks.is_empty();
let member_live_mutation_pending = !self.member_live_mutation_tasks.is_empty();
let identity_reconcile_pending = !self.identity_reconcile_queue.is_empty();
let identity_reconcile_backoff_wait = self.identity_reconcile_next_backoff_wait();
let deferred_command_pending = !deferred_commands.is_empty();
// Task completions are the actor's fault boundary. The outer
// biased select observes every ready JoinError before either a
// deferred or newly received command can be admitted. The inner
// select retains fairness for the always-ready identity lane
// versus new commands and timers.
let regular_wake = async {
tokio::select! {
routed = command_rx.recv() => RegularActorWake::Routed(routed),
_tick = host_status_poll.tick() => RegularActorWake::HostStatusPoll,
// Queue admission is itself the causal wake. Tokio's
// default select fairness arbitrates this always-ready
// branch with commands and timers, so a self-requeued
// identity cannot monopolize the actor.
_ready = std::future::ready(()), if identity_reconcile_pending => {
RegularActorWake::IdentityReconcile
}
_deadline = async move {
match identity_reconcile_backoff_wait {
Some(wait) => tokio::time::sleep(wait).await,
None => std::future::pending::<()>().await,
}
} => RegularActorWake::IdentityBackoffDeadline,
_tick = identity_reconcile_safety_scan.tick() => {
RegularActorWake::IdentitySafetyScan
}
}
};
let deferred_command = async { deferred_commands.pop_front() };
let routed = tokio::select! {
biased;
joined = self.actor_io_tasks.join_next(), if actor_io_pending => {
if let Some(joined) = joined {
self.reconcile_actor_io_task_join(joined);
}
continue;
}
joined = self.peer_delivery_tasks.join_next(), if peer_delivery_pending => {
if let Some(joined) = joined {
match joined {
Ok(completion) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Err(error) => {
let _ = self.reconcile_peer_delivery_join_error(
error,
"peer delivery completion",
ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop,
);
}
}
}
continue;
}
joined = self.lifecycle_tasks.join_next(), if lifecycle_pending => {
if let Some(joined) = joined
&& let Err(error) = self.surface_lifecycle_task_outcome(joined)
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"orchestrator lifecycle completion reconciliation failed"
);
self.retain_lifecycle_delivery_error(error);
}
continue;
}
joined = self.member_live_mutation_tasks.join_next(),
if member_live_mutation_pending =>
{
if let Some(joined) = joined
&& let Err(error) =
self.reconcile_joined_member_live_mutation(
joined,
MemberLiveReconcileMode::Background,
).await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"mutating member-live completion reconciliation failed"
);
}
continue;
}
deferred = deferred_command, if deferred_command_pending => {
let Some(routed) = deferred else {
continue;
};
routed
}
wake = regular_wake => {
match wake {
RegularActorWake::Routed(Some(routed)) => routed,
RegularActorWake::Routed(None) => break,
RegularActorWake::HostStatusPoll => {
self.spawn_periodic_host_status_polls(
&mut host_status_polls_in_flight,
)
.await;
continue;
}
RegularActorWake::IdentityReconcile => {
self.reconcile_next_identity(&mut identity_session_witnesses).await;
continue;
}
RegularActorWake::IdentityBackoffDeadline => {
self.enqueue_due_identity_reconcile_backoffs();
continue;
}
RegularActorWake::IdentitySafetyScan => {
self.enqueue_next_identity_intent_safety_page().await;
continue;
}
}
}
};
// Chokepoint (a): resolve+require BEFORE any handler logic, on
// the actor's own serialized machine state (DEC-P5E-4). Deferred
// commands keep their authority and re-enter the gate here.
let RoutedMobCommand { authority, cmd } = routed;
tracing::debug!(command_kind = cmd.kind(), "MobActor received command");
let cmd = match self.admit_command_scope(&authority, cmd) {
ScopeAdmission::Admitted(cmd) => cmd,
// The typed denial was already sent down the command's own
// reply channel by `reject_scope_denied`.
ScopeAdmission::Denied => continue,
};
match self
.dispatch_command_boxed(
&authority,
cmd,
&mut command_rx,
&mut deferred_commands,
&mut host_status_polls_in_flight,
)
.await
{
ActorLoopControl::ProceedBoundary => {}
ActorLoopControl::SkipBoundary => continue,
ActorLoopControl::BreakActor => break,
}
if self.durable_uncertainty_fail_stop {
tracing::error!(
mob_id = %self.definition.id,
"durable mutation outcome remained uncertain; crash-quiescing and terminating mob actor for cold replay"
);
self.quiesce_volatile_producers_after_fail_stop().await;
// Publish channel closure only after volatile producers and
// listeners are fully quiescent. A caller observing
// `ActorCommandChannelClosed` can therefore safely construct
// the cold replacement without racing the dying actor's
// listener ownership.
command_rx.close();
break;
}
// Drain routed effects at every loop boundary. Machine-closed
// consumer refusals remove exactly their rejected head, so retry
// this boundary immediately until every independent tail has
// progressed; never wait for an unrelated command to wake it.
// Structural routing failures retain their head and remain fatal.
if self.destroy_admitted()
&& !self.destroy_cleanup_active
&& !self.pending_routed_effects.is_empty()
{
tracing::debug!(
mob_id = %self.definition.id,
queued_remaining = self.pending_routed_effects.len(),
"retaining destroy cleanup routed effects for explicit destroy retry"
);
continue;
}
loop {
let result = self.flush_routed_effects().await;
match Self::classify_actor_boundary_flush(result) {
ActorBoundaryFlushDisposition::Drained => break,
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
queued_remaining = self.pending_routed_effects.len(),
"composition consumer refusal was machine-closed; draining independent routed-effect tails"
);
}
ActorBoundaryFlushDisposition::Fatal(error) => {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
queued_remaining = self.pending_routed_effects.len(),
"composition dispatch failed; terminating mob actor task"
);
break 'actor;
}
}
}
}
// Unconditional crash-style epilogue: command-channel EOF, explicit
// shutdown, fail-stop, and internal dispatch failure all share the
// same join barrier. Graceful paths have already drained their keyed
// execution tables, making this idempotent; abnormal paths must also
// abort run, pending-spawn, and autonomous-turn producers before the
// actor object (and its command receiver) is dropped.
self.quiesce_volatile_producers_after_fail_stop().await;
}
fn pending_spawn_cleanup_anchor_for_slot(
slot: &super::pending_spawn_lineage::PendingSpawnSlot,
reason: &str,
retire_incarnation: Option<&RetirePendingSpawnCleanupIncarnation>,
) -> Result<Option<PendingSpawnCleanupAnchor>, MobError> {
let snapshot = {
let progress = slot
.spawn
.progress
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
progress
.bridge_session_id
.clone()
.zip(progress.operation_id.clone())
};
let Some((session_id, operation_id)) = snapshot else {
return Ok(None);
};
if let Some(incarnation) = retire_incarnation
&& incarnation.pending_spawn_session_id.0 != session_id.to_string()
{
return Err(MobError::Internal(format!(
"MobMachine-authorized retire cleanup session '{}' did not match pending spawn cleanup capability '{}' for '{}'",
incarnation.pending_spawn_session_id.0, session_id, slot.spawn.agent_identity
)));
}
Ok(Some(PendingSpawnCleanupAnchor {
spawn_ticket: slot.ticket,
agent_identity: slot.spawn.agent_identity.clone(),
session_id,
operation_id,
reason: reason.to_string(),
retire_incarnation: retire_incarnation.cloned(),
}))
}
async fn cleanup_pending_spawn_anchor(
&self,
anchor: &PendingSpawnCleanupAnchor,
) -> Result<(), MobError> {
self.provisioner
.abort_member_provision(
&MemberRef::from_bridge_session_id(anchor.session_id.clone()),
&anchor.operation_id,
&anchor.reason,
)
.await
}
fn pending_spawn_cleanup_error(context: &str, errors: Vec<String>) -> MobError {
MobError::Internal(format!(
"{context}: pending spawn cleanup incomplete: {}",
errors.join("; ")
))
}
async fn drain_pending_spawn_cleanup_anchor_set(
&mut self,
anchors: Vec<PendingSpawnCleanupAnchor>,
context: &str,
) -> Result<(), MobError> {
if anchors.is_empty() {
return Ok(());
}
let mut errors = Vec::new();
for anchor in anchors {
match self.cleanup_pending_spawn_anchor(&anchor).await {
Ok(()) => {
self.pending_spawn_cleanup_anchors
.remove(&anchor.spawn_ticket);
tracing::info!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
session_id = %anchor.session_id,
operation_id = %anchor.operation_id,
retire_incarnation = ?anchor.retire_incarnation,
"retried pending spawn cleanup anchor successfully"
);
}
Err(error) => {
errors.push(format!(
"{} ticket {} session {} operation {}: {error}",
anchor.agent_identity,
anchor.spawn_ticket,
anchor.session_id,
anchor.operation_id
));
tracing::warn!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
session_id = %anchor.session_id,
operation_id = %anchor.operation_id,
retire_incarnation = ?anchor.retire_incarnation,
error = %error,
"pending spawn cleanup anchor still failed"
);
}
}
}
if errors.is_empty() {
Ok(())
} else {
Err(Self::pending_spawn_cleanup_error(context, errors))
}
}
async fn drain_pending_spawn_cleanup_anchors(&mut self, context: &str) -> Result<(), MobError> {
let anchors = self
.pending_spawn_cleanup_anchors
.values()
.cloned()
.collect::<Vec<_>>();
self.drain_pending_spawn_cleanup_anchor_set(anchors, context)
.await
}
/// Drain only retained mechanical cleanup capabilities for the requested
/// stable identity. Each anchor remains exact-session/operation scoped;
/// this never consumes a live pending-spawn slot or mutates MobMachine's
/// current pending incarnation.
async fn drain_pending_spawn_cleanup_anchors_for_member(
&mut self,
agent_identity: &AgentIdentity,
context: &str,
) -> Result<(), MobError> {
let anchors = self
.pending_spawn_cleanup_anchors
.values()
.filter(|anchor| &anchor.agent_identity == agent_identity)
.cloned()
.collect::<Vec<_>>();
self.drain_pending_spawn_cleanup_anchor_set(anchors, context)
.await
}
#[cfg(feature = "runtime-adapter")]
fn authorize_restored_member_operation_owner(
&mut self,
entry: &RosterEntry,
bridge_session_id: &SessionId,
) -> Result<SessionId, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let dsl_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::RecoverMemberSessionBinding {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
bridge_session_id: dsl_session_id.clone(),
replacing,
},
"restore_member_operation_owner_binding",
)?;
let authorized = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::SessionProvisionOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id,
} if effect_identity == &dsl_identity && session_id == &dsl_session_id
)
});
if !authorized {
return Err(MobError::Internal(format!(
"MobMachine restore did not authorize operation owner for '{}'",
entry.agent_identity
)));
}
Ok(bridge_session_id.clone())
}
#[cfg(feature = "runtime-adapter")]
fn authorize_peer_only_operation_owner_from_machine(
&mut self,
agent_identity: &AgentIdentity,
peer_id: &str,
context: &'static str,
) -> Result<(SessionId, mob_dsl::MemberPeerEndpoint), MobError> {
let owner_bridge_session_id = self
.dsl_authority
.state()
.owner_bridge_session_id
.clone()
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MobMachine owner bridge authority"
))
})?;
let owner_bridge_session_id =
SessionId::parse(&owner_bridge_session_id.0).map_err(|error| {
MobError::Internal(format!(
"{context}: MobMachine has invalid owner bridge session for peer-only member '{agent_identity}': {error}"
))
})?;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let expected_peer_endpoint = self
.dsl_authority
.state()
.member_peer_endpoints
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MobMachine member peer endpoint authority"
))
})?;
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerRebind {
agent_identity: dsl_identity.clone(),
expected_peer_endpoint: expected_peer_endpoint.clone(),
},
context,
)?;
let authorized_peer_endpoint = transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::MemberPeerRebindAuthorized {
agent_identity: effect_identity,
peer_id: effect_peer_id,
peer_endpoint,
} if effect_identity == &dsl_identity
&& effect_peer_id.0 == peer_id
&& peer_endpoint == &expected_peer_endpoint =>
{
Some(peer_endpoint.clone())
}
_ => None,
})
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: MobMachine did not authorize peer-only operation owner for '{agent_identity}'"
))
})?;
Ok((owner_bridge_session_id, authorized_peer_endpoint))
}
#[cfg(feature = "runtime-adapter")]
async fn generated_peer_only_operation_owner_context(
&mut self,
agent_identity: &AgentIdentity,
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<
(
SessionId,
Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
),
MobError,
> {
// W-F.2: External bindings carry observed peer material to check
// against the machine authority; HostMaterialized bindings carry
// none — the machine's `member_peer_ids`/`member_peer_endpoints`
// maps (ack-folded at the remote commit) ARE the single owner, so
// the peer id is read from them and only the machine authorization
// runs.
let (peer_id, observed_peer) = match binding {
crate::RuntimeBinding::External { peer_id, .. } => (
peer_id.clone(),
Some(Self::peer_only_spec_for_binding(binding, context)?),
),
crate::RuntimeBinding::HostMaterialized { .. } => {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let peer_id = self
.dsl_authority
.state()
.member_peer_ids
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: placed member '{agent_identity}' has no MobMachine peer identity"
))
})?;
(peer_id.0, None)
}
_ => {
return Err(MobError::Internal(format!(
"{context}: peer-only operation owner requested for non-external binding"
)));
}
};
let (owner_bridge_session_id, authorized_endpoint) = self
.authorize_peer_only_operation_owner_from_machine(agent_identity, &peer_id, context)?;
if let Some(observed_peer) = observed_peer {
let authorized_peer =
Self::peer_only_spec_from_member_endpoint(&authorized_endpoint, context)?;
if authorized_peer.name != observed_peer.name
|| authorized_peer.peer_id != observed_peer.peer_id
|| authorized_peer.address != observed_peer.address
|| authorized_peer.pubkey != observed_peer.pubkey
{
return Err(MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' does not match MobMachine peer endpoint authority"
)));
}
}
let adapter = self.runtime_adapter.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MeerkatMachine runtime authority"
))
})?;
let bindings = adapter
.prepare_local_session_bindings(owner_bridge_session_id.clone())
.await
.map_err(|error| {
MobError::Internal(format!(
"{context}: failed to prepare MeerkatMachine operation bindings for peer-only member '{agent_identity}': {error}"
))
})?;
if bindings.session_id() != &owner_bridge_session_id {
return Err(MobError::Internal(format!(
"{context}: MeerkatMachine operation bindings for peer-only member '{agent_identity}' returned session '{}' but expected '{owner_bridge_session_id}'",
bindings.session_id()
)));
}
if !meerkat_runtime::session_runtime_bindings_have_machine_authority(&bindings) {
return Err(MobError::Internal(format!(
"{context}: MeerkatMachine operation bindings for peer-only member '{agent_identity}' lacked machine authority"
)));
}
Ok((
owner_bridge_session_id,
Arc::clone(bindings.ops_lifecycle()),
))
}
#[cfg(feature = "runtime-adapter")]
fn authorize_restored_peer_only_operation_owner(
&mut self,
entry: &RosterEntry,
) -> Result<Option<SessionId>, MobError> {
// W-F.2: entry selection is PLACEMENT-aware — a roster entry the
// machine places on a member host enters ops-owner anchoring
// regardless of ref shape, and missing owner-bridge authority is a
// typed error, never a skip (§19.L5 invariant; the spawn-side
// `owner_bridge_present` guard is the suspender, this is the belt,
// reachable only via corrupted stores).
let placed =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity);
let peer_id = match &entry.member_ref {
MemberRef::BackendPeer {
peer_id,
session_id: None,
..
} => peer_id.clone(),
_ if placed => {
return Err(MobError::Internal(format!(
"restore: placed member '{}' carries a non-peer member ref; the ops owner \
for placed members is the owner bridge session (§19.L5 invariant)",
entry.agent_identity
)));
}
_ => return Ok(None),
};
if self.dsl_authority.state().owner_bridge_session_id.is_none() {
if placed {
return Err(MobError::Internal(format!(
"restore: placed member '{}' requires MobMachine owner-bridge authority \
(§19.L5 invariant)",
entry.agent_identity
)));
}
// Legacy peer-only mobs without an owner bridge keep the
// load-bearing silent skip verbatim (gotcha 13).
return Ok(None);
}
let (owner_bridge_session_id, _) = self.authorize_peer_only_operation_owner_from_machine(
&entry.agent_identity,
&peer_id,
"restore_peer_only_operation_owner_binding",
)?;
Ok(Some(owner_bridge_session_id))
}
#[cfg(feature = "runtime-adapter")]
async fn restore_generated_member_operation_bindings(&mut self) -> Result<(), MobError> {
let Some(adapter) = self.runtime_adapter.clone() else {
return Ok(());
};
let entries = self.roster.read().await.list().cloned().collect::<Vec<_>>();
for entry in entries {
// W-F.2: placement decides the anchoring lane FIRST — a placed
// member's ref-carried session id (if any) names a REMOTE
// session and must never enter the local
// `RecoverMemberSessionBinding` path.
let placed = super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&entry.agent_identity,
);
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let (generated_owner_session_id, placed_operation) = if placed {
let operation_id_text = self
.dsl_authority
.state()
.current_placed_spawn_provision_operation_ids
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"restore: placed member '{}' has no exact current provision operation",
entry.agent_identity
))
})?;
let anchored_owner = self
.dsl_authority
.state()
.current_placed_spawn_operation_owner_session_ids
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"restore: placed member '{}' has no exact current operation owner",
entry.agent_identity
))
})?;
let operation_id = meerkat_core::ops::OperationId(
uuid::Uuid::parse_str(&operation_id_text).map_err(|error| {
MobError::Internal(format!(
"restore: placed member '{}' has invalid provision operation '{}': {error}",
entry.agent_identity, operation_id_text
))
})?,
);
let anchored_owner = SessionId::parse(&anchored_owner.0).map_err(|error| {
MobError::Internal(format!(
"restore: placed member '{}' has invalid operation owner '{}': {error}",
entry.agent_identity, anchored_owner.0
))
})?;
let Some(generated_owner_session_id) =
self.authorize_restored_peer_only_operation_owner(&entry)?
else {
return Err(MobError::Internal(format!(
"restore: placed member '{}' did not authorize its owner bridge session",
entry.agent_identity
)));
};
if generated_owner_session_id != anchored_owner {
return Err(MobError::Internal(format!(
"restore: placed member '{}' operation owner '{}' diverges from generated owner bridge '{}'",
entry.agent_identity, anchored_owner, generated_owner_session_id
)));
}
let carrier = self
.runtime_metadata
.load_placed_spawn(&self.definition.id, entry.agent_identity.as_str())
.await?
.ok_or_else(|| {
MobError::Internal(format!(
"restore: placed member '{}' has no durable committed carrier",
entry.agent_identity
))
})?;
if !matches!(
&carrier.phase,
crate::store::PlacedSpawnCarrierPhase::Committed(_)
) || carrier.provision_operation_id != operation_id
|| carrier.operation_owner_session_id != anchored_owner
{
return Err(MobError::Internal(format!(
"restore: placed member '{}' machine operation tuple diverges from its committed carrier",
entry.agent_identity
)));
}
let display_name = render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
entry.agent_identity.as_str(),
)?;
let recovery_expectation = match self
.dsl_authority
.state()
.member_state_markers
.get(&mob_dsl::AgentRuntimeId::from_domain(
&entry.agent_runtime_id,
)) {
Some(mob_dsl::MobMemberState::Active) => {
super::provisioner::PlacedOperationRecoveryExpectation::Active
}
Some(mob_dsl::MobMemberState::Retiring) => {
super::provisioner::PlacedOperationRecoveryExpectation::Retiring
}
marker => {
return Err(MobError::Internal(format!(
"restore: placed member '{}' has incompatible machine state marker {marker:?}",
entry.agent_identity
)));
}
};
(
generated_owner_session_id,
Some((operation_id, display_name, recovery_expectation)),
)
} else if let Some(bridge_session_id) = entry.member_ref.bridge_session_id().cloned() {
(
self.authorize_restored_member_operation_owner(&entry, &bridge_session_id)?,
None,
)
} else {
let Some(owner_bridge_session_id) =
self.authorize_restored_peer_only_operation_owner(&entry)?
else {
continue;
};
(owner_bridge_session_id, None)
};
if matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
) {
let machine_owner = self
.dsl_authority
.state()
.owner_bridge_session_id
.as_ref()
.ok_or_else(|| {
MobError::Internal(format!(
"peer-only operation owner restore for '{}' lost MobMachine owner bridge authority",
entry.agent_identity
))
})?;
let machine_owner = SessionId::parse(&machine_owner.0).map_err(|error| {
MobError::Internal(format!(
"peer-only operation owner restore for '{}' found invalid MobMachine owner bridge session '{}': {error}",
entry.agent_identity, machine_owner.0
))
})?;
if machine_owner != generated_owner_session_id {
return Err(MobError::Internal(format!(
"peer-only operation owner restore for '{}' lost MobMachine owner bridge authority",
entry.agent_identity
)));
}
}
let bindings = adapter
.prepare_local_session_bindings(generated_owner_session_id.clone())
.await
.map_err(|error| {
MobError::Internal(format!(
"restore operation owner binding failed for member '{}': {error}",
entry.agent_identity
))
})?;
if bindings.session_id() != &generated_owner_session_id {
return Err(MobError::Internal(format!(
"restore operation owner binding returned session '{}' for member '{}' generated owner '{}'",
bindings.session_id(),
entry.agent_identity,
generated_owner_session_id
)));
}
if !meerkat_runtime::session_runtime_bindings_have_machine_authority(&bindings) {
return Err(MobError::Internal(format!(
"restore operation owner binding lacked MeerkatMachine authority for member '{}'",
entry.agent_identity
)));
}
if let Some((operation_id, display_name, recovery_expectation)) = placed_operation {
self.provisioner
.bind_placed_member_owner_context_exact(
&entry.member_ref,
generated_owner_session_id,
Arc::clone(bindings.ops_lifecycle()),
display_name,
operation_id,
recovery_expectation,
)
.await?;
} else {
self.provisioner
.bind_member_owner_context(
&entry.member_ref,
generated_owner_session_id,
Arc::clone(bindings.ops_lifecycle()),
)
.await?;
}
}
Ok(())
}
#[cfg(not(feature = "runtime-adapter"))]
async fn restore_generated_member_operation_bindings(&self) -> Result<(), MobError> {
Ok(())
}
/// NOTE (dogma row R044): aborting a pending spawn mid-provision does NOT
/// close its `pending_recipient_trust` obligation. The provisioning task
/// may have installed recipient trust without reaching an authorization
/// terminality verdict, so the machine truthfully keeps the window open;
/// the obligation is volatile (like the trust it tracks) and clears on
/// restart. Only a provision RESULT (success or failure) closes it.
async fn fail_all_pending_spawns(&mut self, reason: &str) -> Result<(), MobError> {
self.drain_pending_spawn_cleanup_anchors(reason).await?;
if self.pending_spawns.is_empty() {
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
reason,
message = %message,
"pending spawn alignment violated with no local pending slots to drain"
);
}
return Ok(());
}
let mut cleanup_errors = Vec::new();
for mut slot in self.pending_spawns.drain_all() {
let spawn_ticket = slot.ticket;
let agent_identity = slot.spawn.agent_identity.clone();
let respawn_origin = slot.spawn.respawn_origin.clone();
let dsl_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
// Extract the provisioning task handle before `fail()` consumes the
// slot so we can abort AND await it (abort is the cancellation signal;
// await ensures the task has fully stopped before destroy commits).
let task_handle = slot.task.take();
if let Some(handle) = task_handle {
handle.abort();
let _ = handle.await;
}
// Realize the RequestPendingSpawnQuiesceForDestroy obligation with an
// unconditional CancelPendingSpawn input. The input is total: the
// machine self-loops with Present/Absent/Destroyed variants, so no
// shell-side probe is needed (decision 6, Stage-A notes).
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::CancelPendingSpawn {
agent_identity: dsl_identity,
},
"fail_all_pending_spawns",
) {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
} else {
let generated_failed =
generated_mob_command_capabilities::CommandPlanKind::FailSpawn;
debug_assert_eq!(
generated_failed,
generated_mob_command_capabilities::CommandPlanKind::FailSpawn
);
}
if let Err(error) = self.abort_pending_spawn_slot(&slot, reason, None).await {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
}
// Multi-host: a remote pending spawn opened the machine ladder
// at enqueue (rung MaterializePending) — reset it so destroy /
// later respawns are not wedged; the host-side row (if any) is
// a documented orphan seed for the HostStatus sweep (I10).
if let Some(remote) = slot.spawn.remote.as_ref() {
let identity = AgentIdentity::from(agent_identity.as_str());
let pending_carrier = remote.pending_carrier.clone();
let cleanup_result = self
.fail_remote_spawn_exec(
&identity,
&pending_carrier,
true,
MobError::Internal(format!(
"spawn canceled for '{agent_identity}': {reason}"
)),
"materialize_canceled".to_string(),
"fail_all_pending_spawns_remote",
)
.await;
if self.durable_uncertainty_fail_stop {
cleanup_errors.push(format!(
"{agent_identity} ticket {spawn_ticket}: {cleanup_result}"
));
}
}
let mut may_reply = true;
if let Some(respawn_origin) = respawn_origin
&& let Err(error) = self
.durably_abandon_respawn_topology_if_terminal_exact(
&agent_identity,
&respawn_origin,
)
.await
{
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
cleanup_errors.push(format!(
"{agent_identity} ticket {spawn_ticket}: durable respawn-topology abandonment failed; actor is fail-stopping for cold recovery: {error}"
));
}
if may_reply {
slot.fail(&format!("spawn canceled for '{agent_identity}': {reason}"));
}
tracing::debug!(
spawn_ticket,
agent_identity = %agent_identity,
"failed pending spawn due to lifecycle transition"
);
}
debug_assert!(
self.pending_spawns.is_empty(),
"all pending spawn slots should be drained during lifecycle transition"
);
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
message = %message,
"pending spawn alignment still violated after lifecycle drain"
);
}
if !cleanup_errors.is_empty() {
return Err(Self::pending_spawn_cleanup_error(reason, cleanup_errors));
}
self.drain_pending_spawn_cleanup_anchors(reason).await?;
Ok(())
}
async fn abort_pending_spawn_slot(
&mut self,
slot: &super::pending_spawn_lineage::PendingSpawnSlot,
reason: &str,
retire_incarnation: Option<&RetirePendingSpawnCleanupIncarnation>,
) -> Result<(), MobError> {
let Some(anchor) =
Self::pending_spawn_cleanup_anchor_for_slot(slot, reason, retire_incarnation)?
else {
return Ok(());
};
self.cleanup_or_retain_pending_spawn_anchor(anchor).await
}
async fn cleanup_or_retain_pending_spawn_anchor(
&mut self,
anchor: PendingSpawnCleanupAnchor,
) -> Result<(), MobError> {
if let Err(error) = self.cleanup_pending_spawn_anchor(&anchor).await {
self.pending_spawn_cleanup_anchors
.insert(anchor.spawn_ticket, anchor.clone());
tracing::warn!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
operation_id = %anchor.operation_id,
error = %error,
"failed to abort pending member provision during lifecycle drain; retained cleanup anchor"
);
return Err(MobError::Internal(format!(
"pending spawn cleanup failed for '{}': {error}",
anchor.agent_identity
)));
}
self.pending_spawn_cleanup_anchors
.remove(&anchor.spawn_ticket);
Ok(())
}
/// Stage a real shell pending-spawn capability for an already committed
/// member. The test then invokes the public Retire command, so production
/// classification, machine cancellation, exact-witness capture, abort, and
/// retention all execute unchanged.
#[cfg(test)]
async fn stage_pending_spawn_for_retire_test(
&mut self,
agent_identity: AgentIdentity,
pending_spawn_session_id: SessionId,
operation_id: meerkat_core::ops::OperationId,
) -> Result<(), MobError> {
let entry = self
.roster
.read()
.await
.get(&agent_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"test pending-spawn fixture requires committed roster entry '{agent_identity}'"
))
})?;
let mut profile = if let Some(profile) = entry.effective_profile_override.clone() {
profile
} else {
self.definition
.resolve_profile(&entry.role, self.realm_profile_store.as_ref())
.await?
};
if let Some(model) = entry.effective_model_override.as_ref() {
profile.model.clone_from(model);
}
let (_, authorized_profile_material) =
authorize_spawn_profile_input(&agent_identity, &entry.role, &profile, None)?;
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let started = self.stage_orchestrator_spawn(&agent_identity, &pending_spawn_session_id)?;
let (reply_tx, _reply_rx) = oneshot::channel();
let pending = PendingSpawn {
profile_name: entry.role,
agent_identity,
admitted_bridge_session_id: pending_spawn_session_id.clone(),
prompt: ContentInput::from("test pending spawn awaiting retire"),
initial_turn_prompt: None,
suppress_autonomous_initial_prompt: false,
identity_member_permit: None,
runtime_mode: entry.runtime_mode,
labels: entry.labels,
owner_bridge_session_id: None,
auto_wire_parent: false,
restore_wiring: None,
respawn_origin: None,
effective_profile_override: entry.effective_profile_override,
effective_model_override: entry.effective_model_override,
objective_id: None,
per_spawn_external_tools: None,
authorized_profile_material,
continuity_intent: super::handle::SpawnContinuityIntent::Ephemeral,
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress {
bridge_session_id: Some(pending_spawn_session_id),
operation_id: Some(operation_id),
})),
pending_recipient_trust_peer_id: None,
observations: SpawnExecObservations::default(),
remote: None,
enqueued_at: Instant::now(),
reply_tx,
};
let started = started.start(&pending)?;
let task = tokio::spawn(async { std::future::pending::<()>().await });
self.insert_pending_spawn(spawn_ticket, pending, task, started)
.await?;
self.ensure_pending_spawn_alignment("stage_pending_spawn_for_retire_test")
}
async fn cancel_pending_spawns_for_member(
&mut self,
agent_identity: &AgentIdentity,
reason: &str,
) -> Result<usize, MobError> {
self.drain_pending_spawn_cleanup_anchors_for_member(agent_identity, reason)
.await?;
let slots = self.pending_spawns.take_for_member(agent_identity);
self.cancel_pending_spawn_slots(agent_identity, slots, reason, None)
.await
}
async fn cancel_pending_spawn_for_retire_incarnation(
&mut self,
agent_identity: &AgentIdentity,
retire_incarnation: RetirePendingSpawnCleanupIncarnation,
reason: &str,
) -> Result<usize, MobError> {
let domain_session_id = SessionId::parse(&retire_incarnation.pending_spawn_session_id.0)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine authorized retire cancellation with invalid pending session '{}': {error}",
retire_incarnation.pending_spawn_session_id.0
))
})?;
let slots = self
.pending_spawns
.take_for_member_session(agent_identity, &domain_session_id);
if slots.is_empty() {
return Err(MobError::Internal(format!(
"MobMachine authorized retire cancellation for '{agent_identity}' pending session '{domain_session_id}', but no matching shell capability exists"
)));
}
self.cancel_pending_spawn_slots(agent_identity, slots, reason, Some(&retire_incarnation))
.await
}
async fn cancel_pending_spawn_slots(
&mut self,
agent_identity: &AgentIdentity,
slots: Vec<super::pending_spawn_lineage::PendingSpawnSlot>,
reason: &str,
retire_incarnation: Option<&RetirePendingSpawnCleanupIncarnation>,
) -> Result<usize, MobError> {
if slots.is_empty() {
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
agent_identity = %agent_identity,
reason,
message = %message,
"pending spawn alignment violated while canceling member-specific pending spawns"
);
}
return Ok(0);
}
let canceled = slots.len();
let mut cleanup_errors = Vec::new();
for mut slot in slots {
let spawn_ticket = slot.ticket;
let respawn_origin = slot.spawn.respawn_origin.clone();
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
slot.spawn.agent_identity.as_str(),
));
if let Some(task) = slot.task.take() {
task.abort();
let _ = task.await;
}
// Replace the machine-probe + CompleteSpawn laundering with an
// unconditional CancelPendingSpawn (total input — decision 6,
// Stage-A notes).
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::CancelPendingSpawn {
agent_identity: dsl_identity,
},
"cancel_pending_spawn_slots",
) {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
} else {
let generated_failed =
generated_mob_command_capabilities::CommandPlanKind::FailSpawn;
debug_assert_eq!(
generated_failed,
generated_mob_command_capabilities::CommandPlanKind::FailSpawn
);
}
if let Err(error) = self
.abort_pending_spawn_slot(&slot, reason, retire_incarnation)
.await
{
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
}
if let Some(remote) = slot.spawn.remote.as_ref() {
let pending_carrier = remote.pending_carrier.clone();
let cleanup_result = self
.fail_remote_spawn_exec(
agent_identity,
&pending_carrier,
true,
MobError::Internal(format!(
"spawn canceled for '{agent_identity}': {reason}"
)),
"materialize_canceled".to_string(),
"cancel_pending_spawns_for_member_remote",
)
.await;
if self.durable_uncertainty_fail_stop {
cleanup_errors.push(format!(
"{agent_identity} ticket {spawn_ticket}: {cleanup_result}"
));
}
}
let mut may_reply = true;
if let Some(respawn_origin) = respawn_origin
&& let Err(error) = self
.durably_abandon_respawn_topology_if_terminal_exact(
agent_identity,
&respawn_origin,
)
.await
{
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
cleanup_errors.push(format!(
"{agent_identity} ticket {spawn_ticket}: durable respawn-topology abandonment failed; actor is fail-stopping for cold recovery: {error}"
));
}
if may_reply {
slot.fail(&format!("spawn canceled for '{agent_identity}': {reason}"));
}
tracing::debug!(
spawn_ticket,
agent_identity = %agent_identity,
"canceled pending spawn for member lifecycle command"
);
}
self.debug_assert_pending_spawn_alignment();
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
agent_identity = %agent_identity,
message = %message,
"pending spawn alignment violated after member-specific cancellation"
);
}
if cleanup_errors.is_empty() {
Ok(canceled)
} else {
Err(Self::pending_spawn_cleanup_error(reason, cleanup_errors))
}
}
fn customize_spawn_spec(
&self,
spawn_source: super::handle::SpawnSource,
spawner: Option<&(AgentIdentity, AgentRuntimeId)>,
spec: &mut super::handle::SpawnMemberSpec,
) -> Result<(), MobError> {
if matches!(spawn_source, super::handle::SpawnSource::IdentityReconcile) {
// The portable half of customization is already digest-sealed in
// IdentityIntent. Re-running it on cold materialization would let
// process-local state silently rewrite desired content.
return Ok(());
}
if let Some(customizer) = self.spawn_member_customizer.as_ref() {
let ctx = super::handle::SpawnCustomizationContext {
mob_id: self.definition.id.clone(),
spawn_source,
spawner_identity: spawner.map(|(identity, _)| identity.clone()),
spawner_runtime_id: spawner.map(|(_, runtime_id)| runtime_id.clone()),
requested_profile: spec.role_name.clone(),
};
customizer.customize_spawn(&ctx, spec)?;
}
Ok(())
}
/// P1-T04: spawn() creates a real session.
///
/// Provisioning runs in parallel tasks; final actor commit stays serialized.
async fn enqueue_spawn(
&mut self,
mut spec: super::handle::SpawnMemberSpec,
spawn_source: super::handle::SpawnSource,
identity_member_permit: Option<crate::identity::IdentityActuationPermit>,
owner_bridge_session_id: Option<SessionId>,
ops_registry: Option<Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>>,
reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
) {
let requested_identity = AgentIdentity::from(spec.identity.as_str());
macro_rules! reject_spawn_before_custody {
($stage:literal, $error:expr) => {{
let error = $error;
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %requested_identity,
spawn_source = spawn_source.as_str(),
stage = $stage,
error = %error,
"member spawn rejected before asynchronous spawn custody"
);
let _ = reply_tx.send(Err(error));
return;
}};
}
let suppress_autonomous_initial_prompt =
matches!(spawn_source, super::handle::SpawnSource::IdentityReconcile);
if suppress_autonomous_initial_prompt != identity_member_permit.is_some() {
reject_spawn_before_custody!(
"source_permit_contract",
MobError::Internal(
"identity member actuation permits must be carried only by identity reconciliation spawns"
.to_string(),
)
);
}
let spawner = match owner_bridge_session_id.as_ref() {
Some(owner_session_id) => self.spawner_for_bridge_session(owner_session_id).await,
None => None,
};
if let Err(error) = self.customize_spawn_spec(spawn_source, spawner.as_ref(), &mut spec) {
reject_spawn_before_custody!("customize_spawn_spec", error);
}
// Multi-host §7.3: a placed spawn takes the remote materialization
// lane wholesale (compile → digest-authorized ladder open at enqueue
// → bridge dispatch → ack-fed remote commit).
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
if spec.placement.is_some() {
if identity_member_permit.is_some() {
reject_spawn_before_custody!(
"identity_placement_contract",
MobError::WiringError(
"identity reconciliation does not materialize placed members in this release slice"
.to_string(),
)
);
}
self.enqueue_spawn_remote(
spec,
owner_bridge_session_id,
ops_registry,
None,
None,
reply_tx,
)
.await;
return;
}
#[cfg(not(all(feature = "runtime-adapter", not(target_arch = "wasm32"))))]
if spec.placement.is_some() {
reject_spawn_before_custody!(
"placement_capability",
MobError::WiringError(
"placed spawns require the runtime-adapter mob build".to_string(),
)
);
}
let allow_reserved_flow_identity = spawn_source.allows_reserved_flow_identity();
let super::handle::SpawnMemberSpec {
role_name: profile_name,
identity,
initial_message,
runtime_mode,
backend,
binding,
context,
labels,
launch_mode,
tool_access_policy,
budget_limits,
auto_wire_parent,
additional_instructions,
shell_env,
inherited_tool_filter,
override_profile,
model_override,
objective_id,
auth_binding,
external_tools: per_spawn_external_tools,
system_prompt_override,
continuity_intent,
placement: _,
} = spec;
let agent_identity = AgentIdentity::from(identity.as_str());
if let Err(error) = self.preview_spawn_command_admission(&agent_identity) {
reject_spawn_before_custody!("command_admission", error);
}
// Normalize launch-mode resume/fork details for the provisioning path.
let resume_bridge_session_id = launch_mode.resume_bridge_session_id().cloned();
let fork_spec = match launch_mode {
crate::launch::MemberLaunchMode::Fork {
source_member_id,
fork_context,
} => Some((source_member_id, fork_context)),
_ => None,
};
let prepare_result = async {
if agent_identity.is_system_reserved() && !allow_reserved_flow_identity {
return Err(MobError::WiringError(format!(
"meerkat id '{agent_identity}' uses reserved system identifier namespace"
)));
}
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn start"
);
{
let roster = self.roster.read().await;
if roster.get(&agent_identity).is_some() {
return Err(MobError::MemberAlreadyExists(agent_identity.clone()));
}
}
// Always validate role_name exists in definition for roster consistency,
// even when an override profile is provided.
if !self.definition.profiles.contains_key(&profile_name) {
return Err(MobError::ProfileNotFound(profile_name.clone()));
}
// Preserve presence separately while consuming the requested
// profile. If a field-scoped model override is also present, the
// durable full-profile projection must carry the FINAL composed
// profile so live events and cold placed carriers rehydrate the
// same value.
let full_profile_override_requested = override_profile.is_some();
let effective_model_override = model_override.clone();
// Use override_profile if provided (from SpawnTooling::Profile resolution),
// otherwise resolve from the mob definition.
let mut profile = if let Some(p) = override_profile {
p
} else {
self.definition
.resolve_profile(&profile_name, self.realm_profile_store.as_ref())
.await?
};
if let Some(model) = model_override {
profile.model = model;
}
let effective_profile_override =
full_profile_override_requested.then(|| profile.clone());
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn profile resolved"
);
// ADJ-6: the workgraph observation is the EXPLICIT resolved
// profile fact, captured BEFORE the inherited-open mutation.
let explicit_workgraph = profile.tools.workgraph;
if inherited_tool_filter.is_some() && effective_profile_override.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let observations = observe_spawn_exec_facts(
&profile,
explicit_workgraph,
per_spawn_external_tools.is_some(),
shell_env.as_ref(),
inherited_tool_filter.is_some(),
None,
);
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn authorizing profile material"
);
let authorized_profile_material = self.authorize_spawn_profile_material(
&agent_identity,
&profile_name,
&profile,
"enqueue_spawn_profile_authority",
)?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn profile material authorized"
);
let selected_runtime_mode = runtime_mode.unwrap_or(profile.runtime_mode);
let profile_external_addressable = authorized_profile_material.external_addressable;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn resolving external tools"
);
// ---------- Resume bridge-session fast-path ----------
// When resume_bridge_session_id is set, skip provisioning and go
// straight to finalization. The bridge session must already exist
// and be usable.
if let Some(resume_id) = resume_bridge_session_id {
let member_ref = MemberRef::from_bridge_session_id(resume_id.clone());
// Validate the session exists and is active.
let is_active = self
.provisioner
.is_member_active(&member_ref)
.await
.map_err(|e| {
MobError::Internal(format!(
"resume bridge session check failed for '{agent_identity}': {e}"
))
})?;
if is_active.unwrap_or(false) {
// Validate interaction-scoped injection for autonomous mode.
if selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& self.provisioner.interaction_event_injector(&resume_id).await.is_none()
{
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous member resume",
});
}
// Validate comms if wiring rules exist.
let has_wiring = self.definition.wiring.auto_wire_orchestrator
|| !self.definition.wiring.role_wiring.is_empty();
if has_wiring
&& self
.provisioner
.comms_runtime(&member_ref)
.await
.is_none()
{
return Err(MobError::Internal(format!(
"resumed session '{resume_id}' has no comms runtime for '{agent_identity}'"
)));
}
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let resolved_labels = labels.unwrap_or_default();
return Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
Some(member_ref),
None,
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override.clone(),
effective_model_override.clone(),
objective_id,
per_spawn_external_tools.clone(),
authorized_profile_material.clone(),
continuity_intent.clone(),
observations.clone(),
));
}
if self.session_service.supports_persistent_sessions() {
let selected_binding = resolve_binding(
binding.clone(),
backend,
profile.backend,
self.definition.backend.default,
&agent_identity,
)?;
let selected_runtime_mode =
normalize_runtime_mode_for_binding(selected_runtime_mode, &selected_binding);
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let resolved_labels = labels.clone().unwrap_or_default();
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
if matches!(&selected_binding, crate::RuntimeBinding::Session) {
let external_tools = self.external_tools_for_profile(
&profile,
per_spawn_external_tools.clone(),
)?;
let deferred = DeferredResumeProvision {
definition: self.definition.clone(),
profile_name: profile_name.clone(),
agent_identity: agent_identity.clone(),
profile,
external_tools,
context,
labels: labels.clone(),
additional_instructions,
shell_env,
inherited_tool_filter: inherited_tool_filter.clone(),
tool_access_policy: tool_access_policy.clone(),
system_prompt_override: system_prompt_override.clone(),
resume_id,
prompt: prompt.clone(),
budget_limits: budget_limits.clone(),
keep_alive: selected_runtime_mode
== crate::MobRuntimeMode::AutonomousHost,
default_llm_client: self.default_llm_client.clone(),
binding: selected_binding,
peer_name,
owner_bridge_session_id: owner_bridge_session_id.clone(),
ops_registry: ops_registry.clone(),
generated_self_owned_operation_owner: None,
};
return Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
None::<MemberRef>,
Some(SpawnProvisionInput::DeferredResume(Box::new(deferred))),
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override.clone(),
effective_model_override.clone(),
objective_id,
per_spawn_external_tools.clone(),
authorized_profile_material.clone(),
continuity_intent.clone(),
observations.clone(),
));
}
// External durable resumes retain the existing actor-owned
// preparation order. Their provision opens a machine-owned
// recipient-trust obligation, whose cancellation semantics
// are deliberately not widened by the session optimization.
let stored_session = self
.session_service
.materialize_session_for_resume(&resume_id)
.await
.map_err(MobError::from)?
.into_session_or_error(&resume_id)?;
let external_tools = self.external_tools_for_profile(
&profile,
per_spawn_external_tools.clone(),
)?;
let mut config = build::build_resumed_agent_config(
build::BuildResumedAgentConfigParams {
base: build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: labels.clone(),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter: inherited_tool_filter.clone(),
tool_access_policy: tool_access_policy.clone(),
system_prompt_override: system_prompt_override.clone(),
},
expected_session_id: &resume_id,
resumed_session: stored_session,
},
)
.await?;
config.keep_alive =
selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits.clone());
let provision_request = ProvisionMemberRequest {
create_session: req,
session_origin:
super::provisioner::ProvisionSessionOrigin::ResumedDurable,
binding: selected_binding,
peer_name,
owner_bridge_session_id: owner_bridge_session_id.clone(),
ops_registry: ops_registry.clone(),
generated_self_owned_operation_owner: None,
runtime_revival_intent:
super::provisioner::RuntimeRevivalIntent::None,
};
let resolved_labels = labels.unwrap_or_default();
return Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
None::<MemberRef>,
Some(SpawnProvisionInput::Ready(provision_request)),
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override.clone(),
effective_model_override.clone(),
objective_id,
per_spawn_external_tools.clone(),
authorized_profile_material.clone(),
continuity_intent.clone(),
observations.clone(),
));
}
return Err(MobError::Internal(format!(
"resumed session '{resume_id}' not found or inactive for '{agent_identity}'"
)));
}
// ---------- Fork path ----------
// When fork_spec is set, read source member's session and render
// conversation history as context in the initial prompt. §19.L2:
// a PLACED source routes through the same placement switch as
// `member_history` (the `ReadMemberHistory` proxy) — the phase-3
// `RemoteReadUnavailable` shape now means "tried and failed".
let fork_context_text = if let Some((source_member_id, fork_context)) = fork_spec {
let rows = self
.fork_source_history_rows(&source_member_id, &fork_context)
.await?;
Some(render_fork_context(&source_member_id, &rows))
} else {
None
};
let external_tools =
self.external_tools_for_profile(&profile, per_spawn_external_tools.clone())?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn external tools resolved"
);
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: labels.clone(),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
})
.await?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn agent config built"
);
config.keep_alive =
selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
// Deferral §1: per-member auth binding.
if let Some(ref cref) = auth_binding {
config.auth_binding = Some(cref.clone());
}
let base_prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let prompt = if let Some(fork_text) = fork_context_text {
let mut blocks = vec![meerkat_core::types::ContentBlock::Text {
text: format!("{fork_text}\n\n"),
}];
blocks.extend(base_prompt.into_blocks());
ContentInput::Blocks(blocks)
} else {
base_prompt
};
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits);
let selected_binding = resolve_binding(
binding,
backend,
profile.backend,
self.definition.backend.default,
&agent_identity,
)?;
let selected_runtime_mode =
normalize_runtime_mode_for_binding(selected_runtime_mode, &selected_binding);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
let provision_request = ProvisionMemberRequest {
create_session: req,
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
binding: selected_binding,
peer_name,
owner_bridge_session_id: owner_bridge_session_id.clone(),
ops_registry: ops_registry.clone(),
generated_self_owned_operation_owner: None,
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
};
let resolved_labels = labels.unwrap_or_default();
Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
None::<MemberRef>,
Some(SpawnProvisionInput::Ready(provision_request)),
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
))
}
.await;
let (
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
_external_addressable,
resolved_labels,
resume_member_ref,
maybe_provision_input,
spawn_owner_bridge_session_id,
auto_wire_parent,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
) = match prepare_result {
Ok(prepared) => prepared,
Err(error) => {
reject_spawn_before_custody!("prepare", error);
}
};
// ---------- Resume fast-path: skip async provisioning ----------
if let Some(member_ref) = resume_member_ref {
let Some(bridge_session_id) = member_ref.bridge_session_id().cloned() else {
reject_spawn_before_custody!(
"resume_bridge_session",
MobError::Internal(format!(
"resumed member '{agent_identity}' has no bridge session id"
))
);
};
if let Err(error) = self.preview_spawn_admission(
&agent_identity,
&authorized_profile_material,
Some(&bridge_session_id),
) {
reject_spawn_before_custody!("resume_admission", error);
}
if let (Some(owner_bridge_session_id), Some(ops_registry)) =
(owner_bridge_session_id.clone(), ops_registry.clone())
&& let Err(error) = self
.provisioner
.bind_member_owner_context(&member_ref, owner_bridge_session_id, ops_registry)
.await
{
reject_spawn_before_custody!("resume_owner_context", error);
}
let operation_id = self
.provisioner
.active_operation_id_for_member(&member_ref)
.await
.ok_or_else(|| {
MobError::Internal(format!(
"resumed member '{agent_identity}' has no tracked mob child operation"
))
});
let operation_id = match operation_id {
Ok(operation_id) => operation_id,
Err(error) => {
reject_spawn_before_custody!("resume_operation_authority", error);
}
};
// Go straight to finalization — no async provisioning task needed.
let fence = match self.issue_fence_token() {
Ok(fence) => fence,
Err(error) => {
reject_spawn_before_custody!("resume_fence", error);
}
};
// Machine-owned generation mint (ADJ-24): INITIAL when the
// identity has no machine history, prior+1 on reuse.
let generation = match self.mint_spawn_generation(&agent_identity) {
Ok(generation) => generation,
Err(error) => {
reject_spawn_before_custody!("resume_generation", error);
}
};
let rollback_authority = match self
.provisioner
.capture_resumed_member_rollback_authority(&member_ref)
.await
{
Ok(authority) => authority,
Err(error) => {
reject_spawn_before_custody!("resume_rollback_authority", error);
}
};
let provision = PendingProvision::new(
member_ref,
agent_identity.clone(),
self.provisioner.clone(),
operation_id.clone(),
super::provisioner::ProvisionSessionOrigin::ResumedDurable,
Some(rollback_authority),
);
let result = Box::pin(self.finalize_spawn_from_pending(
&profile_name,
&agent_identity,
generation,
fence,
selected_runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
resolved_labels,
provision,
operation_id,
spawn_owner_bridge_session_id,
auto_wire_parent,
None,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
None,
))
.await
.map(|outcome| outcome.receipt);
if let Err(error) = result.as_ref() {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
spawn_source = spawn_source.as_str(),
stage = "resume_finalize",
error = %error,
"member spawn failed before asynchronous spawn custody"
);
}
let _ = reply_tx.send(result);
return;
}
// Normal provisioning path — resume path already returned above.
let Some(mut provision_input) = maybe_provision_input else {
reject_spawn_before_custody!(
"provision_input",
MobError::Internal("provision input missing for normal spawn path".into())
);
};
let admitted_bridge_session_id = provision_input.admitted_bridge_session_id();
let spawn_bridge_session_id = match provision_input.binding() {
crate::RuntimeBinding::Session => Some(&admitted_bridge_session_id),
crate::RuntimeBinding::External { .. }
| crate::RuntimeBinding::HostMaterialized { .. } => None,
};
if let Err(error) = self.preview_spawn_admission(
&agent_identity,
&authorized_profile_material,
spawn_bridge_session_id,
) {
reject_spawn_before_custody!("spawn_admission", error);
}
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let spawn_member_identity = agent_identity.clone();
let spawn_member_identity_for_log = spawn_member_identity.clone();
let spawn_runtime_mode = selected_runtime_mode;
let pending_progress = Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default()));
let generated_self_owned_operation_owner =
match self.stage_orchestrator_spawn(&agent_identity, &admitted_bridge_session_id) {
Ok(authority) => authority,
Err(error) => {
reject_spawn_before_custody!("orchestrator_spawn_stage", error);
}
};
if let Err(error) = provision_input
.install_generated_self_owned_operation_owner(&generated_self_owned_operation_owner)
{
reject_spawn_before_custody!("operation_owner_install", error);
}
// External provisioning installs supervisor-bridge recipient trust
// ahead of bind terminality inside the provisioner (which cannot reach
// MobMachine authority). The actor owns the machine fact: record the
// `pending_recipient_trust` obligation before the provisioning task
// starts; `handle_spawn_provisioned_batch` resolves or rolls it back
// when the provision result lands.
let pending_recipient_trust_peer_id = match provision_input.binding() {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
// Placed members: member trust is post-ack (§4.3); the remote
// lane never constructs a local provision request.
crate::RuntimeBinding::Session | crate::RuntimeBinding::HostMaterialized { .. } => None,
};
if let Some(peer_id) = pending_recipient_trust_peer_id.as_deref()
&& let Err(error) = self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"enqueue_spawn external provision",
)
{
reject_spawn_before_custody!("recipient_trust_obligation", error);
}
let pending = PendingSpawn {
profile_name,
agent_identity,
admitted_bridge_session_id,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
runtime_mode: selected_runtime_mode,
labels: resolved_labels,
owner_bridge_session_id: spawn_owner_bridge_session_id,
auto_wire_parent,
restore_wiring: None,
respawn_origin: None,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
progress: pending_progress.clone(),
pending_recipient_trust_peer_id,
observations,
remote: None,
enqueued_at: Instant::now(),
reply_tx,
};
let spawn_started = match generated_self_owned_operation_owner.start(&pending) {
Ok(started) => started,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %pending.agent_identity,
spawn_source = spawn_source.as_str(),
spawn_ticket,
stage = "operation_owner_start",
error = %error,
"member spawn failed before asynchronous spawn custody"
);
let _ = pending.reply_tx.send(Err(error));
return;
}
};
// Treat pending spawn lifecycle as a single keyed table: pending intent
// and async task handle must be inserted/removed together.
let provisioner = self.provisioner.clone();
let session_service = self.session_service.clone();
let command_tx = self.command_tx.clone();
let panic_log_ledger = Arc::clone(&self.spawn_panic_log_ledger);
let panic_mob_id = self.definition.id.clone();
let task = tokio::spawn(async move {
let panic_member_identity = spawn_member_identity.clone();
// Panic boundary (see `panic_capture` for the 2026-07-29 incident
// WHY): recover + log the payload at bounded repeat checkpoints and
// feed the typed SpawnProvisioned failure path — never a silent
// `Err(_)` that the identity-reconcile requeue can retry into a
// 99%-CPU panic/symbolication furnace.
// The nested result deliberately makes the guarded future itself
// infallible: an outer Err can therefore only be a caught panic,
// while ordinary provisioning failures remain the inner result.
// A panic cannot certify whether external materialization happened.
let guarded_provision = super::panic_capture::run_spawn_provision_guarded(
panic_log_ledger.as_ref(),
&panic_mob_id,
"spawn provisioning task",
&panic_member_identity,
async {
let provision_result: Result<_, MobError> = async {
let provision_request =
provision_input.into_request(session_service).await?;
let spawn_receipt =
provisioner.provision_member(provision_request).await?;
if let Some(bridge_session_id) =
spawn_receipt.member_ref.bridge_session_id().cloned()
{
let mut progress = pending_progress
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
progress.bridge_session_id = Some(bridge_session_id);
progress.operation_id = Some(spawn_receipt.operation_id.clone());
}
#[cfg(test)]
{
let delay_ms = SPAWN_PROVISIONED_COMMAND_DELAY_MS
.load(std::sync::atomic::Ordering::Relaxed);
if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
}
if spawn_runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&provisioner,
&spawn_member_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) =
provisioner.retire_member(&spawn_receipt.member_ref).await
{
return Err(MobError::Internal(format!(
"autonomous capability check failed for '{spawn_member_identity}': {capability_error}; cleanup retire failed for member '{:?}': {retire_error}",
spawn_receipt.member_ref
)));
}
return Err(capability_error);
}
Ok(spawn_receipt)
}
.await;
Ok::<_, MobError>(provision_result)
},
)
.await;
let provision_result = match guarded_provision {
Ok(result) => result,
Err(panic_error) => {
tracing::error!(
spawn_ticket,
member = %panic_member_identity,
disposition = "external_cleanup_uncertain",
error = %panic_error,
"spawn provisioning panic cannot certify external cleanup"
);
Err(MobError::ExternalMemberCleanupUncertain {
reason: panic_error.to_string(),
})
}
};
if let Err(send_error) = command_tx
.send(RoutedMobCommand::internal(MobCommand::SpawnProvisioned {
spawn_ticket,
result: provision_result,
}))
.await
{
match send_error.0.cmd {
MobCommand::SpawnProvisioned {
result: Ok(spawn_receipt),
..
} => match provisioner.retire_member(&spawn_receipt.member_ref).await {
Ok(disposal) => tracing::warn!(
spawn_ticket,
member_ref = ?spawn_receipt.member_ref,
disposal = ?disposal,
"spawn completion dropped because the actor is gone; provisioned member was retired"
),
Err(cleanup_error) => tracing::warn!(
spawn_ticket,
member_ref = ?spawn_receipt.member_ref,
error = %cleanup_error,
"spawn completion dropped; failed cleanup retire for provisioned member"
),
},
MobCommand::SpawnProvisioned {
result: Err(error), ..
} => tracing::warn!(
spawn_ticket,
error = %error,
"spawn completion dropped because the actor is gone; preserving the typed provisioning failure"
),
_ => tracing::error!(
spawn_ticket,
"spawn task recovered a non-spawn command after completion delivery failed"
),
}
}
});
if let Err(error) = self
.insert_pending_spawn(spawn_ticket, pending, task, spawn_started)
.await
{
tracing::error!(
spawn_ticket,
error = %error,
"pending spawn insertion failed; canceling pending spawn lineage"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn insertion failed")
.await
{
tracing::error!(
spawn_ticket,
error = %cleanup_error,
"pending spawn cleanup failed after insertion failure"
);
}
return;
}
if let Err(error) = self.ensure_pending_spawn_alignment("enqueue_spawn post-insert") {
tracing::error!(
spawn_ticket,
error = %error,
"pending spawn alignment check failed after enqueue; canceling all pending spawns"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated after enqueue")
.await
{
tracing::error!(
spawn_ticket,
error = %cleanup_error,
"pending spawn cleanup failed after enqueue alignment violation"
);
}
return;
}
tracing::debug!(
spawn_ticket,
agent_identity = %spawn_member_identity_for_log,
runtime_mode = ?spawn_runtime_mode,
"MobActor::enqueue_spawn queued provisioning task"
);
}
/// Validate the full machine obligation/carrier tuple and mint the only
/// persistence authority that can delete it. Callers must do this before
/// mutating the carrier-anchored lifecycle operation: a stale obligation
/// must never terminalize a successor operation and fail only at CAS.
fn authorize_placed_carrier_cleanup(
&mut self,
expected: &crate::store::MobPlacedSpawnCarrierRecord,
obligation: &mob_dsl::PlacedCarrierCleanupObligation,
context: &str,
) -> Result<crate::store::MobPlacedSpawnCleanupAuthority, MobError> {
let authorize = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizePlacedCarrierCleanup {
obligation: obligation.clone(),
},
context,
)?;
crate::store::MobPlacedSpawnCleanupAuthority::from_transition(expected, &authorize)
.map_err(MobError::from)
}
/// Realize a pre-authorized cleanup. Resolve is fed only after
/// Deleted/AlreadyAbsent; conflict or store failure retains the obligation
/// and therefore blocks an ABA successor for the identity.
async fn realize_authorized_placed_carrier_cleanup(
&mut self,
expected: &crate::store::MobPlacedSpawnCarrierRecord,
obligation: mob_dsl::PlacedCarrierCleanupObligation,
authority: &crate::store::MobPlacedSpawnCleanupAuthority,
context: &str,
) -> Result<(), MobError> {
match self
.runtime_metadata
.compare_and_delete_placed_spawn(&self.definition.id, expected, authority)
.await?
{
crate::store::DeletePlacedSpawnResult::Deleted
| crate::store::DeletePlacedSpawnResult::AlreadyAbsent => {}
crate::store::DeletePlacedSpawnResult::Conflict => {
return Err(MobError::Internal(format!(
"placed carrier cleanup conflict for '{}' spawn_id={} generation={} fence={} ({context}); obligation retained",
expected.agent_identity,
expected.spawn_id,
expected.generation,
expected.fence_token,
)));
}
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::ResolvePlacedCarrierCleanup { obligation },
context,
)
}
/// A sent MaterializeMember may have committed at the host even when its
/// reply was lost. Release the exact tuple, or use an authenticated
/// HostStatus inventory to certify that exact tuple absent. No rejection
/// string is treated as absence.
async fn release_placed_attempt_or_certify_absent(
&mut self,
carrier: &crate::store::MobPlacedSpawnCarrierRecord,
) -> Result<(), MobError> {
let host_id = mob_dsl::HostId::from(carrier.host_id.to_string());
let current_binding_is_carrier_binding = self
.dsl_authority
.state()
.host_binding_generations
.get(&host_id)
.copied()
== Some(carrier.host_binding_generation)
&& self.dsl_authority.state().host_bind_phase.get(&host_id)
== Some(&mob_dsl::HostBindPhase::Bound);
if !current_binding_is_carrier_binding {
return super::placed_carrier_cleanup::release_placed_attempt_or_certify_absent(
&self.definition.id,
carrier,
&self.dsl_authority,
self.provisioner.as_ref(),
Arc::clone(&self.supervisor_bridge),
)
.await;
}
let release_key = super::state::HostOrphanReleaseKey {
binding_incarnation: self.current_host_binding_incarnation(&host_id)?,
host_id,
agent_identity: mob_dsl::AgentIdentity::from(carrier.agent_identity.clone()),
generation: mob_dsl::Generation(carrier.generation),
fence_token: mob_dsl::FenceToken(carrier.fence_token),
};
if !Self::reserve_host_orphan_release(&mut self.orphan_release_reservations, &release_key) {
return Err(MobError::Internal(format!(
"placed-carrier cleanup collided with an existing orphan release for '{}' generation {} fence {}",
carrier.agent_identity, carrier.generation, carrier.fence_token,
)));
}
let result = super::placed_carrier_cleanup::release_placed_attempt_or_certify_absent(
&self.definition.id,
carrier,
&self.dsl_authority,
self.provisioner.as_ref(),
Arc::clone(&self.supervisor_bridge),
)
.await;
if !Self::absorb_host_orphan_release_completion(
&mut self.orphan_release_reservations,
&release_key,
result.is_ok(),
) {
return Err(MobError::Internal(format!(
"placed-carrier cleanup lost its actor-owned orphan-release reservation for '{}' generation {} fence {}",
carrier.agent_identity, carrier.generation, carrier.fence_token,
)));
}
result
}
/// Persist the authenticated ACK by exact Pending->Committed CAS. Store
/// errors are reconciled by bounded exact reads: exact Committed is a
/// lost ACK success, exact Pending proves no write and permits a bounded
/// replay, while unreadable/conflicting state fail-stops for cold recovery.
async fn persist_placed_spawn_commit_exact(
&mut self,
pending: &crate::store::MobPlacedSpawnCarrierRecord,
ack: &super::provisioner::MaterializedMemberAck,
operation_id: &meerkat_core::ops::OperationId,
transition: &mob_dsl::MobMachineTransition,
) -> Result<(), MobError> {
if operation_id != &pending.provision_operation_id {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' attempted to replace pre-minted provision operation '{}' with '{}'; actor is fail-stopping for cold recovery",
pending.agent_identity, pending.provision_operation_id, operation_id,
)));
}
let mut committed = pending.clone();
committed.phase = crate::store::PlacedSpawnCarrierPhase::Committed(
crate::store::PlacedSpawnCommitRecord {
member_session_id: ack.session_id.clone(),
member_peer_endpoint: mob_dsl::MemberPeerEndpoint::from(&ack.member_peer),
ack_engine_version: ack.engine_version.clone(),
},
);
let authority = crate::store::MobPlacedSpawnCommitPersistenceAuthority::from_transition(
&committed, transition,
)?;
let mut last_error = None;
for attempt in 1..=2 {
match self
.runtime_metadata
.compare_and_commit_placed_spawn(
&self.definition.id,
pending,
&committed,
&authority,
)
.await
{
Ok(
crate::store::CommitPlacedSpawnResult::Committed
| crate::store::CommitPlacedSpawnResult::AlreadyCommittedExact,
) => return Ok(()),
Ok(crate::store::CommitPlacedSpawnResult::StillPending) if attempt < 2 => {
tokio::task::yield_now().await;
}
Ok(crate::store::CommitPlacedSpawnResult::StillPending) => {
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' remained exact Pending after bounded CAS retries",
pending.agent_identity
)));
}
Ok(crate::store::CommitPlacedSpawnResult::Conflict) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier commit conflict for '{}' spawn_id={}; actor is fail-stopping for cold recovery",
pending.agent_identity, pending.spawn_id
)));
}
Err(error) => {
last_error = Some(error);
match self
.runtime_metadata
.load_placed_spawn(&self.definition.id, &pending.agent_identity)
.await
{
Ok(Some(current)) if current == committed => return Ok(()),
Ok(Some(current)) if current == *pending && attempt < 2 => {
tokio::task::yield_now().await;
}
Ok(Some(current)) if current == *pending => {
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' failed and exact reread proved it remained Pending: {}",
pending.agent_identity,
last_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown store error".to_string())
)));
}
Ok(Some(current)) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' found conflicting durable phase {:?}; actor is fail-stopping for cold recovery",
pending.agent_identity, current.phase
)));
}
Ok(None) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' lost its Pending carrier; actor is fail-stopping for cold recovery",
pending.agent_identity
)));
}
Err(read_error) if attempt < 2 => {
last_error = Some(read_error);
tokio::task::yield_now().await;
}
Err(read_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed carrier commit for '{}' and exact reread remained uncertain; actor is fail-stopping for cold recovery: {read_error}",
pending.agent_identity
)));
}
}
}
}
}
Err(MobError::Internal(format!(
"placed carrier commit for '{}' exhausted retries: {}",
pending.agent_identity,
last_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown store error".to_string())
)))
}
/// Append an event after an irreversible placed-carrier commit. Each
/// append error is cursor-fenced and reconciled exactly before retrying;
/// persistent uncertainty or repeated proven no-write fail-stops so cold
/// recovery can repair from the committed carrier without duplicating an
/// older same-shaped event.
async fn append_committed_placed_event_exact(
&mut self,
desired: MobEventKind,
) -> Result<(), MobError> {
let mut last_append_error = None;
for attempt in 1..=2 {
let floor = match self.events.latest_cursor().await {
Ok(cursor) => cursor,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"committed placed event could not establish an append cursor floor; actor is fail-stopping for cold recovery: {error}"
)));
}
};
match self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
{
Ok(_) => return Ok(()),
Err(error) => {
last_append_error = Some(error);
match reconcile_exact_mob_event_after_cursor(
self.events.as_ref(),
&self.definition.id,
floor,
&desired,
)
.await
{
Ok(Some(_)) => return Ok(()),
Ok(None) if attempt < 2 => tokio::task::yield_now().await,
Ok(None) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"committed placed event append was proven absent after bounded retries; actor is fail-stopping for cold recovery: {}",
last_append_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown append error".to_string())
)));
}
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"committed placed event append acknowledgement and exact cursor reconciliation remained uncertain; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
}
}
}
}
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"committed placed event append exhausted retries; actor is fail-stopping for cold recovery: {}",
last_append_error
.map(|error| error.to_string())
.unwrap_or_else(|| "unknown append error".to_string())
)))
}
/// §4.4: witness the typed materialization failure, open the exact Pending
/// cleanup obligation, and synchronously discharge it. Callers that may
/// have sent MaterializeMember first certify ReleaseMember/absence before
/// entering this helper.
async fn fail_remote_spawn_exec_outcome(
&mut self,
agent_identity: &AgentIdentity,
pending_carrier: &crate::store::MobPlacedSpawnCarrierRecord,
materialize_may_have_been_sent: bool,
original: MobError,
kind: String,
context: &'static str,
) -> RemoteSpawnFailureOutcome {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
if let Err(record_error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::RecordMemberMaterializationFailure {
agent_identity: dsl_identity.clone(),
kind: kind.clone(),
},
context,
) {
tracing::error!(
agent_identity = %agent_identity,
kind = %kind,
error = %record_error,
"failed to record member materialization failure before abort"
);
}
let abort = match self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AbortSpawnExec {
agent_identity: dsl_identity,
},
context,
) {
Ok(transition) => transition,
Err(abort_error) => {
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; AbortSpawnExec failed, the durable Pending carrier is retained, and the actor is fail-stopping for cold recovery: {abort_error}"
)),
cleanup_completed: false,
};
}
};
let obligation = abort.effects().iter().find_map(|effect| match effect {
mob_dsl::MobMachineEffect::PlacedCarrierCleanupRequested { obligation } => {
Some(obligation.clone())
}
_ => None,
});
let Some(obligation) = obligation else {
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; AbortSpawnExec emitted no placed-carrier cleanup obligation, the durable Pending carrier is retained, and the actor is fail-stopping for cold recovery"
)),
cleanup_completed: false,
};
};
let cleanup_authority = match self.authorize_placed_carrier_cleanup(
pending_carrier,
&obligation,
context,
) {
Ok(authority) => authority,
Err(authority_error) => {
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; cleanup obligation does not authorize the exact Pending carrier, no lifecycle operation was mutated, and the actor is fail-stopping for cold recovery: {authority_error}"
)),
cleanup_completed: false,
};
}
};
if materialize_may_have_been_sent
&& let Err(release_error) = self
.release_placed_attempt_or_certify_absent(pending_carrier)
.await
{
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; exact remote absence could not be certified, the Pending cleanup obligation is retained, and the actor is fail-stopping for cold recovery: {release_error}"
)),
cleanup_completed: false,
};
}
let operation_display_name = match render_member_comms_name(
self.definition.id.as_str(),
&pending_carrier.spec.profile_name,
agent_identity.as_str(),
) {
Ok(display_name) => display_name,
Err(display_error) => {
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; exact operation display tuple could not be reconstructed, the Pending cleanup obligation is retained, and the actor is fail-stopping for cold recovery: {display_error}"
)),
cleanup_completed: false,
};
}
};
// The Pending carrier pre-mints the one durable lifecycle operation.
// Release/certified absence must happen first when the dispatch may
// have reached the remote host; only then may the exact operation be
// terminalized. Deleting the carrier before this exact operation
// cleanup would strand an owner-scoped Provisioning/Running record
// that cold recovery could no longer identify without rediscovery.
if let Err(operation_error) = self
.provisioner
.abort_placed_provision_operation(
&pending_carrier.operation_owner_session_id,
&pending_carrier.provision_operation_id,
&operation_display_name,
)
.await
{
self.durable_uncertainty_fail_stop = true;
return RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; exact provision operation '{}' owned by '{}' could not be terminalized, the Pending cleanup obligation is retained, and the actor is fail-stopping for cold recovery: {operation_error}",
pending_carrier.provision_operation_id,
pending_carrier.operation_owner_session_id,
)),
cleanup_completed: false,
};
}
match self
.realize_authorized_placed_carrier_cleanup(
pending_carrier,
obligation,
&cleanup_authority,
context,
)
.await
{
Ok(()) => RemoteSpawnFailureOutcome {
error: original,
cleanup_completed: true,
},
Err(cleanup_error) => {
self.durable_uncertainty_fail_stop = true;
RemoteSpawnFailureOutcome {
error: MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; exact placed-carrier cleanup failed and the actor is fail-stopping for cold recovery: {cleanup_error}"
)),
cleanup_completed: false,
}
}
}
}
async fn fail_remote_spawn_exec(
&mut self,
agent_identity: &AgentIdentity,
pending_carrier: &crate::store::MobPlacedSpawnCarrierRecord,
materialize_may_have_been_sent: bool,
original: MobError,
kind: String,
context: &'static str,
) -> MobError {
self.fail_remote_spawn_exec_outcome(
agent_identity,
pending_carrier,
materialize_may_have_been_sent,
original,
kind,
context,
)
.await
.error
}
/// Remote-spawn failure after recipient trust was installed. The exact
/// host attempt and carrier are cleaned first; only then may trust newly
/// installed by this attempt be removed. Pre-existing trust is never
/// revoked, and any uncertainty keeps the machine obligation open.
// Keep the exact durable carrier, trust receipt, peer, and failure facts
// explicit; grouping them would add a parallel pre-commit authority type.
#[allow(clippy::too_many_arguments)]
async fn fail_remote_spawn_exec_after_recipient_trust(
&mut self,
agent_identity: &AgentIdentity,
pending_carrier: &crate::store::MobPlacedSpawnCarrierRecord,
original: MobError,
kind: String,
context: &'static str,
peer: &TrustedPeerDescriptor,
install: Option<super::supervisor_bridge::RecipientTrustInstall>,
) -> MobError {
let outcome = self
.fail_remote_spawn_exec_outcome(
agent_identity,
pending_carrier,
true,
original,
kind,
context,
)
.await;
if !outcome.cleanup_completed {
return outcome.error;
}
let Some(install) = install else {
return self.quarantine_uncertain_recipient_trust_install(
peer,
context,
MobError::Internal(format!(
"{}; exact remote cleanup completed but the admission path lost its recipient-trust install receipt",
outcome.error
)),
);
};
if Self::recipient_trust_was_newly_installed(install)
&& let Err(untrust_error) = self.supervisor_bridge.untrust_recipient(peer).await
{
self.durable_uncertainty_fail_stop = true;
return MobError::Internal(format!(
"{}; exact remote cleanup completed but newly installed recipient trust could not be removed, so the actor is fail-stopping for cold recovery: {untrust_error}",
outcome.error
));
}
if let Err(obligation_error) = self.rollback_pending_recipient_trust_obligation(
peer,
"remote_spawn_precommit_cleanup_confirmed",
) {
self.durable_uncertainty_fail_stop = true;
return MobError::Internal(format!(
"{}; exact remote cleanup completed but the pending recipient-trust obligation could not be closed, so the actor is fail-stopping for cold recovery: {obligation_error}",
outcome.error
));
}
outcome.error
}
/// Remote-spawn failure when recipient trust returned an error instead of
/// an install receipt. The host attempt and exact Pending carrier may be
/// cleaned because membership is proven uncommitted, but the trust
/// obligation cannot be closed: `apply_bridge_trust` mutates the DSL
/// before live reconciliation and the missing receipt cannot distinguish
/// a clean no-install from a committed projection gap.
async fn fail_remote_spawn_exec_after_recipient_trust_error(
&mut self,
agent_identity: &AgentIdentity,
pending_carrier: &crate::store::MobPlacedSpawnCarrierRecord,
original: MobError,
peer: &TrustedPeerDescriptor,
) -> MobError {
let outcome = self
.fail_remote_spawn_exec_outcome(
agent_identity,
pending_carrier,
true,
original,
"member_trust_install_failed".to_string(),
"finalize_spawn_admit_remote_trust_install",
)
.await;
if !outcome.cleanup_completed {
return outcome.error;
}
self.quarantine_uncertain_recipient_trust_install(
peer,
"finalize_spawn_admit_remote_trust_install",
MobError::Internal(format!(
"{}; exact remote materialization and Pending carrier cleanup completed",
outcome.error
)),
)
}
/// Multi-host §7.3 remote spawn lane: compile → digest-authorized
/// ladder open (`BeginSpawnExecRemote` at enqueue) → durable operator
/// authority → `RequestMemberMaterialization` realization as a bridge
/// `MaterializeMember` dispatch in the pending-spawn task.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
async fn enqueue_spawn_remote(
&mut self,
spec: super::handle::SpawnMemberSpec,
owner_bridge_session_id: Option<SessionId>,
ops_registry: Option<Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>>,
respawn_origin: Option<RespawnOrigin>,
restore_wiring: Option<RestoreWiringPlan>,
reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
) {
let requested_identity = AgentIdentity::from(spec.identity.as_str());
let abandonment_origin = respawn_origin.clone();
let result = self
.enqueue_spawn_remote_inner(
spec,
owner_bridge_session_id,
ops_registry,
respawn_origin,
restore_wiring,
reply_tx,
)
.await;
if let Err((reply_tx, error)) = result {
let mut may_reply = true;
let error = if let Some(origin) = abandonment_origin.as_ref() {
let identity = origin.old_runtime_id.identity.clone();
match self
.durably_abandon_respawn_topology_if_terminal_exact(&identity, origin)
.await
{
Ok(()) => error,
Err(abandon_error) => {
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
MobError::Internal(format!(
"{error}; durable respawn-topology abandonment failed and the actor is fail-stopping for cold recovery: {abandon_error}"
))
}
}
} else {
error
};
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %requested_identity,
spawn_lane = "remote",
reply_withheld = !may_reply,
error = %error,
"remote member spawn failed before asynchronous spawn custody"
);
if may_reply {
let _ = reply_tx.send(Err(error));
}
}
}
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
#[allow(clippy::type_complexity, clippy::too_many_lines)]
async fn enqueue_spawn_remote_inner(
&mut self,
spec: super::handle::SpawnMemberSpec,
owner_bridge_session_id: Option<SessionId>,
ops_registry: Option<Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>>,
respawn_origin: Option<RespawnOrigin>,
restore_wiring: Option<RestoreWiringPlan>,
reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
) -> Result<
(),
(
oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
MobError,
),
> {
macro_rules! fail {
($error:expr) => {
return Err((reply_tx, $error))
};
}
let super::handle::SpawnMemberSpec {
role_name: profile_name,
identity,
initial_message,
runtime_mode,
backend,
binding,
context,
labels,
launch_mode,
tool_access_policy,
budget_limits,
auto_wire_parent,
additional_instructions,
shell_env,
inherited_tool_filter,
override_profile,
model_override,
objective_id,
auth_binding,
external_tools: per_spawn_external_tools,
system_prompt_override,
continuity_intent,
placement,
} = spec;
let Some(host) = placement else {
fail!(MobError::Internal(
"enqueue_spawn_remote invoked without placement".to_string()
));
};
// DEC-P3-3: one transport story per member — placement and an
// explicit backend/binding are mutually exclusive, BEFORE any
// machine input.
if binding.is_some() || backend.is_some() {
fail!(MobError::WiringError(format!(
"placement and explicit backend/binding are mutually exclusive for '{identity}'"
)));
}
let agent_identity = AgentIdentity::from(identity.as_str());
if let Err(error) = self.preview_spawn_command_admission(&agent_identity) {
fail!(error);
}
if agent_identity.is_system_reserved() {
fail!(MobError::WiringError(format!(
"meerkat id '{agent_identity}' uses reserved system identifier namespace"
)));
}
{
let roster = self.roster.read().await;
if roster.get(&agent_identity).is_some() {
fail!(MobError::MemberAlreadyExists(agent_identity.clone()));
}
}
if !self.definition.profiles.contains_key(&profile_name) {
fail!(MobError::ProfileNotFound(profile_name.clone()));
}
// The digest-covered portable profile is the cold-recovery source of
// truth. For a combined full-profile + model override, persist the
// final composed profile (not the pre-model input) in MemberSpawned.
let full_profile_override_requested = override_profile.is_some();
let effective_model_override = model_override.clone();
let mut profile = match override_profile {
Some(p) => p,
None => {
match self
.definition
.resolve_profile(&profile_name, self.realm_profile_store.as_ref())
.await
{
Ok(profile) => profile,
Err(error) => fail!(error),
}
}
};
if let Some(model) = model_override {
profile.model = model;
}
let effective_profile_override = full_profile_override_requested.then(|| profile.clone());
// ADJ-6: explicit fact, captured before the inherited-open mutation.
let explicit_workgraph = profile.tools.workgraph;
if inherited_tool_filter.is_some() && effective_profile_override.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let selected_runtime_mode = runtime_mode.unwrap_or(profile.runtime_mode);
// §4.1(a) resume/fork normalization: the LOCAL resume fast-path is
// skipped — member-side validation is authoritative (§19.L1). Fork
// with a local source renders context text controlling-side; the
// wire launch is Fresh (Fork unrepresentable, A6).
let resume_session_id = launch_mode.resume_bridge_session_id().cloned();
let fork_spec = match launch_mode {
crate::launch::MemberLaunchMode::Fork {
source_member_id,
fork_context,
} => Some((source_member_id, fork_context)),
_ => None,
};
let fork_context_text = if let Some((source_member_id, fork_context)) = fork_spec {
// §19.L2 placement switch: local and remote SOURCE reads share
// one proxy + one renderer (DEC-P6E-20).
let rows = match self
.fork_source_history_rows(&source_member_id, &fork_context)
.await
{
Ok(rows) => rows,
Err(error) => fail!(error),
};
Some(render_fork_context(&source_member_id, &rows))
} else {
None
};
// Compose and persist the exact first-turn material BEFORE the
// canonical Pending carrier. The carrier is the earliest durable
// boundary for a placed spawn (and can repair a missing
// `MemberSpawned` event), so a later in-memory mint would be lost on
// controller restart.
let base_kickoff = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let prompt = if let Some(fork_text) = fork_context_text {
let mut blocks = vec![meerkat_core::types::ContentBlock::Text {
text: format!("{fork_text}\n\n"),
}];
blocks.extend(base_kickoff.into_blocks());
ContentInput::Blocks(blocks)
} else {
base_kickoff
};
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
// Autonomous kickoff always owns an objective. Preserve a caller's
// inherited objective when present; otherwise mint it once here and
// carry the same value through machine state and bridge delivery.
let objective_id = if selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost {
Some(objective_id.unwrap_or_default())
} else {
objective_id
};
let kickoff_intent = match (selected_runtime_mode, objective_id) {
(crate::MobRuntimeMode::AutonomousHost, Some(objective_id)) => {
Some(crate::store::MobPlacedKickoffIntent {
input_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
objective_id,
prompt: prompt.clone(),
handling_mode: meerkat_core::types::HandlingMode::Queue,
injected_context: Vec::new(),
})
}
_ => None,
};
// §3.2 compile (pure; skill-file reads are small local reads).
let base_prompt = self.spawn_base_prompt_source.clone();
let compiled = match super::spec_compiler::compile_portable_member_spec(
super::spec_compiler::CompileMemberSpecParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
context: context.as_ref(),
labels: labels.as_ref(),
additional_instructions: additional_instructions.as_ref(),
system_prompt_override: system_prompt_override.as_ref(),
tool_access_policy: tool_access_policy.as_ref(),
auth_binding: auth_binding.as_ref(),
budget_limits: budget_limits.as_ref(),
runtime_mode: selected_runtime_mode,
continuity_intent: &continuity_intent,
base_prompt: base_prompt.as_deref(),
non_portable_disabled: Vec::new(),
},
)
.await
{
Ok(compiled) => compiled,
Err(error) => fail!(error),
};
// §3.3 authorize with the compiled digest (machine records it
// single-shot; the remote commit's echo guard consumes it).
let authorized_profile_material = match self.authorize_spawn_profile_material_with_digest(
&agent_identity,
&profile_name,
&profile,
Some(compiled.digest.clone()),
"enqueue_spawn_remote_profile_authority",
) {
Ok(material) => material,
Err(error) => fail!(error),
};
let observations = observe_spawn_exec_facts(
&profile,
explicit_workgraph,
per_spawn_external_tools.is_some(),
shell_env.as_ref(),
inherited_tool_filter.is_some(),
resume_session_id.clone(),
);
// The placed operation owner and operation id are known before the
// Pending carrier is prepared. They are persisted in that carrier
// and echoed by the generated handoff, so a lost materialization
// completion remains exactly abortable on cold recovery.
let machine_owner_session = match self
.dsl_authority
.state()
.owner_bridge_session_id
.as_ref()
{
Some(session_id) => match SessionId::parse(&session_id.0) {
Ok(session_id) => Some(session_id),
Err(error) => fail!(MobError::Internal(format!(
"placed member '{agent_identity}' found invalid MobMachine owner bridge session '{}': {error}",
session_id.0
))),
},
None => None,
};
let owner_session = owner_bridge_session_id.clone().or(machine_owner_session);
let Some(owner_session) = owner_session else {
fail!(MobError::Internal(format!(
"placed member '{agent_identity}' requires a generated owner-bridge ops binding (§19.L5)"
)));
};
let provision_operation_id = meerkat_core::ops::OperationId::new();
// §4.1(e-f) mint + ladder open. Post-apply, fail-closed effect scan
// (the handle_bind_host discipline — never `_ => {}`).
// Machine-owned generation mint (ADJ-24): `INITIAL` for a fresh
// identity, prior+1 for retired-identity reuse — the host admits the
// replacement tuple as Superseding, never StaleFence.
let generation = match self.mint_spawn_generation(&agent_identity) {
Ok(generation) => generation,
Err(error) => fail!(error),
};
let fence = match self.issue_fence_token() {
Ok(fence) => fence,
Err(error) => fail!(error),
};
let placed_spawn_id = crate::ids::PlacedSpawnId::new();
let dsl_placed_spawn_id = mob_dsl::PlacedSpawnId(placed_spawn_id.to_string());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&agent_identity);
let identity_typed = crate::ids::AgentIdentity::from(agent_identity.as_str());
let agent_runtime_id = crate::ids::AgentRuntimeId::new(identity_typed, generation);
// Prepare on an isolated authority. The live machine must never
// publish materialization authority until the exact Pending carrier
// is durably known to exist.
let prepared_begin = match self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(fence),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::from(selected_runtime_mode),
bridge_session_id: None,
replacing: None,
placement: Some(host.clone()),
workgraph_required: observations.workgraph_required,
rust_bundles_present: observations.rust_bundles_present,
per_spawn_external_tools_present: observations.per_spawn_external_tools_present,
mob_default_external_tools_present: self.default_external_tools_provider.is_some(),
default_llm_client_override_present: self.default_llm_client.is_some(),
host_surface_mcp_allowlist_present: observations.host_surface_mcp_allowlist_present,
inherited_tool_filter_present: observations.inherited_tool_filter_present,
shell_env_present: observations.shell_env_present,
mcp_stdio_env_present: observations.mcp_stdio_env_present,
mcp_http_headers_present: observations.mcp_http_headers_present,
memory_required: observations.memory_required,
mcp_required: observations.mcp_required,
resume_session_id: observations
.resume_session_id
.as_ref()
.map(mob_dsl::SessionId::from_domain),
placed_spawn_id: Some(dsl_placed_spawn_id.clone()),
placed_provision_operation_id: Some(provision_operation_id.to_string()),
placed_operation_owner_session_id: Some(mob_dsl::SessionId::from_domain(
&owner_session,
)),
effective_profile_override_present: effective_profile_override.is_some(),
effective_model_override_present: effective_model_override.is_some(),
},
"enqueue_spawn_remote_open",
) {
Ok(prepared) => prepared,
Err(error) => fail!(error),
};
let transition = &prepared_begin.transition;
let handoff = transition.effects().iter().find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RequestMemberMaterialization {
spawn_id,
agent_identity,
generation,
fence_token,
spec_digest,
host,
host_binding_generation,
provision_operation_id,
operation_owner_session_id,
} => Some((
spawn_id.clone(),
agent_identity.clone(),
*generation,
*fence_token,
spec_digest.clone(),
host.clone(),
*host_binding_generation,
provision_operation_id.clone(),
operation_owner_session_id.clone(),
)),
_ => None,
});
let (
effect_spawn_id,
effect_identity,
effect_generation,
effect_fence,
effect_digest,
effect_host,
effect_host_binding_generation,
effect_operation_id,
effect_owner_session_id,
) = match handoff {
Some(handoff) => handoff,
None => {
// Denial arms accept the input, mutate nothing, and emit
// the typed admission verdict.
let admission = transition.effects().iter().find_map(|effect| match effect {
mob_dsl::MobMachineEffect::SpawnMemberAdmissionResolved { admission } => {
Some(*admission)
}
_ => None,
});
match admission {
Some(admission) => fail!(MobError::SpawnMemberAdmissionDenied { admission }),
None => fail!(MobError::Internal(
"BeginSpawnExec(remote) emitted neither materialization handoff nor admission verdict"
.to_string(),
)),
}
}
};
// A12 drift check: the MACHINE value rides the wire; the shell
// asserts equality where it holds both.
if effect_spawn_id != dsl_placed_spawn_id
|| effect_identity != dsl_identity
|| effect_generation != mob_dsl::Generation::from_domain(generation)
|| effect_fence != mob_dsl::FenceToken::from_domain(fence)
|| effect_host != host
|| effect_host_binding_generation == 0
|| effect_digest != compiled.digest
|| effect_operation_id != provision_operation_id.to_string()
|| effect_owner_session_id != mob_dsl::SessionId::from_domain(&owner_session)
{
fail!(MobError::Internal(format!(
"MobMachine RequestMemberMaterialization drift for '{agent_identity}'"
)));
}
// The digest-covered portable spec is also the sole durable owner of
// per-member operator context. There is deliberately no parallel
// operator-authority write on this lane.
let carrier_host = match meerkat_core::comms::PeerId::parse(effect_host.as_str()) {
Ok(host) => host,
Err(error) => fail!(MobError::Internal(format!(
"placed spawn host '{}' is not a canonical peer id: {error}",
effect_host.as_str()
))),
};
let pending_carrier = crate::store::MobPlacedSpawnCarrierRecord::pending(
placed_spawn_id.clone(),
agent_identity.as_str().to_string(),
effect_generation.0,
effect_fence.0,
provision_operation_id.clone(),
owner_session.clone(),
carrier_host,
effect_host_binding_generation,
effect_digest.clone(),
compiled.spec.clone(),
kickoff_intent,
effective_profile_override.is_some(),
effective_model_override.is_some(),
);
let pending_authority =
match crate::store::MobPlacedSpawnPendingPersistenceAuthority::from_transition(
&pending_carrier,
transition,
) {
Ok(authority) => authority,
Err(error) => fail!(MobError::from(error)),
};
let mut begin_error = None;
let mut pending_certain = false;
let mut final_read_proved_absent = false;
for attempt in 1..=2 {
match self
.runtime_metadata
.begin_placed_spawn_if_absent(
&self.definition.id,
&pending_carrier,
&pending_authority,
)
.await
{
Ok(
crate::store::BeginPlacedSpawnResult::Inserted
| crate::store::BeginPlacedSpawnResult::ExistingExactPending,
) => {
pending_certain = true;
break;
}
Ok(crate::store::BeginPlacedSpawnResult::ExistingExactCommitted) => {
self.durable_uncertainty_fail_stop = true;
fail!(MobError::Internal(format!(
"placed spawn begin for '{agent_identity}' encountered an already committed carrier for the same attempt; actor is fail-stopping for cold recovery"
)));
}
Ok(crate::store::BeginPlacedSpawnResult::Conflict) => {
self.durable_uncertainty_fail_stop = true;
fail!(MobError::Internal(format!(
"placed spawn begin conflict for '{agent_identity}' generation={generation} fence={fence} spawn_id={placed_spawn_id}; actor is fail-stopping for cold recovery"
)));
}
Err(error) => {
begin_error = Some(error);
// A write may have committed before its acknowledgement
// was lost. An exact reread proves that case; absence gets
// one bounded replay of the same idempotent request.
match self
.runtime_metadata
.load_placed_spawn(&self.definition.id, agent_identity.as_str())
.await
{
Ok(Some(current)) if current == pending_carrier => {
pending_certain = true;
break;
}
Ok(None) if attempt < 2 => tokio::task::yield_now().await,
Ok(Some(current)) => {
self.durable_uncertainty_fail_stop = true;
fail!(MobError::Internal(format!(
"placed spawn begin acknowledgement was lost and exact reread found conflicting carrier for '{agent_identity}' spawn_id={}; actor is fail-stopping for cold recovery",
current.spawn_id
)));
}
Ok(None) => final_read_proved_absent = true,
Err(read_error) if attempt < 2 => {
begin_error = Some(read_error);
tokio::task::yield_now().await;
}
Err(read_error) => begin_error = Some(read_error),
}
}
}
}
if !pending_certain {
if !final_read_proved_absent {
self.durable_uncertainty_fail_stop = true;
}
fail!(MobError::Internal(format!(
"placed spawn pending carrier for '{agent_identity}' was not established after bounded begin/reread retries{}: {}",
if final_read_proved_absent {
" (final exact read proved no write)"
} else {
"; actor is fail-stopping for cold recovery"
},
begin_error
.map(|error| error.to_string())
.unwrap_or_else(|| "carrier remained absent".to_string())
)));
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared_begin) {
// Durable Pending exists but the in-memory authority could not be
// published. Cold recovery owns this exact attempt; do not issue
// bridge traffic from the stale actor.
self.durable_uncertainty_fail_stop = true;
fail!(MobError::Internal(format!(
"placed spawn pending carrier for '{agent_identity}' committed but machine publication failed; actor is fail-stopping for cold recovery: {error}"
)));
}
// §4.1(h) payload build (owner realization: the actor constructs
// the command from the EFFECT; host route material from the machine
// host maps — recorded at CommitHostBind, fail-closed on absence).
let host_peer = {
let state = self.dsl_authority.state();
let endpoint = state.host_endpoints.get(&host).cloned();
let pubkey = state.host_public_keys.get(&host).copied();
match (endpoint, pubkey) {
(Some(endpoint), Some(pubkey)) => {
match TrustedPeerDescriptor::unsigned_with_pubkey(
host.as_str(),
host.as_str(),
pubkey.0,
endpoint.0.as_str(),
) {
Ok(peer) => peer,
Err(error) => {
let error = MobError::Internal(format!(
"bound host '{}' facts do not form a canonical peer descriptor: {error}",
host.as_str()
));
fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_host_descriptor",
)
.await
);
}
}
}
_ => {
let error = MobError::Internal(format!(
"bound host '{}' has no recorded endpoint/signing key",
host.as_str()
));
fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_host_facts",
)
.await
);
}
}
};
let authority = self.supervisor_bridge.authority().await;
let binding_generation = match self.current_host_binding_generation(&host) {
Ok(binding_generation) if binding_generation == effect_host_binding_generation => {
effect_host_binding_generation
}
Ok(binding_generation) => fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
MobError::Internal(format!(
"placed spawn host-binding generation drift for '{agent_identity}': carrier={effect_host_binding_generation} current={binding_generation}"
)),
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_binding_generation_drift",
)
.await
),
Err(error) => fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_binding_generation",
)
.await
),
};
let supervisor_spec = match self
.supervisor_bridge
.supervisor_spec_for_recipient(&host_peer)
.await
{
Ok(spec) => spec,
Err(error) => fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_supervisor_spec",
)
.await
),
};
let launch = match resume_session_id.as_ref() {
Some(session_id) => super::bridge_protocol::MaterializeLaunchMode::Resume {
session_id: session_id.to_string(),
},
None => super::bridge_protocol::MaterializeLaunchMode::Fresh {},
};
let payload = Box::new(super::bridge_protocol::BridgeMaterializePayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
generation: effect_generation.0,
fence_token: effect_fence.0,
spec: compiled.spec,
spec_digest: effect_digest,
launch,
});
// Callers that don't carry an owner ops context (the embedder
// `spawn_spec` path) re-derive the registry from the MobMachine-owned
// owner bridge session — the owner-bridge REQUIRED arm, never a
// generated-self-owned fallback (§19.L5 / design-S §1).
let ops_registry = match ops_registry {
Some(ops_registry) => ops_registry,
None => {
let derived = match self.runtime_adapter.as_ref() {
Some(adapter) => adapter
.prepare_local_session_bindings(owner_session.clone())
.await
.map_err(|error| {
MobError::Internal(format!(
"placed member '{agent_identity}': failed to prepare \
MeerkatMachine ops bindings for the owner bridge session: {error}"
))
})
.and_then(|bindings| {
if bindings.session_id() != &owner_session {
return Err(MobError::Internal(format!(
"placed member '{agent_identity}': owner bridge ops bindings \
returned session '{}' but expected '{owner_session}'",
bindings.session_id()
)));
}
if !meerkat_runtime::session_runtime_bindings_have_machine_authority(
&bindings,
) {
return Err(MobError::Internal(format!(
"placed member '{agent_identity}': owner bridge ops bindings \
lacked machine authority"
)));
}
Ok(Arc::clone(bindings.ops_lifecycle()))
}),
None => Err(MobError::Internal(format!(
"placed member '{agent_identity}' requires MeerkatMachine runtime \
authority for the owner-bridge ops binding (§19.L5)"
))),
};
match derived {
Ok(ops_registry) => ops_registry,
Err(error) => {
fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_ops_owner",
)
.await
);
}
}
}
};
let peer_name = match render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
) {
Ok(peer_name) => peer_name,
Err(error) => fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_peer_name",
)
.await
),
};
// Pending is now durable and the live machine has published the
// exact owner tuple. Register the pre-minted operation synchronously
// before the bridge task exists, so any registration failure is a
// certified-no-send cleanup and every later transport error may
// conservatively have reached the host.
if let Err(error) =
super::ops_adapter::MobOpsAdapter::register_placed_provision_operation_exact(
ops_registry.as_ref(),
owner_session.clone(),
provision_operation_id.clone(),
&peer_name,
)
{
fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_operation_anchor",
)
.await
);
}
let resolved_labels = labels.unwrap_or_default();
// Pending-spawn machinery (DEC-P3-4): the bridge round-trip runs in
// the spawned task; cancellation/panic/alignment machinery is
// inherited, not re-implemented.
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
// Correlation id for the generated spawn-plan authority (the same
// pre-provision mint `admit_bridge_session_for_spawn` performs for
// local spawns; never a session claim).
let admitted_bridge_session_id = SessionId::new();
let generated_self_owned_operation_owner =
match self.stage_orchestrator_spawn(&agent_identity, &admitted_bridge_session_id) {
Ok(authority) => authority,
Err(error) => fail!(
self.fail_remote_spawn_exec(
&agent_identity,
&pending_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_stage",
)
.await
),
};
let pending = PendingSpawn {
profile_name,
agent_identity: agent_identity.clone(),
admitted_bridge_session_id,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt: false,
identity_member_permit: None,
runtime_mode: selected_runtime_mode,
labels: resolved_labels,
owner_bridge_session_id: Some(owner_session.clone()),
auto_wire_parent,
restore_wiring,
respawn_origin,
effective_profile_override,
effective_model_override,
objective_id,
authorized_profile_material,
continuity_intent,
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default())),
// Member trust is post-ack (§4.3); host trust exists from bind.
pending_recipient_trust_peer_id: None,
// Non-portable on the remote lane (machine admission denies a
// present overlay); carried verbatim so the single owner flows —
// today always None past admission.
per_spawn_external_tools,
observations,
remote: Some(Box::new(RemoteSpawnExec {
placed_spawn_id,
placement: host,
generation,
fence_token: fence,
resolved_spec_digest: compiled.digest,
pending_carrier: pending_carrier.clone(),
})),
enqueued_at: Instant::now(),
reply_tx,
};
// Publish the operation owner before any bridge I/O can begin. A
// start refusal is therefore a certified-no-send Pending cleanup.
let cleanup_carrier = pending_carrier;
let spawn_started = match generated_self_owned_operation_owner.start(&pending) {
Ok(started) => started,
Err(error) => {
let respawn_origin = pending.respawn_origin.clone();
let error = self
.fail_remote_spawn_exec(
&agent_identity,
&cleanup_carrier,
false,
error,
"pre_dispatch_failed".to_string(),
"enqueue_spawn_remote_start",
)
.await;
let mut may_reply = true;
if let Some(respawn_origin) = respawn_origin.as_ref()
&& let Err(abandon_error) = self
.durably_abandon_respawn_topology_if_terminal_exact(
&agent_identity,
respawn_origin,
)
.await
{
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
tracing::error!(
agent_identity = %agent_identity,
error = %error,
abandonment_error = %abandon_error,
"remote respawn start refusal could not durably abandon topology; withholding reply and fail-stopping for cold recovery"
);
}
if may_reply {
let _ = pending.reply_tx.send(Err(error));
}
return Ok(());
}
};
let provisioner = self.provisioner.clone();
let command_tx = self.command_tx.clone();
let spawn_member_identity = agent_identity.clone();
let panic_log_ledger = Arc::clone(&self.spawn_panic_log_ledger);
let panic_mob_id = self.definition.id.clone();
let task = tokio::spawn(async move {
let panic_member_identity = spawn_member_identity.clone();
// Panic boundary (see `panic_capture` for the 2026-07-29 incident
// WHY): recover + log the payload at bounded repeat checkpoints and
// feed the typed SpawnProvisioned failure path — never a silent
// `Err(_)` that an eager requeue can retry into a furnace.
// As above, the outer result is reserved for caught panics so
// cleanup uncertainty cannot be confused with an ordinary typed
// materialization failure that certified rollback.
let guarded_provision = super::panic_capture::run_spawn_provision_guarded(
panic_log_ledger.as_ref(),
&panic_mob_id,
"remote spawn dispatch task",
&panic_member_identity,
async {
let provision_result: Result<_, MobError> = async {
let materialized = provisioner
.materialize_member(super::provisioner::MaterializeMemberRequest {
host_peer,
payload,
peer_name,
owner_bridge_session_id: owner_session,
ops_registry,
provision_operation_id,
operation_anchor:
super::provisioner::PlacedProvisionOperationAnchor::NewPending,
timeout: super::provisioner::MATERIALIZE_BRIDGE_TIMEOUT,
})
.await?;
Ok(materialized.receipt)
}
.await;
Ok::<_, MobError>(provision_result)
},
)
.await;
let provision_result = match guarded_provision {
Ok(result) => result,
Err(panic_error) => {
tracing::error!(
spawn_ticket,
member = %panic_member_identity,
disposition = "external_cleanup_uncertain",
error = %panic_error,
"remote spawn dispatch panic cannot certify materialization cleanup"
);
Err(MobError::ExternalMemberCleanupUncertain {
reason: panic_error.to_string(),
})
}
};
if let Err(send_error) = command_tx
.send(RoutedMobCommand::internal(MobCommand::SpawnProvisioned {
spawn_ticket,
result: provision_result,
}))
.await
{
match send_error.0.cmd {
MobCommand::SpawnProvisioned { result: Ok(_), .. } => tracing::warn!(
spawn_ticket,
"remote spawn completion dropped because the actor is gone; host row is an orphan seed"
),
MobCommand::SpawnProvisioned {
result: Err(error), ..
} => tracing::warn!(
spawn_ticket,
error = %error,
"remote spawn completion dropped because the actor is gone; preserving the typed materialization failure"
),
_ => tracing::error!(
spawn_ticket,
"remote spawn task recovered a non-spawn command after completion delivery failed"
),
}
}
});
if let Err(error) = self
.insert_pending_spawn(spawn_ticket, pending, task, spawn_started)
.await
{
tracing::error!(
spawn_ticket,
error = %error,
"remote pending spawn insertion failed; canceling pending spawn lineage"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("remote pending spawn insertion failed")
.await
{
tracing::error!(
spawn_ticket,
error = %cleanup_error,
"remote pending spawn cleanup failed after insertion failure"
);
}
return Ok(());
}
if let Err(error) = self.ensure_pending_spawn_alignment("enqueue_spawn_remote post-insert")
{
tracing::error!(
spawn_ticket,
error = %error,
"pending spawn alignment check failed after remote enqueue; canceling all pending spawns"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated after remote enqueue")
.await
{
tracing::error!(
spawn_ticket,
error = %cleanup_error,
"pending spawn cleanup failed after remote enqueue alignment violation"
);
}
}
Ok(())
}
async fn handle_spawn_provisioned_batch(
&mut self,
completions: Vec<(u64, Result<super::handle::MemberSpawnReceipt, MobError>)>,
) {
tracing::debug!(
completion_count = completions.len(),
"MobActor::handle_spawn_provisioned_batch start"
);
if let Err(error) = self.ensure_pending_spawn_alignment("spawn batch preflight") {
tracing::error!(
error = %error,
"pending spawn alignment check failed before spawn completion batch"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated before spawn batch")
.await
{
tracing::error!(
error = %cleanup_error,
"pending spawn cleanup failed after pre-batch alignment violation"
);
}
return;
}
let mut pending_items = Vec::with_capacity(completions.len());
for (spawn_ticket, result) in completions {
tracing::debug!(
spawn_ticket,
"MobActor::handle_spawn_provisioned_batch completing pending slot"
);
let (pending, task_handle) =
self.complete_pending_spawn_slot(spawn_ticket, "spawn provisioned batch");
let Some(pending) = pending else {
tracing::warn!(spawn_ticket, "received spawn completion for unknown ticket");
if let Some(handle) = task_handle {
handle.abort();
tracing::warn!(
spawn_ticket,
"received spawn completion for unknown pending metadata but found task handle"
);
}
if let Ok(spawn_receipt) = result {
let orphan = PendingProvision::new(
spawn_receipt.member_ref,
AgentIdentity::from("__unknown_ticket__"),
self.provisioner.clone(),
spawn_receipt.operation_id,
spawn_receipt.session_origin,
spawn_receipt.rollback_authority,
);
if let Err(error) = orphan.rollback().await {
tracing::warn!(
spawn_ticket,
error = %error,
"unknown spawn completion cleanup failed"
);
}
}
continue;
};
pending_items.push((spawn_ticket, pending, result));
}
for (spawn_ticket, pending, result) in pending_items {
tracing::debug!("MobActor::handle_spawn_provisioned_batch finalizing pending spawn");
let PendingSpawn {
profile_name,
agent_identity,
admitted_bridge_session_id: _,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
runtime_mode,
labels,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
respawn_origin,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
progress: _,
pending_recipient_trust_peer_id,
observations,
remote,
enqueued_at,
reply_tx,
} = pending;
let identity_reconcile_authority = identity_member_permit
.as_ref()
.map(IdentityReconcileCompletionAuthority::from_permit);
if let Err(error) = &result
&& error.external_member_cleanup_is_uncertain()
{
// A caught provisioning panic cannot prove whether an
// external member or host row was created. Keep every durable
// recovery anchor intact and stop this incarnation; it must
// never flow through the ordinary "failure means cleanup"
// path below, even when no recipient-trust peer was involved.
self.durable_uncertainty_fail_stop = true;
tracing::error!(
member = %agent_identity,
error = %error,
"spawn completion cannot certify external cleanup; fail-stopping actor for cold recovery"
);
}
// Close the external recipient-trust obligation window recorded at
// enqueue only when terminality is certified. A successful
// provision resolves it; an ordinary failure means the provisioner
// confirmed full cleanup. Typed external-member cleanup uncertainty
// keeps the obligation open and fail-stops the live actor --
// emitting Rollback there would falsely certify remote retirement,
// trust rollback, and exact-operation cleanup.
if let Some(peer_id) = pending_recipient_trust_peer_id.as_deref() {
let close_result = match &result {
Ok(_) => Some(self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn provisioned batch confirmed",
)),
Err(error) if error.external_member_cleanup_is_uncertain() => {
tracing::error!(
peer_id,
error = %error,
"external member cleanup is uncertain; keeping pending obligation open and fail-stopping actor"
);
None
}
Err(_) => Some(
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn provisioned batch failed with full cleanup confirmed",
),
),
};
if let Some(Err(error)) = close_result {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
peer_id,
error = %error,
"failed to close pending recipient-trust obligation after provision result; fail-stopping actor"
);
}
}
let reply = match result {
Ok(mut spawn_receipt) => {
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
spawn_receipt.operation_id.clone(),
spawn_receipt.session_origin,
spawn_receipt.rollback_authority.take(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Some(remote_exec) = remote.as_ref() {
self.consume_spawn_provision_failure(
provision,
true,
&spawn_receipt.operation_id,
"mob state changed while remote spawn completed",
)
.await;
Err(self
.fail_remote_spawn_exec(
&agent_identity,
&remote_exec.pending_carrier,
true,
error,
"materialize_canceled".to_string(),
"spawn_provisioned_batch_remote_ineligible",
)
.await)
} else if let Err(retire_error) = provision.rollback().await {
Err(MobError::Internal(format!(
"spawn completed while mob state changed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)))
} else {
Err(error)
}
} else if let Some(remote_exec) = remote {
// Remote spawn: NEVER re-mint — the machine already
// holds the enqueue-time tuple from
// `BeginSpawnExecRemote`.
let Some(ack) = spawn_receipt.materialized_ack.take() else {
self.consume_spawn_provision_failure(
provision,
true,
&spawn_receipt.operation_id,
"materialized receipt carried no typed ack",
)
.await;
let error = self.fail_remote_spawn_exec(
&agent_identity,
&remote_exec.pending_carrier,
true,
MobError::Internal(format!(
"materialized receipt for '{agent_identity}' carries no typed ack"
)),
"materialize_decode_failed".to_string(),
"spawn_provisioned_batch_remote_no_ack",
).await;
let mut may_reply = true;
let error = if let Some(respawn_origin) = respawn_origin.as_ref() {
match self
.durably_abandon_respawn_topology_if_terminal_exact(
&agent_identity,
respawn_origin,
)
.await
{
Ok(()) => error,
Err(abandon_error) => {
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
MobError::Internal(format!(
"{error}; durable respawn-topology abandonment failed and the actor is fail-stopping for cold recovery: {abandon_error}"
))
}
}
} else {
error
};
if may_reply {
let _ = reply_tx.send(Err(error));
}
continue;
};
let generation = remote_exec.generation;
let fence = remote_exec.fence_token;
let remote_finalize = Some(Box::new(RemoteSpawnFinalize {
placed_spawn_id: remote_exec.placed_spawn_id.clone(),
placement: remote_exec.placement.clone(),
resolved_spec_digest: remote_exec.resolved_spec_digest.clone(),
pending_carrier: remote_exec.pending_carrier.clone(),
ack,
}));
let classify_respawn_topology = respawn_origin.is_some();
match Box::pin(self.finalize_spawn_from_pending(
&profile_name,
&agent_identity,
generation,
fence,
runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
labels,
provision,
spawn_receipt.operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
remote_finalize,
))
.await
{
Ok(outcome) => {
let mut receipt = outcome.receipt;
if classify_respawn_topology {
match self.resolve_respawn_topology_restore_result(
&agent_identity,
outcome.failed_restore_peer_ids,
) {
Ok(resolution) => {
receipt.failed_restore_peer_ids =
resolution.failed_peer_ids;
Ok(receipt)
}
Err(error) => Err(error),
}
} else {
Ok(receipt)
}
}
Err(error) => Err(error),
}
} else {
let fence = match self.issue_fence_token() {
Ok(fence) => fence,
Err(error) => {
let error = match provision.rollback().await {
Ok(()) => error,
Err(retire_error) => MobError::Internal(format!(
"spawn fence allocation failed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)),
};
let reply = Err(error);
if let Some(authority) = identity_reconcile_authority.as_ref() {
let disposition = identity_member_actuation_disposition(&reply);
self.record_identity_reconcile_disposition(
&agent_identity,
authority,
disposition,
)
.await;
}
let reply_delivered = reply_tx.send(reply).is_ok();
#[cfg(test)]
if identity_reconcile_authority.is_some() && !reply_delivered {
IDENTITY_RECONCILE_REPLY_DELIVERY_FAILURES
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
continue;
}
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_spawn_provisioned_batch calling finalize_spawn_from_pending"
);
// Machine-owned generation mint (ADJ-24): INITIAL
// when the identity has no machine history, prior+1
// on retired-identity reuse.
let generation = match self.mint_spawn_generation(&agent_identity) {
Ok(generation) => generation,
Err(error) => {
let error = match provision.rollback().await {
Ok(()) => error,
Err(retire_error) => MobError::Internal(format!(
"spawn generation allocation failed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)),
};
let reply = Err(error);
if let Some(authority) = identity_reconcile_authority.as_ref() {
let disposition = identity_member_actuation_disposition(&reply);
self.record_identity_reconcile_disposition(
&agent_identity,
authority,
disposition,
)
.await;
}
let reply_delivered = reply_tx.send(reply).is_ok();
#[cfg(test)]
if identity_reconcile_authority.is_some() && !reply_delivered {
IDENTITY_RECONCILE_REPLY_DELIVERY_FAILURES
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
continue;
}
};
Box::pin(self.finalize_spawn_from_pending(
&profile_name,
&agent_identity,
generation,
fence,
runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
labels,
provision,
spawn_receipt.operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
None,
))
.await
.map(|outcome| outcome.receipt)
}
}
Err(error) => {
if let Some(remote) = remote.as_ref() {
// §4.4 failure matrix: witness the typed
// classification, reset the MaterializePending rung,
// delete the durable operator-authority record. Any
// host-side row is a documented orphan seed reclaimed
// by the HostStatus sweep at stale fence.
let kind = materialization_failure_kind(&error);
Err(self
.fail_remote_spawn_exec(
&agent_identity,
&remote.pending_carrier,
true,
error,
kind,
"spawn_provisioned_batch_remote_failed",
)
.await)
} else {
Err(error)
}
}
};
let mut may_reply = true;
let reply = match (reply, respawn_origin.as_ref()) {
(Err(reply_error), Some(respawn_origin)) => match self
.durably_abandon_respawn_topology_if_terminal_exact(
&agent_identity,
respawn_origin,
)
.await
{
Ok(()) => Err(reply_error),
Err(abandon_error) => {
self.durable_uncertainty_fail_stop = true;
self.respawn_topology_reply_withheld = true;
may_reply = false;
Err(MobError::Internal(format!(
"{reply_error}; durable respawn-topology abandonment failed and the actor is fail-stopping for cold recovery: {abandon_error}"
)))
}
},
(reply, _) => reply,
};
// Per-spawn outcome observability: the reply waiter may be
// detached (identity reconcile deliberately drops it), so the
// terminal provisioning outcome is logged here unconditionally.
match &reply {
Ok(_) => tracing::info!(
spawn_ticket,
agent_identity = %agent_identity,
elapsed_ms = enqueued_at.elapsed().as_millis() as u64,
"spawn built"
),
Err(error) => tracing::info!(
spawn_ticket,
agent_identity = %agent_identity,
elapsed_ms = enqueued_at.elapsed().as_millis() as u64,
error = %error,
"spawn failed"
),
}
if let Some(authority) = identity_reconcile_authority.as_ref() {
let disposition = identity_member_actuation_disposition(&reply);
self.record_identity_reconcile_disposition(&agent_identity, authority, disposition)
.await;
}
if may_reply {
let reply_delivered = reply_tx.send(reply).is_ok();
#[cfg(test)]
if identity_reconcile_authority.is_some() && !reply_delivered {
IDENTITY_RECONCILE_REPLY_DELIVERY_FAILURES
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
}
}
if let Err(error) = self.ensure_pending_spawn_alignment("spawn batch completion") {
tracing::error!(
error = %error,
"pending spawn alignment check failed after spawn completion batch"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated after spawn batch")
.await
{
tracing::error!(
error = %cleanup_error,
"pending spawn cleanup failed after spawn batch alignment violation"
);
}
}
}
async fn spawn_from_policy_inline(
&mut self,
agent_identity: &AgentIdentity,
spawn_spec: super::spawn_policy::SpawnSpec,
work_ref: &WorkRef,
origin: WorkOrigin,
) -> Result<super::handle::MemberSpawnReceipt, MobError> {
self.ensure_pending_spawn_alignment("spawn_from_policy_inline preflight")?;
let requested_identity = AgentIdentity::from(agent_identity.as_str());
let mut member_spec =
super::handle::SpawnMemberSpec::new(spawn_spec.profile, requested_identity.clone());
member_spec.runtime_mode = spawn_spec.runtime_mode;
// Policy auto-spawn material is the MobMachine-recorded
// SpawnPolicyResolutionRecorded effect. Build-boundary customizers are
// intentionally skipped here so shell code cannot mutate the
// post-authority profile/runtime/capability material before
// provisioning.
let super::handle::SpawnMemberSpec {
role_name: profile_name,
identity: _,
initial_message,
runtime_mode,
backend,
binding,
context,
labels,
launch_mode: _,
tool_access_policy,
budget_limits,
auto_wire_parent: _,
additional_instructions,
shell_env,
inherited_tool_filter,
override_profile,
model_override,
objective_id: _,
auth_binding,
external_tools: per_spawn_external_tools,
system_prompt_override,
continuity_intent,
placement: _,
} = member_spec;
if agent_identity.is_system_reserved() {
return Err(MobError::WiringError(format!(
"meerkat id '{agent_identity}' uses reserved system identifier namespace"
)));
}
self.probe_member_admission(agent_identity)?;
{
let roster = self.roster.read().await;
if roster.get(agent_identity).is_some() {
return Err(MobError::MemberAlreadyExists(agent_identity.clone()));
}
}
let mut profile = if let Some(p) = override_profile.clone() {
p
} else {
self.definition
.resolve_profile(&profile_name, self.realm_profile_store.as_ref())
.await?
};
if let Some(model) = model_override.as_ref() {
profile.model.clone_from(model);
}
// ADJ-6: explicit fact captured before the inherited-open mutation.
let explicit_workgraph = profile.tools.workgraph;
if inherited_tool_filter.is_some() && override_profile.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let observations = observe_spawn_exec_facts(
&profile,
explicit_workgraph,
per_spawn_external_tools.is_some(),
shell_env.as_ref(),
inherited_tool_filter.is_some(),
None,
);
let authorized_profile_material = self.authorize_spawn_profile_material(
agent_identity,
&profile_name,
&profile,
"policy_spawn_profile_authority",
)?;
self.preview_policy_spawn_submit_work_admission(
&requested_identity,
&authorized_profile_material,
work_ref,
origin,
)?;
let runtime_mode = runtime_mode.unwrap_or(profile.runtime_mode);
let selected_binding = resolve_binding(
binding,
backend,
profile.backend,
self.definition.backend.default,
agent_identity,
)?;
let runtime_mode = normalize_runtime_mode_for_binding(runtime_mode, &selected_binding);
let external_tools =
self.external_tools_for_profile(&profile, per_spawn_external_tools.clone())?;
let labels = labels.unwrap_or_default();
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: Some(labels.clone()),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
})
.await?;
config.keep_alive = runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
if let Some(ref cref) = auth_binding {
config.auth_binding = Some(cref.clone());
}
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
let mut provision_request = ProvisionMemberRequest {
create_session: req,
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
binding: selected_binding,
peer_name,
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: None,
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
};
let admitted_bridge_session_id =
admit_bridge_session_for_spawn(&mut provision_request.create_session);
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let generated_self_owned_operation_owner =
self.stage_orchestrator_spawn(agent_identity, &admitted_bridge_session_id)?;
Self::apply_generated_self_owned_operation_owner(
&mut provision_request,
&generated_self_owned_operation_owner,
)?;
// External provisioning installs supervisor-bridge recipient trust
// ahead of bind terminality inside the provisioner. The inline policy
// spawn runs on the actor task, so the obligation window is recorded
// here (before any pending slot exists), resolved inside the spawn
// future once the provision confirms terminality, and rolled back
// below if the provision failed.
let inline_pending_trust_peer_id = match &provision_request.binding {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
crate::RuntimeBinding::Session | crate::RuntimeBinding::HostMaterialized { .. } => None,
};
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline external provision",
)?;
}
let (pending_reply_tx, _pending_reply_rx) = oneshot::channel();
let pending = PendingSpawn {
profile_name: profile_name.clone(),
agent_identity: agent_identity.clone(),
admitted_bridge_session_id,
prompt: prompt.clone(),
initial_turn_prompt: initial_turn_prompt.clone(),
suppress_autonomous_initial_prompt: false,
identity_member_permit: None,
runtime_mode,
labels: labels.clone(),
owner_bridge_session_id: None,
auto_wire_parent: false,
restore_wiring: None,
respawn_origin: None,
effective_profile_override: override_profile.clone(),
effective_model_override: model_override.clone(),
objective_id: None,
per_spawn_external_tools: per_spawn_external_tools.clone(),
authorized_profile_material: authorized_profile_material.clone(),
continuity_intent: continuity_intent.clone(),
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default())),
pending_recipient_trust_peer_id: None,
observations: observations.clone(),
remote: None,
enqueued_at: Instant::now(),
reply_tx: pending_reply_tx,
};
let pending_task = tokio::spawn(async {
std::future::pending::<()>().await;
});
let spawn_started = generated_self_owned_operation_owner.start(&pending)?;
if let Err(insert_error) = self
.insert_pending_spawn(spawn_ticket, pending, pending_task, spawn_started)
.await
{
let mut errors = vec![insert_error.to_string()];
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref()
&& let Err(error) = self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline insertion failed",
)
{
errors.push(error.to_string());
}
if let Err(error) = self
.fail_all_pending_spawns("inline policy pending spawn insertion failed")
.await
{
errors.push(error.to_string());
}
return Err(MobError::Internal(format!(
"inline policy pending spawn insertion failed: {}",
errors.join("; ")
)));
}
if let Err(error) =
self.ensure_pending_spawn_alignment("spawn_from_policy_inline staged pending")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated while staging inline policy spawn"
);
// The provision never started: close the recorded obligation
// window before the staging cleanup can short-circuit.
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline staging failed",
)?;
}
self.fail_all_pending_spawns(
"pending spawn alignment violated while staging inline policy spawn",
)
.await?;
return Err(error);
}
let spawn_result = Box::pin(async {
let spawn_receipt = self.provisioner.provision_member(provision_request).await?;
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
// Provision success means bind/authorize terminality was
// confirmed for the external peer: close the obligation.
self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline provision confirmed",
)?;
}
if runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
agent_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) = self
.provisioner
.retire_member(&spawn_receipt.member_ref)
.await
{
return Err(MobError::Internal(format!(
"autonomous capability check failed for '{agent_identity}': {capability_error}; cleanup retire failed: {retire_error}"
)));
}
return Err(capability_error);
}
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
spawn_receipt.operation_id.clone(),
spawn_receipt.session_origin,
spawn_receipt.rollback_authority.clone(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Err(retire_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"policy spawn completed while mob state changed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)));
}
return Err(error);
}
let fence = match self.issue_fence_token() {
Ok(fence) => fence,
Err(error) => {
if let Err(retire_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"policy spawn fence allocation failed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)));
}
return Err(error);
}
};
// Machine-owned generation mint (ADJ-24). The policy lane's
// strict admission probe rejects retired-identity reuse, so this
// is INITIAL in practice — the mint still has one owner.
let generation = match self.mint_spawn_generation(agent_identity) {
Ok(generation) => generation,
Err(error) => {
if let Err(retire_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"policy spawn generation mint failed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)));
}
return Err(error);
}
};
Box::pin(self.finalize_spawn_from_pending(
&profile_name,
agent_identity,
generation,
fence,
runtime_mode,
prompt,
initial_turn_prompt,
false,
None,
labels,
provision,
spawn_receipt.operation_id,
None,
false,
None,
override_profile, // policy spawns usually use definition profiles; customizers may supply an override
model_override,
None,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
None,
))
.await
.map(|outcome| outcome.receipt)
})
.await;
let (_pending, task_handle) =
self.complete_pending_spawn_slot(spawn_ticket, "policy inline spawn completion");
if let Some(handle) = task_handle {
handle.abort();
}
if let Err(error) = &spawn_result
&& let Some(peer_id) = inline_pending_trust_peer_id.as_deref()
{
if error.external_member_cleanup_is_uncertain() {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
peer_id,
error = %error,
"inline external provision trust/cleanup is uncertain; retaining pending obligation and fail-stopping actor"
);
} else {
// Confirmed provision failures already restored trust; errors
// after confirmed provision resolved this set entry above, so
// this rollback is idempotent in either safe case.
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline provision failed with cleanup confirmed",
)?;
}
}
if let Err(error) =
self.ensure_pending_spawn_alignment("spawn_from_policy_inline completion")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated after inline policy spawn completion"
);
self.fail_all_pending_spawns(
"pending spawn alignment violated after inline policy spawn completion",
)
.await?;
return Err(error);
}
spawn_result
}
/// Reset the `spawn_exec` phase opened by `BeginSpawnExec` after a
/// pre-`CommitSpawnMembership` failure, so the identity stays respawnable.
/// Returns the original spawn error, folding any `AbortSpawnExec` failure
/// into an `Internal` error — a wedged spawn-exec phase is never silently
/// dropped.
fn fold_spawn_exec_abort(
&mut self,
dsl_identity: &mob_dsl::AgentIdentity,
agent_identity: &AgentIdentity,
original: MobError,
context: &str,
) -> MobError {
match self.apply_dsl_input(
mob_dsl::MobMachineInput::AbortSpawnExec {
agent_identity: dsl_identity.clone(),
},
context,
) {
Ok(()) => original,
Err(abort_error) => MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; spawn-exec phase reset (AbortSpawnExec) failed: {abort_error}"
)),
}
}
/// Machine-driven spawn ladder driver.
///
/// The work is split into two `Box::pin`ned phases so the pre-commit
/// (`finalize_spawn_admit`) future is dropped before the post-commit
/// activation work — including the deep comms drain — is polled. Combined
/// with the boxed parameter carrier this keeps the spawn off tokio's 2MiB
/// worker stack during fleet restore (the old single ~38KiB future could
/// overflow it while nested under `reconcile_resume` and the actor task).
#[allow(clippy::too_many_arguments)]
async fn append_member_spawned_with_identity_fence(
&self,
original_permit: Option<&crate::identity::IdentityActuationPermit>,
mut spawned: crate::event::MemberSpawnedEvent,
) -> Result<(), MobError> {
use crate::store::IdentityMemberEventCommitOutcome;
let Some(original_permit) = original_permit else {
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberSpawned(spawned),
})
.await
.map(|_| ())
.map_err(MobError::from)?;
return Ok(());
};
let Some(store) = self.identity_member.as_ref() else {
return Err(
crate::store::MobStoreError::IdentityMemberAtomicPersistenceUnavailable.into(),
);
};
let Some(write_lease) = self
.renew_identity_actuation_lease(&original_permit.identity)
.await?
else {
return Err(MobError::Internal(
"identity member actuation lost its lease before finalization; reconciliation will re-observe"
.to_string(),
));
};
let permit = Self::refresh_identity_permit_lease(original_permit.clone(), &write_lease);
spawned = spawned
.with_identity_intent_authority_digest(Some(permit.intent_authority_digest.clone()));
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberSpawned(spawned),
};
match store
.commit_identity_member_spawned(&permit, &event)
.await?
{
IdentityMemberEventCommitOutcome::Applied { .. } => {
#[cfg(any(test, feature = "test-support"))]
super::trigger_identity_recovery_fail_stop(
super::IdentityRecoveryFailStopPoint::MemberEventApplied,
);
Ok(())
}
IdentityMemberEventCommitOutcome::AlreadyExact { .. } => Ok(()),
IdentityMemberEventCommitOutcome::Conflict { detail, .. } => {
Err(crate::store::MobStoreError::CasConflict(format!(
"identity member target or authority changed before finalization: {detail}"
))
.into())
}
IdentityMemberEventCommitOutcome::RepairBlocked {
evidence_digest,
detail,
} => Err(crate::store::MobStoreError::IdentityAuthorityBlocked {
evidence_digest,
detail,
}
.into()),
IdentityMemberEventCommitOutcome::Backoff { detail } => {
Err(crate::store::MobStoreError::WriteFailed(detail).into())
}
}
}
#[allow(clippy::too_many_arguments)] // one exact pending spawn carrier
async fn finalize_spawn_from_pending(
&mut self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
fence_token: crate::ids::FenceToken,
runtime_mode: crate::MobRuntimeMode,
prompt: ContentInput,
initial_turn_prompt: Option<ContentInput>,
suppress_autonomous_initial_prompt: bool,
identity_member_permit: Option<crate::identity::IdentityActuationPermit>,
labels: std::collections::BTreeMap<String, String>,
provision: PendingProvision,
operation_id: meerkat_core::ops::OperationId,
owner_bridge_session_id: Option<SessionId>,
auto_wire_parent: bool,
restore_wiring: Option<RestoreWiringPlan>,
effective_profile_override: Option<crate::profile::Profile>,
effective_model_override: Option<String>,
objective_id: Option<meerkat_core::interaction::ObjectiveId>,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
authorized_profile_material: AuthorizedSpawnProfileMaterial,
continuity_intent: super::handle::SpawnContinuityIntent,
observations: SpawnExecObservations,
remote: Option<Box<RemoteSpawnFinalize>>,
) -> Result<FinalizeSpawnOutcome, MobError> {
tracing::debug!(
agent_identity = %agent_identity,
profile = %profile_name,
runtime_mode = ?runtime_mode,
"MobActor::finalize_spawn_from_pending start"
);
let ctx = Box::new(SpawnFinalizeCtx {
profile_name: profile_name.clone(),
agent_identity: agent_identity.clone(),
generation,
fence_token,
runtime_mode,
prompt,
initial_turn_prompt,
suppress_autonomous_initial_prompt,
identity_member_permit,
labels,
operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
effective_model_override,
objective_id,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
observations,
remote,
});
let admitted = Box::pin(self.finalize_spawn_admit(&ctx, provision)).await?;
Box::pin(self.finalize_spawn_activate(ctx, admitted)).await
}
/// Consume a pending provision on a FAILED spawn finalization. Local
/// provisions roll back (session archived). Host-materialized provisions
/// disarm without retire traffic — the host row is a documented orphan
/// seed reclaimed by the HostStatus sweep (§4.4) — and the local
/// ops-provision record is aborted so no live operation lingers.
async fn consume_spawn_provision_failure(
&self,
provision: PendingProvision,
remote: bool,
operation_id: &meerkat_core::ops::OperationId,
reason: &str,
) {
if remote {
match provision.disarm_remote_orphan_seed() {
// Linear guard consumption only. Exact remote cleanup owns
// ordering: Release/HostStatus certification -> carrier-
// anchored owner/op abort -> carrier CAS delete.
Ok(_member_ref) => {}
Err(error) => {
tracing::error!(
error = %error,
"remote spawn provision guard was already consumed at failure"
);
}
}
} else if let Err(error) = provision.rollback().await {
tracing::error!(
error = %error,
"failed to roll back local spawn provision after finalize failure"
);
}
let _ = (operation_id, reason);
}
/// Spawn ladder, pre-commit phase: `BeginSpawnExec` → trust / overlay /
/// event append → `CommitSpawnMembership` → `provision.commit()`. On any
/// pre-membership failure the opened spawn-exec phase is reset via
/// `AbortSpawnExec` and the pending provision is archived, leaving the
/// identity respawnable.
async fn finalize_spawn_admit(
&mut self,
ctx: &SpawnFinalizeCtx,
provision: PendingProvision,
) -> Result<SpawnAdmitted, MobError> {
let profile_name = &ctx.profile_name;
let agent_identity = &ctx.agent_identity;
let generation = ctx.generation;
let fence_token = ctx.fence_token;
let runtime_mode = ctx.runtime_mode;
let authorized_profile_material = &ctx.authorized_profile_material;
let labels = &ctx.labels;
let continuity_intent = &ctx.continuity_intent;
let identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let agent_runtime_id = crate::ids::AgentRuntimeId::new(identity.clone(), generation);
let pending_member_ref = provision.member_ref()?.clone();
// DEC-R1: `HostMaterialized` members never persist an external
// binding overlay — their re-acquire path is machine
// re-materialization, not peer-only rebind material.
let overlay_record = if ctx.remote.is_some() {
None
} else {
self.external_binding_overlay_record(&identity, generation, &pending_member_ref)
};
let external_addressable = authorized_profile_material.external_addressable;
// Feed `Spawn` into the MobMachine DSL so it populates
// `live_runtime_ids` + `externally_addressable_runtime_ids` and
// downstream guards (Retire, SubmitWork, …) operate on authoritative
// membership state.
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
// Remote (host-materialized) finalize: the session binding is the
// ACK's remote session (machine truth only, DEC-P3-6) — the shell
// ref deliberately carries `session_id: None`.
let bridge_session_id = match ctx.remote.as_ref() {
Some(remote) => Some(mob_dsl::SessionId::from_domain(&remote.ack.session_id)),
None => pending_member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain),
};
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let is_replacing = replacing.is_some();
if let Some(remote) = ctx.remote.as_ref()
&& replacing.is_some()
{
// The remote commit arm guards `replacing == None`; failing
// closed here keeps the error shape typed instead of a guard
// rendering.
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"remote spawn found an existing session binding",
)
.await;
return Err(self.fail_remote_spawn_exec(
agent_identity,
&remote.pending_carrier,
true,
MobError::Internal(format!(
"remote spawn of '{agent_identity}' found an existing session binding; placed replacement flows through release + respawn"
)),
"commit_refused_replacing_present".to_string(),
"finalize_spawn_admit_remote_replacing",
).await);
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending preparing DSL Spawn"
);
// Spawn ladder step 1: open the spawn-exec phase. `BeginSpawnExec`
// carries the spawn admission guards, sets the per-identity phase to
// `Opened`, and emits nothing. Any failure between here and
// `CommitSpawnMembership` fires `AbortSpawnExec` to reset the phase.
if ctx.remote.is_none() {
let begin_result = self.apply_dsl_input(
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(fence_token),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::from(runtime_mode),
bridge_session_id: bridge_session_id.clone(),
replacing: replacing.clone(),
// Local arm: placement is absent, so the multi-host
// denial guards are placement-gated vacuous; the
// observations bundle is nonetheless HONEST (DEC-P3-10 —
// captured at the spawn destructure and threaded here).
placement: None,
workgraph_required: ctx.observations.workgraph_required,
rust_bundles_present: ctx.observations.rust_bundles_present,
per_spawn_external_tools_present: ctx
.observations
.per_spawn_external_tools_present,
mob_default_external_tools_present: self
.default_external_tools_provider
.is_some(),
default_llm_client_override_present: self.default_llm_client.is_some(),
host_surface_mcp_allowlist_present: ctx
.observations
.host_surface_mcp_allowlist_present,
inherited_tool_filter_present: ctx.observations.inherited_tool_filter_present,
shell_env_present: ctx.observations.shell_env_present,
mcp_stdio_env_present: ctx.observations.mcp_stdio_env_present,
mcp_http_headers_present: ctx.observations.mcp_http_headers_present,
memory_required: ctx.observations.memory_required,
mcp_required: ctx.observations.mcp_required,
resume_session_id: ctx
.observations
.resume_session_id
.as_ref()
.map(mob_dsl::SessionId::from_domain),
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
},
"finalize_spawn_admit_begin_spawn_exec",
);
if let Err(error) = begin_result {
self.consume_spawn_provision_failure(
provision,
false,
&ctx.operation_id,
"spawn-exec open rejected",
)
.await;
return Err(error);
}
}
// Remote: the ladder was opened at enqueue (`BeginSpawnExecRemote`,
// rung = MaterializePending); the member recipient trust window
// opens BEFORE the membership commit so kickoff/wiring sends can
// route the moment the roster publishes.
let remote_recipient_trust_install = if let Some(remote) = ctx.remote.as_ref() {
if let Err(error) = self.record_pending_recipient_trust_obligation(
&remote.ack.member_peer,
"remote_spawn_member_trust",
) {
let failure = self
.fail_remote_spawn_exec(
agent_identity,
&remote.pending_carrier,
true,
error,
"member_trust_install_failed".to_string(),
"finalize_spawn_admit_remote_trust_obligation",
)
.await;
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"member recipient-trust obligation failed",
)
.await;
return Err(failure);
}
match self
.supervisor_bridge
.trust_recipient(&remote.ack.member_peer)
.await
{
Ok(install) => Some(install),
Err(error) => {
let failure = self
.fail_remote_spawn_exec_after_recipient_trust_error(
agent_identity,
&remote.pending_carrier,
error,
&remote.ack.member_peer,
)
.await;
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"member recipient-trust install failed",
)
.await;
return Err(failure);
}
}
} else {
None
};
// Spawn ladder step 2: prepare `CommitSpawnMembership`. It guards on
// `phase == Opened` and carries the verbatim membership update plus the
// `MemberSpawned` lifecycle journal emit. Prepared (not committed) here
// so the journal effect is validated before the expensive trust /
// overlay / append work; committed once those succeed.
let prepared_spawn = match self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(fence_token),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::from(runtime_mode),
bridge_session_id: bridge_session_id.clone(),
replacing,
// Ack facts are populated from exactly one source — the
// transport-validated `MaterializedMemberAck`; the local
// path has no ack value in scope (DEC-P3-8, machine-
// enforced by the local `ack_fields_absent` guard).
member_peer_endpoint: ctx
.remote
.as_ref()
.map(|remote| mob_dsl::MemberPeerEndpoint::from(&remote.ack.member_peer)),
spec_digest_echo: ctx
.remote
.as_ref()
.map(|remote| remote.ack.spec_digest_echo.clone()),
ack_engine_version: ctx
.remote
.as_ref()
.map(|remote| remote.ack.engine_version.clone()),
placed_spawn_id: ctx
.remote
.as_ref()
.map(|remote| mob_dsl::PlacedSpawnId(remote.placed_spawn_id.to_string())),
provision_operation_id: ctx.remote.as_ref().map(|_| ctx.operation_id.to_string()),
},
"finalize_spawn_admit_commit_membership",
) {
Ok(prepared) => prepared,
Err(error) => {
if let Some(remote) = ctx.remote.as_ref() {
// Guard refusal on the remote arm is proven pre-commit.
// Classify against machine-recorded facts, then release
// the exact host attempt and carrier before rolling back
// only trust installed by this admission attempt.
let kind = if remote.ack.spec_digest_echo != remote.resolved_spec_digest {
"commit_refused_digest_echo".to_string()
} else {
"host_engine_version_changed".to_string()
};
let failure = self
.fail_remote_spawn_exec_after_recipient_trust(
agent_identity,
&remote.pending_carrier,
error,
kind,
"finalize_spawn_admit_commit_membership_remote",
&remote.ack.member_peer,
remote_recipient_trust_install,
)
.await;
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"remote membership commit refused",
)
.await;
return Err(failure);
}
self.consume_spawn_provision_failure(
provision,
false,
&ctx.operation_id,
"membership commit prepare rejected",
)
.await;
return Err(self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error,
"finalize_spawn_admit_commit_membership",
));
}
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit prepared membership commit"
);
if let Err(error) = Self::require_member_lifecycle_journal_effect(
&prepared_spawn.transition,
mob_dsl::MobLifecycleJournalKind::MemberSpawned,
&identity,
&agent_runtime_id,
Some(fence_token),
generation,
bridge_session_id.clone(),
"finalize_spawn_admit_commit_membership",
) {
if let Some(remote) = ctx.remote.as_ref() {
let failure = self
.fail_remote_spawn_exec_after_recipient_trust(
agent_identity,
&remote.pending_carrier,
error,
"commit_authority_missing".to_string(),
"finalize_spawn_admit_commit_authority",
&remote.ack.member_peer,
remote_recipient_trust_install,
)
.await;
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"membership commit journal effect missing",
)
.await;
return Err(failure);
}
self.consume_spawn_provision_failure(
provision,
false,
&ctx.operation_id,
"membership commit journal effect missing",
)
.await;
return Err(self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error,
"finalize_spawn_admit_commit_membership",
));
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit validated lifecycle journal"
);
if let Some(remote) = ctx.remote.as_ref() {
let persist_result = self
.persist_placed_spawn_commit_exact(
&remote.pending_carrier,
&remote.ack,
&ctx.operation_id,
&prepared_spawn.transition,
)
.await;
if let Err(error) = persist_result {
if self.durable_uncertainty_fail_stop {
// CAS may have committed. Preserve durable ownership for
// cold exact reread; never release/disarm an uncertain
// possibly-Committed carrier.
let _member_ref = provision.commit()?;
return Err(error);
}
let failure = self
.fail_remote_spawn_exec_after_recipient_trust(
agent_identity,
&remote.pending_carrier,
error,
"commit_record_write_failed".to_string(),
"finalize_spawn_admit_commit_record",
&remote.ack.member_peer,
remote_recipient_trust_install,
)
.await;
self.consume_spawn_provision_failure(
provision,
true,
&ctx.operation_id,
"placed carrier commit proved Pending",
)
.await;
return Err(failure);
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared_spawn) {
self.durable_uncertainty_fail_stop = true;
// The carrier is already Committed: ownership is durable and
// must never be disarmed/aborted as a precommit orphan.
let _member_ref = provision.commit()?;
return Err(MobError::Internal(format!(
"placed carrier for '{agent_identity}' committed durably but membership machine publication failed; actor is fail-stopping for cold recovery: {error}"
)));
}
// Carrier and machine are now irreversible. Transfer provision
// ownership before the private-linked event append; every later
// failure is recovery/retirement, never AbortSpawnExec.
let session_origin = provision.session_origin();
let member_ref = match provision.commit() {
Ok(member_ref) => member_ref,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"remote spawn of '{agent_identity}' committed its carrier and membership, but the provision guard could not transfer ownership; trust and its pending obligation are retained and the actor is fail-stopping: {error}"
)));
}
};
let member_peer_endpoint = remote.ack.member_peer.clone();
if let Err(error) = self.resolve_pending_recipient_trust_obligation(
&member_peer_endpoint,
"remote_spawn_member_trust_confirmed",
) {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"failed to resolve member recipient-trust obligation after remote commit; fail-stopping actor with trust and obligation retained"
);
return Err(MobError::Internal(format!(
"remote spawn of '{agent_identity}' committed its carrier and membership, but the recipient-trust obligation could not be resolved; actor is fail-stopping for cold recovery: {error}"
)));
}
let mut spawned = crate::event::MemberSpawnedEvent::new(
identity.clone(),
generation,
fence_token,
agent_runtime_id.clone(),
profile_name.clone(),
)
.with_bridge_member_ref(Some(Self::sanitized_member_ref(&member_ref)))
.with_placed_spawn_id(Some(remote.placed_spawn_id.clone()))
.with_member_peer_endpoint(Some(member_peer_endpoint.clone()));
spawned.runtime_mode = runtime_mode;
spawned.labels = labels.clone();
spawned.continuity_intent = continuity_intent.clone();
spawned.effective_profile_override = ctx.effective_profile_override.clone();
spawned.effective_model_override = ctx.effective_model_override.clone();
self.append_committed_placed_event_exact(MobEventKind::MemberSpawned(spawned))
.await?;
self.restore_diagnostics
.write()
.await
.remove(agent_identity);
return Ok(SpawnAdmitted {
member_ref,
session_origin,
agent_runtime_id,
is_replacing,
member_peer_endpoint: Some(member_peer_endpoint),
transport_public_key: None,
});
}
let (member_peer_endpoint, transport_public_key) = match self
.resolve_spawn_member_peer_material(&pending_member_ref, profile_name, &identity)
.await
{
Ok(material) => material,
Err(error) => {
let error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error,
"finalize_spawn_admit_resolve_member_peer",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn peer resolution failed for '{agent_identity}': {error}; archive compensation failed: {rollback_error}"
)));
}
return Err(error);
}
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending resolving supervisor comms"
);
let supervisor_private_trust_install = if agent_identity.is_flow_member_namespace() {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending skipped supervisor private trust for run-scoped flow member"
);
None
} else if let (Some(session_id), Some(comms)) = (
pending_member_ref.bridge_session_id().cloned(),
self.provisioner_comms(&pending_member_ref).await,
) {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %session_id,
"MobActor::finalize_spawn_from_pending installing supervisor private trust"
);
match Box::pin(async {
self.install_supervisor_private_trust_for_session(&session_id, &comms, None)
.await
})
.await
{
Ok(install) => {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %session_id,
"MobActor::finalize_spawn_from_pending installed supervisor private trust"
);
Some((session_id, comms, install))
}
Err(error) => {
let error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error.into(),
"finalize_spawn_admit_trust",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn supervisor private trust failed for '{agent_identity}': {error}; archive compensation failed: {rollback_error}"
)));
}
return Err(error);
}
}
} else {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending skipped supervisor private trust"
);
None
};
if let Some(overlay_record) = overlay_record.as_ref() {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending upserting overlay"
);
if let Err(error) = self
.runtime_metadata
.upsert_external_binding_overlay(&self.definition.id, overlay_record)
.await
{
if let Some((session_id, comms, install)) =
supervisor_private_trust_install.as_ref()
{
Box::pin(
self.cleanup_supervisor_private_trust_for_session(
session_id, comms, install,
),
)
.await;
}
let error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error.into(),
"finalize_spawn_admit_overlay",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn overlay upsert failed for '{agent_identity}': {error}; archive compensation failed: {rollback_error}"
)));
}
return Err(error);
}
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending appending spawn event"
);
let mut spawned_event = crate::event::MemberSpawnedEvent::new(
identity.clone(),
generation,
fence_token,
agent_runtime_id.clone(),
profile_name.clone(),
)
.with_bridge_member_ref(Some(Self::sanitized_member_ref(&pending_member_ref)))
.with_member_peer_endpoint(member_peer_endpoint.clone());
spawned_event.runtime_mode = runtime_mode;
spawned_event.labels = labels.clone();
spawned_event.continuity_intent = continuity_intent.clone();
// Durable per-spawn declarative provenance: replay repopulates
// RosterEntry.effective_profile_override so restarts keep per-spawn
// tooling without a customizer.
spawned_event.effective_profile_override = ctx.effective_profile_override.clone();
spawned_event.effective_model_override = ctx.effective_model_override.clone();
spawned_event = spawned_event.with_placed_spawn_id(None);
if let Err(append_error) = self
.append_member_spawned_with_identity_fence(
ctx.identity_member_permit.as_ref(),
spawned_event,
)
.await
{
if overlay_record.is_some() {
let _ = self
.delete_external_binding_overlay_for_member(&identity, generation)
.await;
}
if let Some((session_id, comms, install)) = supervisor_private_trust_install.as_ref() {
Box::pin(
self.cleanup_supervisor_private_trust_for_session(session_id, comms, install),
)
.await;
}
let append_error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
append_error,
"finalize_spawn_admit_append",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn append failed for '{agent_identity}': {append_error}; archive compensation failed: {rollback_error}"
)));
}
return Err(append_error);
}
// Local spawn ladder step 3. The remote arm returned above after
// canonical carrier CAS -> machine commit -> provision commit ->
// private event append.
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committing membership"
);
if let Err(commit_error) = self.commit_prepared_dsl_transition(prepared_spawn) {
let commit_error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
commit_error,
"finalize_spawn_admit_commit_membership",
);
// MemberSpawned is already durable at this point. Close that
// journal incarnation before releasing its provision so replay
// cannot resurrect a member whose session was returned to its
// pre-spawn lifecycle.
let mut terminal_retry_delay = std::time::Duration::from_millis(25);
loop {
match self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberRetired {
agent_identity: identity.clone(),
generation,
role: profile_name.clone(),
},
})
.await
{
Ok(_) => break,
Err(retire_error) => {
// The durable MemberSpawned record already exists, so
// releasing this provision would make replay
// resurrect a member with no backing continuity. Keep
// ownership in this actor command and fail-stop on the
// event-store boundary until the matching terminal is
// durable. The bounded backoff avoids a hot loop while
// preserving the only safe ordering.
tracing::error!(
agent_identity = %agent_identity,
%retire_error,
"spawn membership commit failed; retaining provision until durable terminal compensation succeeds"
);
tokio::time::sleep(terminal_retry_delay).await;
terminal_retry_delay = terminal_retry_delay
.saturating_mul(2)
.min(std::time::Duration::from_secs(1));
}
}
}
self.retired_event_index
.write()
.await
.insert(Self::retire_event_key(&identity, generation));
if overlay_record.is_some() {
let _ = self
.delete_external_binding_overlay_for_member(&identity, generation)
.await;
}
if let Some((session_id, comms, install)) = supervisor_private_trust_install.as_ref() {
Box::pin(
self.cleanup_supervisor_private_trust_for_session(session_id, comms, install),
)
.await;
}
return match provision.rollback().await {
Ok(()) => Err(commit_error),
Err(rollback_error) => Err(MobError::Internal(format!(
"spawn membership commit failed for '{agent_identity}': {commit_error}; provision rollback failed: {rollback_error}"
))),
};
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committed membership"
);
// Commit the provision: the member is now owned by the roster.
// From this point, rollback_failed_spawn handles cleanup via the
// disposal pipeline.
let session_origin = provision.session_origin();
let member_ref = provision.commit()?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committed provision"
);
self.restore_diagnostics
.write()
.await
.remove(agent_identity);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit cleared diagnostics"
);
Ok(SpawnAdmitted {
member_ref,
session_origin,
agent_runtime_id,
is_replacing,
member_peer_endpoint,
transport_public_key,
})
}
/// Spawn ladder, post-commit phase: roster projection, peer registration,
/// role wiring, runtime start / turn-driven comms drain, respawn topology
/// restore, and finally `CommitSpawnActivation`. Failures here unwind via
/// `rollback_failed_spawn` (which destroys the member and resets the
/// spawn-exec phase), not `AbortSpawnExec`.
async fn finalize_spawn_activate(
&mut self,
ctx: Box<SpawnFinalizeCtx>,
admitted: SpawnAdmitted,
) -> Result<FinalizeSpawnOutcome, MobError> {
// Sequential phase helpers: at opt-level=0 the merged body's poll
// frame reserved a slot for every local in every branch (~700 KiB on
// the actor task, beneath the whole spawn ladder). Splitting keeps
// only the active phase's frame on the 2 MiB production worker
// stack. See `SpawnActivateState`.
let mut state = SpawnActivateState::admit(ctx, admitted);
self.activate_spawn_membership(&mut state).await?;
self.activate_spawn_wiring(&mut state).await?;
self.activate_spawn_kickoff(&mut state).await?;
self.activate_spawn_commit(state).await
}
/// Activation phase 1: kickoff-intent validation, trusted-peer
/// publication, roster/overlay insertion, and the machine-owned
/// membership facts.
async fn activate_spawn_membership(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
let profile_name = &state.profile_name;
let agent_identity = &state.agent_identity;
let generation = state.generation;
let fence_token = state.fence_token;
let runtime_mode = state.runtime_mode;
let suppress_autonomous_initial_prompt = state.suppress_autonomous_initial_prompt;
let placed_kickoff_intent = state
.remote
.as_ref()
.and_then(|remote| remote.pending_carrier.kickoff_intent.clone());
if state.remote.is_some()
&& runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& !suppress_autonomous_initial_prompt
{
let Some(intent) = placed_kickoff_intent.as_ref() else {
return Err(MobError::Internal(format!(
"placed autonomous member '{agent_identity}' has no durable kickoff intent"
)));
};
if intent.prompt != state.prompt || state.objective_id != Some(intent.objective_id) {
return Err(MobError::Internal(format!(
"placed autonomous member '{agent_identity}' kickoff intent drifted after durable carrier commit"
)));
}
}
// Populate the Roster projection AFTER DSL `Spawn` authoritatively
// applies. The pre-DSL roster insert was deleted in Wave-A commit
// `e77ce8797` (running before `MobMachineInput::Spawn` committed, so
// rejected admissions could leave shell state stale); the
// correctly-ordered replacement was never wired until now. Without
// this insert, `start_autonomous_member` below reads an empty roster
// and fails with `"autonomous member '{id}' missing roster entry for
// startup readiness"` (#30 D-spawn-readiness-lookup).
//
// `peer_id` is the canonical comms routing UUID. The MobMachine also
// records the full descriptor so generated member trust authority is
// bound to the exact name/address/signing key that will be installed.
// Session-backed members publish operation readiness after the
// membership commit. Peer-only members never used that readiness
// path: their backend descriptor intentionally keeps the backend
// identity name rather than the rendered local comms name.
if state.member_ref.bridge_session_id().is_some()
&& let Some(endpoint) = state.member_peer_endpoint.as_ref()
{
self.provisioner
.publish_trusted_peer_spec_for_operation(
&state.member_ref,
&state.operation_id,
endpoint.clone(),
)
.await?;
}
let peer_id = state
.member_peer_endpoint
.as_ref()
.map(|descriptor| descriptor.peer_id);
// Host-materialized members: `CommitSpawnMembershipRemote` already
// folded the member peer endpoint FROM THE ACK (single owner); a
// second RegisterMemberPeer here would overwrite the machine fact
// with a shell-derived name.
if state.remote.is_none()
&& let Some(descriptor) = state.member_peer_endpoint.as_ref()
{
self.apply_dsl_input(
mob_dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: state.dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&state.agent_runtime_id),
generation: mob_dsl::Generation::from_domain(generation),
fence_token: mob_dsl::FenceToken::from_domain(fence_token),
peer_endpoint: mob_dsl::MemberPeerEndpoint::from(descriptor),
},
"finalize_spawn_register_member_peer",
)?;
}
{
let mut roster = self.roster.write().await;
roster.add_member(crate::roster::RosterAddEntry {
agent_identity: state.identity.clone(),
generation,
fence_token,
agent_runtime_id: state.agent_runtime_id.clone(),
role: profile_name.clone(),
runtime_mode,
member_ref: Self::sanitized_member_ref(&state.member_ref),
peer_id,
transport_public_key: state.transport_public_key.take(),
labels: state.labels.clone(),
effective_profile_override: state.effective_profile_override.clone(),
effective_model_override: state.effective_model_override.clone(),
});
}
{
// Same commit as the roster insert: retain the per-spawn overlay
// so machine-authorized revival recomposes it. `None` clears any
// prior incarnation's overlay (respawn replacement semantics).
let mut per_spawn = self.per_spawn_external_tools.write().await;
if let Some(dispatcher) = state.per_spawn_external_tools.take() {
per_spawn.insert(state.identity.clone(), dispatcher);
} else {
per_spawn.remove(&state.identity);
}
}
// Row #314: record the machine-owned external-member rebind capability
// from the spawn's member_ref bootstrap proof so the external-member
// projection reads it from machine state instead of re-deriving from the
// roster bootstrap_token.
self.apply_dsl_input(
mob_dsl::MobMachineInput::SetExternalMemberRebindCapability {
agent_identity: state.dsl_identity.clone(),
capability: external_member_rebind_capability_from_member_ref(&state.member_ref),
},
"finalize_spawn_set_external_member_rebind_capability",
)?;
if runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& !suppress_autonomous_initial_prompt
{
if state.is_replacing {
self.clear_kickoff_state(agent_identity).await;
}
let kickoff_objective_id = placed_kickoff_intent
.as_ref()
.map(|intent| intent.objective_id)
.or(state.objective_id)
.unwrap_or_default();
let _ = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffMarkPending {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
objective_id: kickoff_objective_id.to_string(),
},
"finalize_spawn_kickoff_mark_pending",
)
.await?;
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending roster updated"
);
Ok(())
}
/// Activation phase 2: role/auto-wire fan-out and respawn topology
/// restore (with spawn rollback on wiring-contract violations).
async fn activate_spawn_wiring(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
let profile_name = &state.profile_name;
let agent_identity = &state.agent_identity;
let generation = state.generation;
// Wave-A damage restored: `spawn_wiring_targets` computes the
// auto-wire + role-wiring fan-out targets for this spawn, and the
// imperative wire-call loop deleted alongside the pre-DSL
// `roster.add_member` insert (commit `e77ce8797`) needs to run here
// so role-wired profiles actually get wired at spawn time. The
// compensating rollback in `rollback_failed_spawn` expects both
// `wired_spawn_targets` and `planned_wiring_targets` populated so
// partial-wire failures can unwind.
state.planned_wiring_targets =
if state.identity_fenced_member || agent_identity.is_flow_member_namespace() {
Vec::new()
} else {
Box::pin(self.spawn_wiring_targets(profile_name, agent_identity)).await
};
if state.auto_wire_parent
&& let Some(parent_target) = self
.resolve_auto_wire_parent_target(
state.owner_bridge_session_id.as_ref(),
agent_identity,
)
.await
&& !state.planned_wiring_targets.contains(&parent_target)
{
state.planned_wiring_targets.push(parent_target);
}
// Respawn replacement: edges captured in the MobMachine-owned restore
// plan are owned by the topology-restore loop below, whose failures
// are collected per peer and classified by the generated
// `ResolveRespawnTopologyRestore` authority into a typed
// `TopologyRestoreFailed` result that preserves the replacement
// receipt. Running those same edges through the spawn-contract
// role-wiring fan-out instead made any peer-side wiring failure
// (e.g. a peer whose live session was fail-closed discarded after a
// terminal turn failure) roll back and destroy the replacement
// member ("spawn failed after retire ...: wire requires comms
// runtime for '<peer>'"). Only role-derived edges that the machine
// does NOT already own (i.e. genuinely new wiring) keep the
// fail-fast spawn contract.
if let Some(plan) = state.restore_wiring.as_ref() {
state
.planned_wiring_targets
.retain(|target| !plan.local_peers.contains(target));
}
for target in &state.planned_wiring_targets {
let target_identity = crate::ids::AgentIdentity::from(target.as_str());
let local_meerkat = agent_identity.clone();
match Box::pin(self.handle_wire(
local_meerkat,
super::handle::PeerTarget::Local(target_identity),
))
.await
{
Ok(()) => state.wired_spawn_targets.push(target.clone()),
Err(wire_error) => {
let surfaced_wire_error = match wire_error {
MobError::WiringError(_) => wire_error,
other => MobError::WiringError(other.to_string()),
};
// Rollback the spawn: the member is in the DSL + roster
// but the role-wiring contract was violated. Surface the
// failure to the caller so they can decide how to
// compensate (tests assert this path at e.g.
// `test_role_wiring_failure_is_returned_to_spawn_caller`).
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
FailedSpawnRollback {
generation,
profile_name,
member_ref: &state.member_ref,
operation_id: &state.operation_id,
session_origin: state.session_origin,
successful_wiring_targets: &state.wired_spawn_targets,
planned_wiring_targets: &state.planned_wiring_targets,
},
))
.await
{
return Err(MobError::Internal(format!(
"spawn wire fan-out failed for '{agent_identity}': {surfaced_wire_error}; rollback failed: {rollback_error}"
)));
}
return Err(surfaced_wire_error);
}
}
}
// Respawn restore is repair-only and precedes every kickoff/initial
// work dispatch. The saved plan identifies candidates, while the
// current MobMachine graph remains the authority: an edge removed
// while a placed replacement was materializing must not be recreated
// from a stale snapshot.
if let Some(plan) = state.restore_wiring.take() {
for peer_identity in plan.local_peers {
if peer_identity == *agent_identity {
continue;
}
let desired_now = self.dsl_authority.state().wiring_edges.iter().any(|edge| {
(edge.a.0.as_str() == agent_identity.as_str()
&& edge.b.0.as_str() == peer_identity.as_str())
|| (edge.b.0.as_str() == agent_identity.as_str()
&& edge.a.0.as_str() == peer_identity.as_str())
});
if !desired_now {
tracing::debug!(
agent_identity = %agent_identity,
peer = %peer_identity,
"respawn: skipped stale local restore candidate absent from machine graph"
);
continue;
}
let peer_agent_identity = crate::ids::AgentIdentity::from(peer_identity.as_str());
if let Err(error) = Box::pin(self.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::Local(peer_agent_identity),
))
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_identity,
%error,
"respawn: failed to restore machine-owned local peer edge"
);
state
.failed_restore_peer_ids
.push(RespawnTopologyPeerId::from(peer_identity.as_str()));
}
}
for peer_spec in plan.external_peers {
let desired_edge = Self::external_peer_edge(agent_identity, &peer_spec);
let desired_key = Self::external_peer_key(agent_identity, &peer_spec.name);
// NB: `state` is the phase carrier here; read the machine
// authority under its own name to avoid shadowing it.
let machine_state = self.dsl_authority.state();
let desired_now = machine_state.external_peer_edges.contains(&desired_edge)
&& machine_state.external_peer_edges_by_key.get(&desired_key)
== Some(&desired_edge);
if !desired_now {
tracing::debug!(
agent_identity = %agent_identity,
peer = %peer_spec.name,
"respawn: skipped stale external restore candidate absent from machine graph"
);
continue;
}
let peer_id = RespawnTopologyPeerId::from(peer_spec.peer_id.as_str());
if let Err(error) = Box::pin(self.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::External(peer_spec.clone()),
))
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_spec.name,
%error,
"respawn: failed to restore machine-owned external peer edge"
);
state.failed_restore_peer_ids.push(peer_id);
}
}
}
Ok(())
}
/// Activation phase 3: placed/local kickoff dispatch, autonomous member
/// startup, and the turn-driven comms-drain + initial-turn seams.
async fn activate_spawn_kickoff(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
// Placed kickoff custody may start only after the spawn ladder has
// settled. Commit activation after wiring/restore but before opening
// the record-before-send obligation. Local members retain the older
// ordering because their runtime binding/start is part of activation.
#[cfg(feature = "runtime-adapter")]
{
state.spawn_activation_committed = state.runtime_mode
== crate::MobRuntimeMode::AutonomousHost
&& super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&state.agent_identity,
);
}
#[cfg(feature = "runtime-adapter")]
if state.spawn_activation_committed {
self.apply_dsl_input(
mob_dsl::MobMachineInput::CommitSpawnActivation {
agent_identity: state.dsl_identity.clone(),
},
"finalize_placed_spawn_activate_before_kickoff",
)?;
}
// Per-branch helpers keep each kickoff lane's locals out of this
// phase's opt-level=0 poll frame (see `SpawnActivateState`).
#[cfg(feature = "runtime-adapter")]
if state.runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&state.agent_identity,
)
&& !state.suppress_autonomous_initial_prompt
{
self.kickoff_placed_member(state).await?;
} else if state.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
self.kickoff_local_autonomous(state).await?;
}
if state.runtime_mode == crate::MobRuntimeMode::TurnDriven
&& !state.agent_identity.is_flow_member_namespace()
{
self.kickoff_turn_driven(state).await?;
}
Ok(())
}
/// Kickoff lane: PLACED autonomous member (loop runs on the member host).
#[cfg(feature = "runtime-adapter")]
async fn kickoff_placed_member(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
let agent_identity = &state.agent_identity;
// PLACED autonomous member: the loop runs on the MEMBER host
// (ADJ-23 residency) — the local kickoff ladder and injector
// capability are local-lane mechanisms and never apply. The
// kickoff prompt rides the placed delivery lane (the
// host_materialize contract: "kickoff arrives later via
// DeliverMemberInput"), detached off the actor loop (ADJ-P4-12).
let entry = self
.roster
.read()
.await
.get(agent_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"placed autonomous kickoff has no roster incarnation for '{agent_identity}'"
))
})?;
let expected_member = self.placed_member_incarnation(&entry)?;
let kickoff_intent = state
.remote
.as_ref()
.and_then(|remote| remote.pending_carrier.kickoff_intent.clone())
.ok_or_else(|| {
MobError::Internal(format!(
"placed autonomous kickoff has no durable intent for '{agent_identity}'"
))
})?;
let obligation_event = crate::event::PlacedKickoffObligationEvent {
agent_identity: agent_identity.clone(),
host_id: expected_member.host_id.clone(),
host_binding_generation: expected_member.binding_generation,
member_session_id: expected_member.member_session_id.clone(),
generation: crate::ids::Generation::new(expected_member.generation),
fence_token: crate::ids::FenceToken::new(expected_member.fence_token),
input_id: kickoff_intent.input_id.clone(),
objective_id: kickoff_intent.objective_id,
};
self.start_placed_kickoff_obligation_in_actor(obligation_event)
.await?;
self.ensure_member_event_pump(agent_identity).await?;
Ok(())
}
/// Kickoff lane: locally hosted autonomous member (runtime binding,
/// host-loop start, and spawn rollback on failure).
#[cfg(feature = "runtime-adapter")]
async fn kickoff_local_autonomous(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
let profile_name = &state.profile_name;
let agent_identity = &state.agent_identity;
let generation = state.generation;
if !state.suppress_autonomous_initial_prompt {
let _ = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffMarkStarting {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"finalize_spawn_kickoff_mark_starting",
)
.await?;
}
// Spawn emits RequestRuntimeBinding. Drain it before startup can
// publish RuntimeBound, otherwise the session may emit a fallback
// runtime id that MobMachine correctly rejects as not live.
if let Err(binding_error) = Box::pin(self.flush_routed_effects()).await {
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
FailedSpawnRollback {
generation,
profile_name,
member_ref: &state.member_ref,
operation_id: &state.operation_id,
session_origin: state.session_origin,
successful_wiring_targets: &state.wired_spawn_targets,
planned_wiring_targets: &state.planned_wiring_targets,
},
))
.await
{
return Err(MobError::Internal(format!(
"spawn runtime binding failed for '{agent_identity}': {binding_error}; rollback failed: {rollback_error}"
)));
}
return Err(binding_error);
}
let kickoff_prompt =
(!state.suppress_autonomous_initial_prompt).then_some(state.prompt.clone());
if let Err(start_error) = Box::pin(self.start_autonomous_member(
agent_identity,
&state.member_ref,
kickoff_prompt,
))
.await
{
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
FailedSpawnRollback {
generation,
profile_name,
member_ref: &state.member_ref,
operation_id: &state.operation_id,
session_origin: state.session_origin,
successful_wiring_targets: &state.wired_spawn_targets,
planned_wiring_targets: &state.planned_wiring_targets,
},
))
.await
{
return Err(MobError::Internal(format!(
"spawn host-loop start failed for '{agent_identity}': {start_error}; rollback failed: {rollback_error}"
)));
}
return Err(start_error);
}
Ok(())
}
/// Kickoff lane: turn-driven member (persistent comms drain + optional
/// initial turn, with spawn rollback on failure).
async fn kickoff_turn_driven(
&mut self,
state: &mut SpawnActivateState,
) -> Result<(), MobError> {
let profile_name = &state.profile_name;
let agent_identity = &state.agent_identity;
let generation = state.generation;
let fence_token = state.fence_token;
// Turn-driven mob members still need a persistent comms drain:
// async peer requests/responses arrive between user turns (think
// realtime audio operators calling `send_request` and waiting for
// `send_response`). Without a drain running, the
// `peer_response_terminal` notice never reaches the session's
// runtime queue, so the wake path is dead. The drain-spawn seam
// is independent of `config.keep_alive` (which the mock session
// services overload as "block on start_turn"), so we drive it
// explicitly here for turn-driven members that have a bridge
// session and a comms runtime.
#[cfg(all(not(target_arch = "wasm32"), feature = "runtime-adapter"))]
if !super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity)
&& let (Some(adapter), Some(bridge_session_id)) = (
self.runtime_adapter.clone(),
state.member_ref.bridge_session_id(),
)
{
let comms_runtime = self.provisioner.comms_runtime(&state.member_ref).await;
if std::env::var_os("RKAT_TRACE_COMMS_DRAIN_BIND").is_some()
&& let Some(runtime) = comms_runtime.as_ref()
{
tracing::info!(
agent_identity = %agent_identity,
session_id = %bridge_session_id,
comms_ptr = ?Arc::as_ptr(runtime),
"mob turn-driven spawn binding comms drain"
);
}
// W2-G: route through the mob-owned spawn seam so peer-ingress
// ownership transitions to `MobOwned { comms_runtime_id, mob_id }`.
if let Some(comms_runtime) = comms_runtime {
let mob_id =
meerkat_runtime::meerkat_machine::dsl::MobId::from(self.definition.id.as_ref());
Box::pin(adapter.maybe_spawn_mob_comms_drain(
bridge_session_id,
comms_runtime,
mob_id,
))
.await
.map_err(|err| {
MobError::Internal(format!(
"mob comms drain spawn failed for session {bridge_session_id}: {err}"
))
})?;
}
}
if let Some(initial_turn_prompt) = state.initial_turn_prompt.take() {
if let Err(start_error) = Box::pin(self.dispatch_turn_driven_spawn_initial_turn(
agent_identity,
&state.agent_runtime_id,
fence_token,
&state.operation_id,
initial_turn_prompt,
state.objective_id,
))
.await
{
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
FailedSpawnRollback {
generation,
profile_name,
member_ref: &state.member_ref,
operation_id: &state.operation_id,
session_origin: state.session_origin,
successful_wiring_targets: &state.wired_spawn_targets,
planned_wiring_targets: &state.planned_wiring_targets,
},
))
.await
{
return Err(MobError::Internal(format!(
"turn-driven spawn initial turn failed for '{agent_identity}': {start_error}; rollback failed: {rollback_error}"
)));
}
return Err(start_error);
}
}
Ok(())
}
/// Activation phase 4: `CommitSpawnActivation`, event-pump
/// re-materialization, and the finalize receipt.
async fn activate_spawn_commit(
&mut self,
state: SpawnActivateState,
) -> Result<FinalizeSpawnOutcome, MobError> {
let SpawnActivateState {
agent_identity,
dsl_identity,
member_ref,
operation_id,
session_origin,
failed_restore_peer_ids,
spawn_activation_committed,
..
} = state;
let agent_identity = &agent_identity;
// Spawn ladder step 4: finalize. `CommitSpawnActivation` advances the
// phase past `MembershipCommitted` and clears the per-identity
// spawn-exec entry — the member is fully live and the ladder is
// settled, so a future respawn of this identity can `BeginSpawnExec`
// again. Best-effort respawn topology-restore failures
// (`failed_restore_peer_ids`) do not block activation.
if !spawn_activation_committed {
self.apply_dsl_input(
mob_dsl::MobMachineInput::CommitSpawnActivation {
agent_identity: dsl_identity.clone(),
},
"finalize_spawn_activate_commit_activation",
)?;
}
// ADJ-24 + A17: a re-materialized incarnation rotates its comms
// identity, so a LIVE pump (obligation- or tap-kept) still polls the
// OLD transport and could never observe the bumped generation — the
// §18.8:1004 fail-fast source. Replace it with fresh material from
// the new roster incarnation (covers every finalize caller: spawn
// batch, respawn, revival).
let flow_obligation_outstanding = self
.dsl_authority
.state()
.pending_remote_turn_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.committed_remote_turn_outcomes
.iter(),
)
.chain(
self.dsl_authority
.state()
.resolved_remote_turn_outcomes
.iter(),
)
.any(|obligation| obligation.agent_identity.0 == agent_identity.as_str());
let kickoff_obligation_outstanding = self
.dsl_authority
.state()
.pending_placed_kickoff_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.resolved_placed_kickoff_outcomes
.iter(),
)
.any(|obligation| obligation.agent_identity.0 == agent_identity.as_str());
let obligation_outstanding = flow_obligation_outstanding || kickoff_obligation_outstanding;
if self.member_event_pumps.pump_exists(agent_identity) || obligation_outstanding {
tracing::debug!(
agent_identity = %agent_identity,
obligation_outstanding,
"replacing live member event pump after re-materialization"
);
if let Err(error) = self.ensure_member_event_pump(agent_identity).await {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
"pump replacement after re-materialization failed; \
stale-transport polls back off until liveness lapses"
);
}
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_activate done"
);
Ok(FinalizeSpawnOutcome {
receipt: super::handle::MemberSpawnReceipt {
member_ref,
operation_id,
session_origin,
rollback_authority: None,
materialized_ack: None,
failed_restore_peer_ids: Vec::new(),
},
failed_restore_peer_ids,
})
}
async fn spawn_wiring_targets(
&self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
) -> Vec<AgentIdentity> {
let mut targets = Vec::new();
if self.definition.wiring.auto_wire_orchestrator
&& let Some(orchestrator) = &self.definition.orchestrator
&& profile_name != &orchestrator.profile
{
let orchestrator_ids =
self.active_machine_member_ids_for_profile(&orchestrator.profile, agent_identity);
for orchestrator_id in orchestrator_ids {
if orchestrator_id != *agent_identity && !targets.contains(&orchestrator_id) {
targets.push(orchestrator_id);
}
}
}
for rule in &self.definition.wiring.role_wiring {
let target_profile = if &rule.a == profile_name {
Some(&rule.b)
} else if &rule.b == profile_name {
Some(&rule.a)
} else {
None
};
if let Some(target_profile) = target_profile {
let target_ids =
self.active_machine_member_ids_for_profile(target_profile, agent_identity);
for target_id in target_ids {
if !targets.contains(&target_id) {
targets.push(target_id);
}
}
}
}
targets
}
async fn resolve_auto_wire_parent_target(
&self,
owner_bridge_session_id: Option<&SessionId>,
spawned_member_identity: &AgentIdentity,
) -> Option<AgentIdentity> {
let owner_bridge_session_id = owner_bridge_session_id?;
let dsl_session_id = mob_dsl::SessionId::from_domain(owner_bridge_session_id);
let dsl = self.dsl_authority.state();
dsl.member_session_bindings
.iter()
.find(|(identity, bound_session_id)| {
**bound_session_id == dsl_session_id
&& identity.0.as_str() != spawned_member_identity.as_str()
&& !dsl.member_restore_failures.contains_key(*identity)
&& MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
})
.map(|(identity, _)| AgentIdentity::from(identity.0.as_str()))
}
async fn spawner_for_bridge_session(
&self,
owner_bridge_session_id: &SessionId,
) -> Option<(AgentIdentity, AgentRuntimeId)> {
let dsl_session_id = mob_dsl::SessionId::from_domain(owner_bridge_session_id);
let dsl = self.dsl_authority.state();
dsl.member_session_bindings
.iter()
.find_map(|(identity, bound_session_id)| {
if *bound_session_id != dsl_session_id
|| !MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
{
return None;
}
let agent_identity = AgentIdentity::from(identity.0.as_str());
let (agent_runtime_id, _) = dsl
.member_runtime_material_for_identity(identity)?
.to_domain_for_identity(&agent_identity);
Some((agent_identity, agent_runtime_id))
})
}
/// P1-T05: force-cancel a member's in-flight turn cooperatively.
///
/// Does NOT retire the member — the member remains in the roster and can
/// receive new turns. Use [`handle_retire`] to fully remove a member.
async fn handle_force_cancel(&mut self, agent_identity: AgentIdentity) -> Result<(), MobError> {
// An identity the roster has never seen must answer MemberNotFound;
// letting it reach machine admission collapses into the operator-
// hostile "invalid state transition: Running -> Running".
{
let roster = self.roster.read().await;
if roster.get(&agent_identity).is_none() {
return Err(MobError::MemberNotFound(agent_identity));
}
}
let placed = self
.ensure_placed_carrier_binding_active(&agent_identity, "force cancel")?
.is_some();
let prepared = self.prepare_command_admission(
mob_dsl::MobMachineInput::ForceCancel {
agent_identity: mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
agent_identity.as_str(),
)),
},
MobState::Running,
"force_cancel",
)?;
if Self::force_cancel_interrupt_authorized(&prepared) {
let entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
if let Some(entry) = entry {
let member_ref = self.machine_member_ref_for_behavior(&entry, "force cancel")?;
let expected_member = placed
.then(|| self.placed_member_incarnation(&entry))
.transpose()?;
self.provisioner
.interrupt_member(&member_ref, expected_member.as_ref())
.await?;
} else {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"MobMachine admitted force-cancel without a roster projection; committing machine authority without mechanical interrupt"
);
}
} else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"MobMachine converged force-cancel as an idempotent no-op (no live runtime or member already retiring)"
);
}
self.commit_prepared_dsl_input(prepared)?;
Ok(())
}
/// Machine-owned interrupt authority for the Cancel-class verbs: the
/// active `ForceCancelRunning` arm is the only ForceCancel arm that emits
/// (`FlowTerminalized`); the wedge arms (`ForceCancelRunningRuntimeNotLive`,
/// `ForceCancelRunningAlreadyRetiring`) admit as idempotent no-ops with no
/// emission and therefore authorize no mechanical interrupt dispatch.
fn force_cancel_interrupt_authorized(prepared: &PreparedDslInput) -> bool {
prepared
.effects
.iter()
.any(|effect| matches!(effect, mob_dsl::MobMachineEffect::FlowTerminalized))
}
/// Phase 6 (DEC-P6E-8): the explicit HARD cancel verb — the immediate
/// user-interrupt authority, distinct from [`Self::handle_force_cancel`]'s
/// cooperative boundary cancel. Placement present ⇒ gate on the
/// machine-recorded host `hard_cancel_member` capability fact BEFORE any
/// bridge dispatch (FLAG-P6E-8; the member drain's fail-closed arm is the
/// backstop for drift), then dispatch on a DETACHED task (ADJ-P4-12).
async fn handle_hard_cancel_member(
&mut self,
agent_identity: AgentIdentity,
reason: String,
reply_tx: oneshot::Sender<Result<(), MobError>>,
) {
if let Err(error) =
self.ensure_placed_carrier_binding_active(&agent_identity, "hard cancel")
{
let _ = reply_tx.send(Err(error));
return;
}
// Admission preview: the same Cancel-class machine gate the
// cooperative cancel walks (member known + phase admits).
let prepared = match self.prepare_command_admission(
mob_dsl::MobMachineInput::ForceCancel {
agent_identity: mob_dsl::AgentIdentity::from_domain(&agent_identity),
},
MobState::Running,
"hard_cancel_member",
) {
Ok(prepared) => prepared,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
// Cancel-class no-op admission (runtime not live / already retiring):
// nothing is running to hard-cancel, so the level-triggered verb
// converges idempotently without any bridge or runtime dispatch.
if !Self::force_cancel_interrupt_authorized(&prepared) {
let result = self.commit_prepared_dsl_input(prepared);
let _ = reply_tx.send(result);
return;
}
let entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
let Some(entry) = entry else {
let _ = reply_tx.send(Err(MobError::MemberNotFound(agent_identity)));
return;
};
let member_ref = entry.member_ref.clone();
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&agent_identity);
let placement = self
.dsl_authority
.state()
.member_placement
.get(&dsl_identity)
.cloned();
let placed = if let Some(host) = &placement {
// Capability gate on the recorded host fact — no bridge traffic
// when the host never advertised hard cancel (ADJ-P6-5).
let advertised = self
.dsl_authority
.state()
.host_hard_cancel_member
.get(host)
.copied()
.unwrap_or(false);
if !advertised {
let _ = reply_tx.send(Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unsupported,
reason: "host does not advertise hard_cancel_member".to_string(),
}));
return;
}
true
} else {
false
};
let expected_member = placed
.then(|| self.placed_member_incarnation(&entry))
.transpose();
let expected_member = match expected_member {
Ok(expected_member) => expected_member,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
if let Err(error) = self.commit_prepared_dsl_input(prepared) {
let _ = reply_tx.send(Err(error));
return;
}
// Detached dispatch: the actor loop never awaits a bridge
// round-trip (the spawn_turn_completed_reply pattern). Lane choice
// is the machine placement fact (ADJ-24), never the ref shape.
let provisioner = self.provisioner.clone();
self.actor_io_tasks.spawn(async move {
let result = match expected_member.as_ref() {
Some(expected_member) => {
provisioner
.hard_cancel_placed_member(&member_ref, expected_member, &reason)
.await
}
None => provisioner.hard_cancel_member(&member_ref, &reason).await,
};
let _ = reply_tx.send(result);
});
}
/// Phase 6 (DEC-P6E-20/21): the placement-switched member history read.
/// Local members serve the local session page through THE shared wire
/// projection; placed members proxy `ReadMemberHistory` on a detached
/// task (ADJ-P4-12).
async fn handle_member_history(
&mut self,
agent_identity: AgentIdentity,
from_index: Option<u64>,
limit: Option<u32>,
reply_tx: oneshot::Sender<
Result<super::member_history_proxy::MemberHistoryPageDomain, MobError>,
>,
) {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&agent_identity);
let placement = self
.dsl_authority
.state()
.member_placement
.get(&dsl_identity)
.cloned();
if let Some(placement) = placement {
let (peer, expected_member) = {
let roster = self.roster.read().await;
let Some(entry) = roster.get(&agent_identity) else {
let _ = reply_tx.send(Err(MobError::MemberNotFound(agent_identity)));
return;
};
match self.member_pump_material(entry) {
Ok(material) => match self.placed_member_incarnation(entry) {
Ok(expected_member) => (material.peer, expected_member),
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
},
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
}
};
let bridge = Arc::clone(&self.supervisor_bridge);
let command_tx = self.command_tx.clone();
let completion_identity = agent_identity.clone();
let completion_incarnation = expected_member.clone();
self.actor_io_tasks.spawn(async move {
let result = super::member_history_proxy::read_remote_member_history_page(
&bridge,
&peer,
placement,
expected_member,
from_index,
limit,
)
.await;
let _ = command_tx
.send(RoutedMobCommand::internal(
MobCommand::PlacedBehaviorCompleted {
agent_identity: completion_identity,
expected_member: completion_incarnation,
completion: super::state::PlacedBehaviorCompletion::MemberHistory {
result,
reply_tx,
},
},
))
.await;
});
return;
}
// Local branch: session read → the ONE wire projection.
let session_id = {
let roster = self.roster.read().await;
let Some(entry) = roster.get(&agent_identity) else {
let _ = reply_tx.send(Err(MobError::MemberNotFound(agent_identity)));
return;
};
match entry.member_ref.bridge_session_id().cloned() {
Some(session_id) => session_id,
None => {
let _ = reply_tx.send(Err(MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
reason: "member has no session to read history from".to_string(),
}));
return;
}
}
};
let generation = {
let roster = self.roster.read().await;
roster
.get(&agent_identity)
.map(|entry| entry.generation.get())
.unwrap_or(0)
};
let session_service = self.session_service.clone();
self.actor_io_tasks.spawn(async move {
let result = local_member_history_page(
session_service.as_ref(),
&session_id,
generation,
from_index,
limit,
)
.await;
let _ = reply_tx.send(result);
});
}
/// Resolve one live verb's dispatch lane (phase 6b, DEC-P6B-C3 order:
/// roster → placement → `host_live_endpoints` capability gate). The
/// capability reject is produced SYNCHRONOUSLY on the actor with ZERO
/// bridge traffic (§16.6 row 1) — absence of the bind-time endpoint
/// fact IS live-incapability (DL5, no boolean shadow).
async fn member_live_lane(
&self,
agent_identity: &AgentIdentity,
) -> Result<MemberLiveLane, MobError> {
let roster = self.roster.read().await;
let Some(entry) = roster.get(agent_identity) else {
return Err(MobError::MemberNotFound(agent_identity.clone()));
};
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let placement = self
.dsl_authority
.state()
.member_placement
.get(&dsl_identity)
.cloned();
match placement {
Some(host) => {
let advertised = self
.dsl_authority
.state()
.host_live_endpoints
.contains_key(&host);
if !advertised {
return Err(MobError::BridgeCommandRejected {
cause:
super::bridge_protocol::BridgeRejectionCause::LiveTransportUnavailable,
reason: "host does not advertise a live endpoint".to_string(),
});
}
let material = self.member_pump_material(entry)?;
let expected_member = self.placed_member_incarnation(entry)?;
Ok(MemberLiveLane::Placed {
peer: material.peer,
expected_member,
})
}
None => {
// The member_history local-branch session read: the bound
// bridge session is the gateway's addressing fact.
let Some(session_id) = entry.member_ref.bridge_session_id().cloned() else {
return Err(MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
reason: "member has no session to open a live channel on".to_string(),
});
};
Ok(MemberLiveLane::Local {
session_id,
generation: entry.generation.get(),
})
}
}
}
async fn require_member_live_mutation_admissible(
&self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
if self
.member_live_open_cleanup_obligations
.values()
.any(|obligation| {
&obligation.agent_identity == agent_identity
&& !obligation.delivery_pending_ack
&& !obligation.caller_acknowledged
})
{
return Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unavailable,
reason: format!(
"member '{agent_identity}' has an unresolved live Open cleanup; mutating live operations remain fenced until absence is proved"
),
});
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let state = self.dsl_authority.state();
if state
.identity_to_runtime
.get(&dsl_identity)
.and_then(|runtime_id| state.member_state_markers.get(runtime_id))
== Some(&mob_dsl::MobMemberState::Retiring)
{
return Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unavailable,
reason: format!(
"member '{agent_identity}' is retiring; mutating live operations are fenced"
),
});
}
let placement = state.member_placement.get(&dsl_identity).cloned();
if let Some(host_id) = placement {
let revoke_pending = self
.current_pending_host_revoke_anchors()
.await?
.into_iter()
.any(|anchor| anchor.host_id == host_id.0);
if revoke_pending {
return Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unavailable,
reason: format!(
"member '{agent_identity}' is on host '{}' with a pending revoke; mutating live operations are fenced",
host_id.as_str()
),
});
}
let supervisor_rotation_pending = self
.load_supervisor_authority_snapshot()
.await?
.is_some_and(|snapshot| snapshot.durable.pending_rotation.is_some());
if supervisor_rotation_pending {
return Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unavailable,
reason: format!(
"member '{agent_identity}' is on host '{}' while supervisor rebind authority is pending; mutating live operations are fenced",
host_id.as_str()
),
});
}
}
Ok(())
}
fn member_live_cleanup_proves_absent(error: &MobError) -> bool {
matches!(
error,
MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::LiveChannelNotFound
| super::bridge_protocol::BridgeRejectionCause::LiveTransportUnavailable,
..
}
)
}
fn member_live_status_proves_absent(error: &MobError) -> bool {
// Generic Unavailable/timeout is not negative channel evidence: the
// owning service may be temporarily unreachable while a previously
// admitted channel still exists. LiveTransportUnavailable is
// different: it is the authenticated responder's structural report
// that this machine has no MemberLiveHost slot. The slot starts empty
// and can only be installed (never removed), so it proves that no
// channel can remain on this host. Accept the same proof after a
// status -> close race: if the exact close observes the absent slot,
// there is likewise no owning substrate left to retain the channel.
Self::member_live_cleanup_proves_absent(error)
}
/// Cleanup deliberately does not reuse `member_live_lane`'s advertised-live
/// capability gate. A host may have admitted a channel and subsequently
/// refreshed that capability away; lifecycle must still address the exact
/// placed incarnation over its supervisor bridge and ask the owning side to
/// prove the channel absent.
async fn member_live_cleanup_target(
&self,
agent_identity: &AgentIdentity,
) -> Result<Option<MemberLiveMutationTarget>, MobError> {
let roster = self.roster.read().await;
let Some(entry) = roster.get(agent_identity) else {
return Err(MobError::MemberNotFound(agent_identity.clone()));
};
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
if self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity)
{
if self.confirmed_revoked_placed_host(agent_identity).is_some() {
// The only unreachable placed lane that counts as absent is
// one backed by the validated HostRevoked receipt described
// in `confirmed_revoked_placed_host`. Mere transport loss or
// NotBound never reaches this arm.
return Ok(None);
}
let material = self.member_pump_material(entry)?;
let expected_member = self.placed_member_incarnation(entry)?;
return Ok(Some(MemberLiveMutationTarget::Placed {
peer: material.peer,
expected_member,
}));
}
let Some(session_id) = entry.member_ref.bridge_session_id().cloned() else {
return Ok(None);
};
if self.member_live_host.is_none() {
return Ok(None);
}
Ok(Some(MemberLiveMutationTarget::Local {
session_id,
generation: entry.generation.get(),
}))
}
async fn member_live_mutation_target_current(
&self,
agent_identity: &AgentIdentity,
target: &MemberLiveMutationTarget,
) -> Result<(), MobError> {
match target {
MemberLiveMutationTarget::Placed {
expected_member, ..
} => {
let current = {
let entry = self.roster.read().await.get(agent_identity).cloned();
entry
.as_ref()
.map(|entry| self.placed_member_incarnation(entry))
.transpose()?
};
if current.as_ref() == Some(expected_member) {
Ok(())
} else {
Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleFence,
reason: format!(
"placed live operation completed for superseded member incarnation {expected_member:?}; current is {current:?}"
),
})
}
}
MemberLiveMutationTarget::Local {
session_id,
generation,
} => {
let entry = self.roster.read().await.get(agent_identity).cloned();
let current_session = entry
.as_ref()
.and_then(|entry| entry.member_ref.bridge_session_id());
let is_local = !self
.dsl_authority
.state()
.member_placement
.contains_key(&mob_dsl::AgentIdentity::from_domain(agent_identity));
if is_local
&& entry
.as_ref()
.is_some_and(|entry| entry.generation.get() == *generation)
&& current_session == Some(session_id)
{
Ok(())
} else {
Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleFence,
reason: format!(
"local live operation completed for superseded member '{}' generation {generation} session '{}'; current generation/session is {:?}/{current_session:?}",
agent_identity,
session_id,
entry.as_ref().map(|entry| entry.generation.get()),
),
})
}
}
}
}
async fn close_exact_member_live_channel_owned(
supervisor_bridge: Arc<super::MobSupervisorBridge>,
member_live_host: Option<Arc<dyn meerkat_runtime::member_live::MemberLiveHost>>,
target: MemberLiveMutationTarget,
channel_id: String,
) -> Result<(), MobError> {
let result = match target {
MemberLiveMutationTarget::Placed {
peer,
expected_member,
} => super::member_live_proxy::close_remote_member_live_channel(
&supervisor_bridge,
&peer,
expected_member,
channel_id,
)
.await
.map(|_| ()),
MemberLiveMutationTarget::Local { session_id, .. } => {
let Some(live_host) = member_live_host.as_ref() else {
return Err(Self::local_live_transport_unavailable());
};
let timeout = super::member_live_proxy::LIVE_CHANNEL_BRIDGE_TIMEOUT;
tokio::time::timeout(timeout, live_host.close(&session_id, &channel_id))
.await
.map_err(|_| Self::member_live_timeout_error("cleanup-close", timeout))?
.map(|_| ())
.map_err(Self::member_live_error_to_mob_error)
}
};
match result {
Ok(()) => Ok(()),
Err(error) if Self::member_live_cleanup_proves_absent(&error) => Ok(()),
Err(error) => Err(error),
}
}
async fn member_live_status_for_target_owned(
supervisor_bridge: &Arc<super::MobSupervisorBridge>,
member_live_host: Option<&Arc<dyn meerkat_runtime::member_live::MemberLiveHost>>,
target: &MemberLiveMutationTarget,
) -> Result<super::member_live_proxy::MemberLiveStatusDomain, MobError> {
match target {
MemberLiveMutationTarget::Placed {
peer,
expected_member,
} => {
super::member_live_proxy::remote_member_live_status(
supervisor_bridge,
peer,
expected_member.clone(),
None,
)
.await
}
MemberLiveMutationTarget::Local { session_id, .. } => {
let Some(live_host) = member_live_host else {
return Err(Self::local_live_transport_unavailable());
};
let timeout = super::member_live_proxy::LIVE_CHANNEL_BRIDGE_TIMEOUT;
tokio::time::timeout(timeout, live_host.status(session_id, None))
.await
.map_err(|_| Self::member_live_timeout_error("cleanup-status", timeout))?
.map(|status| super::member_live_proxy::MemberLiveStatusDomain {
channel_id: status.channel_id,
status: status.status,
})
.map_err(Self::member_live_error_to_mob_error)
}
}
}
async fn close_member_live_cleanup_target_owned(
supervisor_bridge: Arc<super::MobSupervisorBridge>,
member_live_host: Option<Arc<dyn meerkat_runtime::member_live::MemberLiveHost>>,
target: MemberLiveMutationTarget,
channel_id: String,
) -> Result<(), MobError> {
Self::close_exact_member_live_channel_owned(
supervisor_bridge,
member_live_host,
target,
channel_id,
)
.await
}
fn member_live_task_panic_log_key(
operation: &str,
agent_identity: &AgentIdentity,
target: &MemberLiveMutationTarget,
disposition: MemberLivePanicDisposition,
) -> String {
let target_kind = match target {
MemberLiveMutationTarget::Placed { .. } => "placed",
MemberLiveMutationTarget::Local { .. } => "local",
};
format!(
"member-live:{operation}:{agent_identity}:{target_kind}:{}",
disposition.as_str()
)
}
fn member_live_task_panic_log_gate() -> &'static meerkat_core::panic_payload::PanicPayloadLogGate
{
static PANIC_LOG_GATE: std::sync::OnceLock<
meerkat_core::panic_payload::PanicPayloadLogGate,
> = std::sync::OnceLock::new();
PANIC_LOG_GATE.get_or_init(meerkat_core::panic_payload::PanicPayloadLogGate::default)
}
fn clear_member_live_task_panic_log(
operation: &str,
agent_identity: &AgentIdentity,
target: &MemberLiveMutationTarget,
disposition: MemberLivePanicDisposition,
) {
Self::member_live_task_panic_log_gate().clear(&Self::member_live_task_panic_log_key(
operation,
agent_identity,
target,
disposition,
));
}
fn member_live_task_panic_error(
operation: &str,
agent_identity: &AgentIdentity,
target: &MemberLiveMutationTarget,
disposition: MemberLivePanicDisposition,
payload: Box<dyn std::any::Any + Send>,
) -> MobError {
let detail = super::panic_capture::panic_payload_detail(payload.as_ref());
let log_decision = Self::member_live_task_panic_log_gate().observe(
&Self::member_live_task_panic_log_key(operation, agent_identity, target, disposition),
&detail,
);
if log_decision.should_log {
let target_kind = match target {
MemberLiveMutationTarget::Placed { .. } => "placed",
MemberLiveMutationTarget::Local { .. } => "local",
};
tracing::error!(
member = %agent_identity,
operation,
target_kind,
disposition = disposition.as_str(),
panic = %detail,
repeated_sightings = log_decision.repeated_sightings,
"actor-owned member-live task panicked; payload recovered, sanitized, and routed through its typed completion"
);
}
MobError::Internal(format!(
"actor-owned member-live {operation} panicked for member '{agent_identity}' at \
{target:?}; disposition={}: {detail}",
disposition.as_str()
))
}
fn member_live_timeout_error(operation: &str, timeout: std::time::Duration) -> MobError {
MobError::BridgeRequestTimedOut {
request_envelope_id: format!("local-member-live:{operation}"),
timeout_ms: timeout.as_millis().min(u64::MAX as u128) as u64,
}
}
fn member_live_open_error_is_ambiguous(
result: &Result<super::bridge_protocol::LiveOpenResult, MobError>,
) -> bool {
matches!(result, Err(MobError::BridgeRequestTimedOut { .. }))
}
fn member_live_cleanup_retry_delay(attempts: u32) -> std::time::Duration {
let shift = attempts.saturating_sub(1).min(5);
std::time::Duration::from_millis(100_u64.saturating_mul(1_u64 << shift))
}
fn member_live_cleanup_panic_retry_delay(attempts: u32) -> std::time::Duration {
// A caught panic already ran the process panic hook (including
// symbolication). If durable custody is not yet confirmed, retain the
// volatile fence and retry slowly rather than feeding an invariant
// fault into the ordinary sub-second transport cadence.
let shift = attempts.saturating_sub(1).min(5);
std::time::Duration::from_secs(30_u64.saturating_mul(1_u64 << shift))
}
fn member_live_cleanup_target_record(
target: &MemberLiveMutationTarget,
) -> crate::store::MobMemberLiveCleanupTargetRecord {
match target {
MemberLiveMutationTarget::Local {
session_id,
generation,
} => crate::store::MobMemberLiveCleanupTargetRecord::Local {
session_id: session_id.to_string(),
generation: *generation,
},
MemberLiveMutationTarget::Placed {
expected_member, ..
} => crate::store::MobMemberLiveCleanupTargetRecord::Placed {
expected_member: expected_member.clone(),
},
}
}
fn member_live_open_cleanup_record(
agent_identity: &AgentIdentity,
target: &MemberLiveMutationTarget,
channel_id: String,
reason: &str,
) -> crate::store::MobMemberLiveCleanupRecord {
crate::store::MobMemberLiveCleanupRecord {
cleanup_id: uuid::Uuid::new_v4().to_string(),
agent_identity: agent_identity.clone(),
target: Self::member_live_cleanup_target_record(target),
channel_id,
reason: reason.to_string(),
}
}
async fn confirm_member_live_open_cleanup_persistence(
runtime_metadata: &dyn crate::store::MobRuntimeMetadataStore,
mob_id: &MobId,
record: &crate::store::MobMemberLiveCleanupRecord,
) -> Result<(), MobError> {
let mut last_error = None;
for attempt in 0..3u32 {
match runtime_metadata
.put_member_live_cleanup_record_if_absent(mob_id, record)
.await
{
Ok(_) => return Ok(()),
Err(write_error) => {
match runtime_metadata
.list_member_live_cleanup_records(mob_id)
.await
{
Ok(records) => {
let mut matching = records
.iter()
.filter(|existing| existing.cleanup_id == record.cleanup_id);
if let Some(existing) = matching.next() {
if matching.next().is_some() || existing != record {
return Err(MobError::Internal(format!(
"member-live cleanup id '{}' conflicts after ambiguous persistence: {write_error}",
record.cleanup_id
)));
}
return Ok(());
}
last_error = Some(format!(
"write failed and exact reread proved cleanup '{}' absent: {write_error}",
record.cleanup_id
));
}
Err(read_error) => {
last_error = Some(format!(
"write failed ({write_error}) and exact cleanup reread failed ({read_error})"
));
}
}
}
}
if attempt < 2 {
tokio::time::sleep(Self::member_live_cleanup_retry_delay(attempt + 1)).await;
}
}
Err(MobError::Internal(last_error.unwrap_or_else(|| {
format!(
"member-live cleanup '{}' persistence could not be confirmed",
record.cleanup_id
)
})))
}
async fn enqueue_member_live_open_cleanup(
&mut self,
agent_identity: AgentIdentity,
target: MemberLiveMutationTarget,
channel_id: String,
reason: &'static str,
terminal_reply: Option<(
oneshot::Sender<Result<super::state::MemberLiveOpenDelivery, MobError>>,
MobError,
)>,
) -> Result<(), MobError> {
let durable_record = Self::member_live_open_cleanup_record(
&agent_identity,
&target,
channel_id.clone(),
reason,
);
let persistence = Self::confirm_member_live_open_cleanup_persistence(
self.runtime_metadata.as_ref(),
&self.definition.id,
&durable_record,
)
.await;
let durable_persistence_confirmed = persistence.is_ok();
let last_error = persistence.err().map(|error| error.to_string());
let ticket = self
.next_member_live_open_cleanup_ticket
.checked_add(1)
.ok_or_else(|| {
MobError::Internal("member-live open cleanup ticket space exhausted".to_string())
})?;
self.next_member_live_open_cleanup_ticket = ticket;
self.member_live_open_cleanup_obligations.insert(
ticket,
MemberLiveOpenCleanupObligation {
durable_record,
durable_persistence_confirmed,
agent_identity,
target,
channel_id,
reason: reason.to_string(),
attempts: 0,
last_error,
panic_quarantined: false,
panic_backoff: false,
delivery_pending_ack: false,
caller_acknowledged: false,
delivery_confirmation: None,
terminal_reply,
},
);
self.spawn_member_live_open_cleanup_attempt(ticket, std::time::Duration::ZERO)
}
async fn deliver_member_live_open_with_acknowledged_custody(
&mut self,
agent_identity: AgentIdentity,
target: MemberLiveMutationTarget,
open: super::bridge_protocol::LiveOpenResult,
reply_tx: oneshot::Sender<Result<super::state::MemberLiveOpenDelivery, MobError>>,
) -> Result<(), MobError> {
let reason = "successful Open result was not acknowledged by its caller";
let durable_record = Self::member_live_open_cleanup_record(
&agent_identity,
&target,
open.channel_id.clone(),
reason,
);
if let Err(persist_error) = Self::confirm_member_live_open_cleanup_persistence(
self.runtime_metadata.as_ref(),
&self.definition.id,
&durable_record,
)
.await
{
let ticket = self
.next_member_live_open_cleanup_ticket
.checked_add(1)
.ok_or_else(|| {
MobError::Internal(
"member-live open cleanup ticket space exhausted".to_string(),
)
})?;
self.next_member_live_open_cleanup_ticket = ticket;
self.member_live_open_cleanup_obligations.insert(
ticket,
MemberLiveOpenCleanupObligation {
durable_record,
durable_persistence_confirmed: false,
agent_identity,
target,
channel_id: open.channel_id,
reason: reason.to_string(),
attempts: 0,
last_error: Some(persist_error.to_string()),
panic_quarantined: false,
panic_backoff: false,
delivery_pending_ack: false,
caller_acknowledged: false,
delivery_confirmation: None,
terminal_reply: Some((reply_tx, persist_error)),
},
);
return self.spawn_member_live_open_cleanup_attempt(ticket, std::time::Duration::ZERO);
}
let ticket = self
.next_member_live_open_cleanup_ticket
.checked_add(1)
.ok_or_else(|| {
MobError::Internal("member-live open cleanup ticket space exhausted".to_string())
})?;
self.next_member_live_open_cleanup_ticket = ticket;
self.member_live_open_cleanup_obligations.insert(
ticket,
MemberLiveOpenCleanupObligation {
durable_record,
durable_persistence_confirmed: true,
agent_identity,
target,
channel_id: open.channel_id.clone(),
reason: reason.to_string(),
attempts: 0,
last_error: None,
panic_quarantined: false,
panic_backoff: false,
delivery_pending_ack: true,
caller_acknowledged: false,
delivery_confirmation: None,
terminal_reply: None,
},
);
if !self.member_live_open_cleanup_inflight.insert(ticket) {
return Err(MobError::Internal(format!(
"new member-live delivery ticket {ticket} was already in flight"
)));
}
let (delivery_ack, delivery_ack_rx) = oneshot::channel();
self.member_live_mutation_tasks.spawn(async move {
let confirmation = match tokio::time::timeout(
std::time::Duration::from_secs(5),
delivery_ack_rx,
)
.await
{
Ok(Ok(confirmation)) => Some(confirmation),
Ok(Err(_)) | Err(_) => None,
};
MemberLiveMutationCompletion::OpenDeliveryAck {
ticket,
confirmation,
}
});
// If the receiver was cancelled before this send, the returned
// delivery is dropped here, closing `delivery_ack`; the tracked task
// then converts the retained obligation into an exact cleanup.
let _ = reply_tx.send(Ok(super::state::MemberLiveOpenDelivery {
open,
delivery_ack,
}));
Ok(())
}
fn spawn_member_live_open_cleanup_attempt(
&mut self,
ticket: u64,
delay: std::time::Duration,
) -> Result<(), MobError> {
if !self.member_live_open_cleanup_inflight.insert(ticket) {
return Ok(());
}
let Some(obligation) = self.member_live_open_cleanup_obligations.get(&ticket) else {
self.member_live_open_cleanup_inflight.remove(&ticket);
return Err(MobError::Internal(format!(
"member-live cleanup ticket {ticket} has no actor-owned obligation"
)));
};
if obligation.panic_quarantined {
self.member_live_open_cleanup_inflight.remove(&ticket);
return Ok(());
}
let supervisor_bridge = Arc::clone(&self.supervisor_bridge);
let member_live_host = self.member_live_host.clone();
let agent_identity = obligation.agent_identity.clone();
let target = obligation.target.clone();
let panic_target = target.clone();
let channel_id = obligation.channel_id.clone();
let caller_acknowledged = obligation.caller_acknowledged;
let durable_record = obligation.durable_record.clone();
let durable_persistence_confirmed = obligation.durable_persistence_confirmed;
let runtime_metadata = Arc::clone(&self.runtime_metadata);
let mob_id = self.definition.id.clone();
let panic_runtime_metadata = Arc::clone(&runtime_metadata);
let panic_mob_id = mob_id.clone();
let panic_durable_record = durable_record.clone();
self.member_live_mutation_tasks.spawn(async move {
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
let cleanup = async move {
let persistence = if durable_persistence_confirmed {
Ok(())
} else {
Self::confirm_member_live_open_cleanup_persistence(
runtime_metadata.as_ref(),
&mob_id,
&durable_record,
)
.await
};
let persistence_confirmed = persistence.is_ok();
let result = if caller_acknowledged {
persistence
} else {
let close = Self::close_member_live_cleanup_target_owned(
supervisor_bridge,
member_live_host,
target.clone(),
channel_id,
)
.await;
match (persistence, close) {
(_, Ok(())) => Ok(()),
(Ok(()), Err(close_error)) => Err(close_error),
(Err(persist_error), Err(close_error)) => {
Err(MobError::Internal(format!(
"member-live cleanup persistence remains unconfirmed ({persist_error}) and exact close also failed ({close_error})"
)))
}
}
};
(persistence_confirmed, result)
};
let (durable_persistence_confirmed, result, panicked) =
match std::panic::AssertUnwindSafe(cleanup).catch_unwind().await {
Ok((durable_persistence_confirmed, result)) => {
Self::clear_member_live_task_panic_log(
"open cleanup",
&agent_identity,
&panic_target,
MemberLivePanicDisposition::QuarantinedCleanup,
);
Self::clear_member_live_task_panic_log(
"open cleanup",
&agent_identity,
&panic_target,
MemberLivePanicDisposition::BackoffUntilDurable,
);
(durable_persistence_confirmed, result, false)
}
Err(payload) => {
// The faulting future may have persisted the row before
// unwinding. Re-establish exact durable custody before
// authorizing a cold-recovery fail-stop. This second
// boundary is independently caught because persistence
// code is exactly what may have faulted.
let (durable_persistence_confirmed, confirmation_failure) =
if durable_persistence_confirmed {
(true, None)
} else {
match std::panic::AssertUnwindSafe(
Self::confirm_member_live_open_cleanup_persistence(
panic_runtime_metadata.as_ref(),
&panic_mob_id,
&panic_durable_record,
),
)
.catch_unwind()
.await
{
Ok(Ok(())) => {
Self::clear_member_live_task_panic_log(
"open cleanup custody confirmation",
&agent_identity,
&panic_target,
MemberLivePanicDisposition::BackoffUntilDurable,
);
(true, None)
}
Ok(Err(error)) => {
Self::clear_member_live_task_panic_log(
"open cleanup custody confirmation",
&agent_identity,
&panic_target,
MemberLivePanicDisposition::BackoffUntilDurable,
);
tracing::error!(
member = %agent_identity,
target = ?panic_target,
disposition = MemberLivePanicDisposition::BackoffUntilDurable.as_str(),
error = %error,
"post-panic member-live cleanup custody confirmation failed"
);
(false, Some(error.to_string()))
}
Err(payload) => {
let error = Self::member_live_task_panic_error(
"open cleanup custody confirmation",
&agent_identity,
&panic_target,
MemberLivePanicDisposition::BackoffUntilDurable,
payload,
);
(false, Some(error.to_string()))
}
}
};
let disposition = if durable_persistence_confirmed {
MemberLivePanicDisposition::QuarantinedCleanup
} else {
MemberLivePanicDisposition::BackoffUntilDurable
};
let panic_error = Self::member_live_task_panic_error(
"open cleanup",
&agent_identity,
&panic_target,
disposition,
payload,
);
let panic_error = match confirmation_failure {
None => panic_error,
Some(confirmation_failure) => MobError::Internal(format!(
"{panic_error}; exact durable cleanup custody remains unconfirmed: {confirmation_failure}"
)),
};
(durable_persistence_confirmed, Err(panic_error), true)
}
};
MemberLiveMutationCompletion::OpenCleanup {
ticket,
durable_persistence_confirmed,
result,
panicked,
}
});
Ok(())
}
fn schedule_retained_member_live_open_cleanups(&mut self) -> Result<(), MobError> {
let tickets = self
.member_live_open_cleanup_obligations
.iter()
.filter(|(ticket, obligation)| {
!obligation.panic_quarantined
&& !self.member_live_open_cleanup_inflight.contains(ticket)
})
.map(|(ticket, _)| *ticket)
.collect::<Vec<_>>();
for ticket in tickets {
self.spawn_member_live_open_cleanup_attempt(ticket, std::time::Duration::ZERO)?;
}
Ok(())
}
async fn recover_member_live_cleanup_target(
&self,
record: &crate::store::MobMemberLiveCleanupRecord,
) -> Result<Option<MemberLiveMutationTarget>, MobError> {
match &record.target {
crate::store::MobMemberLiveCleanupTargetRecord::Local {
session_id,
generation,
} => {
let session_id = SessionId::parse(session_id).map_err(|error| {
MobError::Internal(format!(
"durable member-live cleanup '{}' has invalid local session id: {error}",
record.cleanup_id
))
})?;
let target = MemberLiveMutationTarget::Local {
session_id,
generation: *generation,
};
self.member_live_mutation_target_current(&record.agent_identity, &target)
.await?;
Ok(Some(target))
}
crate::store::MobMemberLiveCleanupTargetRecord::Placed { expected_member } => {
if self
.confirmed_revoked_placed_host(&record.agent_identity)
.is_some_and(|host| host.as_str() == expected_member.host_id.as_str())
{
return Ok(None);
}
let entry = self
.roster
.read()
.await
.get(&record.agent_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"durable member-live cleanup '{}' lost roster identity '{}' before absence was proved",
record.cleanup_id, record.agent_identity
))
})?;
let current = self.placed_member_incarnation(&entry)?;
if ¤t != expected_member {
return Err(MobError::Internal(format!(
"durable member-live cleanup '{}' targets superseded incarnation {expected_member:?}; current is {current:?}",
record.cleanup_id
)));
}
let peer = self.member_pump_material(&entry)?.peer;
Ok(Some(MemberLiveMutationTarget::Placed {
peer,
expected_member: expected_member.clone(),
}))
}
}
}
async fn recover_durable_member_live_open_cleanups(&mut self) -> Result<(), MobError> {
let records = self
.runtime_metadata
.list_member_live_cleanup_records(&self.definition.id)
.await?;
for record in records {
let Some(target) = self.recover_member_live_cleanup_target(&record).await? else {
self.runtime_metadata
.delete_member_live_cleanup_record(&self.definition.id, &record)
.await?;
continue;
};
let ticket = self
.next_member_live_open_cleanup_ticket
.checked_add(1)
.ok_or_else(|| {
MobError::Internal(
"member-live cleanup ticket space exhausted during recovery".to_string(),
)
})?;
self.next_member_live_open_cleanup_ticket = ticket;
self.member_live_open_cleanup_obligations.insert(
ticket,
MemberLiveOpenCleanupObligation {
durable_record: record.clone(),
durable_persistence_confirmed: true,
agent_identity: record.agent_identity.clone(),
target,
channel_id: record.channel_id.clone(),
reason: record.reason.clone(),
attempts: 0,
last_error: None,
panic_quarantined: false,
panic_backoff: false,
delivery_pending_ack: false,
caller_acknowledged: false,
delivery_confirmation: None,
terminal_reply: None,
},
);
}
// Exact-channel cleanup recovery is a startup barrier, not merely
// background scheduling. Ambiguous Open failures never create these
// records: without a returned channel id cleanup stays caller-driven.
self.drain_member_live_mutations_for_lifecycle().await
}
async fn reconcile_member_live_mutation_completion(
&mut self,
completion: MemberLiveMutationCompletion,
mode: MemberLiveReconcileMode,
) -> Result<(), MobError> {
match completion {
MemberLiveMutationCompletion::Open {
agent_identity,
target,
result,
ambiguous,
reply_tx,
} => {
let completion_gate = self
.member_live_mutation_target_current(&agent_identity, &target)
.await;
match (completion_gate, result) {
(Ok(()), Ok(open)) => {
self.deliver_member_live_open_with_acknowledged_custody(
agent_identity,
target,
open,
reply_tx,
)
.await
}
(Ok(()), Err(error)) if ambiguous => {
// A timeout/panic carries no exact channel identity.
// Keep reconciliation caller-driven: status(None) is
// an observation primitive, never authority to close
// whichever channel happens to be current.
let _ = reply_tx.send(Err(error));
Ok(())
}
(Ok(()), Err(error)) => {
let _ = reply_tx.send(Err(error));
Ok(())
}
(Err(gate_error), Ok(open)) => {
// The owning side may already have minted the channel.
// Never discard that success under a stale completion
// gate: close the exact returned id before surfacing the
// stale-incarnation rejection. Install actor-owned
// custody BEFORE dispatching the compensating close;
// a failed attempt remains retryable in the table.
self.enqueue_member_live_open_cleanup(
agent_identity,
target,
open.channel_id,
"successful Open completed for a stale incarnation",
Some((reply_tx, gate_error)),
)
.await
}
(Err(gate_error), Err(operation_error)) if ambiguous => {
let _ = reply_tx.send(Err(MobError::Internal(format!(
"stale-incarnation Open outcome remains caller-reconciled because no exact channel id was returned: operation={operation_error}; incarnation={gate_error}"
))));
Ok(())
}
(Err(gate_error), Err(_operation_error)) => {
let _ = reply_tx.send(Err(gate_error));
Ok(())
}
}
}
MemberLiveMutationCompletion::Control {
agent_identity,
target,
result,
ambiguous_effect,
reply_tx,
} => {
let completion_gate = self
.member_live_mutation_target_current(&agent_identity, &target)
.await;
let result = match (completion_gate, result) {
(Ok(()), result) => result,
(Err(gate_error), Err(operation_error)) if ambiguous_effect => {
// Do not let a later incarnation mismatch erase the
// fact that this non-idempotent control may already
// have taken effect on the admitted incarnation.
Err(MobError::Internal(format!(
"stale-incarnation Control outcome has an ambiguous effect and must not be retried blindly: operation={operation_error}; incarnation={gate_error}"
)))
}
(Err(gate_error), _) => Err(gate_error),
};
let _ = reply_tx.send(result);
Ok(())
}
MemberLiveMutationCompletion::OpenCleanup {
ticket,
durable_persistence_confirmed,
result,
panicked,
} => {
self.member_live_open_cleanup_inflight.remove(&ticket);
let obligation = self
.member_live_open_cleanup_obligations
.get_mut(&ticket)
.ok_or_else(|| {
MobError::Internal(format!(
"member-live cleanup ticket {ticket} completed without its actor-owned obligation"
))
})?;
obligation.durable_persistence_confirmed |= durable_persistence_confirmed;
if !panicked {
obligation.panic_backoff = false;
}
if panicked {
let Err(error) = result else {
return Err(MobError::Internal(format!(
"member-live cleanup ticket {ticket} carried a panic disposition with a successful result"
)));
};
let error_text = error.to_string();
obligation.attempts = obligation.attempts.saturating_add(1);
obligation.last_error = Some(error_text.clone());
let attempts = obligation.attempts;
let durable_custody_confirmed = obligation.durable_persistence_confirmed;
obligation.panic_quarantined = durable_custody_confirmed;
obligation.panic_backoff = !durable_custody_confirmed;
// A waiting Open caller has not observed success. Return a
// terminal failure now and restore cleanup ownership so a
// cold actor recovery closes the exact channel instead of
// transferring it after an invariant failure.
if let Some(confirmation) = obligation.delivery_confirmation.take() {
obligation.caller_acknowledged = false;
obligation.delivery_pending_ack = false;
let _ = confirmation.send(Err(MobError::Internal(error_text.clone())));
}
if let Some((reply_tx, _terminal_error)) = obligation.terminal_reply.take() {
let _ = reply_tx.send(Err(MobError::Internal(error_text.clone())));
}
if durable_custody_confirmed {
// Only an exact durable row authorizes process exit.
// Cold recovery reconstructs this obligation; the
// current incarnation must not re-run faulting cleanup.
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
// No durable owner exists yet. Keep the volatile mutation
// fence and retry on a panic-specific slow cadence; once a
// later attempt confirms the row, another panic transfers
// custody to cold recovery through fail-stop.
return match mode {
MemberLiveReconcileMode::Background => {
self.spawn_member_live_open_cleanup_attempt(
ticket,
Self::member_live_cleanup_panic_retry_delay(attempts),
)?;
Ok(())
}
MemberLiveReconcileMode::Lifecycle => Err(error),
};
}
match result {
Ok(()) => {
let Some(durable_record) = self
.member_live_open_cleanup_obligations
.get(&ticket)
.map(|obligation| obligation.durable_record.clone())
else {
return Err(MobError::Internal(format!(
"member-live cleanup ticket {ticket} completed without its actor-owned obligation"
)));
};
if let Err(error) = self
.runtime_metadata
.delete_member_live_cleanup_record(&self.definition.id, &durable_record)
.await
{
let attempts = {
let obligation = self
.member_live_open_cleanup_obligations
.get_mut(&ticket)
.ok_or_else(|| {
MobError::Internal(format!(
"member-live cleanup ticket {ticket} disappeared after durable deletion failed"
))
})?;
obligation.attempts = obligation.attempts.saturating_add(1);
obligation.last_error = Some(error.to_string());
obligation.attempts
};
return match mode {
MemberLiveReconcileMode::Background => {
self.spawn_member_live_open_cleanup_attempt(
ticket,
Self::member_live_cleanup_retry_delay(attempts),
)?;
Ok(())
}
MemberLiveReconcileMode::Lifecycle => Err(error.into()),
};
}
let mut obligation = self
.member_live_open_cleanup_obligations
.remove(&ticket)
.ok_or_else(|| {
MobError::Internal(format!(
"member-live cleanup ticket {ticket} disappeared after durable deletion"
))
})?;
if let Some(confirmation) = obligation.delivery_confirmation.take()
&& confirmation.send(Ok(())).is_err()
{
// The handle was cancelled while waiting for
// durable custody deletion. It never returned the
// public Open, so reclaim exact channel custody.
// The row is already absent; retain this volatile
// obligation until the exact close succeeds.
obligation.caller_acknowledged = false;
obligation.delivery_pending_ack = false;
obligation.attempts = obligation.attempts.saturating_add(1);
obligation.last_error = Some(
"durable custody deletion completed after the delivery waiter closed"
.to_string(),
);
self.member_live_open_cleanup_obligations
.insert(ticket, obligation);
self.spawn_member_live_open_cleanup_attempt(
ticket,
std::time::Duration::ZERO,
)?;
return Ok(());
}
if let Some((reply_tx, terminal_error)) = obligation.terminal_reply.take() {
let _ = reply_tx.send(Err(terminal_error));
}
Ok(())
}
Err(error) => {
let attempts = {
let Some(obligation) =
self.member_live_open_cleanup_obligations.get_mut(&ticket)
else {
return Err(MobError::Internal(format!(
"failed member-live cleanup ticket {ticket} has no actor-owned obligation"
)));
};
obligation.attempts = obligation.attempts.saturating_add(1);
let previous_error = obligation.last_error.clone();
obligation.last_error = Some(error.to_string());
tracing::debug!(
ticket,
member = %obligation.agent_identity,
channel_id = ?obligation.channel_id,
reason = %obligation.reason,
attempt = obligation.attempts,
previous_error = ?previous_error,
error = %error,
"retaining failed member-live Open cleanup for retry"
);
obligation.attempts
};
match mode {
MemberLiveReconcileMode::Background => {
self.spawn_member_live_open_cleanup_attempt(
ticket,
Self::member_live_cleanup_retry_delay(attempts),
)?;
Ok(())
}
// Preserve BridgeRequestTimedOut and typed bridge
// rejection classification. Exact obligation
// context remains retained in the actor table for
// the next retry.
MemberLiveReconcileMode::Lifecycle => Err(error),
}
}
}
}
MemberLiveMutationCompletion::OpenDeliveryAck {
ticket,
confirmation,
} => {
self.member_live_open_cleanup_inflight.remove(&ticket);
let obligation = self
.member_live_open_cleanup_obligations
.get_mut(&ticket)
.ok_or_else(|| {
MobError::Internal(format!(
"member-live Open delivery ticket {ticket} completed without its actor-owned cleanup obligation"
))
})?;
obligation.delivery_pending_ack = false;
if let Some(confirmation) = confirmation {
// This is only transfer intent. The public handle waits;
// it does not return success until OpenCleanup deletes the
// durable row and sends this confirmation.
obligation.caller_acknowledged = true;
obligation.delivery_confirmation = Some(confirmation);
}
self.spawn_member_live_open_cleanup_attempt(ticket, std::time::Duration::ZERO)
}
}
}
async fn reconcile_joined_member_live_mutation(
&mut self,
joined: Result<MemberLiveMutationCompletion, tokio::task::JoinError>,
mode: MemberLiveReconcileMode,
) -> Result<(), MobError> {
match joined {
Ok(completion) => {
self.reconcile_member_live_mutation_completion(completion, mode)
.await
}
Err(error) if error.is_cancelled() => {
let task_id = actor_task_join_error_task_id(&error);
self.durable_uncertainty_fail_stop = true;
tracing::error!(
task_id = %task_id,
disposition = "cancelled_ambiguous_effect_fail_stop",
"actor-owned mutating member-live task was cancelled without a completion; effect outcome is unknown"
);
Err(MobError::Internal(format!(
"actor-owned mutating member-live task {task_id} was cancelled without a \
completion; disposition=cancelled_ambiguous_effect_fail_stop"
)))
}
Err(error) => {
let disposition = ActorTaskJoinPanicDisposition::AmbiguousEffectFailStop;
let error = actor_task_join_panic_error(
"mutating member-live completion",
disposition,
error,
);
self.durable_uncertainty_fail_stop = true;
Err(error)
}
}
}
async fn drain_completed_member_live_mutations(&mut self) {
while let Some(joined) = self.member_live_mutation_tasks.try_join_next() {
if let Err(error) = self
.reconcile_joined_member_live_mutation(joined, MemberLiveReconcileMode::Background)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"mutating member-live completion reconciliation failed"
);
}
}
}
async fn drain_member_live_mutations_for_lifecycle(&mut self) -> Result<(), MobError> {
self.schedule_retained_member_live_open_cleanups()?;
let mut first_error = None;
while let Some(joined) = self.member_live_mutation_tasks.join_next().await {
if let Err(error) = self
.reconcile_joined_member_live_mutation(joined, MemberLiveReconcileMode::Lifecycle)
.await
&& first_error.is_none()
{
first_error = Some(error);
}
}
if first_error.is_none() && !self.member_live_open_cleanup_obligations.is_empty() {
first_error = Some(MobError::Internal(format!(
"{} actor-owned member-live Open cleanup obligation(s) remain without a successful exact close",
self.member_live_open_cleanup_obligations.len()
)));
}
if first_error.is_some() && !self.durable_uncertainty_fail_stop {
let retries = self
.member_live_open_cleanup_obligations
.iter()
.filter(|(ticket, obligation)| {
!obligation.panic_quarantined
&& !self.member_live_open_cleanup_inflight.contains(ticket)
})
.map(|(ticket, obligation)| {
let delay = if obligation.panic_backoff {
Self::member_live_cleanup_panic_retry_delay(obligation.attempts.max(1))
} else {
Self::member_live_cleanup_retry_delay(obligation.attempts.max(1))
};
(*ticket, delay)
})
.collect::<Vec<_>>();
for (ticket, delay) in retries {
self.spawn_member_live_open_cleanup_attempt(ticket, delay)?;
}
}
first_error.map_or(Ok(()), Err)
}
async fn close_member_live_channels_for_lifecycle(
&self,
identities: Vec<AgentIdentity>,
context: &'static str,
) -> Result<(), MobError> {
let mut identities = identities;
identities.sort();
identities.dedup();
let mut cleanup_tasks = tokio::task::JoinSet::new();
for agent_identity in identities {
let target = match self.member_live_cleanup_target(&agent_identity).await {
Ok(Some(target)) => target,
Ok(None) => continue,
Err(error) => {
return Err(MobError::Internal(format!(
"{context}: cannot resolve member '{agent_identity}' live cleanup lane: {error}"
)));
}
};
let supervisor_bridge = Arc::clone(&self.supervisor_bridge);
let member_live_host = self.member_live_host.clone();
cleanup_tasks.spawn(async move {
let panic_identity = agent_identity.clone();
let panic_target = target.clone();
let attempted = std::panic::AssertUnwindSafe(async move {
let status = match Self::member_live_status_for_target_owned(
&supervisor_bridge,
member_live_host.as_ref(),
&target,
)
.await
{
Ok(status) => status,
Err(error) if Self::member_live_status_proves_absent(&error) => {
return Ok(());
}
// Preserve typed timeout/rejection classification so
// the caller can retry the still-pre-marker lifecycle.
Err(error) => return Err(error),
};
Self::close_exact_member_live_channel_owned(
supervisor_bridge,
member_live_host,
target,
status.channel_id,
)
.await
})
.catch_unwind()
.await;
match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"lifecycle cleanup",
&panic_identity,
&panic_target,
MemberLivePanicDisposition::TerminalFailure,
);
result
}
Err(payload) => Err(Self::member_live_task_panic_error(
"lifecycle cleanup",
&panic_identity,
&panic_target,
MemberLivePanicDisposition::TerminalFailure,
payload,
)),
}
});
}
let mut first_error = None;
while let Some(joined) = cleanup_tasks.join_next().await {
match joined {
Ok(Ok(())) => {}
Ok(Err(error)) => {
if first_error.is_none() {
first_error = Some(error);
}
}
Err(error) => {
let task_error = if error.is_cancelled() {
let task_id = actor_task_join_error_task_id(&error);
tracing::error!(
task_id = %task_id,
disposition = "cancelled_retryable_idempotent_cleanup",
"member-live lifecycle cleanup task was cancelled"
);
MobError::Internal(format!(
"{context}: member-live lifecycle cleanup task {task_id} was \
cancelled; disposition=cancelled_retryable_idempotent_cleanup"
))
} else {
actor_task_join_panic_error(
"member-live lifecycle cleanup",
ActorTaskJoinPanicDisposition::RetryableIdempotentCleanup,
error,
)
};
if first_error.is_none() {
first_error = Some(task_error);
}
}
}
}
first_error.map_or(Ok(()), Err)
}
async fn close_all_member_live_channels_for_lifecycle(
&self,
context: &'static str,
) -> Result<(), MobError> {
let identities = {
let roster = self.roster.read().await;
roster
.list_all()
.map(|entry| entry.agent_identity.clone())
.collect::<Vec<_>>()
};
self.close_member_live_channels_for_lifecycle(identities, context)
.await
}
fn member_live_ref_is_controller_local(member_ref: &MemberRef) -> bool {
matches!(member_ref, MemberRef::Session { .. })
}
/// Close only live channels owned by this controlling process during actor
/// shutdown.
///
/// Placed live channels are owned by their member hosts, and MobMachine
/// deliberately carries no controlling-side channel map. An unnamed status
/// probe for every placed roster member would therefore turn ordinary
/// process shutdown into a remote liveness gate. Exact channels retained by
/// an admitted Open cleanup remain covered by
/// `drain_member_live_mutations_for_lifecycle` before this method runs; all
/// other placed channel state remains with its owning host across a
/// controlling-process restart. `BackendPeer` is likewise never local
/// session ownership, even when legacy replay projects a stale bridge
/// session ID onto it.
async fn close_controller_local_member_live_channels_for_shutdown(
&self,
context: &'static str,
) -> Result<(), MobError> {
let identities = {
let roster = self.roster.read().await;
roster
.list_all()
.filter(|entry| Self::member_live_ref_is_controller_local(&entry.member_ref))
.map(|entry| entry.agent_identity.clone())
.collect::<Vec<_>>()
};
self.close_member_live_channels_for_lifecycle(identities, context)
.await
}
async fn quiesce_member_live_for_identities(
&mut self,
identities: Vec<AgentIdentity>,
context: &'static str,
) -> Result<(), MobError> {
self.drain_member_live_mutations_for_lifecycle().await?;
self.close_member_live_channels_for_lifecycle(identities, context)
.await
}
/// Phase 6b (DEC-P6B-C3): the identity-addressed live OPEN. Handler
/// order per ADJ-P6B-11: scope gate (dispatch loop) → roster →
/// placement → capability gate (zero bridge traffic) → webrtc gate
/// (zero dispatch, BOTH branches — the family is placement-blind,
/// DEC-P6B-C4) → DETACHED round-trip (bridge AND local gateway — the
/// phase-3 deadlock class; the local open does provider connects +
/// per-open credential resolution, the same starvation class).
async fn handle_member_live_open(
&mut self,
agent_identity: AgentIdentity,
turning_mode: Option<super::bridge_protocol::RealtimeTurningMode>,
transport: Option<super::bridge_protocol::LiveOpenTransport>,
reply_tx: oneshot::Sender<Result<super::state::MemberLiveOpenDelivery, MobError>>,
) {
// Opening a live channel can connect a provider and create a fresh
// realtime input origin. Drive the generated MobMachine lifecycle
// classifier before resolving either local or placed dispatch lanes.
if let Err(error) = self.require_member_operation_eligible() {
let _ = reply_tx.send(Err(error));
return;
}
if let Err(error) = self
.require_member_live_mutation_admissible(&agent_identity)
.await
{
let _ = reply_tx.send(Err(error));
return;
}
let lane = match self.member_live_lane(&agent_identity).await {
Ok(lane) => lane,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
// DEC-P6B-C4 / ADJ-P6B-12: webrtc is a typed reject at this verb
// family REGARDLESS of placement — a family that succeeded for
// local members and rejected for placed ones would leak placement
// through behavior (DL3). Session-scoped `live/open` keeps local
// webrtc; nothing is removed.
if matches!(
transport,
Some(super::bridge_protocol::LiveOpenTransport::Webrtc)
) {
let _ = reply_tx.send(Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::LiveTransportUnsupported {
requested: "webrtc".to_string(),
},
reason: "webrtc is not supported for mob member live channels; \
use the session-scoped live/open"
.to_string(),
}));
return;
}
match lane {
MemberLiveLane::Placed {
peer,
expected_member,
} => {
// Detached dispatch: the actor loop never awaits a bridge
// round-trip (ADJ-P4-12). Unlike read/cleanup I/O, mutating
// live effects return through the dedicated actor-owned lane
// so lifecycle can drain and reconcile them without waiting on
// unrelated actor I/O.
let bridge = Arc::clone(&self.supervisor_bridge);
let completion_identity = agent_identity.clone();
let target = MemberLiveMutationTarget::Placed {
peer: peer.clone(),
expected_member: expected_member.clone(),
};
self.member_live_mutation_tasks.spawn(async move {
let attempted = std::panic::AssertUnwindSafe(
super::member_live_proxy::open_remote_member_live_channel(
&bridge,
&peer,
expected_member,
turning_mode,
transport,
),
)
.catch_unwind()
.await;
let (result, ambiguous) = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Open",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
let ambiguous = Self::member_live_open_error_is_ambiguous(&result);
(result, ambiguous)
}
Err(payload) => (
Err(Self::member_live_task_panic_error(
"Open",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
true,
),
};
MemberLiveMutationCompletion::Open {
agent_identity: completion_identity,
target,
result,
ambiguous,
reply_tx,
}
});
}
MemberLiveLane::Local {
session_id,
generation,
} => {
let Some(live_host) = self.member_live_host.clone() else {
let _ = reply_tx.send(Err(Self::local_live_transport_unavailable()));
return;
};
let completion_identity = agent_identity.clone();
let target = MemberLiveMutationTarget::Local {
session_id: session_id.clone(),
generation,
};
self.member_live_mutation_tasks.spawn(async move {
let timeout = super::member_live_proxy::LIVE_OPEN_BRIDGE_TIMEOUT;
let attempted = std::panic::AssertUnwindSafe(async {
match tokio::time::timeout(
timeout,
live_host.open(&session_id, turning_mode, transport),
)
.await
{
Ok(result) => {
(result.map_err(Self::member_live_error_to_mob_error), false)
}
Err(_) => (Err(Self::member_live_timeout_error("open", timeout)), true),
}
})
.catch_unwind()
.await;
let (result, ambiguous) = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Open",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
result
}
Err(payload) => (
Err(Self::member_live_task_panic_error(
"Open",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
true,
),
};
MemberLiveMutationCompletion::Open {
agent_identity: completion_identity,
target,
result,
ambiguous,
reply_tx,
}
});
}
}
}
/// Phase 6b: close-what-you-name (DEC-P6B-C9's CAS property).
async fn handle_member_live_close(
&mut self,
agent_identity: AgentIdentity,
channel_id: String,
reply_tx: oneshot::Sender<Result<super::bridge_protocol::LiveCloseStatus, MobError>>,
) {
let lane = match self.member_live_lane(&agent_identity).await {
Ok(lane) => lane,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
match lane {
MemberLiveLane::Placed {
peer,
expected_member,
} => {
let bridge = Arc::clone(&self.supervisor_bridge);
let command_tx = self.command_tx.clone();
let completion_identity = agent_identity.clone();
let completion_incarnation = expected_member.clone();
let target = MemberLiveMutationTarget::Placed {
peer: peer.clone(),
expected_member: expected_member.clone(),
};
self.actor_io_tasks.spawn(async move {
let attempted = std::panic::AssertUnwindSafe(
super::member_live_proxy::close_remote_member_live_channel(
&bridge,
&peer,
expected_member,
channel_id,
),
)
.catch_unwind()
.await;
let result = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Close",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
result
}
Err(payload) => Err(Self::member_live_task_panic_error(
"Close",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
};
let _ = command_tx
.send(RoutedMobCommand::internal(
MobCommand::PlacedBehaviorCompleted {
agent_identity: completion_identity,
expected_member: completion_incarnation,
completion: super::state::PlacedBehaviorCompletion::LiveClose {
result,
reply_tx,
},
},
))
.await;
});
}
MemberLiveLane::Local {
session_id,
generation,
} => {
let Some(live_host) = self.member_live_host.clone() else {
let _ = reply_tx.send(Err(Self::local_live_transport_unavailable()));
return;
};
let target = MemberLiveMutationTarget::Local {
session_id: session_id.clone(),
generation,
};
self.actor_io_tasks.spawn(async move {
let timeout = super::member_live_proxy::LIVE_CHANNEL_BRIDGE_TIMEOUT;
let attempted = std::panic::AssertUnwindSafe(async {
tokio::time::timeout(timeout, live_host.close(&session_id, &channel_id))
.await
.map_err(|_| Self::member_live_timeout_error("close", timeout))?
.map_err(Self::member_live_error_to_mob_error)
})
.catch_unwind()
.await;
let result = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Close",
&agent_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
result
}
Err(payload) => Err(Self::member_live_task_panic_error(
"Close",
&agent_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
};
let _ = reply_tx.send(result);
});
}
}
}
/// Phase 6b (§16.9 / DEC-P6B-C10): the dedicated live point read —
/// `channel_id: None` resolves the member's active channel on the
/// owning side (ADJ-P6B-2). `handle_member_status` stays untouched: no
/// live enrichment, no bridge fan-out, no controlling-side channel
/// cache; this read is the ONLY remote channel-state path.
async fn handle_member_live_status(
&mut self,
agent_identity: AgentIdentity,
channel_id: Option<String>,
reply_tx: oneshot::Sender<
Result<super::member_live_proxy::MemberLiveStatusDomain, MobError>,
>,
) {
let lane = match self.member_live_lane(&agent_identity).await {
Ok(lane) => lane,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
match lane {
MemberLiveLane::Placed {
peer,
expected_member,
} => {
let bridge = Arc::clone(&self.supervisor_bridge);
let command_tx = self.command_tx.clone();
let completion_identity = agent_identity.clone();
let completion_incarnation = expected_member.clone();
let target = MemberLiveMutationTarget::Placed {
peer: peer.clone(),
expected_member: expected_member.clone(),
};
self.actor_io_tasks.spawn(async move {
let attempted = std::panic::AssertUnwindSafe(
super::member_live_proxy::remote_member_live_status(
&bridge,
&peer,
expected_member,
channel_id,
),
)
.catch_unwind()
.await;
let result = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Status",
&completion_identity,
&target,
MemberLivePanicDisposition::TerminalFailure,
);
result
}
Err(payload) => Err(Self::member_live_task_panic_error(
"Status",
&completion_identity,
&target,
MemberLivePanicDisposition::TerminalFailure,
payload,
)),
};
let _ = command_tx
.send(RoutedMobCommand::internal(
MobCommand::PlacedBehaviorCompleted {
agent_identity: completion_identity,
expected_member: completion_incarnation,
completion: super::state::PlacedBehaviorCompletion::LiveStatus {
result,
reply_tx,
},
},
))
.await;
});
}
MemberLiveLane::Local {
session_id,
generation,
} => {
let Some(live_host) = self.member_live_host.clone() else {
let _ = reply_tx.send(Err(Self::local_live_transport_unavailable()));
return;
};
let target = MemberLiveMutationTarget::Local {
session_id: session_id.clone(),
generation,
};
self.actor_io_tasks.spawn(async move {
let timeout = super::member_live_proxy::LIVE_CHANNEL_BRIDGE_TIMEOUT;
let attempted = std::panic::AssertUnwindSafe(async {
tokio::time::timeout(timeout, live_host.status(&session_id, channel_id))
.await
.map_err(|_| Self::member_live_timeout_error("status", timeout))?
.map(|status| super::member_live_proxy::MemberLiveStatusDomain {
channel_id: status.channel_id,
status: status.status,
})
.map_err(Self::member_live_error_to_mob_error)
})
.catch_unwind()
.await;
let result = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Status",
&agent_identity,
&target,
MemberLivePanicDisposition::TerminalFailure,
);
result
}
Err(payload) => Err(Self::member_live_task_panic_error(
"Status",
&agent_identity,
&target,
MemberLivePanicDisposition::TerminalFailure,
payload,
)),
};
let _ = reply_tx.send(result);
});
}
}
}
/// Phase 6b: turn-level control verbs (DL10's closed vocabulary; live
/// interrupt is media-plane barge-in through the gateway/bridge, never
/// `hard_cancel_current_run` — no effect-authority reach).
async fn handle_member_live_control(
&mut self,
agent_identity: AgentIdentity,
channel_id: String,
verb: super::bridge_protocol::BridgeLiveControlVerb,
reply_tx: oneshot::Sender<
Result<super::bridge_protocol::BridgeLiveControlOutcome, MobError>,
>,
) {
// CommitInput, Refresh, Truncate, and Interrupt all mutate a live
// realtime channel. None has a machine-proved cleanup-only authority,
// so fail closed for the whole closed vocabulary while lifecycle
// cleanup owns work origin. Close and Status remain separately allowed.
if let Err(error) = self.require_member_operation_eligible() {
let _ = reply_tx.send(Err(error));
return;
}
if let Err(error) = self
.require_member_live_mutation_admissible(&agent_identity)
.await
{
let _ = reply_tx.send(Err(error));
return;
}
let lane = match self.member_live_lane(&agent_identity).await {
Ok(lane) => lane,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
match lane {
MemberLiveLane::Placed {
peer,
expected_member,
} => {
let bridge = Arc::clone(&self.supervisor_bridge);
let completion_identity = agent_identity.clone();
let target = MemberLiveMutationTarget::Placed {
peer: peer.clone(),
expected_member: expected_member.clone(),
};
self.member_live_mutation_tasks.spawn(async move {
let attempted = std::panic::AssertUnwindSafe(
super::member_live_proxy::control_remote_member_live_channel(
&bridge,
&peer,
expected_member,
channel_id,
verb,
),
)
.catch_unwind()
.await;
let (result, ambiguous_effect) = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Control",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
let ambiguous_effect =
matches!(&result, Err(MobError::BridgeRequestTimedOut { .. }));
(result, ambiguous_effect)
}
Err(payload) => (
Err(Self::member_live_task_panic_error(
"Control",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
true,
),
};
MemberLiveMutationCompletion::Control {
agent_identity: completion_identity,
target,
result,
ambiguous_effect,
reply_tx,
}
});
}
MemberLiveLane::Local {
session_id,
generation,
} => {
let Some(live_host) = self.member_live_host.clone() else {
let _ = reply_tx.send(Err(Self::local_live_transport_unavailable()));
return;
};
let completion_identity = agent_identity.clone();
let target = MemberLiveMutationTarget::Local {
session_id: session_id.clone(),
generation,
};
self.member_live_mutation_tasks.spawn(async move {
let timeout = super::member_live_proxy::LIVE_CHANNEL_BRIDGE_TIMEOUT;
let attempted = std::panic::AssertUnwindSafe(async {
tokio::time::timeout(
timeout,
live_host.control(&session_id, &channel_id, verb),
)
.await
.map_err(|_| Self::member_live_timeout_error("control", timeout))?
.map_err(Self::member_live_error_to_mob_error)
})
.catch_unwind()
.await;
let (result, ambiguous_effect) = match attempted {
Ok(result) => {
Self::clear_member_live_task_panic_log(
"Control",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
);
let ambiguous_effect =
matches!(&result, Err(MobError::BridgeRequestTimedOut { .. }));
(result, ambiguous_effect)
}
Err(payload) => (
Err(Self::member_live_task_panic_error(
"Control",
&completion_identity,
&target,
MemberLivePanicDisposition::AmbiguousEffect,
payload,
)),
true,
),
};
MemberLiveMutationCompletion::Control {
agent_identity: completion_identity,
target,
result,
ambiguous_effect,
reply_tx,
}
});
}
}
}
/// The local branch's absent-gateway reject: honest degradation for a
/// process without a live transport (DEC-P6B-C3 step 7); zero cost.
fn local_live_transport_unavailable() -> MobError {
MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::LiveTransportUnavailable,
reason: "this process has no live transport".to_string(),
}
}
/// DEC-P6B-C5 / ADJ-P6B-13: ONE cause taxonomy, ONE carrier — a local
/// pipeline failure converts through the ADJ-P6B-1 conversion
/// (`MemberLiveError::to_bridge_rejection`, the SAME impl the member
/// responder uses for its `Rejected` replies — cause parity across
/// placements) and wraps in the carrier the remote decode produces.
/// The "bridge"-named carrier on a local failure is a recorded naming
/// wart, not dual truth.
fn member_live_error_to_mob_error(
error: meerkat_runtime::member_live::MemberLiveError,
) -> MobError {
MobError::BridgeCommandRejected {
cause: error.to_bridge_rejection(),
reason: error.to_string(),
}
}
fn bound_placement_host_id(
&self,
identity: &AgentIdentity,
) -> Result<mob_dsl::HostId, MobError> {
let Some(host_id) =
self.ensure_placed_carrier_binding_active(identity, "placed remote behavior")?
else {
return Err(MobError::Internal(format!(
"member '{identity}' has no authoritative MobMachine placement"
)));
};
Ok(host_id)
}
/// One admission predicate for every ordinary host-addressed behavior.
/// Logical membership can survive a revoked G1 carrier, but no remote
/// command is exposed until authenticated revival/status has durably
/// promoted that carrier to the exact active host generation.
fn ensure_placed_carrier_binding_active(
&self,
identity: &AgentIdentity,
context: &str,
) -> Result<Option<mob_dsl::HostId>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let state = self.dsl_authority.state();
let Some(host_id) = state.member_placement.get(&dsl_identity).cloned() else {
return Ok(None);
};
let carrier_generation = state
.current_placed_spawn_host_binding_generations
.get(&dsl_identity)
.copied();
let active_generation = state.host_binding_generations.get(&host_id).copied();
let bound =
state.host_bind_phase.get(&host_id).copied() == Some(mob_dsl::HostBindPhase::Bound);
if !state.placed_carrier_binding_active_for_identity(&dsl_identity) {
return Err(MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
reason: format!(
"{context}: placed member '{identity}' is dormant on host '{}' (carrier binding generation {carrier_generation:?}, active {active_generation:?}, bound={bound})",
host_id.as_str()
),
});
}
Ok(Some(host_id))
}
fn confirmed_revoked_placed_host(&self, identity: &AgentIdentity) -> Option<mob_dsl::HostId> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let state = self.dsl_authority.state();
let host = state.member_placement.get(&dsl_identity)?.clone();
if state.host_bind_phase.get(&host) == Some(&mob_dsl::HostBindPhase::Bound) {
return None;
}
let generation = state
.current_placed_spawn_host_binding_generations
.get(&dsl_identity)
.copied()?;
// `RemoteHostRevokeConfirmed` is appended only after validating a
// HostRevoked receipt whose released-members membership is emitted
// after the owning disposal arc's live status/exact-close barrier.
// Thus this tombstone is cleanup-backed negative evidence, not mere
// loss of reachability.
(generation > 0
&& state.confirmed_host_binding_revocations.contains(
&mob_dsl::HostBindingGenerationTombstone {
host_id: host.clone(),
binding_generation: generation,
},
))
.then_some(host)
}
fn placed_member_incarnation(
&self,
entry: &crate::roster::RosterEntry,
) -> Result<super::bridge_protocol::BridgeMemberIncarnation, MobError> {
let host_authority = self
.ensure_placed_carrier_binding_active(
&entry.agent_identity,
"placed member incarnation",
)?
.ok_or_else(|| {
MobError::Internal(format!(
"placed member '{}' has no authoritative placement",
entry.agent_identity
))
})?;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let (host_id, generation, fence_token, host_binding_generation, machine_session_id) = {
let state = self.dsl_authority.state();
let missing = |fact: &str| {
MobError::Internal(format!(
"placed member '{}' has no authoritative MobMachine {fact}",
entry.agent_identity
))
};
let host_id = state
.member_placement
.get(&dsl_identity)
.cloned()
.ok_or_else(|| missing("placement"))?;
if host_id != host_authority {
return Err(MobError::Internal(format!(
"member '{}' placement host '{}' drifted after active carrier admission",
entry.agent_identity, host_id.0
)));
}
(
host_id,
state
.identity_runtime_generations
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("generation"))?,
state
.identity_runtime_fence_tokens
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("fence token"))?,
state
.current_placed_spawn_host_binding_generations
.get(&dsl_identity)
.copied()
.ok_or_else(|| missing("host binding generation"))?,
state
.member_session_bindings
.get(&dsl_identity)
.cloned()
.ok_or_else(|| missing("member session binding"))?,
)
};
let MemberRef::BackendPeer {
session_id: roster_session_id,
..
} = &entry.member_ref
else {
return Err(MobError::Internal(format!(
"placed member '{}' has no peer transport route",
entry.agent_identity
)));
};
if entry.agent_runtime_id.generation.get() != generation.0
|| entry.fence_token.get() != fence_token.0
|| !optional_route_session_matches_machine(
roster_session_id.as_ref(),
&machine_session_id.0,
)
{
return Err(MobError::Internal(format!(
"placed member '{}' roster route is stale against MobMachine incarnation authority",
entry.agent_identity
)));
}
Ok(super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: entry.agent_identity.to_string(),
host_id: host_id.0,
binding_generation: host_binding_generation,
member_session_id: machine_session_id.0,
generation: generation.0,
fence_token: fence_token.0,
})
}
/// Derive the static pump material for one placed roster entry
/// (DEC-P6E-9: machine-recorded identity facts + the roster's shell
/// transport composition — the placed-delivery peer derivation).
///
/// Shape note (ADJ-24): a placed member's ref is peer-shaped, but its
/// optional session copy may be absent. Placement and session identity
/// are CALLER-owned machine facts; this helper only extracts transport.
fn member_pump_material(
&self,
entry: &crate::roster::RosterEntry,
) -> Result<super::event_pump::MemberPumpMaterial, MobError> {
let host_id = self.bound_placement_host_id(&entry.agent_identity)?;
let MemberRef::BackendPeer {
peer_id,
address,
pubkey,
..
} = &entry.member_ref
else {
return Err(MobError::Internal(format!(
"member '{}' has no placed peer binding for event pumping",
entry.agent_identity
)));
};
let peer_name = address
.strip_prefix("inproc://")
.map(|value| value.split('?').next().unwrap_or(value).to_string())
.unwrap_or_else(|| format!("mob_member/backend_peer/{peer_id}"));
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
peer_name,
peer_id.to_string(),
*pubkey,
address.to_string(),
)
.map_err(|error| MobError::WiringError(format!("invalid member peer spec: {error}")))?;
Ok(super::event_pump::MemberPumpMaterial {
agent_identity: entry.agent_identity.clone(),
host_id: host_id.0,
expected_member: self.placed_member_incarnation(entry)?,
runtime_id: entry.agent_runtime_id.clone(),
fence_token: entry.fence_token,
role: entry.role.clone(),
peer,
})
}
/// §19.L2 fork-context read: ONE placement switch, ONE row vocabulary.
/// Placed sources proxy `ReadMemberHistory` (`LastMessages` = a single
/// tail-addressed round trip); local sources read the session and
/// project through the SAME wire page projection. Runs under spawn
/// admission — no `ReadHistory` scope check on this internal path.
async fn fork_source_history_rows(
&self,
source_member_id: &AgentIdentity,
fork_context: &crate::launch::ForkContext,
) -> Result<Vec<meerkat_contracts::wire::WireHistoryRow>, MobError> {
let remote_unavailable = || MobError::ForkSourceUnavailable {
source_member_id: source_member_id.to_string(),
cause: crate::error::ForkSourceUnavailableCause::RemoteReadUnavailable,
};
let source_dsl = mob_dsl::AgentIdentity::from_domain(source_member_id);
let placement = self
.dsl_authority
.state()
.member_placement
.get(&source_dsl)
.cloned();
if let Some(placement) = placement {
let (peer, expected_member) = {
let roster = self.roster.read().await;
let entry = roster
.get(source_member_id)
.ok_or_else(|| MobError::MemberNotFound(source_member_id.clone()))?;
let peer = self
.member_pump_material(entry)
.map_err(|_| remote_unavailable())?
.peer;
let expected_member = self
.placed_member_incarnation(entry)
.map_err(|_| remote_unavailable())?;
(peer, expected_member)
};
let page = match fork_context {
crate::launch::ForkContext::FullHistory => {
super::member_history_proxy::read_remote_member_full_history(
&self.supervisor_bridge,
&peer,
placement,
expected_member,
)
.await
}
crate::launch::ForkContext::LastMessages { count } => {
super::member_history_proxy::read_remote_member_history_tail(
&self.supervisor_bridge,
&peer,
placement,
expected_member,
*count,
)
.await
}
}
.map_err(|error| {
tracing::warn!(
source_member_id = %source_member_id,
error = %error,
"remote fork-source history read failed"
);
remote_unavailable()
})?;
return Ok(page.page.messages);
}
let source_session_id = {
let roster = self.roster.read().await;
let source_entry = roster
.get(source_member_id)
.ok_or_else(|| MobError::MemberNotFound(source_member_id.clone()))?;
source_entry
.member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| MobError::ForkSourceUnavailable {
source_member_id: source_member_id.to_string(),
cause: crate::error::ForkSourceUnavailableCause::NoSession,
})?
};
let query = match fork_context {
crate::launch::ForkContext::FullHistory => {
meerkat_core::service::SessionHistoryQuery::default()
}
crate::launch::ForkContext::LastMessages { count } => {
let view = self
.session_service
.read(&source_session_id)
.await
.map_err(|e| {
MobError::Internal(format!(
"failed to read source session metadata for fork from '{source_member_id}': {e}"
))
})?;
let total = view.state.message_count;
let count = *count as usize;
meerkat_core::service::SessionHistoryQuery {
offset: total.saturating_sub(count),
limit: Some(count),
}
}
};
let history = meerkat_core::service::SessionServiceHistoryExt::read_history(
self.session_service.as_ref(),
&source_session_id,
query,
)
.await
.map_err(|e| {
MobError::Internal(format!(
"failed to read source session history for fork from '{source_member_id}': {e}"
))
})?;
Ok(
meerkat_contracts::wire::WireMemberHistoryPageBody::try_from_history_page(&history)
.map_err(|error| {
MobError::Internal(format!(
"failed to project source member history page: {error}"
))
})?
.messages,
)
}
/// Ensure the member's event pump runs (A17 liveness: subscription
/// authorized OR obligation outstanding — DEC-P6E-11). Idempotent.
/// Derive validated pump material for a PLACED member (the shared gate
/// for `ensure_member_event_pump` and the atomic ensure+tap arm).
async fn member_pump_tap_material(
&self,
agent_identity: &AgentIdentity,
) -> Result<super::event_pump::MemberPumpMaterial, MobError> {
// Placement is the machine fact, never the ref shape (ADJ-24):
// pumps exist only for placed members, so gate structurally here
// rather than trusting every caller's context.
let placed =
super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity);
if !placed {
return Err(MobError::Internal(format!(
"member event pump requested for unplaced member '{agent_identity}'"
)));
}
let roster = self.roster.read().await;
let Some(entry) = roster.get(agent_identity) else {
return Err(MobError::MemberNotFound(agent_identity.clone()));
};
self.member_pump_material(entry)
}
pub(super) async fn ensure_member_event_pump(
&mut self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
let material = self.member_pump_tap_material(agent_identity).await?;
self.member_event_pumps.ensure_pump(material).await;
Ok(())
}
/// D-wire-handler (#26) + #31 D-trust-reconciliation (Wave D): forward a
/// wire command to the MobMachine DSL and install bidirectional comms
/// trust + peer notifications.
///
/// Shell-mechanical steps on a successful local-local wire:
///
/// 1. Normalize `(local, target)` into a canonical `WiringEdge`.
/// 2. Resolve both endpoints' comms runtimes + trusted-peer specs. Any
/// missing comms runtime / public key fails fast as
/// [`MobError::WiringError`] with **zero side effects**.
/// 3. Submit `MobMachineInput::WireMembers { edge }` to the DSL
/// authority. Already-wired idempotency is a generated no-op
/// transition; only `WiringGraphChanged` means the machine graph
/// actually mutated.
/// 4. On DSL acceptance, bidirectionally install trust on both
/// runtimes (A trusts B, B trusts A) and emit `mob.peer_added`
/// notifications from both sides. Any failure mid-step rolls back
/// the trust installs and reverts the DSL wire so failure leaves
/// no observable side effect. This replaces the pre-DSL
/// trust-install loop that Wave-A commit `0ad584cde` deleted.
/// 5. Append `MembersWired` event and project it through the roster.
/// Append failure also rolls back trust + DSL wire.
///
/// External peer targets are routed to [`Self::handle_wire_external`],
/// whose descriptor-bearing trust edge is admitted by
/// `MobMachineInput::WireExternalPeer` instead of being coerced into a
/// member `WiringEdge`. Public surfaces should pass
/// [`PeerTarget::ExternalBinding`], which this actor resolves before trust
/// installation; pre-resolved [`PeerTarget::External`] is retained for
/// internal callers and tests.
async fn handle_wire(
&mut self,
local: AgentIdentity,
target: super::handle::PeerTarget,
) -> Result<(), MobError> {
let peer_identity = match target {
super::handle::PeerTarget::Local(id) => id,
super::handle::PeerTarget::ExternalName(name) => {
return Err(MobError::WiringError(format!(
"wire external peer '{name}' requires external_binding"
)));
}
super::handle::PeerTarget::ExternalBinding(binding) => {
let descriptor = Self::trusted_peer_descriptor_from_external_binding(binding)?;
return self.handle_wire_external(local, descriptor).await;
}
super::handle::PeerTarget::External(descriptor) => {
return self.handle_wire_external(local, descriptor).await;
}
};
let local_identity = AgentIdentity::from(local.as_str());
let peer_member_identity = AgentIdentity::from(peer_identity.as_str());
let dsl_a = mob_dsl::AgentIdentity::from_domain(&local_identity);
let dsl_b = mob_dsl::AgentIdentity::from_domain(&peer_identity);
let edge = mob_dsl::WiringEdge::new(dsl_a, dsl_b);
self.probe_command_admission(
mob_dsl::MobMachineInput::WireMembers { edge: edge.clone() },
MobState::Running,
"wire_members_command_admission",
)?;
if local_identity == peer_identity {
return Err(MobError::WiringError(format!(
"wire requires distinct members (got '{local}')"
)));
}
self.ensure_member_not_broken(&local).await?;
self.ensure_member_not_broken(&peer_member_identity).await?;
// Pre-flight: roster lookups. Missing members fail fast before
// any authority mutation.
let (local_entry, peer_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&local)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?,
roster
.get(&peer_member_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(peer_member_identity.clone()))?,
)
};
// Idempotent repair path when the MobMachine already owns the edge.
// The DSL emits a local repair effect; without a graph-change effect,
// this path may reinstall live trust but must not synthesize a public
// roster/event projection.
let dsl_has_edge = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
// Resolve both endpoints' comms runtimes + specs BEFORE mutating
// any authority state. Missing comms / missing public key yields
// WiringError with zero side effects.
let local_endpoint = self.resolve_wiring_endpoint(&local_entry, "wire").await?;
let peer_endpoint = self.resolve_wiring_endpoint(&peer_entry, "wire").await?;
if dsl_has_edge {
let authority = self.apply_wire_members_idempotent(&edge)?;
if !authority.is_repair() {
return Err(MobError::WiringError(
"idempotent wire repair did not produce generated repair authority".to_string(),
));
}
let handoff = authority.member_handoff()?;
match (&local_endpoint, &peer_endpoint) {
(
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) => {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.repair_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
handoff.require_peer_id_for(&local, &local_key)?;
if let Err(error) = self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.repair_authority_for(
&local,
&local_key,
&self.dsl_authority,
)?,
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_repair_rollback_trust_authority",
)
.await;
}
return Err(MobError::from(error));
}
}
(
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) => {
self.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
.map_err(|error| {
tracing::debug!(
mob_id = %self.definition.id,
%error,
"peer-only trust repair failed before reciprocal side"
);
error
})?;
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
false,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
}
(
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) => {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.repair_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_repair_rollback_trust_authority",
)
.await;
}
return Err(error);
}
}
(
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) => {
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&local, &local_key)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.repair_authority_for(
&local,
&local_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_repair_rollback_trust_authority",
)
.await;
}
return Err(error);
}
}
(
WiringEndpoint::Local {
comms: local_comms, ..
},
WiringEndpoint::Placed {
identity: placed_identity,
spec: placed_spec,
..
},
) => {
// LOCAL-side reinstall only: the placed side's trust
// repair rides the route-install obligations re-driven
// below.
let placed_key = Self::trusted_peer_removal_key(placed_spec);
handoff.require_peer_id_for(placed_identity, &placed_key)?;
self.apply_trusted_peer_add_report(
local_comms.as_ref(),
placed_spec.clone(),
handoff.repair_authority_for(
placed_identity,
&placed_key,
&self.dsl_authority,
)?,
)
.await
.map_err(MobError::from)?;
}
(
WiringEndpoint::Placed {
identity: placed_identity,
spec: placed_spec,
..
},
WiringEndpoint::Local {
comms: peer_comms, ..
},
) => {
let placed_key = Self::trusted_peer_removal_key(placed_spec);
handoff.require_peer_id_for(placed_identity, &placed_key)?;
self.apply_trusted_peer_add_report(
peer_comms.as_ref(),
placed_spec.clone(),
handoff.repair_authority_for(
placed_identity,
&placed_key,
&self.dsl_authority,
)?,
)
.await
.map_err(MobError::from)?;
}
(WiringEndpoint::Placed { .. }, WiringEndpoint::Placed { .. }) => {
// Both endpoints remote: every trust lane rides the
// route-install obligations re-driven below.
}
(WiringEndpoint::Placed { .. }, WiringEndpoint::PeerOnly { .. })
| (WiringEndpoint::PeerOnly { .. }, WiringEndpoint::Placed { .. }) => {
return Err(MobError::WiringError(format!(
"wire between a placed member and a legacy external (peer-only) member is unsupported ('{local}' <-> '{peer_identity}')"
)));
}
}
// Repair is the operator retry lane (§9 "retry drains
// obligations idempotently"): re-record + re-drive the
// cross-host obligations for this edge (set-insert idempotent,
// no epoch bump; the machine re-emits `RouteInstallRequested`).
self.fold_route_install_obligations_after_wire(&edge).await;
return Ok(());
}
if matches!(local_endpoint, WiringEndpoint::Placed { .. })
|| matches!(peer_endpoint, WiringEndpoint::Placed { .. })
{
return self
.wire_new_edge_with_placement(
&edge,
&local,
&peer_member_identity,
local_endpoint,
peer_endpoint,
)
.await;
}
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::both(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
if let (
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let peer_key = Self::trusted_peer_removal_key(peer_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.wiring_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_event_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::peer(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&local, &local_key)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.wiring_authority_for(&local, &local_key, &self.dsl_authority)?,
)
.await
{
Ok(created) => created,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_event_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
let (local_comms, local_spec) = match local_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire requires local session comms runtime for '{local}'"
)));
}
WiringEndpoint::Placed { .. } => {
return Err(MobError::Internal(format!(
"wire placed lane for '{local}' bypassed its dedicated handler"
)));
}
};
let (peer_comms, peer_spec) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::Placed { .. } => {
return Err(MobError::Internal(format!(
"wire placed lane for '{peer_identity}' bypassed its dedicated handler"
)));
}
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire requires local session comms runtime for '{peer_identity}'"
)));
}
};
// Submit the DSL input. A new edge emits `WiringGraphChanged`; an
// existing edge emits generated local repair authority. Only a graph
// change means rollback must undo a machine graph mutation.
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let local_peer_id = Self::trusted_peer_removal_key(&local_spec);
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
// A-side trust install.
handoff.require_peer_id_for(&peer_member_identity, &peer_peer_id)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.wiring_authority_for(
&peer_member_identity,
&peer_peer_id,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
false,
false,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
// B-side trust install.
handoff.require_peer_id_for(&local, &local_peer_id)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.wiring_authority_for(&local, &local_peer_id, &self.dsl_authority)?,
)
.await
{
Ok(created) => created,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
false,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
// Notify A that B is now wired.
if let Err(err) = self
.notify_peer_added(&peer_comms, &local_spec, &peer_member_identity, &peer_entry)
.await
{
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(err);
}
// Notify B that A is now wired.
if let Err(err) = self
.notify_peer_added(&local_comms, &peer_spec, &local, &local_entry)
.await
{
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(err);
}
// Append MembersWired — rollback on failure.
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
/// New-edge wire lanes involving a placed endpoint (multi-host §10.4).
///
/// Ordering (ADJ-P4-13): the LOCAL side's trust row installs pre-commit
/// (a local-lane failure fails the wire inline with a full DSL revert —
/// a `RouteInstallObligation` can never name the controlling host, so
/// the local lane is never obligation-tracked), then the durable
/// `MembersWired` event commits, and only THEN are the cross-host
/// obligations recorded and realized (the machine's
/// `edge_currently_wired` guard admits `RecordRouteInstall` strictly
/// post-commit). A failed realization leaves the edge committed and the
/// obligation pending — Ok + observable, never fail-quiet (ADJ-P4-9c).
///
/// Like the peer-only lanes, placed lanes send no `peer_added`
/// notifications (the cross-host notification lane is not part of
/// phase 4).
async fn wire_new_edge_with_placement(
&mut self,
edge: &mob_dsl::WiringEdge,
local: &AgentIdentity,
peer_member_identity: &AgentIdentity,
local_endpoint: WiringEndpoint,
peer_endpoint: WiringEndpoint,
) -> Result<(), MobError> {
// (local-session runtime that must trust the placed side, placed
// side's identity, placed side's machine spec) — `None` for a
// placed↔placed edge (both lanes ride obligations).
let local_lane: Option<(
Arc<dyn CoreCommsRuntime>,
AgentIdentity,
TrustedPeerDescriptor,
)> = match (&local_endpoint, &peer_endpoint) {
(
WiringEndpoint::Local { comms, .. },
WiringEndpoint::Placed { identity, spec, .. },
)
| (
WiringEndpoint::Placed { identity, spec, .. },
WiringEndpoint::Local { comms, .. },
) => Some((Arc::clone(comms), identity.clone(), spec.clone())),
(WiringEndpoint::Placed { .. }, WiringEndpoint::Placed { .. }) => None,
(WiringEndpoint::PeerOnly { .. }, _) | (_, WiringEndpoint::PeerOnly { .. }) => {
return Err(MobError::WiringError(format!(
"wire between a placed member and a legacy external (peer-only) member is unsupported ('{local}' <-> '{peer_member_identity}')"
)));
}
(WiringEndpoint::Local { .. }, WiringEndpoint::Local { .. }) => {
return Err(MobError::Internal(
"wire_new_edge_with_placement invoked without a placed endpoint".into(),
));
}
};
let local_spec = local_endpoint.spec().clone();
let peer_spec = peer_endpoint.spec().clone();
let authority = self.apply_wire_members_idempotent(edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let mut local_trust_created = false;
if let Some((local_comms, placed_identity, placed_spec)) = local_lane.as_ref() {
let placed_key = Self::trusted_peer_removal_key(placed_spec);
if let Err(error) = handoff.require_peer_id_for(placed_identity, &placed_key) {
self.rollback_peer_only_wire(
edge,
dsl_added,
WiringSides::empty(),
local,
peer_member_identity,
&local_spec,
&peer_spec,
)
.await;
return Err(error);
}
let trust_authority = match handoff.wiring_authority_for(
placed_identity,
&placed_key,
&self.dsl_authority,
) {
Ok(trust_authority) => trust_authority,
Err(error) => {
self.rollback_peer_only_wire(
edge,
dsl_added,
WiringSides::empty(),
local,
peer_member_identity,
&local_spec,
&peer_spec,
)
.await;
return Err(error);
}
};
match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
placed_spec.clone(),
trust_authority,
)
.await
{
Ok(created) => local_trust_created = created,
Err(error) => {
self.rollback_peer_only_wire(
edge,
dsl_added,
WiringSides::empty(),
local,
peer_member_identity,
&local_spec,
&peer_spec,
)
.await;
return Err(MobError::from(error));
}
}
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
if local_trust_created
&& let Some((local_comms, placed_identity, placed_spec)) = local_lane.as_ref()
{
let placed_key = Self::trusted_peer_removal_key(placed_spec);
self.rollback_peer_only_trust(
edge,
local_comms.as_ref(),
placed_identity,
&placed_key,
"wire_placed_event_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
edge,
dsl_added,
WiringSides::empty(),
local,
peer_member_identity,
&local_spec,
&peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
// Cross-host installs ride the obligations, recorded strictly after
// the durable commit (ADJ-P4-13).
self.fold_route_install_obligations_after_wire(edge).await;
Ok(())
}
/// Unwire lanes involving a placed endpoint (multi-host §10.4 unwire
/// symmetry). Ordering: delivery cancel → supersession rollback of
/// pending Installs (DEC-P4C-10) → synchronous, machine-authorized Remove
/// ACKs from every surviving placed lane → machine unwire commit → LOCAL
/// trust removal → durable `MembersUnwired` append. Any pre-commit Remove
/// failure leaves the graph wired and triggers best-effort Install
/// compensation; no Remove ever enters the pending ledger.
#[allow(clippy::too_many_arguments)]
async fn unwire_edge_with_placement(
&mut self,
edge: &mob_dsl::WiringEdge,
local: &AgentIdentity,
peer_member_identity: &AgentIdentity,
local_endpoint: WiringEndpoint,
peer_endpoint: WiringEndpoint,
dsl_has_edge: bool,
retiring_identity: Option<&AgentIdentity>,
preserve_machine_topology: bool,
) -> Result<(), MobError> {
let local_lane: Option<(
Arc<dyn CoreCommsRuntime>,
AgentIdentity,
TrustedPeerDescriptor,
)> = match (&local_endpoint, &peer_endpoint) {
(
WiringEndpoint::Local { comms, .. },
WiringEndpoint::Placed { identity, spec, .. },
)
| (
WiringEndpoint::Placed { identity, spec, .. },
WiringEndpoint::Local { comms, .. },
) => Some((Arc::clone(comms), identity.clone(), spec.clone())),
(WiringEndpoint::Placed { .. }, WiringEndpoint::Placed { .. }) => None,
(WiringEndpoint::PeerOnly { .. }, _) | (_, WiringEndpoint::PeerOnly { .. }) => {
return Err(MobError::WiringError(format!(
"unwire between a placed member and a legacy external (peer-only) member is unsupported ('{local}' <-> '{peer_member_identity}')"
)));
}
(WiringEndpoint::Local { .. }, WiringEndpoint::Local { .. }) => {
return Err(MobError::Internal(
"unwire_edge_with_placement invoked without a placed endpoint".into(),
));
}
};
// The local member of a mixed edge (for reverse-lane registry
// hygiene after the commit).
let local_member_identity = local_lane.as_ref().map(|(_, placed_identity, _)| {
if placed_identity == local {
peer_member_identity.clone()
} else {
local.clone()
}
});
if !dsl_has_edge {
// Idempotent absence is a no-op only when live trust agrees;
// stale local trust needs MobMachine wiring authority (mirror of
// the local↔local absence rule).
if let Some((local_comms, _, placed_spec)) = local_lane.as_ref() {
let placed_key = Self::trusted_peer_removal_key(placed_spec);
let stale = local_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == placed_key);
if stale {
return Err(MobError::WiringError(format!(
"unwire for '{local}' <-> '{peer_member_identity}' requires MobMachine wiring authority"
)));
}
}
return Ok(());
}
self.cancel_peer_deliveries_for_edge(local, peer_member_identity, "members are unwiring")
.await?;
// Cursor-fence the exact durable occurrence before any remote trust
// mutation. If append later returns an error after writing, only a
// matching event strictly above this cursor proves that THIS unwire
// committed; an older identical A-B cycle cannot be mistaken for it.
let unwire_event_floor = self.events.latest_cursor().await?;
// Supersession (DEC-P4C-10): once the edge is unwired a pending
// Install can never re-validate — roll stale Installs back BEFORE
// the unwire commit.
self.rollback_superseded_install_obligations(edge)?;
// Endpoint migration can leave older generation peer ids in a
// surviving member's trust store. Placement retirement unwires this
// edge before the ordinary disposal fan-out runs, so sweep those
// historical rows here while both the edge and the exact retiring
// runtime still authorize cleanup. A local survivor consumes the
// generated comms authority directly; a placed survivor receives an
// authenticated host-addressed RemovePeerTrust and must ACK before we
// may continue.
if let Some(retiring_identity) = retiring_identity {
let retiring_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(retiring_identity).cloned()
}
.ok_or_else(|| {
MobError::RetirementTopologyIncomplete(format!(
"placed-edge retirement lost roster incarnation for '{retiring_identity}'"
))
})?;
let retained_peer_endpoints = self
.dsl_authority
.state()
.member_prior_peer_endpoints
.get(&mob_dsl::AgentIdentity::from_domain(retiring_identity))
.cloned()
.unwrap_or_default();
let surviving_local_comms = match (
local_member_identity.as_ref(),
local_lane.as_ref().map(|(comms, _, _)| comms),
) {
(Some(local_identity), Some(comms)) if local_identity != retiring_identity => {
Some(Arc::clone(comms))
}
_ => None,
};
for retained_peer_endpoint in retained_peer_endpoints {
let retained_spec = Self::peer_only_spec_from_member_endpoint(
&retained_peer_endpoint,
"placed-edge historical retirement cleanup",
)?;
let retained_peer_id = retained_peer_endpoint.peer_id.0.clone();
let handoff = self.authorize_member_endpoint_migration_trust_cleanup(
edge,
&retiring_entry,
&retained_peer_endpoint,
"placed_edge_historical_retire_trust_authority",
)?;
let authority =
handoff.unwiring_authority_for(retiring_identity, &retained_peer_id)?;
if let Some(comms) = surviving_local_comms.as_ref() {
self.apply_trusted_peer_remove(
comms.as_ref(),
retained_peer_id.clone(),
authority,
)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to remove historical retiring member trust '{retained_peer_id}' from local survivor: {error}"
))
})?;
} else if !self
.realize_historical_route_removals_before_unwire(
edge,
retiring_identity,
&retained_spec,
authority,
)
.await?
{
return Err(MobError::RetirementTopologyIncomplete(format!(
"historical retiring member trust '{retained_peer_id}' has no surviving local or placed cleanup lane"
)));
}
}
}
// Remote trust removal is a synchronous precondition of the durable
// unwire. No post-commit volatile Remove row is needed, eliminating
// the commit-unwire/crash hole. Retirement excludes the retiring
// placed target itself; ReleaseMember destroys that trust store.
self.realize_route_removals_before_unwire(edge, retiring_identity)
.await?;
if preserve_machine_topology {
let unwire_handoff = self.authorize_member_trust_unwiring(
edge,
"retiring_placed_edge_preserve_desired_topology",
)?;
if let Some((local_comms, placed_identity, placed_spec)) = local_lane.as_ref() {
let placed_key = Self::trusted_peer_removal_key(placed_spec);
let removal_authority = unwire_handoff
.require_peer_id_for(placed_identity, &placed_key)
.and_then(|()| {
unwire_handoff.unwiring_authority_for(placed_identity, &placed_key)
})?;
if let Err(error) = self
.apply_trusted_peer_remove(local_comms.as_ref(), placed_key, removal_authority)
.await
{
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(MobError::from(error));
}
}
// The old incarnation's physical trust has been removed from
// every surviving local/placed lane, while the MobMachine edge
// remains the desired topology authority for replacement repair.
// No MembersUnwired event is valid on this path.
return Ok(());
}
let unwire_handoff = match self.apply_unwire_members_idempotent(edge) {
Ok(handoff) => handoff,
Err(error) => {
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
};
// LOCAL-side trust removal; every surviving placed-side lane already
// ACKed its synchronous pre-unwire Remove above.
if let Some((local_comms, placed_identity, placed_spec)) = local_lane.as_ref() {
let placed_key = Self::trusted_peer_removal_key(placed_spec);
let removal_authority = match unwire_handoff
.require_peer_id_for(placed_identity, &placed_key)
.and_then(|()| unwire_handoff.unwiring_authority_for(placed_identity, &placed_key))
{
Ok(authority) => authority,
Err(error) => {
if let Err(rollback_error) = self.apply_wire_members_idempotent(edge) {
tracing::error!(
mob_id = %self.definition.id,
%rollback_error,
"placed unwire authorization rollback failed to restore machine edge"
);
}
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(error);
}
};
if let Err(error) = self
.apply_trusted_peer_remove(
local_comms.as_ref(),
placed_key.clone(),
removal_authority,
)
.await
{
// Restore the machine edge so failure leaves no
// half-unwired graph (the local trust row is still
// present).
if let Err(rollback_error) = self.apply_wire_members_idempotent(edge) {
tracing::warn!(
mob_id = %self.definition.id,
%rollback_error,
"placed unwire rollback: failed to restore MobMachine wire"
);
}
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(MobError::from(error));
}
}
let desired_kind = MobEventKind::MembersUnwired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
};
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired_kind.clone(),
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
let reconciled = match reconcile_exact_mob_event_after_cursor(
self.events.as_ref(),
&self.definition.id,
unwire_event_floor,
&desired_kind,
)
.await
{
Ok(reconciled) => reconciled,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"MembersUnwired append failed ({error}) and exact cursor reconciliation remained uncertain after bounded read retries ({reconcile_error}); the actor is fail-stopping and cold replay is required"
)));
}
};
if let Some(stored) = reconciled {
tracing::warn!(
mob_id = %self.definition.id,
a = %edge.a.0,
b = %edge.b.0,
cursor = stored.cursor,
%error,
"MembersUnwired append acknowledgement was lost; exact new cursor proves commit"
);
stored
} else {
// Restore the machine edge and reinstall the local trust row
// (best-effort; compensation faults are warn-logged, never
// dropped silently).
match self.apply_wire_members_idempotent(edge) {
Ok(rewire_authority) => {
if let Some((local_comms, placed_identity, placed_spec)) =
local_lane.as_ref()
&& let Ok(rewire_handoff) = rewire_authority.member_handoff()
{
let placed_key = Self::trusted_peer_removal_key(placed_spec);
let reinstall = rewire_handoff
.require_peer_id_for(placed_identity, &placed_key)
.and_then(|()| {
rewire_handoff.wiring_authority_for(
placed_identity,
&placed_key,
&self.dsl_authority,
)
});
match reinstall {
Ok(trust_authority) => {
if let Err(reinstall_error) = self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
placed_spec.clone(),
trust_authority,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %reinstall_error,
"placed unwire rollback: failed to reinstall local trust"
);
}
}
Err(authority_error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %authority_error,
"placed unwire rollback: failed to derive trust reinstall authority"
);
}
}
}
}
Err(rollback_error) => {
tracing::warn!(
mob_id = %self.definition.id,
%rollback_error,
"placed unwire rollback: failed to restore MobMachine wire"
);
}
}
self.restore_route_installs_after_unwire_abort(edge).await;
return Err(MobError::from(error));
}
}
};
self.roster.write().await.apply_event(&stored);
// Reverse-lane registry hygiene: drop the local member's acceptor
// registration when it retains no cross-host edges.
#[cfg(not(target_arch = "wasm32"))]
if let Some(local_member) = local_member_identity {
self.deregister_local_member_reverse_lane_if_unused(&local_member)
.await?;
}
#[cfg(target_arch = "wasm32")]
let _ = local_member_identity;
Ok(())
}
/// Dense local-member topology materialization path.
///
/// Dogma shape:
/// - MobMachine remains the single owner of `wiring_edges`; each new edge
/// is staged through `WireMembersWithTrust` on an isolated authority and
/// published only after the durable batch event is stored.
/// - Generated prepared-topology handoffs authorize new comms trust before
/// commit, but the trust writes run only after the live machine edge and
/// durable marker commit.
/// - Existing edges are retried through generated trust-repair authority
/// rather than being treated as projection-only `already_wired` rows.
/// - The actor owns only shell mechanics: roster snapshot validation,
/// comms trust installation, trust cleanup, and compact projection event.
/// - External peers stay on the interactive single-edge path because
/// descriptor exchange and rollback are per-peer semantics.
async fn handle_wire_members_batch(
&mut self,
requested_edges: Vec<(AgentIdentity, AgentIdentity)>,
) -> Result<super::handle::MobWireMembersBatchReport, MobError> {
let requested = requested_edges.len();
let mut normalized_edges = BTreeSet::new();
let mut endpoint_ids = BTreeSet::new();
for (left, right) in requested_edges {
if left == right {
return Err(MobError::WiringError(format!(
"wire_members_batch requires distinct members (got '{left}')"
)));
}
let (a, b) = if left <= right {
(left, right)
} else {
(right, left)
};
endpoint_ids.insert(a.clone());
endpoint_ids.insert(b.clone());
normalized_edges.insert((a, b));
}
if normalized_edges.is_empty() {
return Ok(super::handle::MobWireMembersBatchReport {
requested,
already_wired: Vec::new(),
wired: Vec::new(),
});
}
let broken_members = self
.dsl_authority
.state()
.member_restore_failures
.keys()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<BTreeSet<_>>();
for identity in &endpoint_ids {
if broken_members.contains(identity) {
return Err(MobError::WiringError(format!(
"wire_members_batch cannot wire broken member '{identity}'"
)));
}
}
let entries = {
let roster = self.roster.read().await;
let mut entries = BTreeMap::new();
for identity in &endpoint_ids {
let entry = roster
.get(identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(identity.clone()))?;
entries.insert(identity.clone(), entry);
}
entries
};
let mut endpoints = BTreeMap::new();
for (identity, entry) in entries {
match self
.resolve_wiring_endpoint(&entry, "wire_members_batch")
.await?
{
WiringEndpoint::Local { comms, spec, .. } => {
let removal_key = Self::trusted_peer_removal_key(&spec);
endpoints.insert(
identity,
BatchWiringEndpoint::Local(LocalBatchWiringEndpoint {
comms,
spec,
removal_key,
}),
);
}
// Placed members ride the batch by identity (multi-host
// §10.4): the machine spec feeds the LOCAL counterpart's
// trust row; the remote side's install rides the
// route-install obligations recorded after the batch commit.
WiringEndpoint::Placed { spec, .. } => {
let removal_key = Self::trusted_peer_removal_key(&spec);
endpoints.insert(identity, BatchWiringEndpoint::Placed { spec, removal_key });
}
// V3 legacy-external members were never batch-wireable
// (descriptor exchange and rollback are per-peer
// semantics); this reject names that class only — placed
// members are accepted above.
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire_members_batch does not support legacy external (peer-only) members (got '{identity}')"
)));
}
}
}
let mut already_wired = Vec::new();
let mut repair_edges = Vec::new();
let mut to_add = Vec::new();
for (a, b) in normalized_edges {
let event_edge = crate::event::MemberWireEdge {
a: a.clone(),
b: b.clone(),
};
let dsl_edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&a),
mob_dsl::AgentIdentity::from_domain(&b),
);
if self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &dsl_edge)
{
already_wired.push(event_edge.clone());
repair_edges.push((event_edge, dsl_edge));
} else {
to_add.push((event_edge, dsl_edge));
}
}
let mut wired = Vec::with_capacity(to_add.len());
let mut trust_applications = Vec::with_capacity((to_add.len() + repair_edges.len()) * 2);
if !to_add.is_empty() {
let prepared_batch_authority =
crate::generated::protocol_mob_member_trust_wiring::MobTopologyPreparedBatchAuthority::from_live_authority(
&self.dsl_authority,
);
let mut prepared_authority = self.dsl_authority.prepare_authority();
let transitions = prepared_authority
.apply_batch(to_add.iter().map(|(_, dsl_edge)| {
mob_dsl::MobMachineInput::WireMembersWithTrust {
edge: dsl_edge.clone(),
a_identity: dsl_edge.a.clone(),
b_identity: dsl_edge.b.clone(),
}
}))
.map_err(|error| {
MobError::Internal(format!(
"DSL authority (wire_members_batch) rejected prepared batch: {error}"
))
})?;
let mut batch_effects = Vec::new();
let mut phase_changed = false;
for ((event_edge, dsl_edge), transition) in to_add.iter().zip(transitions) {
let freshness_authority = prepared_batch_authority
.freshness_for_prepared_transitions(&self.dsl_authority, [&transition])
.map_err(MobError::WiringError)?;
let handoff = match self.member_trust_wiring_handoff_from_prepared_batch_transition(
&transition,
dsl_edge,
freshness_authority,
"wire_members_batch",
MemberTrustOperation::Wiring,
) {
Ok(handoff) => handoff,
Err(error) => return Err(error),
};
trust_applications.extend(self.batch_wire_trust_applications_from_handoff(
&endpoints, event_edge, dsl_edge, &handoff,
)?);
if transition.from_phase != transition.to_phase {
phase_changed = true;
}
for effect in transition.effects() {
if let mob_dsl::MobMachineEffect::EmitWiringLifecycleNotice {
kind: mob_dsl::WiringLifecycleKind::Wired,
edge,
} = effect
{
wired.push(crate::event::MemberWireEdge {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
});
}
}
batch_effects.extend(transition.into_effects());
}
let prepared = PreparedDslInput {
authority: prepared_authority,
effects: batch_effects,
phase_changed,
};
let events = self.events.clone();
let mob_id = self.definition.id.clone();
let wired_for_event = wired.clone();
let stored = self
.commit_prepared_dsl_input_after(prepared, move || async move {
let event = NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MembersWiredBatch {
edges: wired_for_event,
},
};
events.append(event).await.map_err(MobError::from)
})
.await?;
self.roster.write().await.apply_event(&stored);
}
for (event_edge, dsl_edge) in &repair_edges {
let handoff = self.authorize_batch_member_trust_repair(dsl_edge)?;
trust_applications.extend(self.batch_wire_trust_applications_from_handoff(
&endpoints, event_edge, dsl_edge, &handoff,
)?);
}
let owner_token = self.dsl_authority.generated_authority_owner_token();
self.apply_batch_wire_trust_applications(trust_applications, &owner_token)
.await?;
// Cross-host installs for every committed edge with a placed
// endpoint — new AND repair edges (repair = the retry lane, §9) —
// recorded strictly after the durable batch commit (ADJ-P4-13).
// The BTreeSet dedupes obligations so a dense batch against one
// host does not re-send identical installs.
let placed_obligations: BTreeSet<mob_dsl::RouteInstallObligation> = to_add
.iter()
.chain(repair_edges.iter())
.flat_map(|(_, dsl_edge)| {
self.route_install_obligations_for_edge(
dsl_edge,
mob_dsl::RouteObligationKind::Install,
)
})
.collect();
self.record_and_realize_route_install_obligations(
placed_obligations.into_iter().collect(),
"wire_members_batch_route_install",
)
.await;
tracing::info!(
mob_id = %self.definition.id,
requested,
wired = wired.len(),
already_wired = already_wired.len(),
participants = endpoints.len(),
"wire_members_batch materialized local topology"
);
Ok(super::handle::MobWireMembersBatchReport {
requested,
already_wired,
wired,
})
}
/// Install (or clear) the host-owned outbound content-taint declaration
/// on a member's comms runtime.
///
/// Carrier config, not machine state: the HOST owns the "this member's
/// session ingested untrusted content" fact (its content-trust tracker);
/// this operation makes the member's runtime the authenticated carrier of
/// that declaration — stamped inside the signed region of every outbound
/// content-bearing envelope until changed. The declaration is in-memory
/// runtime state: respawn/reset mint a fresh runtime with no declaration,
/// which aligns with fresh-context taint semantics (hosts re-declare when
/// their tracker re-marks the new context).
///
/// Local members install directly on their session comms runtime (typed
/// failure when the runtime does not support the declaration — never a
/// silent no-op). External members relay the declaration to the remote
/// runtime over the supervisor bridge.
async fn declare_member_outbound_taint(
&mut self,
identity: AgentIdentity,
taint: Option<meerkat_core::comms::SenderContentTaint>,
) -> Result<(), MobError> {
self.require_member_operation_eligible()?;
self.ensure_member_not_broken(&identity).await?;
let entry = {
let roster = self.roster.read().await;
roster
.get(&identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(identity.clone()))?
};
let placed = super::member_runtime_is_host_owned(self.dsl_authority.state(), &identity);
if !placed {
if let Some(comms) = self.provisioner_comms(&entry.member_ref).await {
return comms.set_outbound_content_taint(taint).map_err(|error| {
// "This runtime cannot carry the declaration" is the SAME
// semantic condition as the no-runtime/no-binding branch
// below — one typed terminal class, regardless of which
// mechanical branch detected it. Other send faults keep
// their typed comms error.
match error {
meerkat_core::comms::SendError::Unsupported(_) => {
MobError::MissingMemberCapability {
member_id: identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "declare member outbound taint",
}
}
other => MobError::CommsError(other),
}
});
}
}
// A placed relay is admitted only from the exact active carrier and
// carries that full incarnation to the receiver. Bound alone is not
// enough: a retained G1 route must not mutate a revived G2 runtime.
let routing_target = if placed {
super::bridge_protocol::BridgeOutboundTaintTarget::Placed(
self.placed_member_incarnation(&entry)?,
)
} else {
super::bridge_protocol::BridgeOutboundTaintTarget::PeerOnly
};
// External-bound member: the outbound comms runtime lives in the
// remote process — relay the declaration over the supervisor bridge.
let Some(binding) = Self::runtime_binding_for_entry(&entry) else {
return Err(MobError::MissingMemberCapability {
member_id: identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "declare member outbound taint",
});
};
let peer = Self::peer_only_spec_for_binding(&binding, "declare_member_outbound_taint")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(&binding))
.await?;
let sup_payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let target = if sup_payload.protocol_version
< super::bridge_protocol::BridgeProtocolVersion::V4
{
match routing_target {
super::bridge_protocol::BridgeOutboundTaintTarget::PeerOnly => None,
super::bridge_protocol::BridgeOutboundTaintTarget::Placed(_) => {
return Err(MobError::Internal(format!(
"placed outbound-taint declaration for '{identity}' requires supervisor bridge protocol V4"
)));
}
}
} else {
Some(routing_target)
};
let payload = super::bridge_protocol::BridgeOutboundTaintPayload {
supervisor: sup_payload.supervisor,
epoch: sup_payload.epoch,
protocol_version: sup_payload.protocol_version,
target,
taint,
};
let command = super::bridge_protocol::BridgeCommand::DeclareMemberOutboundTaint(payload);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(&peer, &command, std::time::Duration::from_secs(10))
.await?;
Ok(())
}
async fn prepare_send_peer_message(
&mut self,
from: AgentIdentity,
to: AgentIdentity,
content: ContentInput,
handling_mode: meerkat_core::types::HandlingMode,
) -> Result<PeerMessageDeliveryPlan, MobError> {
self.require_member_operation_eligible()?;
if from == to {
return Err(MobError::WiringError(format!(
"peer message requires distinct members (got '{from}')"
)));
}
self.ensure_member_not_broken(&from).await?;
self.ensure_member_not_broken(&to).await?;
let (sender_entry, recipient_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&from)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(from.clone()))?,
roster
.get(&to)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(to.clone()))?,
)
};
let edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(from.as_str())),
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(to.as_str())),
);
let wired_by_machine = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
if !wired_by_machine {
return Err(MobError::WiringError(format!(
"peer message requires wired members '{from}' and '{to}'"
)));
}
if super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&sender_entry.agent_identity,
) {
// This API emits from the sender's own comms runtime. A placed
// sender's runtime is owned by its member host; there is no
// controller-side proxy command for impersonating that sender.
return Err(MobError::WiringError(format!(
"peer message from placed member '{from}' must originate on its member host"
)));
}
let sender_comms = self
.provisioner_comms(&sender_entry.member_ref)
.await
.ok_or_else(|| {
MobError::WiringError(format!(
"peer message requires sender comms runtime for '{from}'"
))
})?;
let recipient_endpoint = self
.resolve_wiring_endpoint(&recipient_entry, "peer_message")
.await?;
let expected_recipient = match &recipient_endpoint {
WiringEndpoint::Placed { .. } => {
let expected = self.placed_member_incarnation(&recipient_entry)?;
Some(meerkat_core::comms::PeerRecipientIncarnation {
mob_id: expected.mob_id,
agent_identity: expected.agent_identity,
host_id: expected.host_id,
binding_generation: expected.binding_generation,
member_session_id: expected.member_session_id,
generation: expected.generation,
fence_token: expected.fence_token,
})
}
WiringEndpoint::Local { .. } | WiringEndpoint::PeerOnly { .. } => None,
};
let recipient_spec = recipient_endpoint.spec().clone();
let route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
let (body, blocks) = match content {
ContentInput::Text(body) => (body, None),
ContentInput::Blocks(blocks) => {
(meerkat_core::types::text_content(&blocks), Some(blocks))
}
};
let command = match expected_recipient {
Some(expected_recipient) => CommsCommand::IncarnationFencedPeerMessage {
to: route,
body,
blocks,
content_taint: None,
handling_mode,
objective_id: None,
expected_recipient,
},
None => CommsCommand::PeerMessage {
to: route,
body,
blocks,
content_taint: None,
handling_mode,
objective_id: None,
},
};
Ok(PeerMessageDeliveryPlan {
from,
to,
sender_comms,
command,
})
}
fn apply_wire_members_idempotent(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<WireTrustAuthority, MobError> {
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::WireMembers { edge: edge.clone() },
"wire_members",
)?;
let graph_added =
Self::wire_members_disposition_from_effects(&effects, edge, "wire_members")?;
let handoff = self.authorize_member_trust_wiring(
edge,
"wire_members_trust_authority",
if graph_added {
MemberTrustOperation::Wiring
} else {
MemberTrustOperation::Repair
},
)?;
if graph_added {
Ok(WireTrustAuthority::GraphAdded(handoff))
} else {
Ok(WireTrustAuthority::RepairRequested(handoff))
}
}
fn authorize_batch_member_trust_repair(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::WireMembersWithTrust {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
"wire_members_batch_repair_request",
)?;
let graph_added = Self::wire_members_disposition_from_effects(
transition.effects(),
edge,
"wire_members_batch_repair_request",
)?;
if graph_added {
return Err(MobError::WiringError(
"wire_members_batch repair unexpectedly produced a generated graph change"
.to_string(),
));
}
self.authorize_member_trust_wiring(
edge,
"wire_members_batch_repair_trust_authority",
MemberTrustOperation::Repair,
)
}
fn batch_wire_trust_applications_from_handoff(
&self,
endpoints: &BTreeMap<AgentIdentity, BatchWiringEndpoint>,
event_edge: &crate::event::MemberWireEdge,
dsl_edge: &mob_dsl::WiringEdge,
handoff: &MemberTrustHandoff,
) -> Result<Vec<BatchWireTrustApplication>, MobError> {
let left = endpoints.get(&event_edge.a).ok_or_else(|| {
MobError::WiringError(format!(
"wire_members_batch missing endpoint '{}'",
event_edge.a
))
})?;
let right = endpoints.get(&event_edge.b).ok_or_else(|| {
MobError::WiringError(format!(
"wire_members_batch missing endpoint '{}'",
event_edge.b
))
})?;
// A LOCAL side yields its trust application against the
// counterpart's spec (the machine spec when the counterpart is
// placed); a PLACED side yields NO application — its install rides
// the route-install obligation recorded after the batch commit.
let mut applications = Vec::new();
if let BatchWiringEndpoint::Local(left_local) = left {
let right_peer_id = right.removal_key().to_string();
handoff.require_peer_id_for(&event_edge.b, &right_peer_id)?;
let left_trust_authority =
handoff.add_authority_for(&event_edge.b, &right_peer_id, &self.dsl_authority)?;
applications.push(BatchWireTrustApplication {
edge: dsl_edge.clone(),
identity: event_edge.b.clone(),
peer_id: right_peer_id,
comms: left_local.comms.clone(),
peer: right.spec().clone(),
authority: left_trust_authority,
});
}
if let BatchWiringEndpoint::Local(right_local) = right {
let left_peer_id = left.removal_key().to_string();
handoff.require_peer_id_for(&event_edge.a, &left_peer_id)?;
let right_trust_authority =
handoff.add_authority_for(&event_edge.a, &left_peer_id, &self.dsl_authority)?;
applications.push(BatchWireTrustApplication {
edge: dsl_edge.clone(),
identity: event_edge.a.clone(),
peer_id: left_peer_id,
comms: right_local.comms.clone(),
peer: left.spec().clone(),
authority: right_trust_authority,
});
}
Ok(applications)
}
async fn apply_batch_wire_trust_applications(
&mut self,
applications: Vec<BatchWireTrustApplication>,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), MobError> {
let mut installed = Vec::new();
for application in applications {
let rollback = BatchWireTrustRollback {
edge: application.edge.clone(),
identity: application.identity.clone(),
peer_id: application.peer_id.clone(),
comms: application.comms.clone(),
};
match Self::apply_trusted_peer_add_with_owner_token_report(
application.comms.as_ref(),
application.peer,
application.authority,
owner_token,
)
.await
{
Ok(created) => {
if created {
installed.push(rollback);
}
}
Err(error) => {
self.rollback_batch_wire_trust_applications(installed, owner_token)
.await;
return Err(MobError::from(error));
}
}
}
Ok(())
}
async fn rollback_batch_wire_trust_applications(
&mut self,
installed: Vec<BatchWireTrustRollback>,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) {
for rollback in installed.into_iter().rev() {
let handoff = match self.authorize_member_trust_unwiring(
&rollback.edge,
"wire_members_batch_trust_failure_rollback",
) {
Ok(handoff) => handoff,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch could not authorize trust rollback after trust failure"
);
continue;
}
};
let authority =
match handoff.unwiring_authority_for(&rollback.identity, &rollback.peer_id) {
Ok(authority) => authority,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch could not derive generated trust rollback authority"
);
continue;
}
};
if let Err(error) = Self::apply_trusted_peer_remove_with_owner_token(
rollback.comms.as_ref(),
rollback.peer_id,
authority,
owner_token,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch failed to rollback previously installed trust after trust failure"
);
}
}
}
/// Compensate the comms-trust half of a failed peer-only wire.
///
/// Mirrors [`Self::rollback_wire_side_effects`] for the peer-only paths:
/// the rollback-authority derivation never `?`-propagates (a derivation
/// failure must not abort the caller before its DSL `UnwireMembers` runs),
/// and every compensation fault is warn-logged rather than dropped via
/// `let _ =`. Callers invoke this strictly before `rollback_peer_only_wire`
/// so the DSL edge is always unwound regardless of the trust outcome.
async fn rollback_peer_only_trust(
&mut self,
edge: &mob_dsl::WiringEdge,
comms: &(dyn CoreCommsRuntime + '_),
identity: &AgentIdentity,
removal_key: &str,
context: &str,
) {
let rollback_handoff = match self.authorize_member_trust_unwiring(edge, context) {
Ok(handoff) => handoff,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to obtain generated trust removal authority"
);
return;
}
};
let authority = match rollback_handoff.unwiring_authority_for(identity, removal_key) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to build generated trust removal authority"
);
return;
}
};
if let Err(err) = self
.apply_trusted_peer_remove(comms, removal_key.to_string(), authority)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to remove trust"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn rollback_peer_only_wire(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_added: bool,
installed_sides: WiringSides,
_local_identity: &AgentIdentity,
_peer_identity: &AgentIdentity,
local_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
) {
if !dsl_added {
return;
}
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"peer_only_wire_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to revert DSL wire"
);
return;
}
if installed_sides.has_local()
&& let Err(error) = self
.unwire_peer_only_recipient(
local_spec,
None,
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to unwire local recipient"
);
}
if installed_sides.has_peer()
&& let Err(error) = self
.unwire_peer_only_recipient(
peer_spec,
None,
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to unwire peer recipient"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn rollback_peer_only_unwire(
&mut self,
edge: &mob_dsl::WiringEdge,
sides_to_rewire: WiringSides,
_local_identity: &AgentIdentity,
_peer_identity: &AgentIdentity,
local_spec: &TrustedPeerDescriptor,
local_binding: &crate::RuntimeBinding,
peer_spec: &TrustedPeerDescriptor,
peer_binding: &crate::RuntimeBinding,
) {
if let Err(error) = self.apply_wire_members_idempotent(edge) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to restore MobMachine wire"
);
return;
}
if sides_to_rewire.has_local() {
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to rewire local recipient"
);
}
}
if sides_to_rewire.has_peer() {
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to rewire peer recipient"
);
}
}
}
/// Unwind side effects from a failed local-local wire. Best-effort:
/// logs on compensation failure since we've already decided to surface
/// the original error to the caller.
#[allow(clippy::too_many_arguments)]
async fn rollback_wire_side_effects(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_added: bool,
installed_local_trust: bool,
installed_peer_trust: bool,
local_comms: &Arc<dyn CoreCommsRuntime>,
peer_comms: &Arc<dyn CoreCommsRuntime>,
local_peer_id: &str,
peer_peer_id: &str,
_handoff: &MemberTrustHandoff,
) {
let local_identity = AgentIdentity::from(edge.a.0.as_str());
let peer_identity = AgentIdentity::from(edge.b.0.as_str());
let rollback_handoff = if installed_local_trust || installed_peer_trust {
match self
.authorize_member_trust_unwiring(edge, "wire_members_rollback_trust_authority")
{
Ok(handoff) => Some(handoff),
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to obtain generated trust removal authority"
);
None
}
}
} else {
None
};
if installed_local_trust {
if let Err(err) = self.apply_trusted_peer_remove(
local_comms.as_ref(),
peer_peer_id.to_string(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"wire rollback has no generated local trust removal authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.unwiring_authority_for(&peer_identity, peer_peer_id)
}) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to build generated local trust removal authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to remove local trust"
);
}
}
if installed_peer_trust {
if let Err(err) = self.apply_trusted_peer_remove(
peer_comms.as_ref(),
local_peer_id.to_string(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"wire rollback has no generated peer trust removal authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.unwiring_authority_for(&local_identity, local_peer_id)
}) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to build generated peer trust removal authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to remove peer trust"
);
}
}
if dsl_added {
if let Err(err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"wire_members_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to revert DSL wire"
);
}
}
}
/// D-wire-handler (#26): forward an unwire command to the MobMachine DSL.
///
/// Mirror of [`handle_wire`]: submits
/// `MobMachineInput::UnwireMembers { edge }` and records
/// `MobEventKind::MembersUnwired { a, b }` on acceptance. Already-absent
/// idempotency is a generated no-op transition.
async fn handle_unwire(
&mut self,
local: AgentIdentity,
target: super::handle::PeerTarget,
) -> Result<(), MobError> {
self.handle_unwire_inner(local, target, None, false).await
}
async fn handle_unwire_for_retirement(
&mut self,
retiring: AgentIdentity,
survivor: AgentIdentity,
preserve_machine_topology: bool,
) -> Result<(), MobError> {
self.handle_unwire_inner(
retiring.clone(),
super::handle::PeerTarget::Local(survivor),
Some(retiring),
preserve_machine_topology,
)
.await
}
async fn handle_unwire_inner(
&mut self,
local: AgentIdentity,
target: super::handle::PeerTarget,
retiring_identity: Option<AgentIdentity>,
preserve_machine_topology: bool,
) -> Result<(), MobError> {
let peer_identity = match target {
super::handle::PeerTarget::Local(id) => {
// Callers commonly unwire an external peer by name projected
// as a Local target. The machine-owned external edge decides
// that route; the roster projection is display-only.
let target_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(id.as_str()));
let target_is_member = self
.dsl_authority
.state()
.identity_to_runtime
.contains_key(&target_identity);
let external_peer_name =
meerkat_core::comms::PeerName::new(id.as_str().to_string())
.ok()
.filter(|peer_name| {
let local_identity = AgentIdentity::from(local.as_str());
self.external_peer_edge_for_name(&local_identity, peer_name)
.is_some()
});
if !target_is_member && let Some(peer_name) = external_peer_name {
return self.handle_unwire_external(local, peer_name, None).await;
}
id
}
super::handle::PeerTarget::ExternalName(peer_name) => {
return self.handle_unwire_external(local, peer_name, None).await;
}
super::handle::PeerTarget::ExternalBinding(binding) => {
let descriptor = Self::trusted_peer_descriptor_from_external_binding(binding)?;
let peer_name = descriptor.name.clone();
return self
.handle_unwire_external(local, peer_name, Some(descriptor))
.await;
}
super::handle::PeerTarget::External(descriptor) => {
let peer_name = descriptor.name.clone();
return self
.handle_unwire_external(local, peer_name, Some(descriptor))
.await;
}
};
let local_identity = AgentIdentity::from(local.as_str());
if local_identity == peer_identity {
return Err(MobError::WiringError(format!(
"unwire requires distinct peers (got '{local}')"
)));
}
let peer_member_identity = AgentIdentity::from(peer_identity.as_str());
let dsl_a = mob_dsl::AgentIdentity::from_domain(&local_identity);
let dsl_b = mob_dsl::AgentIdentity::from_domain(&peer_identity);
let edge = mob_dsl::WiringEdge::new(dsl_a, dsl_b);
let (local_entry, peer_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&local)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?,
roster
.get(&peer_member_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(peer_member_identity.clone()))?,
)
};
let dsl_has_edge = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
// Resolve endpoints. Failing here leaves the DSL + roster
// untouched — matches the zero-side-effect expectations.
let (local_endpoint, peer_endpoint) = if retiring_identity.is_some() {
(
self.resolve_wiring_endpoint_for_retirement(&local_entry)
.await?,
self.resolve_wiring_endpoint_for_retirement(&peer_entry)
.await?,
)
} else {
(
self.resolve_wiring_endpoint(&local_entry, "unwire").await?,
self.resolve_wiring_endpoint(&peer_entry, "unwire").await?,
)
};
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
if !dsl_has_edge {
return Err(MobError::WiringError(format!(
"peer-only unwire for '{local}' <-> '{peer_identity}' requires MobMachine wiring authority"
)));
}
self.cancel_peer_deliveries_for_edge(
&local,
&peer_member_identity,
"members are unwiring",
)
.await?;
let _unwire_handoff = self.apply_unwire_members_idempotent(&edge)?;
if let Err(error) = self
.unwire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_unwire(
&edge,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(error);
}
if let Err(error) = self
.unwire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_unwire(
&edge,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersUnwired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
self.rollback_peer_only_unwire(
&edge,
WiringSides::both(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
if matches!(local_endpoint, WiringEndpoint::Placed { .. })
|| matches!(peer_endpoint, WiringEndpoint::Placed { .. })
{
return self
.unwire_edge_with_placement(
&edge,
&local,
&peer_member_identity,
local_endpoint,
peer_endpoint,
dsl_has_edge,
retiring_identity.as_ref(),
preserve_machine_topology,
)
.await;
}
let (local_comms, local_spec) = match local_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"unwire requires local session comms runtime for '{local}'"
)));
}
WiringEndpoint::Placed { .. } => {
return Err(MobError::Internal(format!(
"unwire placed lane for '{local}' bypassed its dedicated handler"
)));
}
};
let (peer_comms, peer_spec) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::Placed { .. } => {
return Err(MobError::Internal(format!(
"unwire placed lane for '{peer_identity}' bypassed its dedicated handler"
)));
}
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"unwire requires local session comms runtime for '{peer_identity}'"
)));
}
};
let local_peer_id = Self::trusted_peer_removal_key(&local_spec);
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
// Idempotent absence stays a no-op only when live comms agrees.
// Stale trust cleanup needs MobMachine authority instead of using the
// roster projection as a behavior owner.
if !dsl_has_edge {
let local_has_stale_trust = local_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == peer_peer_id);
let peer_has_stale_trust = peer_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == local_peer_id);
if local_has_stale_trust || peer_has_stale_trust {
return Err(MobError::WiringError(format!(
"unwire for '{local}' <-> '{peer_identity}' requires MobMachine wiring authority"
)));
}
return Ok(());
}
self.cancel_peer_deliveries_for_edge(&local, &peer_member_identity, "members are unwiring")
.await?;
// Submit DSL input first. Already-absent idempotency is a generated
// no-op transition; only `WiringGraphChanged` means rollback must
// re-submit the wire.
let unwire_handoff = self.apply_unwire_members_idempotent(&edge)?;
let dsl_removed = true;
let mut removed_local_trust = false;
let mut removed_peer_trust = false;
let mut sent_unwired_from_local = false;
let mut sent_unwired_from_peer = false;
// Notify peer_unwired on both sides BEFORE trust removal so the
// notifications can still be delivered (the send path resolves
// the recipient by name in the comms' trusted-peers table). If a
// notification fails, compensate the prior side's notification
// by re-sending peer_added so the observable intent stream stays
// balanced.
if let Err(err) = self
.notify_peer_event(
"mob.peer_unwired",
&peer_spec,
&peer_member_identity,
&peer_entry,
&local_comms,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(err);
}
sent_unwired_from_local = true;
if let Err(err) = self
.notify_peer_event(
"mob.peer_unwired",
&local_spec,
&local,
&local_entry,
&peer_comms,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(err);
}
sent_unwired_from_peer = true;
// A-side trust removal (after notifications succeeded).
unwire_handoff.require_peer_id_for(&peer_member_identity, &peer_peer_id)?;
if let Err(err) = self
.apply_trusted_peer_remove(
local_comms.as_ref(),
peer_peer_id.clone(),
unwire_handoff.unwiring_authority_for(&peer_member_identity, &peer_peer_id)?,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
removed_local_trust = true;
// B-side trust removal.
unwire_handoff.require_peer_id_for(&local, &local_peer_id)?;
if let Err(err) = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
local_peer_id.clone(),
unwire_handoff.unwiring_authority_for(&local, &local_peer_id)?,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
removed_peer_trust = true;
// Append MembersUnwired — rollback on failure.
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersUnwired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(err) => {
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
fn apply_unwire_members_idempotent(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<MemberTrustHandoff, MobError> {
let handoff =
self.authorize_member_trust_unwiring(edge, "unwire_members_trust_authority")?;
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"unwire_members",
)?;
if Self::effects_include_wiring_graph_change(&effects) {
Ok(handoff)
} else {
Err(MobError::WiringError(format!(
"unwire_members produced no generated wiring graph authority for edge {edge:?}"
)))
}
}
fn external_peer_edge(
local_identity: &AgentIdentity,
spec: &TrustedPeerDescriptor,
) -> mob_dsl::ExternalPeerEdge {
mob_dsl::ExternalPeerEdge::new(
mob_dsl::AgentIdentity::from_domain(local_identity),
mob_dsl::ExternalPeerEndpoint::from(spec),
)
}
fn external_peer_key(
local_identity: &AgentIdentity,
peer_name: &meerkat_core::comms::PeerName,
) -> mob_dsl::ExternalPeerKey {
mob_dsl::ExternalPeerKey::new(
mob_dsl::AgentIdentity::from_domain(local_identity),
mob_dsl::PeerName::from(peer_name.as_str()),
)
}
fn external_peer_key_for_edge(edge: &mob_dsl::ExternalPeerEdge) -> mob_dsl::ExternalPeerKey {
mob_dsl::ExternalPeerKey::new(edge.local.clone(), edge.endpoint.name.clone())
}
fn external_peer_edge_for_name(
&self,
local_identity: &AgentIdentity,
peer_name: &meerkat_core::comms::PeerName,
) -> Option<mob_dsl::ExternalPeerEdge> {
let key = Self::external_peer_key(local_identity, peer_name);
self.dsl_authority
.state()
.external_peer_edges_by_key
.get(&key)
.cloned()
}
fn apply_wire_external_peer_idempotent(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<WireTrustAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"wire_external_peer",
)?;
self.wire_external_authority_from_transition(&transition, edge, "wire_external_peer")
}
async fn apply_external_peer_reciprocal_trust(
&mut self,
key: mob_dsl::ExternalPeerKey,
target_comms: Arc<dyn CoreCommsRuntime>,
peer: TrustedPeerDescriptor,
) -> Result<(), MobError> {
let local_identity = AgentIdentity::from(key.local.0.as_str());
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeExternalPeerReciprocalTrust {
key: key.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&local_identity),
},
"apply_external_peer_reciprocal_trust",
)?;
let Some(obligation) =
crate::generated::protocol_mob_external_peer_reciprocal_trust::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_reciprocal_trust::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.key() == &key)
else {
return Err(MobError::WiringError(
"MobMachine produced no external reciprocal trust obligation".to_string(),
));
};
let target_peer_id = target_comms.peer_id().ok_or_else(|| {
MobError::WiringError(
"external reciprocal trust target runtime did not expose peer_id".to_string(),
)
})?;
if target_peer_id.to_string() != obligation.edge().endpoint.peer_id.0 {
return Err(MobError::WiringError(format!(
"external reciprocal trust target peer_id {target_peer_id} does not match MobMachine external edge peer_id {}",
obligation.edge().endpoint.peer_id.0
)));
}
let peer_id = obligation.peer_id().0.clone();
let authority = crate::generated::protocol_mob_external_peer_reciprocal_trust::reciprocal_wiring_authority_for_peer(
&obligation,
&peer_id,
)
.map_err(MobError::WiringError)?;
self.apply_trusted_peer_add(target_comms.as_ref(), peer, authority)
.await
.map_err(MobError::CommsError)
}
fn apply_unwire_external_peer_idempotent(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<Option<CommsTrustMutationAuthority>, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::UnwireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"unwire_external_peer",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects) {
return Ok(None);
}
let obligation =
crate::generated::protocol_mob_external_peer_trust_unwiring::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(
"unwire_external_peer produced graph authority without generated unwiring trust obligation"
.to_string(),
)
})?;
Ok(Some(
crate::generated::protocol_mob_external_peer_trust_unwiring::unwiring_authority_for_peer(
&obligation,
edge.endpoint.peer_id.0.as_str(),
)
.map_err(MobError::WiringError)?,
))
}
fn apply_cleanup_retiring_external_peer(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<CommsTrustMutationAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::CleanupRetiringExternalPeer {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"cleanup_retiring_external_peer",
)?;
let obligation = crate::generated::protocol_mob_external_peer_trust_unwiring::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(
self.dsl_topology_epoch.clone(),
Arc::clone(&self.dsl_authority_owner_token),
),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(
"retiring external cleanup produced no generated unwiring trust obligation"
.to_string(),
)
})?;
crate::generated::protocol_mob_external_peer_trust_unwiring::unwiring_authority_for_peer(
&obligation,
edge.endpoint.peer_id.0.as_str(),
)
.map_err(MobError::WiringError)
}
fn apply_restore_retiring_external_peer(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<WireTrustAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RestoreRetiringExternalPeer {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"restore_retiring_external_peer",
)?;
self.wire_external_authority_from_transition(
&transition,
edge,
"restore_retiring_external_peer",
)
}
fn apply_cleanup_retiring_external_peer_observed_absent(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<(), MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::CleanupRetiringExternalPeerObservedAbsent {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"cleanup_retiring_external_peer_observed_absent",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects)
|| effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::ExternalPeerTrustUnwiringRequested { .. }
)
})
{
return Err(MobError::WiringError(
"observed-absent external cleanup must change topology without publishing live trust work"
.to_string(),
));
}
Ok(())
}
fn apply_restore_retiring_external_peer_observed_absent(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<(), MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RestoreRetiringExternalPeerObservedAbsent {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"restore_retiring_external_peer_observed_absent",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects)
|| effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::ExternalPeerTrustWiringRequested { .. }
)
})
{
return Err(MobError::WiringError(
"observed-absent external restore must change topology without publishing live trust work"
.to_string(),
));
}
Ok(())
}
/// Unwind side effects from a failed local-local unwire. Best-effort.
///
/// Re-installs trust on sides where trust was removed, and emits
/// compensating `mob.peer_added` notifications on sides that already
/// sent `mob.peer_unwired` so the observable intent stream stays
/// balanced. The DSL wire is re-submitted if it was removed.
#[allow(clippy::too_many_arguments)]
async fn rollback_unwire_side_effects(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_removed: bool,
removed_local_trust: bool,
removed_peer_trust: bool,
sent_unwired_from_local: bool,
sent_unwired_from_peer: bool,
local_comms: &Arc<dyn CoreCommsRuntime>,
peer_comms: &Arc<dyn CoreCommsRuntime>,
local_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
local_member_identity: &AgentIdentity,
peer_member_identity: &AgentIdentity,
local_entry: &RosterEntry,
peer_entry: &RosterEntry,
_handoff: &MemberTrustHandoff,
) {
let rollback_handoff = if dsl_removed {
match self.apply_wire_members_idempotent(edge) {
Ok(authority) => match authority.member_handoff() {
Ok(handoff) => Some(handoff.clone()),
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: generated wire authority did not include member handoff"
);
None
}
},
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-submit DSL wire for generated trust authority"
);
None
}
}
} else {
None
};
if removed_local_trust {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
if let Err(err) = self.apply_trusted_peer_add(
local_comms.as_ref(),
peer_spec.clone(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"unwire rollback has no generated local trust add authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.add_authority_for(
peer_member_identity,
&peer_key,
&self.dsl_authority,
)
})
{
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to build generated local trust add authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-install local trust"
);
}
}
if removed_peer_trust {
let local_key = Self::trusted_peer_removal_key(local_spec);
if let Err(err) = self.apply_trusted_peer_add(
peer_comms.as_ref(),
local_spec.clone(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"unwire rollback has no generated peer trust add authority".to_string(),
)
})
.and_then(|handoff| {
handoff.add_authority_for(
local_member_identity,
&local_key,
&self.dsl_authority,
)
})
{
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to build generated peer trust add authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-install peer trust"
);
}
}
// Compensate peer_unwired notifications by re-sending peer_added.
if sent_unwired_from_local {
if let Err(err) = self
.notify_peer_added(local_comms, peer_spec, peer_member_identity, peer_entry)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to send compensating peer_added from local side"
);
}
}
if sent_unwired_from_peer {
if let Err(err) = self
.notify_peer_added(peer_comms, local_spec, local_member_identity, local_entry)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to send compensating peer_added from peer side"
);
}
}
}
fn trusted_peer_descriptor_from_external_binding(
binding: super::handle::ExternalPeerBindingSpec,
) -> Result<TrustedPeerDescriptor, MobError> {
let name = PeerName::new(binding.name.clone()).map_err(|error| {
MobError::WiringError(format!(
"external binding has invalid peer name '{}': {error}",
binding.name
))
})?;
let resolved = binding
.identity
.resolve()
.map_err(|error| MobError::WiringError(error.to_string()))?;
let _address = PeerAddress::parse(&binding.address).map_err(|error| {
MobError::WiringError(format!(
"external binding has invalid peer address '{}': {error}",
binding.address
))
})?;
TrustedPeerDescriptor::unsigned_with_pubkey(
name.as_str().to_string(),
resolved.peer_id.to_string(),
resolved.pubkey,
binding.address,
)
.map_err(|error| MobError::WiringError(format!("external binding is invalid: {error}")))
}
/// Wire a local member to an external trusted peer.
///
/// `MobMachineInput::WireExternalPeer` owns the descriptor-bearing trust
/// edge; the shell mechanically installs the admitted descriptor into the
/// local comms runtime and projects the event only after the machine
/// admits the edge.
///
/// Idempotent: a second wire of the same (local, external_name) edge
/// with the same descriptor is treated as a no-op success.
async fn handle_wire_external(
&mut self,
local: AgentIdentity,
spec: TrustedPeerDescriptor,
) -> Result<(), MobError> {
TrustedPeerDescriptor::validate_pubkey_for_peer_id(spec.peer_id, &spec.pubkey).map_err(
|error| MobError::WiringError(format!("external peer descriptor is invalid: {error}")),
)?;
let local_identity = AgentIdentity::from(local.as_str());
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &local_identity) {
return Err(MobError::WiringError(format!(
"wire between placed member '{local}' and an external peer is unsupported"
)));
}
let external_identity = AgentIdentity::from(spec.name.as_str());
if local_identity == external_identity {
return Err(MobError::WiringError(format!(
"wire requires distinct members (got '{local}')"
)));
}
let edge = Self::external_peer_edge(&local_identity, &spec);
let key = Self::external_peer_key_for_edge(&edge);
self.probe_command_admission(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
MobState::Running,
"wire_external_peer_command_admission",
)?;
// Look up the local member's roster entry and session binding.
let member_ref = {
let roster = self.roster.read().await;
let entry = roster
.get(&local)
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?;
entry.member_ref.clone()
};
// Resolve the local session's comms runtime for trust install.
let comms = self.provisioner_comms(&member_ref).await.ok_or_else(|| {
MobError::WiringError(format!(
"wire requires comms runtime for '{local}' (external peer wire)"
))
})?;
let authority = self.apply_wire_external_peer_idempotent(&key, &edge)?;
let removal_key = Self::trusted_peer_removal_key(&spec);
if authority.is_repair() {
self.apply_trusted_peer_add(
comms.as_ref(),
spec.clone(),
authority.external_authority()?.clone(),
)
.await?;
return Ok(());
}
let dsl_added = authority.dsl_added();
// Install trust on the local's session comms runtime.
if let Err(error) = self
.apply_trusted_peer_add(
comms.as_ref(),
spec.clone(),
authority.external_authority()?.clone(),
)
.await
{
self.rollback_external_wire_dsl(&key, &edge, dsl_added)
.await;
return Err(MobError::from(error));
}
// Append ExternalPeerWired — if the append fails, compensate by
// rolling back the trust install so failure leaves no side effect.
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerWired {
local: local_identity,
spec: spec.clone(),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_err) => {
let rollback_handoff = if dsl_added {
match self.apply_unwire_external_peer_idempotent(&key, &edge) {
Ok(Some(handoff)) => Some(handoff),
Ok(None) => None,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
%error,
"failed to obtain generated external unwiring authority after event append failure"
);
None
}
}
} else {
None
};
if let Some(rollback_handoff) = rollback_handoff {
if let Err(rollback_err) = self
.apply_trusted_peer_remove(
comms.as_ref(),
removal_key.clone(),
rollback_handoff,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external trust install after event append failure"
);
}
} else {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
"external trust install rollback skipped without generated unwiring authority"
);
}
return Err(MobError::from(append_err));
}
};
// Mirror the event through the roster projection — same pattern
// as MembersWired in `handle_wire`.
self.roster.write().await.apply_event(&stored);
Ok(())
}
async fn rollback_external_wire_dsl(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
dsl_added: bool,
) {
if dsl_added
&& let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"external_wire_rollback",
)
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"external wire rollback: failed to revert DSL wire"
);
}
}
/// D-external-peer (#31): unwire a local member from an external trusted peer.
///
/// Mirror of [`Self::handle_wire_external`]: removes trust from the
/// local session comms runtime, appends `ExternalPeerUnwired`, and
/// projects through the roster.
///
/// Idempotent: unwiring an already-absent external peer is a no-op
/// success.
async fn handle_unwire_external(
&mut self,
local: AgentIdentity,
peer_name: meerkat_core::comms::PeerName,
stale_cleanup_spec: Option<TrustedPeerDescriptor>,
) -> Result<(), MobError> {
let local_identity = AgentIdentity::from(local.as_str());
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &local_identity) {
return Err(MobError::WiringError(format!(
"unwire between placed member '{local}' and an external peer is unsupported"
)));
}
// The machine-owned external edge supplies the prior descriptor.
// The roster mirror may lag and is display-only.
let member_ref = {
let roster = self.roster.read().await;
let entry = roster
.get(&local)
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?;
entry.member_ref.clone()
};
let authority_edge = self
.external_peer_edge_for_name(&local_identity, &peer_name)
.or_else(|| {
stale_cleanup_spec
.as_ref()
.map(|spec| Self::external_peer_edge(&local_identity, spec))
});
let prior_spec = authority_edge
.as_ref()
.and_then(|edge| {
self.external_peer_edge_for_name(&local_identity, &peer_name)
.filter(|machine_edge| machine_edge == edge)
})
.map(|edge| Self::trusted_peer_descriptor_from_machine_endpoint(&edge.endpoint))
.transpose()?;
let Some(prior_spec) = prior_spec else {
let unwire_handoff = match authority_edge.as_ref() {
Some(edge) => {
let key = Self::external_peer_key_for_edge(edge);
self.apply_unwire_external_peer_idempotent(&key, edge)?
}
None => None,
};
let dsl_removed = unwire_handoff.is_some();
if let Some(spec) = stale_cleanup_spec
&& let Some(comms) = self.provisioner_comms(&member_ref).await
{
let removal_key = Self::trusted_peer_removal_key(&spec);
let has_stale_trust = comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == removal_key);
if !dsl_removed {
if has_stale_trust {
return Err(MobError::WiringError(format!(
"external unwire for '{local}' -> '{peer_name}' requires MobMachine external peer authority"
)));
}
return Ok(());
}
if let Err(error) = self.apply_trusted_peer_remove(
comms.as_ref(),
removal_key.clone(),
unwire_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(format!(
"external stale cleanup for '{local}' -> '{peer_name}' has no generated unwiring handoff"
))
})?
.clone(),
)
.await
{
if dsl_removed && let Some(edge) = authority_edge.as_ref() {
let key = Self::external_peer_key_for_edge(edge);
if let Err(rollback_err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::WireExternalPeer {
key,
edge: edge.clone(),
},
"external_unwire_stale_cleanup_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external DSL unwire after stale cleanup failure"
);
}
}
return Err(MobError::from(error));
}
}
return Ok(());
};
let edge = authority_edge
.unwrap_or_else(|| Self::external_peer_edge(&local_identity, &prior_spec));
let key = Self::external_peer_key_for_edge(&edge);
let comms = self.provisioner_comms(&member_ref).await.ok_or_else(|| {
MobError::WiringError(format!(
"unwire requires comms runtime for '{local}' (external peer unwire)"
))
})?;
let Some(unwire_handoff) = self.apply_unwire_external_peer_idempotent(&key, &edge)? else {
return Err(MobError::WiringError(format!(
"external unwire for '{local}' -> '{peer_name}' was not authorized by MobMachine"
)));
};
let dsl_removed = true;
// Remove trust on the local session runtime.
let prior_removal_key = Self::trusted_peer_removal_key(&prior_spec);
if let Err(error) = self
.apply_trusted_peer_remove(comms.as_ref(), prior_removal_key.clone(), unwire_handoff)
.await
{
if dsl_removed
&& let Err(rollback_err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"external_unwire_trust_remove_rollback",
)
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external DSL unwire after trust removal failure"
);
}
return Err(MobError::from(error));
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerUnwired {
local: local_identity,
peer_name: peer_name.clone(),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_err) => {
// Restore trust on append failure so the unwire is a full
// no-op from the caller's perspective.
if dsl_removed {
match self.apply_wire_external_peer_idempotent(&key, &edge) {
Ok(rollback_authority) => {
let rollback_authority =
rollback_authority.external_authority()?.clone();
if let Err(rollback_err) = self
.apply_trusted_peer_add(
comms.as_ref(),
prior_spec.clone(),
rollback_authority,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external trust removal after event append failure"
);
}
}
Err(rollback_err) => {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to obtain generated external wiring authority after event append failure"
);
}
}
}
return Err(MobError::from(append_err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
///
/// Mark-then-cleanup: event first, mark Retiring, disposal pipeline
/// (policy-driven), then roster removal only after critical archive cleanup
/// succeeds.
async fn handle_retire(&mut self, agent_identity: AgentIdentity) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_retire preflight")?;
self.handle_retire_inner(&agent_identity, false, false, false, false, None)
.await?;
self.ensure_pending_spawn_alignment("handle_retire completion")
}
/// Observe the archive authority's terminal truth before re-emitting a
/// pending runtime-retire route. This covers the crash window where routed
/// retirement and archive both committed but appending `MemberRetired`
/// failed: the durable start marker still replays, while the runtime has
/// correctly been unregistered and must not be treated as a fresh consumer
/// refusal. Unknown/host-owned absence stays fail-closed and retries the
/// generated route.
async fn retirement_archive_already_complete(
&self,
session_id: &SessionId,
) -> Result<bool, MobError> {
if self.session_service.has_live_session(session_id).await? {
return Ok(false);
}
if self
.session_service
.load_persisted_session(session_id)
.await?
.is_some()
{
return Ok(false);
}
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = self.runtime_adapter.as_ref()
&& adapter
.archive_runtime_residue_present(session_id)
.await
.map_err(|error| {
MobError::Internal(format!(
"pending retirement runtime-residue observation failed for '{session_id}': {error}"
))
})?
{
return Ok(false);
}
self.session_service
.session_known_to_archive_authority(session_id)
.await
.map_err(MobError::from)
}
/// Prove that the exact controller-local runtime retire opened by an
/// explicit respawn has no consumer-side work left to perform.
///
/// This is deliberately weaker than archive completion: a missing
/// durable snapshot without an archive tombstone remains host-owned
/// absence. It authorizes dropping only the exact queued runtime-retire
/// effect for the old incarnation. The ordinary disposal path still runs
/// afterwards and re-captures the machine attachment/sidecar pair, so a
/// replacement that appears after this observation is retired or rejected
/// fail-closed instead of being silently abandoned.
async fn respawn_runtime_retire_target_is_quiescent(
&self,
entry: &RosterEntry,
session_id: &SessionId,
) -> Result<bool, MobError> {
let MemberRef::Session {
session_id: roster_session_id,
} = &entry.member_ref
else {
return Ok(false);
};
if roster_session_id != session_id {
return Ok(false);
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let dsl_session_id = mob_dsl::SessionId::from_domain(session_id);
let exact_machine_target = {
let state = self.dsl_authority.state();
state.member_session_bindings.get(&dsl_identity) == Some(&dsl_session_id)
&& state.runtime_retire_pending_sessions.get(&dsl_runtime_id)
== Some(&dsl_session_id)
&& !state.member_placement.contains_key(&dsl_identity)
};
if !exact_machine_target
|| !self.has_exact_queued_runtime_retire(
&dsl_identity,
&dsl_runtime_id,
&dsl_session_id,
)?
{
return Ok(false);
}
if self.session_service.has_live_session(session_id).await?
|| self
.session_service
.load_persisted_session(session_id)
.await?
.is_some()
{
return Ok(false);
}
#[cfg(feature = "runtime-adapter")]
{
let Some(adapter) = self.runtime_adapter.as_ref() else {
return Ok(false);
};
if adapter
.current_executor_attachment_witness(session_id)
.await
.is_some()
|| adapter
.archive_runtime_residue_present(session_id)
.await
.map_err(|error| {
MobError::Internal(format!(
"respawn runtime-quiescence observation failed for '{session_id}': {error}"
))
})?
{
return Ok(false);
}
Ok(true)
}
#[cfg(not(feature = "runtime-adapter"))]
Ok(false)
}
async fn detach_session_ingress_for_mob_destroy(
&mut self,
session_id: &SessionId,
obligation: MobDestroyingSessionIngressObligation,
) -> Result<(), MobError> {
use crate::generated::protocol_mob_destroying_session_ingress::{
submit_session_ingress_detach_failed_for_mob_destroy,
submit_session_ingress_detached_for_mob_destroy,
};
let detach_result = self.detach_runtime_session_ingress(session_id).await;
match detach_result {
Ok(()) => {
submit_session_ingress_detached_for_mob_destroy(&mut self.dsl_authority, obligation)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine SessionIngressDetachedForMobDestroy transition rejected: {error}"
))
})?;
self.publish_machine_state_projection();
Ok(())
}
Err(error) => {
let reason = error.to_string();
let _ = submit_session_ingress_detach_failed_for_mob_destroy(
&mut self.dsl_authority,
obligation,
reason.clone(),
);
self.publish_machine_state_projection();
Err(MobError::Internal(format!(
"mob_destroying_session_ingress detach failed for {session_id}: {reason}"
)))
}
}
}
fn acknowledge_absent_session_ingress_for_mob_destroy(
&mut self,
agent_identity: &AgentIdentity,
obligation: MobDestroyingSessionIngressObligation,
) -> Result<(), MobError> {
crate::generated::protocol_mob_destroying_session_ingress::submit_session_ingress_detached_for_mob_destroy(
&mut self.dsl_authority,
obligation,
)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine SessionIngressDetachedForMobDestroy transition rejected for member '{agent_identity}' without bridge session: {error}"
))
})?;
self.publish_machine_state_projection();
Ok(())
}
fn destroy_ingress_detach_session_id(
entry: &RosterEntry,
releasing: &mob_dsl::SessionId,
) -> Result<SessionId, MobError> {
let session_id = SessionId::parse(&releasing.0).map_err(|_| {
MobError::Internal(format!(
"destroy retire for member '{}' has invalid bridge session binding '{}'",
entry.agent_identity, releasing.0
))
})?;
let roster_session_id = entry.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"destroy retire for member '{}' has a pending machine detach but no roster bridge session",
entry.agent_identity
))
})?;
if roster_session_id != &session_id {
return Err(MobError::Internal(format!(
"destroy retire for member '{}' has mismatched machine/roster bridge sessions ('{}' != '{}')",
entry.agent_identity, session_id, roster_session_id
)));
}
Ok(session_id)
}
async fn detach_runtime_session_ingress(&self, session_id: &SessionId) -> Result<(), MobError> {
#[cfg(test)]
if let Ok(mut target) = FAIL_SESSION_INGRESS_DETACH_FOR_SESSION.lock()
&& target.as_ref() == Some(session_id)
{
target.take();
return Err(MobError::Internal(format!(
"fault-injected session-ingress detach failure for {session_id}"
)));
}
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
match adapter
.update_peer_ingress_context(session_id, false, None)
.await
{
Ok(_) => {}
// The session's runtime may already be absent or terminal
// (e.g. a stale routed retire after the bridge session was
// destroyed). Those machine verdicts prove no ingress remains
// attached, which is exactly the detach post-condition; the
// owner check below still verifies it.
Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(err) => {
return Err(MobError::Internal(format!(
"failed to detach peer ingress for session {session_id}: {err}"
)));
}
}
let owner = adapter.peer_ingress_owner(session_id).await;
if !matches!(owner, meerkat_runtime::PeerIngressOwner::Unattached) {
return Err(MobError::Internal(format!(
"peer ingress owner remained attached after detach: {owner:?}"
)));
}
}
let _ = session_id;
Ok(())
}
/// Realize the generated ingress-detach obligation for an admitted member
/// retirement. A retry whose original Retire already cleared the machine
/// session binding re-requests the still-pending obligation and targets
/// the retained roster session, so archive can never bypass attached
/// ingress.
async fn realize_member_retire_ingress_detach(
&mut self,
entry: &RosterEntry,
mut obligations: Vec<MobDestroyingSessionIngressObligation>,
context: &'static str,
) -> Result<(), MobError> {
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
if self
.dsl_authority
.state()
.pending_session_ingress_detach_runtime_ids
.contains(&dsl_runtime_id)
&& obligations.is_empty()
{
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RequestPendingSessionIngressDetachForMobDestroy {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: dsl_runtime_id.clone(),
},
context,
)?;
obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&transition,
);
}
let obligation = match obligations.as_slice() {
[] => return Ok(()),
[obligation] => obligation.clone(),
_ => {
return Err(MobError::Internal(format!(
"MobMachine retire for member '{}' produced multiple session ingress detach obligations",
entry.agent_identity
)));
}
};
if obligation.mob_id() != &mob_dsl::MobId::from_domain(&self.definition.id)
|| obligation.agent_runtime_id() != &dsl_runtime_id
{
return Err(MobError::Internal(format!(
"MobMachine retire for member '{}' produced an ingress detach obligation for a different mob/runtime",
entry.agent_identity
)));
}
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity) {
return Err(MobError::Internal(format!(
"MobMachine placed retire for member '{}' unexpectedly produced a controller-local session ingress obligation",
entry.agent_identity
)));
}
if let Some(session_id) = entry.member_ref.bridge_session_id().cloned() {
self.detach_session_ingress_for_mob_destroy(&session_id, obligation)
.await
} else {
self.acknowledge_absent_session_ingress_for_mob_destroy(
&entry.agent_identity,
obligation,
)
}
}
async fn admit_member_retire_for_destroy(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await?
{
// A prior destroy attempt may have committed the terminal member
// journal and then crashed before pruning the shell roster anchor.
// The durable receipt is exact-generation authority to skip a new
// retirement admission; later destroy cleanup removes the residue.
return Ok(());
}
let agent_identity = &entry.agent_identity;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let bound_session_id = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let pending_retire_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned();
// A releasing ordinary retirement removes the live binding when it
// opens detach authority, but its pending-session correlation remains
// the durable route identity. Destroy takeover must carry that exact
// session instead of misclassifying the member as peer-only.
let session_id_for_route = bound_session_id.or_else(|| pending_retire_session.clone());
let placed_remote =
super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity);
if self
.dsl_authority
.state()
.pending_session_ingress_detach_runtime_ids
.contains(&dsl_runtime_id)
{
if placed_remote {
return Err(MobError::Internal(format!(
"MobMachine placed destroy retry for member '{agent_identity}' retained a controller-local session ingress obligation"
)));
}
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RequestPendingSessionIngressDetachForMobDestroy {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: dsl_runtime_id.clone(),
},
"destroy_request_pending_session_ingress_detach",
)?;
let obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&transition,
);
let obligation = match obligations.as_slice() {
[obligation] => obligation.clone(),
[] => {
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' produced no generated session ingress obligation"
)));
}
_ => {
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' produced multiple generated session ingress obligations"
)));
}
};
if obligation.mob_id() != &mob_dsl::MobId::from_domain(&self.definition.id)
|| obligation.agent_runtime_id() != &dsl_runtime_id
{
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' generated an obligation for a different mob/runtime"
)));
}
let pending_retire_session = pending_retire_session.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' has no runtime-retirement session correlation"
))
})?;
let detach_session_id =
Self::destroy_ingress_detach_session_id(entry, pending_retire_session)?;
self.detach_session_ingress_for_mob_destroy(&detach_session_id, obligation)
.await?;
self.ensure_destroy_runtime_retire_route_after_detach(
&dsl_identity,
&dsl_runtime_id,
pending_retire_session,
&detach_session_id,
"destroy_retry_runtime_retire_after_detach_ack",
)
.await?;
return self.flush_routed_effects().await;
}
let already_retiring = matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyMemberRetire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
session_id: session_id_for_route.clone(),
},
"destroy_mark_member_retiring",
)?;
if !already_retiring {
let journal_kind = if session_id_for_route.is_some() {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding
} else {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
};
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"destroy_mark_member_retiring",
)?;
self.append_retirement_started_event_for_entry(
entry,
journal_kind,
session_id_for_route.clone(),
false,
)
.await?;
}
let obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared.transition,
);
self.commit_prepared_dsl_transition(prepared)?;
match (
placed_remote,
session_id_for_route.as_ref(),
obligations.as_slice(),
) {
// A placed member's bridge session lives on the remote host.
// Generated remote destroy authority emits RequestMemberRelease,
// not a local session-ingress detach obligation; release owns the
// remote runtime teardown. Session presence alone must not route
// a placed member through the local detach protocol.
(true, Some(_), []) => {}
(true, None, []) => {
return Err(MobError::Internal(format!(
"MobMachine placed destroy retire for member '{agent_identity}' preserved no remote session route"
)));
}
(true, _, _) => {
return Err(MobError::Internal(format!(
"MobMachine placed destroy retire for member '{agent_identity}' unexpectedly produced a local session ingress obligation"
)));
}
(false, Some(session_id), [obligation]) => {
if obligation.mob_id() != &mob_dsl::MobId::from_domain(&self.definition.id)
|| obligation.agent_runtime_id() != &dsl_runtime_id
{
return Err(MobError::Internal(format!(
"MobMachine fresh destroy detach for member '{agent_identity}' generated an obligation for a different mob/runtime"
)));
}
let detach_session_id = Self::destroy_ingress_detach_session_id(entry, session_id)?;
self.detach_session_ingress_for_mob_destroy(&detach_session_id, obligation.clone())
.await?;
self.ensure_destroy_runtime_retire_route_after_detach(
&dsl_identity,
&dsl_runtime_id,
session_id,
&detach_session_id,
"destroy_runtime_retire_after_fresh_detach_ack",
)
.await?;
}
(false, None, []) => {}
(false, Some(_), []) => {
return Err(MobError::Internal(format!(
"MobMachine session-bound destroy retire for member '{agent_identity}' produced no generated session ingress obligation"
)));
}
(false, None, _) => {
return Err(MobError::Internal(format!(
"MobMachine peer-only destroy retire for member '{agent_identity}' unexpectedly produced a session ingress obligation"
)));
}
(false, Some(_), _) => {
return Err(MobError::Internal(format!(
"MobMachine session-bound destroy retire for member '{agent_identity}' produced multiple session ingress obligations"
)));
}
}
self.flush_routed_effects().await
}
async fn handle_retire_inner(
&mut self,
agent_identity: &AgentIdentity,
bulk: bool,
preserve_realtime_binding: bool,
preserve_machine_topology: bool,
retain_roster_on_archive_failure: bool,
rematerializing_generation: Option<u64>,
) -> Result<(), MobError> {
tracing::debug!(
agent_identity = %agent_identity,
bulk,
preserve_realtime_binding,
"MobActor::handle_retire_inner start"
);
// Idempotent: already retired / never existed is success.
let entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let Some(entry) = entry else {
self.apply_command_admission(
mob_dsl::MobMachineInput::RetireAbsent {
agent_identity: mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
agent_identity.as_str(),
)),
},
MobState::Running,
"handle_retire_inner_absent",
)?;
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"retire requested for unknown meerkat id; MobMachine accepted RetireAbsent"
);
return Ok(());
};
tracing::debug!(
agent_identity = %agent_identity,
member_ref = ?entry.member_ref,
runtime_id = %entry.agent_runtime_id,
"MobActor::handle_retire_inner loaded roster entry"
);
// A previous attempt may have published the exact MemberRetired
// terminal and then failed while deleting its durable placed
// carriers. The roster anchor is deliberately retained in that case;
// retry only the post-final cleanup, never re-drive runtime release or
// ask the already-final machine to admit Retire again.
if self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await?
{
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let has_committed_carrier = self
.runtime_metadata
.load_placed_spawn(&self.definition.id, entry.agent_identity.as_str())
.await?
.is_some_and(|record| {
record.generation == entry.generation.get()
&& matches!(
record.phase,
crate::store::PlacedSpawnCarrierPhase::Committed(_)
)
});
let cleanup_open = self
.dsl_authority
.state()
.pending_placed_carrier_cleanup
.iter()
.any(|obligation| obligation.agent_identity == dsl_identity);
if has_committed_carrier && !cleanup_open {
// The journal append may have succeeded before the prepared
// final machine transition was published. Re-drive that exact
// signal without re-releasing the host member; it observes the
// already-published terminal and opens the Committed cleanup
// obligation.
let ctx = self
.disposal_context_from_entry(
agent_identity,
&entry,
RetireTrustCleanupPlan::empty(),
false,
)
.await;
self.observe_member_retirement_archived(
&ctx,
mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned,
)
.await?;
}
self.delete_retired_placed_member_carriers(&entry.agent_identity)
.await?;
let ctx = self
.disposal_context_from_entry(
agent_identity,
&entry,
RetireTrustCleanupPlan::empty(),
false,
)
.await;
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx, !retain_roster_on_archive_failure)
.await;
self.delete_external_binding_overlay_for_member(
&entry.agent_identity,
entry.generation,
)
.await?;
return Ok(());
}
let retirement_started = self
.retirement_started_event_exists(agent_identity, entry.generation)
.await?;
if !retirement_started {
// Retire and Respawn both rotate or remove the member's live
// effect authority. Drain every admitted mutating call and prove
// this exact identity's owning channel absent before publishing
// the durable retirement-start carrier. A timeout/failure leaves
// the member Running and the command safely retryable.
self.quiesce_member_live_for_identities(
vec![entry.agent_identity.clone()],
"member retire live-channel barrier",
)
.await?;
}
let durable_preserve_machine_topology = retirement_started
&& self
.preserved_respawn_topology_event_exists(agent_identity, entry.generation)
.await;
let durable_respawn_topology_abandoned = self
.dsl_authority
.state()
.abandoned_respawn_topology
.get(&mob_dsl::AgentIdentity::from_domain(agent_identity))
.is_some_and(|generation| generation.0 == entry.generation.get());
if retirement_started
&& preserve_machine_topology
&& !durable_preserve_machine_topology
&& !durable_respawn_topology_abandoned
{
return Err(MobError::WiringError(format!(
"cannot respawn '{agent_identity}' after an ordinary retirement start is already durable; retry or complete that retirement first"
)));
}
// Once Started exists, its private stored bit is the exact original
// intent. A later exact-generation abandonment is a durable terminal
// policy transition and therefore dominates that preserving intent
// on every retry.
let durable_effective_preserve_machine_topology =
durable_preserve_machine_topology && !durable_respawn_topology_abandoned;
let preserve_machine_topology = if retirement_started {
durable_effective_preserve_machine_topology
} else {
preserve_machine_topology
};
let preserve_realtime_binding = if retirement_started {
durable_effective_preserve_machine_topology
&& !super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity)
} else {
preserve_realtime_binding
};
// Mark as Retiring in the DSL (blocks re-spawn with same ID).
// Shell roster does not carry authoritative state; `member_state_markers`
// in the DSL is the source of truth and overlays the read-only projection
// on snapshot construction.
//
// The DSL guards reject Retire when the runtime_id is absent from
// `live_runtime_ids` or the phase forbids it.
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let bound_session = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let pending_retire_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned();
let releasing = if preserve_realtime_binding {
None
} else {
bound_session.clone()
};
let session_id_for_route = bound_session
.clone()
.or_else(|| pending_retire_session.clone());
let member_is_placed = self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity);
// A releasing retirement removes the live member-session binding as
// part of its first committed transition, while retaining the exact
// session correlation in `runtime_retire_pending_sessions`. A retry
// must continue from that authority instead of reclassifying the
// member as peer-only and erasing the pending correlation.
let released_ordinary_retirement =
!member_is_placed && bound_session.is_none() && pending_retire_session.is_some();
let retirement_started_journal_kind = if released_ordinary_retirement {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing
} else {
match (
member_is_placed,
releasing.is_some(),
session_id_for_route.is_some(),
) {
// Placed retirement sends a releasing input to the host-owned
// runtime, but the controlling session binding remains durable
// until ReleaseMember is acknowledged. Its journal must therefore
// replay through the placed/preserving arm, not the local
// releasing arm that removes session ownership.
(true, _, true) => {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding
}
(true, _, false) => {
return Err(MobError::Internal(format!(
"placed retirement for '{agent_identity}' has no host-owned session target"
)));
}
(false, true, true) => {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing
}
(false, false, true) => {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding
}
(false, false, false) => {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
}
(false, true, false) => {
return Err(MobError::Internal(format!(
"retirement for '{agent_identity}' releases a session without a routed session target"
)));
}
}
};
let retire_input = mob_dsl::MobMachineInput::Retire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
agent_identity: dsl_identity.clone(),
generation: mob_dsl::Generation::from_domain(entry.generation),
releasing,
session_id: session_id_for_route.clone(),
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner checking prior retire event"
);
tracing::debug!(
agent_identity = %agent_identity,
retirement_started,
"MobActor::handle_retire_inner checked prior retire event"
);
let preserve_topology_for_respawn = preserve_machine_topology;
#[cfg(not(target_arch = "wasm32"))]
let retiring_reverse_lane_pubkey = if preserve_topology_for_respawn && !member_is_placed {
self.machine_member_peer_spec_for(
agent_identity,
"retiring local reverse-lane endpoint",
)?
.map(|spec| meerkat_comms::PubKey::new(spec.pubkey))
} else {
None
};
// K3 (#26): the Retiring marker is owned by MobMachine
// (`member_state_markers`). Trust that machine fact alone; it has no
// second source of truth.
let cleanup_retry = matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
if cleanup_retry {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"retrying member retire cleanup from retained roster anchor"
);
}
let runtime_live = self
.dsl_authority
.state()
.live_runtime_ids
.contains(&dsl_runtime_id);
if cleanup_retry && (released_ordinary_retirement || !runtime_live) {
if self
.dsl_authority
.state()
.pending_session_ingress_detach_runtime_ids
.contains(&dsl_runtime_id)
{
// A crash may land after the durable retirement-start journal
// but before the live ingress-detach obligation is discharged.
// Reuse the ordinary retirement protocol before retrying the
// runtime route; destroy admission is neither held nor needed.
self.realize_member_retire_ingress_detach(
&entry,
Vec::new(),
"retry_member_retire_pending_session_ingress_detach",
)
.await?;
}
if let Some(pending_session) = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned()
{
let pending_session = SessionId::parse(&pending_session.0).map_err(|error| {
MobError::Internal(format!(
"pending runtime retirement for '{agent_identity}' has invalid session id '{}': {error}",
pending_session.0
))
})?;
if self
.retirement_archive_already_complete(&pending_session)
.await?
{
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %pending_session,
"pending retirement route already discharged by archived session authority"
);
self.discard_queued_runtime_retire_for(&dsl_runtime_id);
} else {
self.ensure_runtime_retire_route_after_detach(
&dsl_identity,
&dsl_runtime_id,
&mob_dsl::SessionId::from_domain(&pending_session),
"retry_runtime_retire_after_consumer_refusal",
)?;
if preserve_topology_for_respawn
&& self
.respawn_runtime_retire_target_is_quiescent(&entry, &pending_session)
.await?
{
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %pending_session,
"explicit respawn proved the exact old runtime-retire target quiescent"
);
self.discard_queued_runtime_retire_for(&dsl_runtime_id);
}
}
}
self.flush_routed_effects().await?;
}
// Cold replay restores the Retiring marker while the recovered runtime
// is still live, so re-submit Retire to re-realize generated
// consequences unless the releasing arm has already removed the live
// session binding. That arm must continue through RetryRuntimeRetire
// above: re-submitting Retire would select the peer-only branch and
// erase its exact pending-session correlation. A same-process retry
// after ObserveRuntimeRetired has no live runtime and likewise
// continues from the exact Started+Retiring anchor.
let prepared_retire = if runtime_live && !released_ordinary_retirement {
let prepared_retire = self
.prepare_dsl_input_transition(retire_input, "handle_retire_inner_mark_retiring")?;
Self::require_member_lifecycle_journal_effect(
&prepared_retire.transition,
retirement_started_journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"handle_retire_inner_mark_retiring",
)?;
Some(prepared_retire)
} else if !cleanup_retry || !retirement_started {
return Err(MobError::Internal(format!(
"member '{agent_identity}' has a non-live runtime without an exact durable retirement anchor"
)));
} else {
None
};
// Persist the crash/retry anchor before every externally visible
// consequence, including placed-respawn flow attribution. Publish the
// exact prepared Retire authority immediately after that append: every
// fallible topology/transport cleanup below must observe the member as
// Retiring and non-routable, even when it fails and the caller retries
// in this same process.
if !retirement_started {
self.append_retirement_started_event_for_entry(
&entry,
retirement_started_journal_kind,
session_id_for_route.clone(),
preserve_topology_for_respawn,
)
.await?;
}
// The durable retirement-start event above is the retry anchor. Stop
// any active provider turn before publishing Retire and detaching its
// ingress: DetachIngress takes runtime authority that a wedged turn
// may retain indefinitely. The ordinary archive step repeats this
// idempotent quiesce before terminal disposal.
if let Some(session_id) = entry.member_ref.bridge_session_id() {
super::provisioner::MemberSessionDisposalArc::cancel_active_runtime_turn_before_retire_with_adapter(
self.runtime_adapter.as_ref(),
session_id,
)
.await
.map_err(MobError::from)?;
}
let detach_obligations = if let Some(prepared_retire) = prepared_retire {
let obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared_retire.transition,
);
self.commit_prepared_dsl_transition(prepared_retire)?;
obligations
} else {
Vec::new()
};
// The durable Retiring marker above fences this identity from new
// SubmitWork without globally quiescing healthy members. Exact
// completion cancellation is actor-recorded only; remote I/O belongs
// to the fair reconciler after this command returns.
if member_is_placed {
self.drive_placed_completion_lifecycle_cleanup(Some(agent_identity), false, None)
.await?;
}
if preserve_topology_for_respawn {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ObserveRespawnTopologyPreservationStarted {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"record_respawn_topology_preservation_start",
)?;
}
// Retire dispatch can terminalize and unregister the session runtime.
// Snapshot the exact old incarnation's comms handle before detach and
// routed-retire flush, but keep generated trust-cleanup authorization
// after that refusal boundary. The later cleanup plan uses this only
// as a transport fallback when the terminal runtime can no longer be
// resolved; topology/spec authority still comes from MobMachine.
let retiring_comms_before_detach = self.sender_runtime_for_entry(&entry).await;
self.realize_member_retire_ingress_detach(
&entry,
detach_obligations,
"retire_request_pending_session_ingress_detach",
)
.await?;
// A Broken member may be retired after recovery has already proved
// either durable archive completion or, for explicit respawn only,
// that the exact old local runtime target has no actor, snapshot,
// attachment, or nonterminal residue. Dispatching the freshly queued
// request in either case can only produce
// `routed_session_not_registered`. Drop only this runtime's exact
// queued request; the ordinary disposal path still revalidates the
// attachment/sidecar pair and preserves honest host-owned archive
// disposition before publishing the terminal member transition.
if let Some(session_id) = entry.member_ref.bridge_session_id() {
let archive_complete = self.retirement_archive_already_complete(session_id).await?;
let exact_respawn_target_quiescent = !archive_complete
&& preserve_topology_for_respawn
&& self
.respawn_runtime_retire_target_is_quiescent(&entry, session_id)
.await?;
if archive_complete || exact_respawn_target_quiescent {
self.discard_queued_runtime_retire_for(&dsl_runtime_id);
}
}
// Flush session-backed routed effects before the disposal pipeline
// tears down the runtime session. A consumer refusal is closed back
// into MobMachine as a typed retirement retry anchor and returned to
// the caller; disposal must not proceed past a refused runtime
// retirement.
if let Err(error) = self.flush_routed_effects().await {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
%error,
"pre-disposal routed-effect flush failed; retaining member for retry"
);
return Err(error);
}
let canceled = self
.cancel_pending_spawns_for_member(
agent_identity,
"durable member retirement superseded pending spawn",
)
.await?;
if canceled > 0 {
tracing::info!(
agent_identity = %agent_identity,
canceled,
"retirement canceled same-identity pending spawn after durable start"
);
}
self.cancel_peer_deliveries_for_member(agent_identity, "member is retiring")
.await?;
if let Some(old_generation) = rematerializing_generation {
self.remote_flow_tickets
.note_member_rematerializing(agent_identity, old_generation);
}
// A final retirement transition cannot merely erase an incident
// placed edge: the remote survivor would retain its trust row. This
// also applies to binding-preserving respawn: the respawn snapshot
// owns desired-topology restoration, while the old peer key must be
// removed before the replacement is wired.
self.cleanup_retiring_placed_member_edges(&domain_identity, preserve_topology_for_respawn)
.await?;
// A respawn preserves the logical machine edge, so the ordinary
// unwire-driven cleanup never runs for the retiring local endpoint.
// Remove its exact process-acceptor lease after every remote trust
// lane has been removed and before the replacement can be published.
#[cfg(not(target_arch = "wasm32"))]
if let Some(pubkey) = retiring_reverse_lane_pubkey.as_ref()
&& let Some(state) = self.controlling_acceptor.as_mut()
{
state.remove_registration(pubkey).await?;
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner planning trust cleanup"
);
let mut trust_cleanup_plan = self
.member_retire_trust_cleanup_plan(agent_identity, &entry)
.await?;
if trust_cleanup_plan.retiring_comms.is_none() {
trust_cleanup_plan.retiring_comms = retiring_comms_before_detach;
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner planned trust cleanup"
);
// Snapshot context and run disposal pipeline.
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner building disposal context"
);
let ctx = self
.disposal_context_from_entry(
agent_identity,
&entry,
trust_cleanup_plan,
preserve_topology_for_respawn,
)
.await;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner built disposal context"
);
let mut policy: Box<dyn ErrorPolicy> = if bulk {
Box::new(BulkBestEffort)
} else {
Box::new(WarnAndContinue)
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner disposing member"
);
let (mut report, archive_disposal) = self.dispose_member(&ctx, policy.as_mut()).await;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner disposed member"
);
if let Some((step, error)) = report.aborted_at.take() {
if step == DisposalStep::CleanupMachineTopology {
return Err(error);
}
report.aborted_at = Some((step, error));
}
if let Some((step, MobError::RetirementTopologyIncomplete(reason))) = &report.aborted_at
&& *step == DisposalStep::NotifyPeers
{
return Err(MobError::RetirementTopologyIncomplete(reason.clone()));
}
// ArchiveSession is critical: a skipped archive means an orphan session
// the caller believes was cleaned up. Surface the error.
// Comms steps (NotifyPeers, RemoveTrustEdges) remain best-effort.
if let Some(index) = report
.skipped
.iter()
.position(|(step, _)| *step == DisposalStep::ArchiveSession)
{
// ArchiveSession is the remote ReleaseMember boundary for a
// placed member. Preserve its typed bridge/session failure so the
// caller can distinguish a certified rejection from transport or
// internal failure and decide whether an exact retry is safe.
let (_, error) = report.skipped.swap_remove(index);
return Err(error);
}
if let Some((step, error)) = report.aborted_at.take()
&& step == DisposalStep::ArchiveSession
{
return Err(error);
}
let is_placed =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity);
if !is_placed && let Some(binding) = Self::runtime_binding_for_entry(&entry) {
if !self.remote_runtime_retired_for_entry(&entry) {
return Err(MobError::Internal(format!(
"peer-only retirement for '{}' reached supervisor revoke without a durable remote-runtime checkpoint",
entry.agent_identity
)));
}
self.revoke_supervisor_for_retiring_entry(
&entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to revoke supervisor authority from retiring peer-only member '{}': {error}",
entry.agent_identity
))
})?;
}
self.delete_external_binding_overlay_for_member(&entry.agent_identity, entry.generation)
.await?;
// `MemberRetired` is the durable completion boundary. It is appended
// only after the routed retire, archive, and every critical local
// cleanup above has succeeded; failures retain the durable
// `MemberRetirementStarted` roster anchor for retry.
let archive_disposal = archive_disposal.ok_or_else(|| {
MobError::Internal(format!(
"retirement for '{}' completed without a typed archive disposition",
entry.agent_identity
))
})?;
self.observe_member_retirement_archived(&ctx, archive_disposal)
.await?;
// Placement/operator carriers remain until the terminal journal and
// machine publication both hold; cold replay needs their exact tuple.
self.delete_retired_placed_member_carriers(&ctx.agent_identity)
.await?;
self.dispose_remove_from_roster(&ctx, !retain_roster_on_archive_failure)
.await;
Ok(())
}
async fn delete_retired_placed_member_carriers(
&mut self,
identity: &AgentIdentity,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let obligation = self
.dsl_authority
.state()
.pending_placed_carrier_cleanup
.iter()
.find(|obligation| obligation.agent_identity == dsl_identity)
.cloned();
let record = self
.runtime_metadata
.load_placed_spawn(&self.definition.id, identity.as_str())
.await?;
let (obligation, record) = match (obligation, record) {
(None, None) => return Ok(()),
(None, Some(record)) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"retirement cleanup for '{}' found canonical carrier {} without a machine cleanup obligation; actor is fail-stopping for cold recovery",
identity, record.spawn_id
)));
}
(Some(obligation), Some(record)) => (obligation, record),
(Some(_), None) => {
// A prior exact delete may have committed before Resolve failed.
// The full expected carrier is no longer reconstructible from the
// obligation alone, so live code must not mint an absent-row
// authority. Cold recovery sees neither carrier nor obligation.
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"retirement cleanup for '{identity}' retained an obligation but the exact carrier is absent; actor is fail-stopping for cold recovery"
)));
}
};
if !matches!(
&record.phase,
crate::store::PlacedSpawnCarrierPhase::Committed(_)
) {
return Err(MobError::Internal(format!(
"retirement cleanup for '{identity}' found a Pending placed carrier; obligation retained"
)));
}
let cleanup_authority = match self.authorize_placed_carrier_cleanup(
&record,
&obligation,
"retired_placed_carrier_cleanup_authorize",
) {
Ok(authority) => authority,
Err(authority_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"retirement cleanup for '{identity}' found an obligation/carrier tuple mismatch; no lifecycle operation was mutated, the obligation and carrier are retained, and the actor is fail-stopping for cold recovery: {authority_error}"
)));
}
};
let display_name = match render_member_comms_name(
self.definition.id.as_str(),
&record.spec.profile_name,
identity.as_str(),
) {
Ok(display_name) => display_name,
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"retirement cleanup for '{identity}' could not reconstruct the exact operation display tuple; cleanup obligation and carrier are retained, and the actor is fail-stopping for cold recovery: {error}"
)));
}
};
if let Err(operation_error) = self
.provisioner
.retire_committed_placed_provision_operation(
&record.operation_owner_session_id,
&record.provision_operation_id,
&display_name,
)
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"retirement cleanup for '{}' could not terminalize exact provision operation '{}' owned by '{}'; cleanup obligation and carrier are retained, and the actor is fail-stopping for cold recovery: {operation_error}",
identity, record.provision_operation_id, record.operation_owner_session_id,
)));
}
let result = self
.realize_authorized_placed_carrier_cleanup(
&record,
obligation,
&cleanup_authority,
"retired_placed_carrier_cleanup",
)
.await;
if result.is_err() {
self.durable_uncertainty_fail_stop = true;
}
result
}
async fn require_placed_spawn_carriers_empty(&self, context: &str) -> Result<(), MobError> {
let carriers = self
.runtime_metadata
.list_placed_spawns(&self.definition.id)
.await?;
if carriers.is_empty() {
return Ok(());
}
let remaining = carriers
.iter()
.map(|carrier| {
format!(
"{}:{}:{}:{:?}",
carrier.agent_identity, carrier.generation, carrier.spawn_id, carrier.phase
)
})
.collect::<Vec<_>>()
.join(", ");
Err(MobError::Internal(format!(
"{context}: canonical placed-spawn carriers remain for mob '{}': {remaining}",
self.definition.id
)))
}
/// Respawn a member: retire the old session and spawn a fresh one with the
/// same identity, profile, wiring, and labels. The old session is archived;
/// the new session gets a fresh session ID.
///
/// This is helper composition over primitive mob behavior. No rollback is
/// attempted after retire. Returns a receipt on full success, or a
/// structured error on failure.
/// ADJ-24 — placed respawn = re-materialization. The old incarnation is
/// retired through the remote release lane (W-C); the replacement rides
/// the ordinary remote spawn exec ladder, which re-materializes the SAME
/// identity at the replacement tuple (machine generation bump + fresh
/// fence — the host admits it as Superseding, never StaleFence).
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
async fn handle_respawn_placed(
&mut self,
agent_identity: AgentIdentity,
replacement_spec: super::handle::SpawnMemberSpec,
snapshot: RespawnSnapshot,
) -> Result<RespawnProgress, super::handle::MobRespawnError> {
use super::handle::MobRespawnError;
if !snapshot.cleanup_retry {
self.preview_dsl_input(
mob_dsl::MobMachineInput::Respawn {
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&snapshot.old_runtime_id,
),
},
"handle_respawn_admission",
)
.map_err(|_| MobRespawnError::from(self.invalid_transition_to(MobState::Running)))?;
}
// Retire with the FULL remote release (never binding-preserving: the
// frozen Retire vocabulary has no placed binding-preserving arm, and
// the replacement re-materializes rather than rebinding). The flow
// lane's rematerializing mark is installed inside the admitted retire
// seam after MemberRetirementStarted is durable and before release.
self.handle_retire_inner(
&agent_identity,
false,
false,
true,
true,
Some(snapshot.old_runtime_id.generation.get()),
)
.await
.map_err(MobRespawnError::from)?;
// Replacement: a fresh placed spawn through the remote
// materialization lane, carrying the machine placement fact threaded
// by the caller. Enqueue runs inline on the actor task and mints
// (generation, fence) through the machine-owned counters — but the
// receipt is NEVER awaited here: the lane finalizes through this
// actor's own mailbox, so completion is deferred to a detached task
// via [`RespawnProgress::DeferredPlaced`].
let (reply_tx, reply_rx) = oneshot::channel();
let restore_wiring = (!snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty())
.then_some(snapshot.restore_wiring.clone());
self.enqueue_spawn_remote(
replacement_spec,
None,
None,
Some(RespawnOrigin {
old_runtime_id: snapshot.old_runtime_id.clone(),
old_fence_token: snapshot.old_fence_token,
}),
restore_wiring,
reply_tx,
)
.await;
Ok(RespawnProgress::DeferredPlaced {
identity: AgentIdentity::from(agent_identity.as_str()),
old_fence_token: snapshot.old_fence_token,
spawn_reply_rx: reply_rx,
})
}
/// Off-actor completion of a placed respawn: awaits the remote spawn
/// lane's receipt (which the actor finalizes through its own mailbox)
/// and builds the respawn receipt from the committed roster entry. Runs
/// on a detached task — never on the actor loop.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
async fn complete_placed_respawn(
roster: Arc<RwLock<RosterAuthority>>,
identity: AgentIdentity,
old_fence_token: crate::ids::FenceToken,
spawn_reply_rx: oneshot::Receiver<Result<super::handle::MemberSpawnReceipt, MobError>>,
) -> Option<Result<super::handle::MemberRespawnReceipt, super::handle::MobRespawnError>> {
use super::handle::{MemberRespawnReceipt, MobRespawnError};
let spawn_receipt = match spawn_reply_rx.await {
Ok(Ok(receipt)) => receipt,
Ok(Err(error)) => {
return Some(Err(MobRespawnError::SpawnAfterRetireWithCause {
identity: AgentIdentity::from(identity.as_str()),
cause: error,
}));
}
Err(_) => {
// The actor deliberately drops this internal sender when a
// durable abandonment or placed-cleanup outcome is unsafe to
// acknowledge. Propagate cancellation by dropping the public
// sender too; fabricating an ordinary SpawnAfterRetire here
// would launder fail-stop uncertainty into a definitive reply.
return None;
}
};
let entry = {
let roster = roster.read().await;
roster.get(&identity).cloned()
};
let Some(entry) = entry else {
return Some(Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(identity.as_str()),
reason: "remote respawn replacement committed but the roster entry is missing"
.to_string(),
}));
};
let receipt = MemberRespawnReceipt::new(
AgentIdentity::from(identity.as_str()),
entry.agent_runtime_id,
old_fence_token,
entry.fence_token,
);
if spawn_receipt.failed_restore_peer_ids.is_empty() {
Some(Ok(receipt))
} else {
Some(Err(MobRespawnError::TopologyRestoreFailed {
receipt,
failed_peer_ids: spawn_receipt.failed_restore_peer_ids,
}))
}
}
async fn handle_respawn(
&mut self,
agent_identity: AgentIdentity,
initial_message: Option<ContentInput>,
) -> Result<RespawnProgress, super::handle::MobRespawnError> {
use super::handle::{MemberRespawnReceipt, MobRespawnError};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn start"
);
self.ensure_pending_spawn_alignment("handle_respawn preflight")
.map_err(MobRespawnError::from)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn preflight aligned"
);
// ADJ-24: the machine `member_placement` fact is the placed-member
// discriminator — never the MemberRef shape. Its peer transport shell
// may omit the redundant session copy; the machine binding remains
// authoritative (DEC-R2).
let placed_host = self
.dsl_authority
.state()
.member_placement
.get(&mob_dsl::AgentIdentity::from_domain(&agent_identity))
.cloned();
// 1. Snapshot all replacement inputs before retiring. Topology comes
// from the MobMachine authority; the roster is only the read model
// that supplies profile/runtime binding details for the old member.
let snapshot = {
let roster = self.roster.read().await;
let entry = roster
.get(&agent_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(agent_identity.clone()))?;
let restore_wiring = self
.machine_restore_wiring_plan(&agent_identity)
.map_err(MobRespawnError::from)?;
let binding = match placed_host.as_ref() {
Some(host) => crate::RuntimeBinding::HostMaterialized { host: host.clone() },
None => match &entry.member_ref {
crate::event::MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
..
} => crate::RuntimeBinding::External {
peer_id: peer_id.clone(),
address: address.clone(),
bootstrap_token: bootstrap_token.clone(),
pubkey: *pubkey,
},
crate::event::MemberRef::Session { .. } => crate::RuntimeBinding::Session,
},
};
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
// K3 (#26): trust the MobMachine-owned Retiring marker alone; it
// has no second source of truth.
let cleanup_retry = matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
RespawnSnapshot {
profile_name: entry.role.clone(),
runtime_mode: entry.runtime_mode,
labels: entry.labels.clone(),
old_runtime_id: entry.agent_runtime_id.clone(),
old_fence_token: entry.fence_token,
restore_wiring,
binding,
effective_profile_override: entry.effective_profile_override,
effective_model_override: entry.effective_model_override,
cleanup_retry,
}
};
tracing::debug!(
agent_identity = %agent_identity,
old_runtime_id = %snapshot.old_runtime_id,
runtime_mode = ?snapshot.runtime_mode,
cleanup_retry = snapshot.cleanup_retry,
"MobActor::handle_respawn captured snapshot"
);
let original_identity = AgentIdentity::from(agent_identity.as_str());
let mut replacement_spec = super::handle::SpawnMemberSpec::new(
snapshot.profile_name.clone(),
original_identity.clone(),
);
replacement_spec.initial_message = initial_message;
replacement_spec.runtime_mode = Some(snapshot.runtime_mode);
// ADJ-24: a placed member's replacement carries the machine placement
// fact and NO explicit binding (placement and binding are mutually
// exclusive on the remote lane — one transport story per member).
match placed_host.as_ref() {
Some(host) => replacement_spec.placement = Some(host.clone()),
None => replacement_spec.binding = Some(snapshot.binding.clone()),
}
replacement_spec.labels = Some(snapshot.labels.clone());
replacement_spec.override_profile = snapshot.effective_profile_override.clone();
replacement_spec.model_override = snapshot.effective_model_override.clone();
self.customize_spawn_spec(
super::handle::SpawnSource::Respawn,
None,
&mut replacement_spec,
)
.map_err(MobRespawnError::from)?;
if replacement_spec.identity != original_identity {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn identity from '{original_identity}' to '{}'",
replacement_spec.identity
))));
}
if replacement_spec.role_name != snapshot.profile_name {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn profile for '{original_identity}' from '{}' to '{}'",
snapshot.profile_name, replacement_spec.role_name
))));
}
if placed_host.is_none() && replacement_spec.binding.as_ref() != Some(&snapshot.binding) {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn runtime binding for '{original_identity}'"
))));
}
if placed_host.is_some() && replacement_spec.binding.is_some() {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot bind respawn of placed member '{original_identity}' \
to an explicit runtime binding"
))));
}
if replacement_spec.placement != placed_host {
// Placement is the machine `member_placement` fact; a customizer
// cannot re-place an existing member mid-respawn (re-placement is
// an operator/machine decision, §9 — fail closed, never silently
// dropped).
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot re-place respawn of '{original_identity}'"
))));
}
// ADJ-24: placed respawn = re-materialization — retire releases the
// remote materialization (W-C), the replacement rides the ordinary
// remote spawn exec ladder at the replacement tuple.
#[cfg(all(feature = "runtime-adapter", not(target_arch = "wasm32")))]
if placed_host.is_some() {
return self
.handle_respawn_placed(agent_identity, replacement_spec, snapshot)
.await;
}
#[cfg(not(all(feature = "runtime-adapter", not(target_arch = "wasm32"))))]
if placed_host.is_some() {
return Err(MobRespawnError::from(MobError::WiringError(
"placed respawn requires the runtime-adapter mob build".to_string(),
)));
}
let super::handle::SpawnMemberSpec {
role_name: _,
identity: _,
initial_message: replacement_initial_message,
runtime_mode: _,
backend: _,
binding: _,
context: replacement_context,
labels: replacement_labels,
launch_mode: _,
tool_access_policy: replacement_tool_access_policy,
budget_limits: replacement_budget_limits,
auto_wire_parent: _,
additional_instructions: replacement_additional_instructions,
shell_env: replacement_shell_env,
inherited_tool_filter: replacement_inherited_tool_filter,
override_profile: replacement_profile_override,
model_override: replacement_model_override,
objective_id: _,
auth_binding: replacement_auth_binding,
external_tools: replacement_external_tools,
system_prompt_override: replacement_system_prompt_override,
continuity_intent: replacement_continuity_intent,
placement: _,
} = replacement_spec;
let replacement_labels = replacement_labels.unwrap_or_default();
if !snapshot.cleanup_retry {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn previewing Respawn admission"
);
self.preview_dsl_input(
mob_dsl::MobMachineInput::Respawn {
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&snapshot.old_runtime_id,
),
},
"handle_respawn_admission",
)
.map_err(|_| MobRespawnError::from(self.invalid_transition_to(MobState::Running)))?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn previewed Respawn admission"
);
}
#[cfg(feature = "runtime-adapter")]
let respawn_peer_only_owner_context =
if matches!(snapshot.binding, crate::RuntimeBinding::External { .. }) {
Some(
self.generated_peer_only_operation_owner_context(
&original_identity,
&snapshot.binding,
"respawn_peer_only_operation_owner",
)
.await
.map_err(MobRespawnError::from)?,
)
} else {
None
};
#[cfg(not(feature = "runtime-adapter"))]
let respawn_peer_only_owner_context: Option<(
SessionId,
Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
)> = None;
let replacement_generation = self
.compute_respawn_generation(&agent_identity)
.map_err(MobRespawnError::from)?;
// 2. Retire the existing member (archives the session, removes from roster).
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn retiring previous member"
);
if let Err(error) = self
.handle_retire_inner(&agent_identity, false, true, true, true, None)
.await
{
let roster_still_contains_member = {
let roster = self.roster.read().await;
roster.get(&agent_identity).is_some()
};
if roster_still_contains_member {
return Err(MobRespawnError::from(error));
}
let mut cleanup_report = super::handle::PreviousMemberCleanupReport {
identity: AgentIdentity::from(agent_identity.as_str()),
agent_runtime_id: snapshot.old_runtime_id.clone(),
fence_token: snapshot.old_fence_token,
retire_attempted: true,
retire_error: Some(error.to_string()),
confirmatory_observation_attempted: false,
confirmatory_observation: None,
destroy_attempted: false,
destroy_error: None,
};
match &snapshot.binding {
crate::RuntimeBinding::External { .. } => {
cleanup_report.confirmatory_observation_attempted = true;
match self
.observe_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_millis(750),
)
.await
{
Ok(observation) => {
cleanup_report.confirmatory_observation =
Some(format!("state={}", observation.state));
let observation_is_terminal = self
.observation_is_terminal(&observation)
.map_err(MobRespawnError::from)?;
if !observation_is_terminal {
cleanup_report.destroy_attempted = true;
if let Err(destroy_error) = self
.destroy_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_secs(5),
)
.await
{
cleanup_report.destroy_error = Some(destroy_error.to_string());
return Err(MobRespawnError::PreviousMemberCleanupAmbiguous {
report: cleanup_report,
});
}
}
}
Err(observe_error) => {
cleanup_report.confirmatory_observation =
Some(observe_error.to_string());
cleanup_report.destroy_attempted = true;
if let Err(destroy_error) = self
.destroy_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_secs(5),
)
.await
{
cleanup_report.destroy_error = Some(destroy_error.to_string());
return Err(MobRespawnError::PreviousMemberCleanupAmbiguous {
report: cleanup_report,
});
}
}
}
}
crate::RuntimeBinding::Session | crate::RuntimeBinding::HostMaterialized { .. } => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
retire_error = %error,
"respawn proceeding after retire removed the stale roster anchor"
);
}
}
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn retired previous member"
);
// 3. Rebuild the replacement spawn preserving identity, profile, labels, mode, and peer intent.
let (prompt, initial_turn_prompt) = match replacement_initial_message {
Some(message) => {
let prompt = message;
(prompt.clone(), Some(prompt))
}
None => (
ContentInput::from(
self.fallback_spawn_prompt(&snapshot.profile_name, &agent_identity),
),
None,
),
};
// Prefer roster's effective_profile_override on respawn for lifecycle safety.
let mut profile = if let Some(p) = replacement_profile_override.clone() {
p
} else {
self.definition
.resolve_profile(&snapshot.profile_name, self.realm_profile_store.as_ref())
.await?
};
if let Some(model) = replacement_model_override.as_ref() {
profile.model.clone_from(model);
}
if replacement_inherited_tool_filter.is_some() && replacement_profile_override.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let replacement_authorized_profile_material = self
.authorize_spawn_profile_material(
&agent_identity,
&snapshot.profile_name,
&profile,
"respawn_profile_authority",
)
.map_err(MobRespawnError::from)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn authorized replacement profile material"
);
let external_tools =
self.external_tools_for_profile(&profile, replacement_external_tools.clone())?;
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &snapshot.profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context: replacement_context,
labels: Some(replacement_labels.clone()),
additional_instructions: replacement_additional_instructions,
shell_env: replacement_shell_env,
mob_tool_authority_context: None,
inherited_tool_filter: replacement_inherited_tool_filter,
tool_access_policy: replacement_tool_access_policy,
system_prompt_override: replacement_system_prompt_override,
})
.await?;
config.keep_alive = snapshot.runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
if let Some(ref cref) = replacement_auth_binding {
config.auth_binding = Some(cref.clone());
}
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, replacement_budget_limits);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
snapshot.profile_name.as_str(),
agent_identity.as_str(),
)?;
let mut provision_request = ProvisionMemberRequest {
create_session: req,
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
binding: snapshot.binding.clone(),
peer_name,
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: None,
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
};
if let Some((owner_bridge_session_id, ops_registry)) = respawn_peer_only_owner_context {
provision_request.owner_bridge_session_id = Some(owner_bridge_session_id);
provision_request.ops_registry = Some(ops_registry);
}
let admitted_bridge_session_id =
admit_bridge_session_for_spawn(&mut provision_request.create_session);
tracing::debug!(
agent_identity = %agent_identity,
bridge_session_id = %admitted_bridge_session_id,
"MobActor::handle_respawn admitted replacement bridge session"
);
let respawn_spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let generated_self_owned_operation_owner = self
.stage_orchestrator_spawn(&agent_identity, &admitted_bridge_session_id)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to stage respawn replacement spawn: {error}"),
})?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn staged replacement spawn"
);
Self::apply_generated_self_owned_operation_owner(
&mut provision_request,
&generated_self_owned_operation_owner,
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to authorize respawn replacement operation owner: {error}"),
})?;
// External provisioning installs supervisor-bridge recipient trust
// ahead of bind terminality inside the provisioner. The respawn
// replacement provisions inline on the actor task, so the obligation
// window is recorded here (before any pending slot exists), resolved
// once the provision confirms terminality, and rolled back below if
// the provision failed.
let respawn_pending_trust_peer_id = match &snapshot.binding {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
crate::RuntimeBinding::Session | crate::RuntimeBinding::HostMaterialized { .. } => None,
};
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn external provision",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
let (respawn_inline_reply_tx, _respawn_inline_reply_rx) = oneshot::channel();
let respawn_pending = PendingSpawn {
profile_name: snapshot.profile_name.clone(),
agent_identity: agent_identity.clone(),
admitted_bridge_session_id,
prompt: prompt.clone(),
initial_turn_prompt: initial_turn_prompt.clone(),
suppress_autonomous_initial_prompt: false,
identity_member_permit: None,
runtime_mode: snapshot.runtime_mode,
labels: replacement_labels.clone(),
owner_bridge_session_id: None,
auto_wire_parent: false,
restore_wiring: (!snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty())
.then_some(snapshot.restore_wiring.clone()),
respawn_origin: Some(RespawnOrigin {
old_runtime_id: snapshot.old_runtime_id.clone(),
old_fence_token: snapshot.old_fence_token,
}),
effective_profile_override: replacement_profile_override.clone(),
effective_model_override: replacement_model_override.clone(),
objective_id: None,
per_spawn_external_tools: replacement_external_tools.clone(),
authorized_profile_material: replacement_authorized_profile_material.clone(),
continuity_intent: replacement_continuity_intent.clone(),
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default())),
pending_recipient_trust_peer_id: None,
// Respawn is local-only vocabulary (a placed member's respawn
// routes through release + a fresh placed spawn); the local
// arms' multi-host guards are placement-gated, so the default
// (phase-2 hardwired) observation set is preserved here.
observations: SpawnExecObservations::default(),
remote: None,
enqueued_at: Instant::now(),
reply_tx: respawn_inline_reply_tx,
};
let respawn_inline_task = tokio::spawn(async {
std::future::pending::<()>().await;
});
let spawn_started = generated_self_owned_operation_owner
.start(&respawn_pending)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to start respawn replacement spawn: {error}"),
})?;
if let Err(insert_error) = self
.insert_pending_spawn(
respawn_spawn_ticket,
respawn_pending,
respawn_inline_task,
spawn_started,
)
.await
{
let cleanup_error = self
.fail_all_pending_spawns("respawn replacement pending insertion failed")
.await
.err();
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: match cleanup_error {
Some(cleanup_error) => format!(
"failed to insert respawn replacement spawn: {insert_error}; cleanup failed: {cleanup_error}"
),
None => format!("failed to insert respawn replacement spawn: {insert_error}"),
},
});
}
if let Err(error) = self.ensure_pending_spawn_alignment("handle_respawn staged replacement")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated while staging respawn replacement"
);
// The provision never started: close the recorded obligation
// window before the staging error propagates.
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn staging failed",
)
.map_err(|rollback_error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: rollback_error.to_string(),
})?;
}
if let Err(cleanup_error) = self
.fail_all_pending_spawns(
"pending spawn alignment violated while staging respawn replacement",
)
.await
{
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: cleanup_error.to_string(),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
});
}
// 4. Provision and finalize the replacement member inline so the receipt reflects
// the committed canonical member/session state before we return.
let mut respawn_trust_cleanup_uncertain = false;
let replacement_result: Result<super::handle::MemberSpawnReceipt, MobRespawnError> = Box::pin(async {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn provisioning replacement member"
);
let spawn_receipt = match self
.provisioner
.provision_member(provision_request)
.await
{
Ok(receipt) => receipt,
Err(error) => {
respawn_trust_cleanup_uncertain =
error.external_member_cleanup_is_uncertain();
return Err(MobRespawnError::SpawnAfterRetireWithCause {
identity: AgentIdentity::from(agent_identity.as_str()),
cause: error,
});
}
};
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
// Provision success means bind/authorize terminality was
// confirmed for the external peer: close the obligation.
self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn provision confirmed",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
tracing::debug!(
agent_identity = %agent_identity,
member_ref = ?spawn_receipt.member_ref,
"MobActor::handle_respawn provisioned replacement member"
);
if snapshot.runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
&agent_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) = self
.provisioner
.retire_member(&spawn_receipt.member_ref)
.await
{
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!(
"autonomous capability check failed: {capability_error}; cleanup retire failed: {retire_error}"
),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: capability_error.to_string(),
});
}
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
spawn_receipt.operation_id.clone(),
spawn_receipt.session_origin,
spawn_receipt.rollback_authority.clone(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Err(retire_error) = provision.rollback().await {
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!(
"mob state changed before respawn finalization: {error}; cleanup retire failed: {retire_error}"
),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
});
}
if !snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty()
{
tracing::info!(
agent_identity = %agent_identity,
local_peers = ?snapshot.restore_wiring.local_peers,
external_peers = ?snapshot.restore_wiring.external_peers,
"respawn: restoring peer wiring during replacement finalization"
);
}
let respawn_fence = match self.issue_fence_token() {
Ok(fence) => fence,
Err(error) => {
let reason = match provision.rollback().await {
Ok(()) => error.to_string(),
Err(retire_error) => format!(
"respawn fence allocation failed: {error}; cleanup retire failed: {retire_error}"
),
};
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason,
});
}
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn finalizing replacement spawn"
);
let finalized = Box::pin(self.finalize_spawn_from_pending(
&snapshot.profile_name,
&agent_identity,
replacement_generation,
respawn_fence,
snapshot.runtime_mode,
prompt,
initial_turn_prompt,
false,
None,
replacement_labels,
provision,
spawn_receipt.operation_id,
None,
false,
(!snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty())
.then_some(snapshot.restore_wiring.clone()),
replacement_profile_override,
replacement_model_override,
None,
replacement_external_tools,
replacement_authorized_profile_material,
replacement_continuity_intent,
SpawnExecObservations::default(),
None,
))
.await
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn finalized replacement spawn"
);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn resolving topology restore"
);
let topology_restore = self
.resolve_respawn_topology_restore_result(
&agent_identity,
finalized.failed_restore_peer_ids,
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
tracing::debug!(
agent_identity = %agent_identity,
result = ?topology_restore.result,
"MobActor::handle_respawn resolved topology restore"
);
match topology_restore.result {
mob_dsl::RespawnTopologyRestoreResultKind::Completed => Ok(finalized.receipt),
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed => {
Err(MobRespawnError::TopologyRestoreFailed {
receipt: super::handle::MemberRespawnReceipt::new(
AgentIdentity::from(agent_identity.as_str()),
crate::ids::AgentRuntimeId::new(
AgentIdentity::from(agent_identity.as_str()),
replacement_generation,
),
snapshot.old_fence_token,
respawn_fence,
),
failed_peer_ids: topology_restore.failed_peer_ids,
})
}
}
})
.await;
let (_respawn_pending, respawn_task) =
self.complete_pending_spawn_slot(respawn_spawn_ticket, "respawn replacement spawn");
if let Some(handle) = respawn_task {
handle.abort();
}
if replacement_result.is_err()
&& let Some(peer_id) = respawn_pending_trust_peer_id.as_deref()
{
if respawn_trust_cleanup_uncertain {
self.durable_uncertainty_fail_stop = true;
tracing::error!(
peer_id,
"respawn external provision trust/cleanup is uncertain; retaining pending obligation and fail-stopping actor"
);
} else {
// Confirmed provision failures restored trust; failures after
// confirmed provision resolved this entry in the future, so
// this is an idempotent safe close.
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn provision failed with cleanup confirmed",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
}
self.ensure_pending_spawn_alignment("handle_respawn completion")
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
let _replacement = replacement_result?;
// 5. Build the receipt from the committed replacement member reference.
Ok(RespawnProgress::Completed(MemberRespawnReceipt::new(
AgentIdentity::from(agent_identity.as_str()),
crate::ids::AgentRuntimeId::new(
AgentIdentity::from(agent_identity.as_str()),
replacement_generation,
),
snapshot.old_fence_token,
self.roster
.read()
.await
.get(&agent_identity)
.map(|entry| entry.fence_token)
.unwrap_or(snapshot.old_fence_token),
)))
}
// -----------------------------------------------------------------------
// Disposal pipeline
// -----------------------------------------------------------------------
fn machine_wired_peer_identities_for(
&self,
identity: &AgentIdentity,
) -> BTreeSet<AgentIdentity> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let state = self.dsl_authority.state();
let mut wired_to = self.machine_member_wired_peer_identities_for(identity);
wired_to.extend(
state
.external_peer_edges
.iter()
.filter(|edge| edge.local == dsl_identity)
.map(|edge| AgentIdentity::from(edge.endpoint.name.0.as_str())),
);
wired_to
}
fn machine_member_wired_peer_identities_for(
&self,
identity: &AgentIdentity,
) -> BTreeSet<AgentIdentity> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.wiring_edges
.iter()
.filter_map(|edge| {
if edge.a == dsl_identity {
Some(AgentIdentity::from(edge.b.0.as_str()))
} else if edge.b == dsl_identity {
Some(AgentIdentity::from(edge.a.0.as_str()))
} else {
None
}
})
.collect()
}
fn machine_external_peer_edges_for(
&self,
identity: &AgentIdentity,
) -> Vec<mob_dsl::ExternalPeerEdge> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.external_peer_edges
.iter()
.filter(|edge| edge.local == dsl_identity)
.cloned()
.collect()
}
async fn durably_abandon_respawn_topology_if_terminal(
&mut self,
identity: &AgentIdentity,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let (pending_hold, current_retiring, old_runtime_id, old_fence_token) = {
let state = self.dsl_authority.state();
let old_runtime_id =
state
.identity_runtime_generations
.get(&dsl_identity)
.map(|generation| {
AgentRuntimeId::new(
identity.clone(),
crate::ids::Generation::new(generation.0),
)
});
let current_retiring =
state
.identity_to_runtime
.get(&dsl_identity)
.is_some_and(|runtime_id| {
state.member_state_markers.get(runtime_id)
== Some(&mob_dsl::MobMemberState::Retiring)
});
let old_fence_token = state
.identity_runtime_fence_tokens
.get(&dsl_identity)
.map(|fence| crate::ids::FenceToken::new(fence.0));
(
state.pending_respawn_topology.contains(&dsl_identity),
current_retiring,
old_runtime_id,
old_fence_token,
)
};
let Some(old_runtime_id) = old_runtime_id else {
if !pending_hold && !current_retiring {
return Ok(());
}
return Err(MobError::Internal(format!(
"respawn topology hold for '{identity}' has no generation history"
)));
};
let durable_preserving_start = self
.preserved_respawn_topology_event_exists(identity, old_runtime_id.generation)
.await;
if !(pending_hold || current_retiring && durable_preserving_start) {
return Ok(());
}
let Some(old_fence_token) = old_fence_token else {
return Err(MobError::Internal(format!(
"respawn topology hold for '{identity}' has no fence-token history"
)));
};
self.durably_abandon_respawn_topology_if_terminal_exact(
identity,
&RespawnOrigin {
old_runtime_id,
old_fence_token,
},
)
.await
}
/// Persist the generation-scoped abandonment marker before exposing a
/// definitive respawn failure. While the exact old runtime is Retiring the
/// machine retains graph authority for physical trust cleanup but switches
/// terminal policy to ordinary unwiring. Once terminal, the same marker
/// atomically prunes retained member and external topology.
async fn durably_abandon_respawn_topology_if_terminal_exact(
&mut self,
identity: &AgentIdentity,
origin: &RespawnOrigin,
) -> Result<(), MobError> {
if origin.old_runtime_id.identity != *identity {
return Err(MobError::Internal(format!(
"respawn origin runtime '{}' does not belong to '{identity}'",
origin.old_runtime_id
)));
}
let desired = MobEventKind::RespawnTopologyAbandoned {
agent_identity: identity.clone(),
generation: origin.old_runtime_id.generation,
agent_runtime_id: Some(origin.old_runtime_id.clone()),
fence_token: Some(origin.old_fence_token),
};
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::ObserveRespawnTopologyAbandoned {
agent_identity: mob_dsl::AgentIdentity::from_domain(identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&origin.old_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(origin.old_fence_token),
generation: mob_dsl::Generation::from_domain(origin.old_runtime_id.generation),
},
"observe_respawn_topology_abandoned",
)?;
let append_authorized = prepared.transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::AppendLifecycleJournal {
kind: mob_dsl::MobLifecycleJournalKind::RespawnTopologyAbandoned,
..
}
)
});
if !append_authorized {
// Exact duplicate and delayed-after-successor signals are total
// machine no-ops and must not append another journal carrier.
return Ok(());
}
let cursor_floor = self.events.latest_cursor().await?;
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await;
if let Err(error) = append {
match reconcile_exact_mob_event_after_cursor(
self.events.as_ref(),
&self.definition.id,
cursor_floor,
&desired,
)
.await
{
Ok(Some(_)) => {}
Ok(None) => return Err(MobError::from(error)),
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"respawn topology abandonment append failed ({error}) and exact reconciliation remained uncertain; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
}
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"respawn topology abandonment event committed but live machine convergence failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.preserved_respawn_topology_event_index
.write()
.await
.remove(&format!(
"{}:{}",
identity,
origin.old_runtime_id.generation.get()
));
Ok(())
}
async fn cleanup_retiring_external_peer_edges(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
let local = AgentIdentity::from(entry.agent_identity.as_str());
let external_edges = self.machine_external_peer_edges_for(&entry.agent_identity);
if external_edges.is_empty() {
return Ok(());
}
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity) {
// Placed↔legacy-external wiring has no remote cleanup protocol.
// Fail closed rather than treating the host-resident session id
// as local or deleting topology while remote trust survives.
return Err(MobError::WiringError(format!(
"retire external-peer cleanup is unsupported for placed member '{}'",
entry.agent_identity
)));
}
let comms = self.provisioner_comms(&entry.member_ref).await;
for edge in external_edges {
let peer_name = meerkat_core::comms::PeerName::new(edge.endpoint.name.0.clone())
.map_err(|error| {
MobError::WiringError(format!(
"retire external cleanup has invalid peer name '{}': {error}",
edge.endpoint.name.0
))
})?;
let prior_spec = Self::trusted_peer_descriptor_from_machine_endpoint(&edge.endpoint)?;
let key = Self::external_peer_key_for_edge(&edge);
match comms.as_ref() {
Some(comms) => {
let unwiring_authority =
self.apply_cleanup_retiring_external_peer(entry, &key, &edge)?;
let prior_removal_key = Self::trusted_peer_removal_key(&prior_spec);
if let Err(error) = self
.apply_trusted_peer_remove(
comms.as_ref(),
prior_removal_key,
unwiring_authority,
)
.await
{
self.apply_restore_retiring_external_peer(entry, &key, &edge)
.map_err(|rollback_error| {
MobError::WiringError(format!(
"retiring external trust removal failed: {error}; machine rollback failed: {rollback_error}"
))
})?;
return Err(MobError::from(error));
}
}
None => {
self.apply_cleanup_retiring_external_peer_observed_absent(entry, &key, &edge)?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %entry.agent_identity,
peer_id = %edge.endpoint.peer_id.0,
"retiring external cleanup observed no surviving local comms runtime; committed no-effect machine topology cleanup"
);
}
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerUnwired {
local: local.clone(),
peer_name,
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_error) => {
match comms.as_ref() {
Some(comms) => {
let rollback =
self.apply_restore_retiring_external_peer(entry, &key, &edge)?;
let rollback_authority = rollback.external_authority()?.clone();
self.apply_trusted_peer_add(
comms.as_ref(),
prior_spec,
rollback_authority,
)
.await
.map_err(|rollback_error| {
MobError::WiringError(format!(
"retiring external event append failed: {append_error}; trust rollback failed: {rollback_error}"
))
})?;
}
None => self.apply_restore_retiring_external_peer_observed_absent(
entry, &key, &edge,
)?,
}
return Err(MobError::from(append_error));
}
};
self.roster.write().await.apply_event(&stored);
}
Ok(())
}
/// Explicitly unwind every incident edge whose route realization reaches
/// a member host. The ordinary unwire path is the only shell realization
/// of the machine's `UnwireMembers` authority. It synchronously removes
/// surviving placed-target trust before the durable graph commit, so
/// retirement composes it rather than pruning graph state at final
/// publication.
async fn cleanup_retiring_placed_member_edges(
&mut self,
retiring_identity: &AgentIdentity,
preserve_machine_topology: bool,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(retiring_identity);
let peers = {
let state = self.dsl_authority.state();
state
.wiring_edges
.iter()
.filter(|edge| edge.a == dsl_identity || edge.b == dsl_identity)
.filter(|edge| {
state.member_placement.contains_key(&edge.a)
|| state.member_placement.contains_key(&edge.b)
})
.filter_map(|edge| {
if edge.a == dsl_identity {
Some(AgentIdentity::from(edge.b.0.as_str()))
} else if edge.b == dsl_identity {
Some(AgentIdentity::from(edge.a.0.as_str()))
} else {
None
}
})
.collect::<Vec<_>>()
};
for peer in peers {
self.handle_unwire_for_retirement(
retiring_identity.clone(),
peer,
preserve_machine_topology,
)
.await?;
}
Ok(())
}
async fn member_retire_trust_cleanup_plan(
&mut self,
agent_identity: &AgentIdentity,
entry: &RosterEntry,
) -> Result<RetireTrustCleanupPlan, MobError> {
let retiring_identity = AgentIdentity::from(agent_identity.as_str());
let machine_wired_peer_identities =
self.machine_member_wired_peer_identities_for(&retiring_identity);
if machine_wired_peer_identities.is_empty() {
return Ok(RetireTrustCleanupPlan::empty());
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let retained_peer_endpoints = self
.dsl_authority
.state()
.member_prior_peer_endpoints
.get(&dsl_identity)
.cloned()
.unwrap_or_default();
let exact_runtime_is_retiring = self
.dsl_authority
.state()
.identity_to_runtime
.get(&dsl_identity)
== Some(&dsl_runtime_id)
&& self
.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id)
== Some(&mob_dsl::MobMemberState::Retiring);
let durable_retiring_spec = if exact_runtime_is_retiring {
self.machine_member_peer_spec_for(
&retiring_identity,
"retire trust authority durable retiring endpoint",
)?
} else {
None
};
let (retiring_spec, retiring_comms) = if let Some(durable_retiring_spec) =
durable_retiring_spec
{
let retiring_comms = match self
.resolve_wiring_endpoint(entry, "retire trust authority retiring member")
.await
{
Ok(WiringEndpoint::Local { comms, .. }) => Some(comms),
Ok(WiringEndpoint::PeerOnly { .. }) => {
Some(self.supervisor_bridge.runtime_core().await)
}
Ok(WiringEndpoint::Placed { .. }) => None,
Err(error) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
error = %error,
"retire trust authority using durable endpoint without a live retiring comms runtime"
);
None
}
};
(durable_retiring_spec, retiring_comms)
} else {
match self
.resolve_wiring_endpoint(entry, "retire trust authority retiring member")
.await
{
Ok(WiringEndpoint::Local { spec, comms, .. }) => (spec, Some(comms)),
Ok(WiringEndpoint::PeerOnly { spec, .. }) => {
(spec, Some(self.supervisor_bridge.runtime_core().await))
}
// A retiring PLACED member's own runtime is remote: peers'
// local rows are cleaned with its machine-recorded spec; its
// own rows die with the host-side release.
Ok(WiringEndpoint::Placed { spec, .. }) => (spec, None),
Err(error) => {
let retained_spec = match self.machine_member_peer_spec_for(
&retiring_identity,
"retire trust authority retiring member",
)? {
Some(spec) => Some(spec),
None => match self.roster_member_peer_spec_for(
entry,
"retire trust authority retiring member",
)? {
Some(spec) => Some(spec),
None => {
self.retained_member_peer_spec_from_wired_peer_trust(
entry,
&machine_wired_peer_identities,
"retire trust authority retiring member",
)
.await?
}
},
};
match retained_spec {
Some(spec) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
"retire trust authority using retained generated peer endpoint for member without live comms runtime"
);
(spec, None)
}
None => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
error = %error,
"retire trust authority found no retained peer trust to clean up for member without live comms runtime"
);
return Ok(RetireTrustCleanupPlan {
retiring_comms: None,
retiring_spec: None,
machine_wired_peer_identities,
trust_unwire_authority_by_peer: BTreeMap::new(),
historical_trust_unwire_authorities_by_peer: BTreeMap::new(),
});
}
}
}
}
};
let retiring_peer_id = Self::trusted_peer_removal_key(&retiring_spec);
let mut authorities = BTreeMap::new();
let mut historical_authorities = BTreeMap::new();
let batch_edges = machine_wired_peer_identities
.iter()
.map(|peer_identity| {
mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&retiring_identity),
mob_dsl::AgentIdentity::from_domain(peer_identity),
)
})
.collect::<Vec<_>>();
let mut batch_unwiring_handoffs = match self.authorize_member_trust_unwiring_batch(
&batch_edges,
"member_retire_trust_authority_batch",
) {
Ok(handoffs) => Some(handoffs),
Err(error) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
%error,
"dense retire trust authorization was not admissible; falling back to exact per-edge recovery authority"
);
None
}
};
for peer_identity in &machine_wired_peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let peer_spec = match self
.resolve_wiring_endpoint(&peer_entry, "member_retire_trust_authority peer")
.await
{
Ok(
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. }
| WiringEndpoint::Placed { spec, .. },
) => spec,
Err(error) => {
let retained_spec = match self.machine_member_peer_spec_for(
peer_identity,
"member_retire_trust_authority peer",
)? {
Some(spec) => Some(spec),
None => self.roster_member_peer_spec_for(
&peer_entry,
"member_retire_trust_authority peer",
)?,
};
retained_spec.ok_or(error)?
}
};
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
let peer_identity_for_cleanup = peer_identity.clone();
let edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&retiring_identity),
mob_dsl::AgentIdentity::from_domain(peer_identity),
);
let mut retained_authorities = Vec::new();
for retained_peer_endpoint in &retained_peer_endpoints {
let retained_peer_id = retained_peer_endpoint.peer_id.0.clone();
let handoff = self.authorize_member_endpoint_migration_trust_cleanup(
&edge,
entry,
retained_peer_endpoint,
"member_retire_historical_trust_authority",
)?;
let authority =
handoff.unwiring_authority_for(&retiring_identity, &retained_peer_id)?;
retained_authorities.push((retained_peer_id, authority));
}
if !retained_authorities.is_empty() {
historical_authorities.insert(peer_identity.clone(), retained_authorities);
}
let handoff = if let Some(handoff) = batch_unwiring_handoffs
.as_mut()
.and_then(|handoffs| handoffs.remove(&edge))
{
handoff
} else {
match self.authorize_member_trust_unwiring(
&edge,
"member_retire_trust_authority",
) {
Ok(handoff) => handoff,
Err(unwiring_error) => match self.authorize_member_trust_cleanup_observed(
&edge,
&retiring_identity,
&retiring_peer_id,
&peer_identity_for_cleanup,
&peer_peer_id,
"member_retire_trust_authority_observed",
) {
Ok(handoff) => handoff,
Err(observed_error) => self
.authorize_retiring_member_trust_cleanup_observed(
&edge,
entry,
&retiring_identity,
&retiring_peer_id,
&peer_identity_for_cleanup,
&peer_peer_id,
"member_retire_trust_authority_retiring_observed",
)
.map_err(|retiring_error| {
MobError::WiringError(format!(
"member retire trust cleanup failed: {unwiring_error}; observed restore-failure cleanup failed: {observed_error}; observed retiring-runtime cleanup failed: {retiring_error}"
))
})?,
},
}
};
let authority =
handoff.unwiring_authority_for(&retiring_identity, &retiring_peer_id)?;
authorities.insert(peer_identity.clone(), authority);
}
Ok(RetireTrustCleanupPlan {
retiring_comms,
retiring_spec: Some(retiring_spec),
machine_wired_peer_identities,
trust_unwire_authority_by_peer: authorities,
historical_trust_unwire_authorities_by_peer: historical_authorities,
})
}
/// Snapshot member state for disposal from a roster entry.
async fn disposal_context_from_entry(
&self,
agent_identity: &AgentIdentity,
entry: &RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
preserve_machine_topology: bool,
) -> DisposalContext {
let placed =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity);
let retiring_key = if placed {
None
} else {
self.provisioner_comms(&entry.member_ref)
.await
.and_then(|comms| comms.public_key())
};
DisposalContext {
agent_identity: agent_identity.clone(),
entry: entry.clone(),
retiring_key,
retiring_comms: trust_cleanup_plan.retiring_comms,
retiring_spec: trust_cleanup_plan.retiring_spec,
preserve_machine_topology,
machine_wired_peer_identities: trust_cleanup_plan.machine_wired_peer_identities,
trust_unwire_authority_by_peer: trust_cleanup_plan.trust_unwire_authority_by_peer,
historical_trust_unwire_authorities_by_peer: trust_cleanup_plan
.historical_trust_unwire_authorities_by_peer,
}
}
/// Execute the disposal pipeline for a member.
///
/// Runs policy-driven steps in order, then removes the member from the
/// roster once archive-confirmed terminal publication succeeds. A failed
/// terminal journal append retains a non-routable Retiring anchor so the
/// same process can retry publication without resurrecting the session.
async fn dispose_member(
&mut self,
ctx: &DisposalContext,
policy: &mut dyn ErrorPolicy,
) -> (DisposalReport, Option<mob_dsl::MemberSessionDisposal>) {
let mut report = DisposalReport::new();
let mut archive_disposal = None;
for &step in &DisposalStep::ORDERED {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
"MobActor::dispose_member executing step"
);
match self.execute_step(step, ctx).await {
Ok(disposal) => {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
"MobActor::dispose_member completed step"
);
if let Some(disposal) = disposal {
archive_disposal = Some(disposal);
}
report.completed.push(step);
}
Err(error) => {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
error = %error,
"MobActor::dispose_member step failed"
);
if policy.on_step_error(step, &error, ctx) {
report.skipped.push((step, error));
} else {
report.aborted_at = Some((step, error));
break;
}
}
}
}
let archive_failed = report
.skipped
.iter()
.any(|(step, _)| *step == DisposalStep::ArchiveSession)
|| matches!(
report.aborted_at.as_ref(),
Some((DisposalStep::ArchiveSession, _))
);
// Edge-lock cleanup is safe after admission, but the roster entry is
// the durable shell realization anchor until archive-confirmed final
// publication succeeds. Any ArchiveSession failure may be transient;
// dropping this non-routable Retiring entry would make both same-
// process retry and cold replay lose the exact session/runtime tuple.
self.dispose_prune_edge_locks(ctx).await;
if archive_failed {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
"retaining durable retiring roster entry after ArchiveSession failure"
);
return (report, archive_disposal);
}
(report, archive_disposal)
}
/// Run destroy disposal mechanics without publishing terminal member
/// authority. Local cleanup finalizes immediately after these steps;
/// remote cleanup must first prove supervisor revocation.
async fn dispose_member_for_destroy_steps(
&mut self,
ctx: &DisposalContext,
) -> (DisposalReport, Option<mob_dsl::MemberSessionDisposal>) {
let mut report = DisposalReport::new();
let mut policy = AbortOnError;
let mut archive_disposal = None;
for &step in &DisposalStep::ORDERED {
match self.execute_destroy_step(step, ctx).await {
Ok(disposal) => {
if let Some(disposal) = disposal {
archive_disposal = Some(disposal);
}
report.completed.push(step);
}
Err(error) => {
if policy.on_step_error(step, &error, ctx) {
report.skipped.push((step, error));
} else {
report.aborted_at = Some((step, error));
break;
}
}
}
}
(report, archive_disposal)
}
/// Destroy cleanup must keep canonical member/session authority until all
/// critical per-member cleanup steps succeed. General member disposal
/// removes roster state in a finally block, but destroy retries rebuild
/// cleanup work from the roster after a partial attempt.
async fn dispose_member_for_destroy(&mut self, ctx: &DisposalContext) -> DisposalReport {
let (mut report, archive_disposal) = self.dispose_member_for_destroy_steps(ctx).await;
if let Some(disposal) = archive_disposal {
if let Err(error) = self
.record_destroy_member_retirement_archived(ctx, disposal)
.await
{
report.aborted_at = Some((DisposalStep::ArchiveSession, error));
} else {
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&ctx.entry.agent_identity)
{
target.take();
report.aborted_at = Some((
DisposalStep::ArchiveSession,
MobError::Internal(format!(
"fault-injected cancellation after destroy member archive commit for '{}'",
ctx.entry.agent_identity
)),
));
}
}
}
report
}
async fn execute_destroy_step(
&mut self,
step: DisposalStep,
ctx: &DisposalContext,
) -> Result<Option<mob_dsl::MemberSessionDisposal>, MobError> {
match step {
DisposalStep::StopHostLoop => self
.dispose_stop_host_loop_for_destroy(ctx)
.await
.map(|()| None),
_ => self.execute_step(step, ctx).await,
}
}
fn destroy_disposal_failure(report: &DisposalReport) -> Option<String> {
if let Some((step, error)) = &report.aborted_at {
return Some(format!("disposal aborted at {step}: {error}"));
}
report
.skipped
.first()
.map(|(step, error)| format!("disposal completed but {step} failed: {error}"))
}
fn cleanup_retired_member_machine_wiring(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
if ctx.machine_wired_peer_identities.is_empty() {
return Ok(());
}
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let retiring_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id);
let cleanup_inputs = ctx
.machine_wired_peer_identities
.iter()
.map(|peer_identity| {
let peer_identity = mob_dsl::AgentIdentity::from_domain(peer_identity);
let edge =
mob_dsl::WiringEdge::new(retiring_identity.clone(), peer_identity.clone());
mob_dsl::MobMachineInput::CleanupRetiringMemberWiring {
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
edge,
agent_identity: retiring_identity.clone(),
agent_runtime_id: retiring_runtime_id.clone(),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
}
})
.collect::<Vec<_>>();
let prepared =
self.prepare_dsl_inputs(&cleanup_inputs, "member_retire_machine_wiring_cleanup")?;
self.commit_prepared_dsl_input(prepared)?;
Ok(())
}
fn cleanup_member_machine_wiring_edge(
&mut self,
a: &AgentIdentity,
b: &AgentIdentity,
context: &'static str,
) -> Result<(), MobError> {
let input = mob_dsl::MobMachineInput::UnwireMembers {
edge: mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(a),
mob_dsl::AgentIdentity::from_domain(b),
),
};
let prepared = self.prepare_dsl_input_transition(input, context)?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
fn cleanup_retiring_member_machine_wiring_edge(
&mut self,
ctx: &DisposalContext,
peer_identity: &AgentIdentity,
context: &'static str,
) -> Result<mob_dsl::WiringEdge, MobError> {
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let peer_identity = mob_dsl::AgentIdentity::from_domain(peer_identity);
let edge = mob_dsl::WiringEdge::new(retiring_identity.clone(), peer_identity.clone());
self.apply_dsl_input(
mob_dsl::MobMachineInput::CleanupRetiringMemberWiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
agent_identity: retiring_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
},
context,
)?;
Ok(edge)
}
fn restore_retiring_member_machine_wiring_edge(
&mut self,
ctx: &DisposalContext,
edge: mob_dsl::WiringEdge,
context: &'static str,
) -> Result<(), MobError> {
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
self.apply_dsl_input(
mob_dsl::MobMachineInput::RestoreRetiringMemberWiring {
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
edge,
agent_identity: retiring_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
},
context,
)?;
Ok(())
}
/// Dispatch a disposal step. Exhaustive match ensures compiler forces new
/// arms when `DisposalStep` variants are added.
/// Only the archive step yields a typed member-session disposal; the
/// other steps report bare completion.
async fn execute_step(
&mut self,
step: DisposalStep,
ctx: &DisposalContext,
) -> Result<Option<mob_dsl::MemberSessionDisposal>, MobError> {
match step {
DisposalStep::StopHostLoop => self.dispose_stop_host_loop(ctx).await.map(|()| None),
DisposalStep::NotifyPeers => self.dispose_notify_peers(ctx).await.map(|()| None),
DisposalStep::CleanupMachineTopology => {
if ctx.preserve_machine_topology {
Ok(None)
} else {
self.cleanup_retiring_external_peer_edges(&ctx.entry)
.await?;
self.cleanup_retired_member_machine_wiring(ctx)?;
Ok(None)
}
}
DisposalStep::ArchiveSession => {
let is_placed = super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&ctx.entry.agent_identity,
);
if Self::runtime_binding_for_entry(&ctx.entry).is_some() && !is_placed {
if self.remote_runtime_retired_for_entry(&ctx.entry) {
return Ok(Some(
mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned,
));
}
let disposal = self.dispose_archive_session(ctx).await?;
self.record_remote_member_runtime_retired(&ctx.entry)
.await?;
Ok(Some(disposal))
} else {
self.dispose_archive_session(ctx).await.map(Some)
}
}
}
}
/// Stop the autonomous member host loop. Session unregister is owned by
/// the archive step after runtime retire/drain quiesces.
async fn dispose_stop_host_loop(&mut self, ctx: &DisposalContext) -> Result<(), MobError> {
if ctx.entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
self.stop_autonomous_member_for_disposal(&ctx.agent_identity, &ctx.entry.member_ref)
.await?;
}
// Discharge the machine-owned kickoff quiesce obligation opened by
// every retire/destroy-retire transition (RequestKickoffQuiesce effect,
// decision 2 in the Stage-A notes). KickoffQuiesced is TOTAL: the
// machine self-loops with no-op when the kickoff is already Idle or the
// member is in the Destroyed phase, so no shell-side guard is needed.
self.apply_kickoff_input(
&ctx.agent_identity,
mob_dsl::MobMachineInput::KickoffQuiesced {
member_id: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
},
"dispose_stop_host_loop",
)
.await?;
Ok(())
}
async fn dispose_stop_host_loop_for_destroy(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
// For AutonomousHost members, stop the host loop unless a prior partial
// destroy already cleaned it up. For all members (including TurnDriven)
// the KickoffQuiesced discharge happens unconditionally at the end of this
// function because RequestKickoffQuiesce is emitted by every
// retire/destroy-retire transition regardless of runtime mode.
if ctx.entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
let placed = super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&ctx.entry.agent_identity,
);
let session_already_gone = {
#[cfg(feature = "runtime-adapter")]
{
!placed
&& match (
&self.runtime_adapter,
ctx.entry.member_ref.bridge_session_id(),
) {
(Some(adapter), Some(session_id)) => {
!adapter.contains_session(session_id).await
}
_ => false,
}
}
#[cfg(not(feature = "runtime-adapter"))]
{
let _ = placed;
false
}
};
if !session_already_gone {
self.stop_autonomous_member_for_disposal(
&ctx.agent_identity,
&ctx.entry.member_ref,
)
.await?;
}
// else: A prior partial destroy stopped and unregistered the
// autonomous runtime; retry continues from the retained roster anchor
// to reach ArchiveSession again. Still need the quiesce discharge
// below, so we fall through rather than returning early.
}
// Unconditional quiesce discharge — covers TurnDriven members (no host
// loop), AutonomousHost members (host loop stopped above), and
// prior-partial-destroy retries (session already gone, loop skipped).
// KickoffQuiesced is total: the machine self-loops with a no-op when the
// kickoff is Idle or the member is Destroyed (decision 2, Stage-A notes).
self.apply_kickoff_input(
&ctx.agent_identity,
mob_dsl::MobMachineInput::KickoffQuiesced {
member_id: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
},
"dispose_stop_host_loop_for_destroy",
)
.await?;
Ok(())
}
/// Notify all machine-wired peers that this member is retiring.
///
/// Iterates the MobMachine-owned wiring snapshot; skips absent peers.
/// Returns the first error encountered, if any.
async fn dispose_notify_peers(&mut self, ctx: &DisposalContext) -> Result<(), MobError> {
if ctx.machine_wired_peer_identities.is_empty() {
return Ok(());
}
let Some(retiring_spec) = ctx.retiring_spec.as_ref() else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
"dispose_notify_peers: skipping lifecycle notice and trust cleanup because retiring endpoint is absent"
);
return Ok(());
};
let peer_identities = ctx
.machine_wired_peer_identities
.iter()
.cloned()
.collect::<Vec<_>>();
let mut local_jobs = Vec::new();
let mut local_peer_specs = Vec::new();
let mut peer_only_jobs = Vec::new();
for peer_identity in peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(&peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: skipping absent peer"
);
continue;
};
let endpoint = match self
.resolve_wiring_endpoint(&peer_entry, "dispose_notify_peers")
.await
{
Ok(endpoint) => endpoint,
Err(error) if Self::runtime_binding_for_entry(&peer_entry).is_none() => {
let retained_spec = self.machine_member_peer_spec_for(
&peer_identity,
"dispose_notify_peers retained local endpoint",
)?;
let retained_spec = match retained_spec {
Some(spec) => Some(spec),
None => self.roster_member_peer_spec_for(
&peer_entry,
"dispose_notify_peers retained local endpoint",
)?,
}
.ok_or_else(|| {
MobError::RetirementTopologyIncomplete(format!(
"failed to resolve machine-wired peer '{peer_identity}' for trust cleanup and no retained endpoint exists: {error}"
))
})?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
%error,
"dispose_notify_peers: retaining machine endpoint for peer without a live local comms runtime"
);
if ctx.preserve_machine_topology {
// Respawn owns this edge through its captured restore
// plan. With no surviving recipient runtime there is
// no physical trust row to revoke here, and deleting
// the machine edge would erase the later typed restore
// failure/retry authority.
continue;
}
local_peer_specs.push((peer_identity, retained_spec));
continue;
}
Err(error) => {
return Err(MobError::RetirementTopologyIncomplete(format!(
"failed to resolve machine-wired peer '{peer_identity}' for trust cleanup: {error}"
)));
}
};
match endpoint {
WiringEndpoint::Local { comms, spec, .. } => {
let Some(authority) = ctx
.trust_unwire_authority_by_peer
.get(&peer_identity)
.cloned()
else {
return Err(MobError::RetirementTopologyIncomplete(format!(
"dispose_notify_peers missing generated retire trust handoff for '{peer_identity}'"
)));
};
let historical_authorities = ctx
.historical_trust_unwire_authorities_by_peer
.get(&peer_identity)
.cloned()
.unwrap_or_default();
local_peer_specs.push((peer_identity.clone(), spec.clone()));
local_jobs.push((
peer_identity,
spec,
comms,
authority,
historical_authorities,
));
}
WiringEndpoint::PeerOnly { spec, binding } => {
peer_only_jobs.push((peer_identity, spec, binding));
}
WiringEndpoint::Placed { .. } => {
// A placed peer's runtime is remote: its trust row for
// the retiring member was synchronously removed before
// the edge's durable unwind, not through this local
// disposal notification fan-out.
tracing::debug!(
mob_id = %self.definition.id,
peer = %peer_identity,
"dispose_notify_peers skipping placed peer (remote runtime)"
);
}
}
}
let actor = &*self;
let retiring_key = Self::trusted_peer_removal_key(retiring_spec);
let retiring_comms = ctx.retiring_comms.clone();
let mut local_tasks = FuturesUnordered::new();
let mut topology_error = None;
for (peer_identity, recipient_spec, recipient_comms, authority, historical_authorities) in
local_jobs
{
let retiring_key = retiring_key.clone();
let retiring_comms = retiring_comms.clone();
let retiring_spec = retiring_spec.clone();
let retired_id = ctx.agent_identity.clone();
let retired_entry = ctx.entry.clone();
local_tasks.push(async move {
if let Some(retiring_comms) = retiring_comms.as_ref() {
if let Err(error) = actor
.notify_peer_retired(
&recipient_spec,
&retired_id,
&retired_entry,
&retiring_spec,
retiring_comms,
)
.await
{
if Self::is_peer_destroying_admission_rejection(&error) {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: peer rejected lifecycle notice (already retiring)"
);
} else {
tracing::warn!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
error = %error,
"dispose_notify_peers: lifecycle notice failed; continuing with critical trust cleanup"
);
}
} else {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: lifecycle notice sent"
);
}
} else {
if super::member_runtime_is_host_owned(
actor.dsl_authority.state(),
&retired_id,
) {
tracing::warn!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: placed retiring member has no remote-sender lifecycle-notice realization; critical trust cleanup continues"
);
} else {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: skipping lifecycle notice because retiring member has no live comms runtime"
);
}
}
for (historical_peer_id, historical_authority) in historical_authorities {
actor
.apply_trusted_peer_remove(
recipient_comms.as_ref(),
historical_peer_id.clone(),
historical_authority,
)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to remove historical retiring member trust '{historical_peer_id}' from '{peer_identity}': {error}"
))
})?;
}
actor
.apply_trusted_peer_remove(recipient_comms.as_ref(), retiring_key, authority)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to remove retiring member trust from '{peer_identity}': {error}"
))
})?;
Ok(())
});
if local_tasks.len() >= RETIRE_LOCAL_TRUST_CLEANUP_CONCURRENCY
&& let Some(result) = local_tasks.next().await
&& let Err(error) = result
&& topology_error.is_none()
{
topology_error = Some(error);
}
}
while let Some(result) = local_tasks.next().await {
if let Err(error) = result
&& topology_error.is_none()
{
topology_error = Some(error);
}
}
drop(local_tasks);
if topology_error.is_none()
&& !ctx.preserve_machine_topology
&& Self::runtime_binding_for_entry(&ctx.entry).is_some()
{
for (peer_identity, peer_spec) in local_peer_specs {
let cleanup_edge = if ctx.preserve_machine_topology {
None
} else {
match self.cleanup_retiring_member_machine_wiring_edge(
ctx,
&peer_identity,
"dispose_notify_peers_local_machine_wiring_cleanup",
) {
Ok(edge) => Some(edge),
Err(error) => {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"failed to reserve machine-wired edge for retiring peer-only cleanup of '{peer_identity}': {error}"
)));
continue;
}
}
};
if let Err(error) = self
.unwire_retiring_peer_only_side(
&ctx.entry,
retiring_spec,
&peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if let Some(cleanup_edge) = cleanup_edge {
if let Err(rollback_error) = self
.restore_retiring_member_machine_wiring_edge(
ctx,
cleanup_edge,
"dispose_notify_peers_local_machine_wiring_rollback",
)
{
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"retiring peer-only unwire failed: {error}; machine wiring rollback failed: {rollback_error}"
)));
} else {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"retiring peer-only member failed to unwire local peer '{peer_identity}': {error}"
)));
}
} else {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"respawn retirement failed to remove old trust for local peer '{peer_identity}' while retaining desired topology: {error}"
)));
}
}
}
}
for (peer_identity, recipient_spec, recipient_binding) in peer_only_jobs {
if let Some(retiring_comms) = ctx.retiring_comms.as_ref() {
if let Err(error) = self
.notify_peer_retired(
&recipient_spec,
&ctx.agent_identity,
&ctx.entry,
retiring_spec,
retiring_comms,
)
.await
{
if Self::is_peer_destroying_admission_rejection(&error) {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: peer rejected lifecycle notice (already retiring)"
);
} else {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
error = %error,
"dispose_notify_peers: lifecycle notice failed; continuing with critical remote trust cleanup"
);
}
} else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: lifecycle notice sent"
);
}
} else {
if super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&ctx.agent_identity,
) {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: placed retiring member has no remote-sender lifecycle-notice realization; critical remote trust cleanup continues"
);
} else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: skipping lifecycle notice because retiring member has no live comms runtime"
);
}
}
if ctx.preserve_machine_topology {
if let Err(error) = self
.unwire_peer_only_survivor_for_respawn(
&recipient_spec,
&recipient_binding,
&ctx.entry,
retiring_spec,
std::time::Duration::from_secs(10),
)
.await
&& topology_error.is_none()
{
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"respawn retirement failed to remove old trust from peer-only survivor '{peer_identity}' while retaining desired topology: {error}"
)));
}
continue;
}
let cleanup_edge = if ctx.preserve_machine_topology {
None
} else {
match self.cleanup_retiring_member_machine_wiring_edge(
ctx,
&peer_identity,
"dispose_notify_peers_peer_only_machine_wiring_cleanup",
) {
Ok(edge) => Some(edge),
Err(error) => {
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"failed to reserve machine-wired edge for peer-only trust cleanup of '{peer_identity}': {error}"
)));
}
continue;
}
}
};
let recipient_error = self
.unwire_peer_only_recipient(
&recipient_spec,
Some(&recipient_binding),
retiring_spec,
std::time::Duration::from_secs(10),
)
.await
.err();
let retiring_error = self
.unwire_retiring_peer_only_side(
&ctx.entry,
retiring_spec,
&recipient_spec,
std::time::Duration::from_secs(10),
)
.await
.err();
if recipient_error.is_some() || retiring_error.is_some() {
let error = match (recipient_error, retiring_error) {
(Some(recipient), Some(retiring)) => format!(
"surviving peer unwire failed: {recipient}; retiring peer unwire failed: {retiring}"
),
(Some(recipient), None) => {
format!("surviving peer unwire failed: {recipient}")
}
(None, Some(retiring)) => {
format!("retiring peer unwire failed: {retiring}")
}
(None, None) => {
"peer-only unwire failure was reported without an error detail".to_string()
}
};
if let Some(cleanup_edge) = cleanup_edge
&& let Err(rollback_error) = self.restore_retiring_member_machine_wiring_edge(
ctx,
cleanup_edge,
"dispose_notify_peers_peer_only_machine_wiring_rollback",
)
{
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"peer-only unwire failed: {error}; machine wiring rollback failed: {rollback_error}"
)));
}
continue;
}
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"peer-only unwire failed for '{peer_identity}': {error}"
)));
}
}
}
match topology_error {
Some(error) => Err(error),
None => Ok(()),
}
}
fn is_peer_destroying_admission_rejection(error: &MobError) -> bool {
matches!(
error,
MobError::CommsError(meerkat_core::comms::SendError::AdmissionDropped {
reason: meerkat_core::comms::AdmissionDropReason::ClassificationRejected
| meerkat_core::comms::AdmissionDropReason::SessionClosed,
})
)
}
/// Archive the member's session, returning the typed disposal the
/// provisioner observed (§19.L4).
async fn dispose_remote_turn_custody_after_host_release(
&mut self,
agent_identity: &AgentIdentity,
host_id: &mob_dsl::HostId,
host_binding_generation: u64,
member_session_id: &mob_dsl::SessionId,
generation: u64,
fence_token: u64,
) -> Result<(), MobError> {
let reason = if self
.remote_flow_tickets
.is_member_rematerializing(agent_identity)
{
super::remote_flow_ticket::REMATERIALIZED_STEP_FAILURE_REASON
} else {
super::remote_flow_ticket::MEMBER_RELEASED_STEP_FAILURE_REASON
};
self.dispose_remote_turn_custody_for_exact_residency(
ExactRemoteTurnResidency {
agent_identity,
host_id,
host_binding_generation,
member_session_id,
generation,
fence_token,
},
reason,
"dispose_remote_turn_obligation_after_exact_host_release",
)
.await?;
self.dispose_placed_completion_custody_for_exact_residency(ExactRemoteTurnResidency {
agent_identity,
host_id,
host_binding_generation,
member_session_id,
generation,
fence_token,
})
.await?;
self.dispose_placed_kickoff_custody_for_exact_residency(ExactRemoteTurnResidency {
agent_identity,
host_id,
host_binding_generation,
member_session_id,
generation,
fence_token,
})
.await
}
async fn dispose_placed_completion_custody_for_exact_residency(
&mut self,
residency: ExactRemoteTurnResidency<'_>,
) -> Result<(), MobError> {
let obligations = {
let matches = |obligation: &&mob_dsl::PlacedCompletionObligation| {
obligation.agent_identity.0 == residency.agent_identity.as_str()
&& obligation.host_id == *residency.host_id
&& obligation.host_binding_generation == residency.host_binding_generation
&& obligation.member_session_id == *residency.member_session_id
&& obligation.generation.0 == residency.generation
&& obligation.fence_token.0 == residency.fence_token
};
let state = self.dsl_authority.state();
state
.pending_placed_completion_outcomes
.iter()
.chain(state.resolved_placed_completion_outcomes.iter())
.filter(matches)
.cloned()
.collect::<BTreeSet<_>>()
};
for obligation in obligations {
let obligation_event =
super::placed_completion_reconciler::obligation_event(&obligation)?;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::DisposePlacedCompletionOutcome { obligation },
"dispose_placed_completion_after_exact_host_release",
)?;
self.ensure_placed_completion_carrier(MobEventKind::PlacedCompletionOutcomeDisposed {
obligation: obligation_event.clone(),
})
.await?;
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
if let Err(error) = self
.dsl_authority
.commit_prepared_authority(prepared.authority)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Dispose carrier is durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.queue_routed_effects_from(&effects);
let expected_member = super::bridge_protocol::BridgeMemberIncarnation {
mob_id: self.definition.id.to_string(),
agent_identity: obligation_event.agent_identity.to_string(),
host_id: obligation_event.host_id,
binding_generation: obligation_event.host_binding_generation,
member_session_id: obligation_event.member_session_id,
generation: obligation_event.generation.get(),
fence_token: obligation_event.fence_token.get(),
};
if let Err(error) = self
.member_event_pumps
.resolve_placed_completion_waiter_disposed(
&expected_member,
&obligation_event.input_id,
)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Dispose committed but waiter handoff failed; actor is fail-stopping for cold recovery: {error}"
)));
}
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
}
Ok(())
}
async fn dispose_placed_kickoff_custody_for_exact_residency(
&mut self,
residency: ExactRemoteTurnResidency<'_>,
) -> Result<(), MobError> {
let obligations = {
let matches = |obligation: &&mob_dsl::PlacedKickoffObligation| {
obligation.agent_identity.0 == residency.agent_identity.as_str()
&& obligation.host_id == *residency.host_id
&& obligation.host_binding_generation == residency.host_binding_generation
&& obligation.member_session_id == *residency.member_session_id
&& obligation.generation.0 == residency.generation
&& obligation.fence_token.0 == residency.fence_token
};
let state = self.dsl_authority.state();
state
.pending_placed_kickoff_outcomes
.iter()
.chain(state.resolved_placed_kickoff_outcomes.iter())
.filter(matches)
.cloned()
.collect::<BTreeSet<_>>()
};
for obligation in obligations {
let obligation_event =
super::remote_flow_ticket::placed_kickoff_obligation_event(&obligation)?;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::DisposePlacedKickoffObligation { obligation },
"dispose_placed_kickoff_after_exact_host_release",
)?;
let effects = prepared.transition.effects().to_vec();
let identity = obligation_event.agent_identity.clone();
let kickoff = self.prepared_kickoff_snapshot(&prepared, &identity).await?;
self.ensure_exact_structural_event_batch(vec![
MobEventKind::PlacedKickoffOutcomeDisposed {
obligation: obligation_event,
},
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: kickoff.clone(),
},
])
.await?;
self.commit_prepared_dsl_transition(prepared)?;
self.roster
.write()
.await
.set_kickoff(&identity, Some(kickoff));
self.emit_prepared_kickoff_notices(&identity, &effects)
.await;
}
Ok(())
}
async fn dispose_remote_turn_custody_for_exact_residency(
&mut self,
residency: ExactRemoteTurnResidency<'_>,
pending_failure_reason: &str,
context: &str,
) -> Result<(), MobError> {
let plan = {
let state = self.dsl_authority.state();
Self::remote_turn_custody_cleanup_plan(
&state.pending_remote_turn_outcomes,
&state.committed_remote_turn_outcomes,
&state.resolved_remote_turn_outcomes,
residency,
)
};
for cleanup in plan {
let obligation = cleanup.obligation;
let carrier = super::remote_flow_ticket::obligation_event(&obligation)?;
if cleanup.phase == RemoteTurnCustodyPhase::Pending {
// The actor is currently inside release/respawn and cannot
// wait for the awakened flow task to round-trip a receipt
// through this same mailbox. Commit the typed step failure
// and durable receipt here, while the exact old-incarnation
// intent still exists, before privacy cleanup deletes it.
let reason = pending_failure_reason.to_string();
self.commit_remote_turn_receipt_in_actor(crate::run::MobRunRemoteTurnReceipt {
obligation: carrier.clone(),
outcome: crate::run::MobRunRemoteTurnReceiptOutcome::Failed {
reason: reason.clone(),
no_effect_proof: None,
},
})
.await?;
self.remote_flow_tickets.fail_armed(
&carrier.agent_identity,
&carrier.input_id,
reason,
);
}
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnOutcomeDisposed {
obligation: carrier.clone(),
})
.await?;
// Privacy rows are deleted while custody still makes this entry
// retry-discoverable. If cleanup fails, the next promotion/release
// pass sees the same phase and retries. The durable Disposed
// carrier makes it safe to delete first and publish machine
// absence last; reversing these operations could strand private
// rows after a crash between machine removal and deletion.
self.run_store
.delete_remote_turn_receipt(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.run_store
.delete_remote_turn_intent(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.apply_dsl_input(
mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { obligation },
context,
)?;
}
Ok(())
}
fn remote_turn_custody_cleanup_plan(
pending: &BTreeSet<mob_dsl::RemoteTurnObligation>,
committed: &BTreeSet<mob_dsl::RemoteTurnObligation>,
resolved: &BTreeSet<mob_dsl::RemoteTurnObligation>,
residency: ExactRemoteTurnResidency<'_>,
) -> Vec<RemoteTurnCustodyCleanup> {
[
(RemoteTurnCustodyPhase::Pending, pending),
(RemoteTurnCustodyPhase::Committed, committed),
(RemoteTurnCustodyPhase::Resolved, resolved),
]
.into_iter()
.flat_map(|(phase, obligations)| {
obligations
.iter()
.filter(move |obligation| {
Self::remote_turn_obligation_matches_exact_residency(obligation, residency)
})
.cloned()
.map(move |obligation| RemoteTurnCustodyCleanup { obligation, phase })
})
.collect()
}
fn remote_turn_obligation_matches_exact_residency(
obligation: &mob_dsl::RemoteTurnObligation,
residency: ExactRemoteTurnResidency<'_>,
) -> bool {
obligation.agent_identity.0 == residency.agent_identity.as_str()
&& obligation.host_id == *residency.host_id
&& obligation.host_binding_generation == residency.host_binding_generation
&& obligation.member_session_id == *residency.member_session_id
&& obligation.generation.0 == residency.generation
&& obligation.fence_token.0 == residency.fence_token
}
pub(super) async fn dispose_archive_session(
&mut self,
ctx: &DisposalContext,
) -> Result<mob_dsl::MemberSessionDisposal, MobError> {
// Multi-host §19.L3: a PLACED member's one disposal verb is the
// host-addressed release. The directive is re-realized from the
// machine-recorded (generation, fence, host) facts — one derivation
// for the first attempt AND the ADJ-10 retry (pending-rotation-
// retry precedent); host-side release admission is dedup-idempotent.
// Reaching the local retire arm with a placed member would fold a
// NotFound into a synthetic `Archived` — an archive lie — so the
// placement read gates BEFORE any MemberRef-shape dispatch (DEC-R2).
{
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity);
let placed_host = self
.dsl_authority
.state()
.member_placement
.get(&dsl_identity)
.cloned();
if let Some(host) = placed_host {
let (generation, fence_token, member_session_id) = {
let state = self.dsl_authority.state();
(
state
.identity_runtime_generations
.get(&dsl_identity)
.copied(),
state
.identity_runtime_fence_tokens
.get(&dsl_identity)
.copied(),
state.member_session_bindings.get(&dsl_identity).cloned(),
)
};
let (Some(generation), Some(fence_token), Some(member_session_id)) =
(generation, fence_token, member_session_id)
else {
return Err(MobError::Internal(format!(
"placed member '{}' has no machine-recorded runtime tuple for release",
ctx.agent_identity
)));
};
let carrier = self
.runtime_metadata
.load_placed_spawn(&self.definition.id, ctx.agent_identity.as_str())
.await?
.filter(|record| {
record.generation == generation.0
&& record.fence_token == fence_token.0
&& record.host_id.to_string() == host.as_str()
&& matches!(
&record.phase,
crate::store::PlacedSpawnCarrierPhase::Committed(_)
)
})
.ok_or_else(|| {
MobError::Internal(format!(
"placed member '{}' has no exact committed carrier for release",
ctx.agent_identity
))
})?;
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
host = %host.as_str(),
"MobActor::dispose_archive_session releasing placed member via host bridge"
);
let carrier_generation_confirmed_revoked = {
let state = self.dsl_authority.state();
state.host_bind_phase.get(&host) != Some(&mob_dsl::HostBindPhase::Bound)
&& state
.current_placed_spawn_host_binding_generations
.get(&dsl_identity)
.copied()
== Some(carrier.host_binding_generation)
&& state.confirmed_host_binding_revocations.contains(
&mob_dsl::HostBindingGenerationTombstone {
host_id: host.clone(),
binding_generation: carrier.host_binding_generation,
},
)
};
if carrier_generation_confirmed_revoked {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
host = %host.as_str(),
binding_generation = carrier.host_binding_generation,
"authenticated revoke tombstone certified dormant placed retirement without ReleaseMember"
);
} else {
self.release_placed_attempt_or_certify_absent(&carrier)
.await?;
}
let disposal = mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned;
self.ensure_exact_structural_event(MobEventKind::RemoteMemberReleaseConfirmed {
agent_identity: ctx.agent_identity.clone(),
host_id: host.0.clone(),
member_session_id: member_session_id.0.clone(),
generation: crate::ids::Generation::new(generation.0),
fence_token: crate::ids::FenceToken::new(fence_token.0),
})
.await?;
// Release admission is exact on identity + generation +
// fence, and host authority prunes every matching journal
// row before replying. Persist that witness locally before
// retirement removes the member or stops its pump.
self.dispose_remote_turn_custody_after_host_release(
&ctx.agent_identity,
&host,
carrier.host_binding_generation,
&member_session_id,
generation.0,
fence_token.0,
)
.await?;
// Keep the durable placement/operator carriers until the
// MemberRetired terminal is published. If the journal append
// fails or the controller crashes here, cold replay needs the
// exact host/session/endpoint tuple to restore Retiring and
// idempotently re-drive ReleaseMember.
return Ok(disposal);
}
}
if let Some(session_id) = ctx.entry.member_ref.bridge_session_id()
&& self.retirement_archive_already_complete(session_id).await?
{
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
session_id = %session_id,
"MobActor::dispose_archive_session accepted existing archive authority"
);
return Ok(mob_dsl::MemberSessionDisposal::Archived);
}
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
member_ref = ?ctx.entry.member_ref,
"MobActor::dispose_archive_session retiring member via provisioner"
);
let disposal = match self.provisioner.retire_member(&ctx.entry.member_ref).await {
Ok(disposal) => disposal,
// NotFound means the durable terminal already holds (the session
// was archived by an earlier attempt); §19.L4 folds
// already-archived into `Archived`.
Err(MobError::SessionError(meerkat_core::service::SessionError::NotFound {
..
})) => return Ok(mob_dsl::MemberSessionDisposal::Archived),
Err(error) => return Err(error),
};
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
disposal = ?disposal,
"MobActor::dispose_archive_session retired member via provisioner"
);
Ok(disposal)
}
async fn observe_member_retirement_archived(
&mut self,
ctx: &DisposalContext,
disposal: mob_dsl::MemberSessionDisposal,
) -> Result<(), MobError> {
let session_id = self
.dsl_authority
.state()
.member_session_bindings
.get(&mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity))
.cloned()
.or_else(|| {
ctx.entry
.member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain)
});
let final_event_exists = self
.retire_event_exists(&ctx.entry.agent_identity, ctx.entry.generation)
.await?;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity);
let is_placed = self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity);
let is_peer_only = Self::runtime_binding_for_entry(&ctx.entry).is_some() && !is_placed;
let terminal_signal = if is_peer_only {
mob_dsl::MobMachineSignal::ObserveRemoteMemberRetirementArchivedAndSupervisorRevoked {
agent_identity: dsl_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
preserve_machine_topology: ctx.preserve_machine_topology,
}
} else {
mob_dsl::MobMachineSignal::ObserveMemberRetirementArchived {
agent_identity: dsl_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id.clone(),
disposal,
preserve_machine_topology: ctx.preserve_machine_topology,
}
};
let prepared = self
.prepare_dsl_signal_transition(terminal_signal, "dispose_member_archive_completed")?;
if !final_event_exists {
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::MemberRetired,
&ctx.entry.agent_identity,
&ctx.entry.agent_runtime_id,
None,
ctx.entry.generation,
if is_peer_only { None } else { session_id },
"dispose_member_archive_completed",
)?;
self.append_retire_event_for_entry(&ctx.entry).await?;
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
// MemberRetired may already be durable while the live admission
// machine still carries the old identity/placement. Stop the
// upcall responder and every volatile producer before another
// command can observe that split; cold replay re-drives the final
// signal from the journal+carrier anchors.
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"MemberRetired journal committed for '{}' but final machine publication failed; actor is fail-stopping for cold recovery: {error}",
ctx.agent_identity
)));
}
Ok(())
}
/// Prune edge locks for the member. Infallible.
async fn dispose_prune_edge_locks(&self, ctx: &DisposalContext) {
self.edge_locks.prune(ctx.agent_identity.as_str()).await;
}
/// Remove the member from the roster. Infallible.
///
/// `drop_flow_lane` is false exactly on the respawn path: the
/// replacement re-materializes under the same identity and the member's
/// remote-flow lane must survive for rematerialization attribution (the
/// generation watch + the respawn mark live on it). Every other removal
/// (retire/release/destroy/rollback) ends the lane with the member.
pub(super) async fn dispose_remove_from_roster(
&self,
ctx: &DisposalContext,
drop_flow_lane: bool,
) {
let mut roster = self.roster.write().await;
roster.remove_member(&ctx.agent_identity);
drop(roster);
// Disposal ends the member's lifetime: drop the retained per-spawn
// overlay so host dispatchers are released and a later spawn of the
// same identity cannot revive with a stale tool surface.
self.per_spawn_external_tools
.write()
.await
.remove(&ctx.agent_identity);
self.restore_diagnostics
.write()
.await
.remove(&ctx.agent_identity);
// Roster removal is a pump stop condition (DEC-P6E-11); the durable
// cursor record is deleted beside the other member cleanup
// (DEC-P6E-10).
self.member_event_pumps.stop_pump(&ctx.agent_identity).await;
self.reachability_observations
.clear_member(&ctx.agent_identity);
if drop_flow_lane {
self.remote_flow_tickets.drop_lane(&ctx.agent_identity);
}
}
async fn handle_complete(&mut self) -> Result<(), MobError> {
self.require_host_authority_anchors_clear_for_action("complete mob")
.await?;
self.ensure_pending_spawn_alignment("handle_complete preflight")?;
self.ensure_flow_tracker_alignment("handle_complete preflight")
.await?;
self.cancel_all_flow_tasks().await?;
// Completion is terminal lifecycle authority, not member work. Never
// lower it into a turn on a member that this same command immediately
// retires: that detached turn can race the idle pre-check and make
// completion wait on work it created. MobCompleted below is the
// durable notification.
self.retire_all_members("complete").await?;
// MobMachine owns both completion admission and the durable journal
// request. The recovery event is appended only after the prepared
// transition is accepted by the live generated authority.
let prepared = self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Complete, "complete_input")
.map_err(|error| {
tracing::debug!(
context = "complete_input",
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(MobState::Completed)
})?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Completed,
"complete_input",
)?;
if let Err(error) = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCompleted,
})
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"Complete marker append was ambiguous; actor is fail-stopping for cold recovery: {error}"
)));
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"MobCompleted is durable but Complete machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
self.ensure_pending_spawn_alignment("handle_complete completion")?;
self.ensure_flow_tracker_alignment("handle_complete completion")
.await?;
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
fn remote_destroy_cleanup_deadline(remote_member_count: usize) -> std::time::Duration {
let batches = remote_member_count
.saturating_add(MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS.saturating_sub(1))
/ MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS.max(1);
let deadline_secs = std::cmp::min(90, 5 + (batches as u64) * 17);
std::time::Duration::from_secs(deadline_secs)
}
fn push_unique_identity(target: &mut Vec<AgentIdentity>, identity: AgentIdentity) {
if !target.iter().any(|existing| existing == &identity) {
target.push(identity);
}
}
fn runtime_binding_for_entry(entry: &RosterEntry) -> Option<crate::RuntimeBinding> {
Self::runtime_binding_for_member_ref(&entry.member_ref)
}
fn runtime_binding_for_member_ref(member_ref: &MemberRef) -> Option<crate::RuntimeBinding> {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id: None,
} => Some(crate::RuntimeBinding::External {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
bootstrap_token: bootstrap_token.clone(),
pubkey: *pubkey,
}),
_ => None,
}
}
fn sanitized_member_ref(member_ref: &MemberRef) -> MemberRef {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id,
..
} => MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
pubkey: *pubkey,
bootstrap_token: bootstrap_token.clone(),
session_id: session_id.clone(),
},
MemberRef::Session { session_id } => MemberRef::Session {
session_id: session_id.clone(),
},
}
}
fn external_binding_overlay_record(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
member_ref: &MemberRef,
) -> Option<crate::store::ExternalBindingOverlayRecord> {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id: None,
} => Some(crate::store::ExternalBindingOverlayRecord {
agent_identity: agent_identity.clone(),
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
pubkey: *pubkey,
bootstrap_token: None,
session_id: None,
}),
bootstrap_token: bootstrap_token.clone(),
status: crate::store::ExternalBindingOverlayStatus::Normalized,
updated_at: chrono::Utc::now(),
}),
_ => None,
}
}
async fn delete_external_binding_overlay_for_member(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<(), MobError> {
self.runtime_metadata
.delete_external_binding_overlay(&self.definition.id, agent_identity, generation)
.await
.map_err(MobError::from)
}
fn remote_runtime_retired_for_entry(&self, entry: &RosterEntry) -> bool {
self.dsl_authority.state().remote_runtime_retired_exact(
&mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
&mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
mob_dsl::Generation::from_domain(entry.generation),
)
}
fn remote_supervisor_revoked_for_entry(&self, entry: &RosterEntry) -> bool {
self.dsl_authority.state().remote_supervisor_revoked_exact(
&mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
&mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
mob_dsl::Generation::from_domain(entry.generation),
)
}
async fn record_remote_member_runtime_retired(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self.remote_runtime_retired_for_entry(entry) {
return Ok(());
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordRemoteMemberRuntimeRetired {
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"record_remote_member_runtime_retired",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::RemoteMemberRuntimeRetired,
&entry.agent_identity,
&entry.agent_runtime_id,
Some(entry.fence_token),
entry.generation,
None,
"record_remote_member_runtime_retired",
)?;
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::RemoteMemberRuntimeRetired {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
fence_token: entry.fence_token,
generation: entry.generation,
},
})
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
async fn record_remote_member_supervisor_revoked(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self.remote_supervisor_revoked_for_entry(entry) {
return Ok(());
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordRemoteMemberSupervisorRevoked {
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"record_remote_member_supervisor_revoked",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::RemoteMemberSupervisorRevoked,
&entry.agent_identity,
&entry.agent_runtime_id,
Some(entry.fence_token),
entry.generation,
None,
"record_remote_member_supervisor_revoked",
)?;
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::RemoteMemberSupervisorRevoked {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
fence_token: entry.fence_token,
generation: entry.generation,
},
})
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
async fn record_destroy_member_retirement_archived(
&mut self,
ctx: &DisposalContext,
disposal: mob_dsl::MemberSessionDisposal,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id);
let bound_session = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let pending_retire_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned();
let released_ordinary_retirement =
bound_session.is_none() && pending_retire_session.is_some();
let session_id_for_journal = bound_session.or(pending_retire_session);
let is_placed = self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity);
let archive_signal = if Self::runtime_binding_for_entry(&ctx.entry).is_some() && !is_placed
{
mob_dsl::MobMachineSignal::ObserveRemoteMemberRetirementArchivedAndSupervisorRevoked {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
preserve_machine_topology: false,
}
} else if released_ordinary_retirement {
mob_dsl::MobMachineSignal::ObserveMemberRetirementArchived {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id_for_journal.clone(),
disposal,
preserve_machine_topology: false,
}
} else {
mob_dsl::MobMachineSignal::ObserveDestroyMemberRetirementArchived {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id_for_journal.clone(),
disposal,
}
};
let prepared =
self.prepare_dsl_signal_transition(archive_signal, "destroy_member_archive_completed")?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::MemberRetired,
&ctx.entry.agent_identity,
&ctx.entry.agent_runtime_id,
None,
ctx.entry.generation,
if Self::runtime_binding_for_entry(&ctx.entry).is_some() && !is_placed {
None
} else {
session_id_for_journal
},
"destroy_member_archive_completed",
)?;
if !self
.retire_event_exists(&ctx.entry.agent_identity, ctx.entry.generation)
.await?
{
self.append_retire_event_for_entry(&ctx.entry).await?;
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"destroy MemberRetired journal committed for '{}' but final machine publication failed; actor is fail-stopping for cold recovery: {error}",
ctx.agent_identity
)));
}
Ok(())
}
async fn record_destroy_member_retired_event(
&self,
entry: &RosterEntry,
) -> Result<(), MobError> {
let retire_event_already_present = self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await?;
if !retire_event_already_present {
return Err(MobError::Internal(format!(
"destroy cleanup for '{}' reached disposal without a generated durable retire journal event",
entry.agent_identity
)));
}
Ok(())
}
async fn record_destroying_event(&mut self) -> Result<(), MobError> {
let destroying_event_exists = self.destroying_event_exists().await?;
if !destroying_event_exists {
if self.destroy_admitted() {
return Err(MobError::Internal(
"destroy cleanup was admitted without a durable MobDestroying journal event"
.to_string(),
));
}
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyCleanup,
"record_destroying_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Destroying,
"record_destroying_event",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobDestroying,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
} else if !self.destroy_admitted() {
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyCleanup,
"record_destroying_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Destroying,
"record_destroying_event",
)?;
self.commit_prepared_dsl_transition(prepared)?;
}
let _ = self.phase_watch_tx.send(self.state());
Ok(())
}
async fn destroying_event_exists(&self) -> Result<bool, MobError> {
let all_events = self.events.replay_all().await?;
let events = all_events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |pos| pos + 1);
Ok(events[epoch_start..]
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)))
}
async fn destroy_storage_finalizing_event_exists(&self) -> Result<bool, MobError> {
let all_events = self.events.replay_all().await?;
let events = all_events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |pos| pos + 1);
Ok(events[epoch_start..]
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroyStorageFinalizing)))
}
async fn record_destroy_storage_finalizing_event(&mut self) -> Result<(), MobError> {
if self.destroy_storage_finalizing_event_exists().await? {
return Ok(());
}
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyStorageFinalizing,
"record_destroy_storage_finalizing_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::DestroyStorageFinalizing,
"record_destroy_storage_finalizing_event",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobDestroyStorageFinalizing,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
Ok(())
}
async fn runtime_metadata_snapshot(&self) -> Result<RuntimeMetadataSnapshot, MobError> {
Ok(RuntimeMetadataSnapshot {
supervisor: self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?,
external_binding_overlays: self
.runtime_metadata
.list_external_binding_overlays(&self.definition.id)
.await?,
member_operator_requests: self
.runtime_metadata
.list_member_operator_requests(&self.definition.id)
.await?,
})
}
async fn restore_runtime_metadata_snapshot(
&mut self,
snapshot: &RuntimeMetadataSnapshot,
) -> Result<(), MobError> {
if let Some(supervisor) = &snapshot.supervisor {
match self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?
{
Some(current) if current == *supervisor => {}
Some(current) => {
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"supervisor authority changed while restoring metadata for mob '{}': current peer={} epoch={}, snapshot peer={} epoch={}",
self.definition.id,
current.public_peer_id,
current.epoch,
supervisor.public_peer_id,
supervisor.epoch,
),
)));
}
None => {
let prepared = self.prepare_supervisor_authority_persistence(
supervisor.dsl_restore_after_destroy_rollback_input(),
supervisor,
"restore_runtime_metadata_snapshot",
)?;
let inserted = self
.runtime_metadata
.put_supervisor_authority_if_absent(
&self.definition.id,
supervisor,
&prepared.authority,
)
.await?;
if !inserted {
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"supervisor authority changed while restoring absent metadata for mob '{}'",
self.definition.id
),
)));
}
self.commit_prepared_dsl_transition(prepared.transition)?;
}
}
}
for overlay in &snapshot.external_binding_overlays {
self.runtime_metadata
.upsert_external_binding_overlay(&self.definition.id, overlay)
.await?;
}
for request in &snapshot.member_operator_requests {
let pending = crate::store::MobMemberOperatorRequestRecord::pending(
request.key(),
request.op_digest.clone(),
);
match self
.runtime_metadata
.begin_member_operator_request(&self.definition.id, &pending)
.await?
{
crate::store::MobMemberOperatorRequestBegin::Started => {
if request.terminal_reply().is_some()
&& !self
.runtime_metadata
.compare_and_put_member_operator_request(
&self.definition.id,
&pending,
request,
)
.await?
{
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"member operator request '{}' changed while restoring metadata for mob '{}'",
request.request_id, self.definition.id
),
)));
}
}
crate::store::MobMemberOperatorRequestBegin::Existing(existing)
if existing == *request => {}
crate::store::MobMemberOperatorRequestBegin::Existing(existing)
if existing == pending && request.terminal_reply().is_some() =>
{
if !self
.runtime_metadata
.compare_and_put_member_operator_request(
&self.definition.id,
&pending,
request,
)
.await?
{
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"member operator request '{}' changed while restoring terminal metadata for mob '{}'",
request.request_id, self.definition.id
),
)));
}
}
crate::store::MobMemberOperatorRequestBegin::Existing(_) => {
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"member operator request '{}' conflicts with metadata snapshot for mob '{}'",
request.request_id, self.definition.id
),
)));
}
}
}
Ok(())
}
async fn incomplete_after_metadata_scrub_error(
&mut self,
mut report: super::handle::MobDestroyReport,
snapshot: &RuntimeMetadataSnapshot,
error: impl std::fmt::Display,
) -> super::handle::MobDestroyError {
report.push_error(error.to_string());
if let Err(restore_error) = self.restore_runtime_metadata_snapshot(snapshot).await {
report.push_error(format!(
"runtime metadata restore failed after incomplete destroy: {restore_error}"
));
}
report.metadata_scrubbed = false;
super::handle::MobDestroyError::Incomplete { report }
}
fn incomplete_destroy_error(
mut report: super::handle::MobDestroyReport,
context: &str,
error: impl std::fmt::Display,
) -> super::handle::MobDestroyError {
report.push_error(format!("{context}: {error}"));
super::handle::MobDestroyError::Incomplete { report }
}
fn verify_destroy_remote_turn_custody_drained(
obligations: &BTreeSet<mob_dsl::RemoteTurnObligation>,
released_remote_authorities: &BTreeSet<(String, u64, u64)>,
) -> Result<(), String> {
if let Some(unwitnessed) = obligations.iter().find(|obligation| {
!released_remote_authorities.contains(&(
obligation.agent_identity.0.clone(),
obligation.generation.0,
obligation.fence_token.0,
))
}) {
return Err(format!(
"remote-turn custody for '{}' generation {} fence {} has no exact successful host-release witness",
unwitnessed.agent_identity.0, unwitnessed.generation.0, unwitnessed.fence_token.0,
));
}
if let Some(residual) = obligations.first() {
return Err(format!(
"remote-turn custody for '{}' sequence {} remained after exact host release; refusing to fabricate Dispose without the canonical public terminal/ticket convergence path",
residual.agent_identity.0, residual.dispatch_sequence,
));
}
Ok(())
}
fn expected_revoke_cleanup_failure(error: &MobError) -> Option<ExpectedRevokeCleanupFailure> {
match error {
MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
..
} => Some(ExpectedRevokeCleanupFailure::BridgeRejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
}),
_ => None,
}
}
#[cfg(not(target_arch = "wasm32"))]
async fn destroy_remote_member_for_destroy(
&mut self,
entry: RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
) -> RemoteDestroyOutcome {
let identity = entry.agent_identity.clone();
let agent_identity = entry.agent_identity.clone();
let mut outcome = RemoteDestroyOutcome {
identity: identity.clone(),
force_destroyed: false,
orphaned: false,
errors: Vec::new(),
};
let placed_host = self
.dsl_authority
.state()
.member_placement
.get(&mob_dsl::AgentIdentity::from_domain(&agent_identity))
.cloned();
let binding = placed_host
.as_ref()
.map(|host| crate::RuntimeBinding::HostMaterialized { host: host.clone() })
.or_else(|| Self::runtime_binding_for_entry(&entry));
let Some(binding) = binding else {
outcome
.errors
.push("remote destroy requested for non peer-only member".to_string());
outcome.orphaned = true;
return outcome;
};
let ctx = self
.disposal_context_from_entry(&agent_identity, &entry, trust_cleanup_plan, false)
.await;
// §19.L3/DEC-R2: a PLACED member's one disposal verb is the
// host-addressed release (dispose_member_for_destroy routes there).
// The peer-only legs below (observe / force-destroy / supervisor
// revoke) address the MEMBER's own endpoint — external-peer
// machinery that does not apply to a host-materialized member,
// whose supervisor authority dies with the host-side release. A
// failed release stays a recorded orphan; the stale-fence sweep is
// the backstop.
let placed = placed_host.is_some();
let disposal_already_complete = match self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await
{
Ok(complete) => complete,
Err(error) => {
outcome.errors.push(format!(
"durable retirement completion observation failed: {error}"
));
outcome.orphaned = true;
return outcome;
}
};
if disposal_already_complete {
if let Err(error) = self
.delete_retired_placed_member_carriers(&entry.agent_identity)
.await
{
outcome.errors.push(format!(
"exact placed carrier cleanup after durable terminal failed: {error}"
));
outcome.orphaned = true;
return outcome;
}
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx, true).await;
return outcome;
}
let (disposal, archive_disposal) = self.dispose_member_for_destroy_steps(&ctx).await;
let disposal_failure = disposal
.aborted_at
.as_ref()
.map(|(step, error)| (*step, error.to_string()))
.or_else(|| {
disposal
.skipped
.first()
.map(|(step, error)| (*step, error.to_string()))
});
let mut remote_cleanup_complete = disposal_failure.is_none();
let mut terminal_disposal = archive_disposal;
if let Some((failed_step, graceful_error)) = disposal_failure {
if placed {
outcome.errors.push(format!(
"placed release failed at {failed_step}: {graceful_error}"
));
outcome.orphaned = true;
return outcome;
}
// Force-destroy can substitute only for the remote archive step.
// Earlier failures leave local trust/topology or kickoff cleanup
// incomplete and must retain the roster retry anchor.
if failed_step != DisposalStep::ArchiveSession {
outcome.errors.push(format!(
"graceful retire failed before remote archive at {failed_step}: {graceful_error}"
));
outcome.orphaned = true;
return outcome;
}
match self
.observe_peer_only_binding(&binding, std::time::Duration::from_millis(750))
.await
{
Ok(observation) => match self.observation_is_terminal(&observation) {
Ok(true) => remote_cleanup_complete = true,
Ok(false) => tracing::debug!(
state = %observation.state,
"confirmatory remote observation remained non-terminal before force destroy"
),
Err(error) => tracing::debug!(
%error,
"confirmatory remote observation terminality classification failed before force destroy"
),
},
Err(error) => tracing::debug!(
%error,
"confirmatory remote observation failed before force destroy"
),
}
if !remote_cleanup_complete {
match self
.destroy_peer_only_binding(&binding, std::time::Duration::from_secs(5))
.await
{
Ok(_) => {
remote_cleanup_complete = true;
outcome.force_destroyed = true;
terminal_disposal = Some(
mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyNoDurableSessions,
);
}
Err(error) => outcome.errors.push(format!(
"graceful retire failed at ArchiveSession: {graceful_error}; force destroy failed: {error}"
)),
}
}
}
if !remote_cleanup_complete {
outcome.orphaned = true;
return outcome;
}
if !placed {
if let Err(error) = self.record_remote_member_runtime_retired(&entry).await {
outcome.errors.push(format!(
"durable remote-runtime retirement checkpoint failed: {error}"
));
outcome.orphaned = true;
return outcome;
}
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&entry.agent_identity)
{
target.take();
outcome.errors.push(format!(
"fault-injected cancellation after remote archive for '{}'",
entry.agent_identity
));
outcome.orphaned = true;
return outcome;
}
if let Err(error) = self
.revoke_supervisor_for_retiring_entry(
&entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
{
outcome
.errors
.push(format!("supervisor revoke failed: {error}"));
outcome.orphaned = true;
return outcome;
}
}
let terminal_disposal = terminal_disposal.unwrap_or({
if placed {
mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned
} else {
mob_dsl::MemberSessionDisposal::RuntimeReleasedOnlyNoDurableSessions
}
});
// Terminal publication is deliberately last: replay may remove the
// roster only after runtime cleanup and, for peer-only members,
// supervisor revocation have reached typed terminal truth.
if let Err(error) = self
.record_destroy_member_retirement_archived(&ctx, terminal_disposal)
.await
{
outcome
.errors
.push(format!("durable retire completion failed: {error}"));
outcome.orphaned = true;
return outcome;
}
if let Err(error) = self
.delete_retired_placed_member_carriers(&entry.agent_identity)
.await
{
outcome
.errors
.push(format!("exact placed carrier cleanup failed: {error}"));
outcome.orphaned = true;
return outcome;
}
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx, true).await;
outcome
}
#[cfg(not(target_arch = "wasm32"))]
async fn destroy_remote_members_for_destroy(
&mut self,
remote_entries: Vec<RosterEntry>,
trust_unwire_authority_by_member: &mut BTreeMap<AgentIdentity, RetireTrustCleanupPlan>,
report: &mut super::handle::MobDestroyReport,
) {
if remote_entries.is_empty() {
return;
}
// Phase 2 sequential expedient: dispose_member currently takes
// `&mut self` (via `dispose_stop_host_loop` / `stop_autonomous_member`).
// Phase 4 will re-introduce FuturesUnordered parallelism once the
// disposal steps are converted to `&self` so a shared actor reference
// can be held across concurrent disposal futures.
let deadline = Self::remote_destroy_cleanup_deadline(remote_entries.len());
let deadline_at = Instant::now() + deadline;
let mut remaining = VecDeque::from(remote_entries);
while let Some(entry) = remaining.pop_front() {
let identity = entry.agent_identity.clone();
if Instant::now() >= deadline_at {
report.remote_cleanup_deadline_exceeded = true;
Self::push_unique_identity(&mut report.orphaned_remote_members, identity);
for entry in remaining {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
return;
}
let trust_cleanup_plan = trust_unwire_authority_by_member
.remove(&identity)
.unwrap_or_else(RetireTrustCleanupPlan::empty);
// Never cancel a member future after it has opened machine-owned
// trust/topology mutations. Each bridge operation is independently
// bounded; the aggregate deadline is enforced only between these
// atomic cleanup units so warm retry authority cannot be dropped.
let outcome = self
.destroy_remote_member_for_destroy(entry, trust_cleanup_plan)
.await;
if outcome.force_destroyed {
Self::push_unique_identity(
&mut report.force_destroyed_members,
outcome.identity.clone(),
);
}
if outcome.orphaned {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
outcome.identity.clone(),
);
}
for error in outcome.errors {
report.push_error(format!("{}: {error}", outcome.identity));
}
if Instant::now() >= deadline_at && !remaining.is_empty() {
report.remote_cleanup_deadline_exceeded = true;
for entry in remaining {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
return;
}
}
}
#[cfg(target_arch = "wasm32")]
async fn destroy_remote_members_for_destroy(
&self,
remote_entries: Vec<RosterEntry>,
_trust_unwire_authority_by_member: &mut BTreeMap<AgentIdentity, RetireTrustCleanupPlan>,
report: &mut super::handle::MobDestroyReport,
) {
for entry in remote_entries {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
}
async fn dispose_local_member_after_destroy_admission(
&mut self,
entry: RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
report: &mut super::handle::MobDestroyReport,
) -> Result<(), super::handle::MobDestroyError> {
let ctx = self
.disposal_context_from_entry(&entry.agent_identity, &entry, trust_cleanup_plan, false)
.await;
let disposal_already_complete = match self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await
{
Ok(complete) => complete,
Err(error) => {
report.push_error(format!(
"{}: durable retirement completion observation failed: {error}",
entry.agent_identity
));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
};
if !disposal_already_complete {
let disposal_report = self.dispose_member_for_destroy(&ctx).await;
if let Some(error) = Self::destroy_disposal_failure(&disposal_report) {
report.push_error(format!("{}: {error}", entry.agent_identity));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
}
if let Err(error) = self.record_destroy_member_retired_event(&entry).await {
report.push_error(format!(
"{}: durable retire event append failed: {error}",
entry.agent_identity
));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
if let Err(error) = self
.delete_retired_placed_member_carriers(&entry.agent_identity)
.await
{
report.push_error(format!(
"{}: exact placed carrier cleanup failed: {error}",
entry.agent_identity
));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx, true).await;
if let Err(error) = self
.delete_external_binding_overlay_for_member(&entry.agent_identity, entry.generation)
.await
{
report.push_error(error.to_string());
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
Ok(())
}
async fn handle_destroy(
&mut self,
) -> Result<super::handle::MobDestroyReport, super::handle::MobDestroyError> {
// A fresh Destroy must not publish its durable work-origin latch until
// every fallible, proved-nonmutating shell preflight has succeeded.
// Admitted/recovered Destroy retries repeat the checks in
// handle_destroy_inner against their recovery-owned anchors.
if !self.destroy_admitted() {
self.require_host_authority_anchors_clear_for_action("begin destroy")
.await
.map_err(super::handle::MobDestroyError::from)?;
self.ensure_pending_spawn_alignment("destroy preflight")
.map_err(super::handle::MobDestroyError::from)?;
self.ensure_flow_tracker_alignment("destroy preflight")
.await
.map_err(super::handle::MobDestroyError::from)?;
}
// Destroy commits the DSL terminal before fallible storage/event
// cleanup. On a later cleanup retry, that committed transition has
// already proved every placed-completion obligation clear; attempting
// to begin the quiesce protocol again from Destroyed is both rejected
// by the DSL and prevents the retry from reaching its remaining work.
if self.dsl_state() != crate::runtime::MobState::Destroyed {
self.drive_placed_completion_lifecycle_cleanup(
None,
false,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Destroy),
)
.await
.map_err(super::handle::MobDestroyError::from)?;
}
let was_active = self.destroy_cleanup_active;
self.destroy_cleanup_active = true;
let result = self.handle_destroy_inner().await;
self.destroy_cleanup_active = was_active;
result
}
async fn handle_destroy_inner(
&mut self,
) -> Result<super::handle::MobDestroyReport, super::handle::MobDestroyError> {
use super::handle::{MobDestroyError, MobDestroyReport};
let mut report = MobDestroyReport::default();
// No destroy phase owns a bind. Keep this check on recovery too so a
// corrupt post-MobDestroying bind cannot be mistaken for destroy work.
match self.current_pending_host_bind_anchors().await {
Ok(anchors) if !anchors.is_empty() => {
for anchor in anchors {
report.push_error(format!(
"host '{}' bind operation '{}' is unfinished (confirmed={}); exact bind convergence is required before destroy",
anchor.request.host_id,
anchor.operation_id,
anchor.confirmed_authority.is_some(),
));
}
return Err(MobDestroyError::Incomplete { report });
}
Ok(_) => {}
Err(error) => {
report.push_error(format!(
"host bind anchor validation before destroy failed: {error}"
));
return Err(MobDestroyError::Incomplete { report });
}
}
// A fresh destroy may not cross either family of remote host-authority
// ceremony. Once MobDestroying is durable, destroy itself may open
// revoke anchors for its tracked hosts; those are recovery work owned
// by this destroy and are fenced again before storage finalization.
// Bind anchors were handled above so callers retain the more precise
// exact-convergence diagnostic; this guard principally catches a
// pre-destroy revoke anchor.
if !self.destroy_admitted()
&& let Err(error) = self
.require_host_authority_anchors_clear_for_action("begin destroy")
.await
{
report.push_error(error.to_string());
return Err(MobDestroyError::Incomplete { report });
}
let destroy_input_needed = self.dsl_state() != crate::runtime::MobState::Destroyed;
self.ensure_pending_spawn_alignment("handle_destroy preflight")
.map_err(|error| {
if self.destroy_admitted() {
Self::incomplete_destroy_error(
report.clone(),
"pending spawn alignment during admitted destroy failed",
error,
)
} else {
MobDestroyError::from(error)
}
})?;
self.ensure_flow_tracker_alignment("handle_destroy preflight")
.await
.map_err(|error| {
if self.destroy_admitted() {
Self::incomplete_destroy_error(
report.clone(),
"flow tracker alignment during admitted destroy failed",
error,
)
} else {
MobDestroyError::from(error)
}
})?;
let entries = {
let roster = self.roster.read().await;
roster.list_all().cloned().collect::<Vec<_>>()
};
let mut trust_cleanup_plan_by_member: BTreeMap<AgentIdentity, RetireTrustCleanupPlan> =
BTreeMap::new();
if destroy_input_needed {
for entry in &entries {
let plan = match self
.member_retire_trust_cleanup_plan(&entry.agent_identity, entry)
.await
{
Ok(plan) => plan,
Err(error) => {
report.push_error(format!(
"{}: destroy retire trust authority failed: {error}",
entry.agent_identity
));
return Err(MobDestroyError::Incomplete { report });
}
};
if plan.has_peers() {
trust_cleanup_plan_by_member.insert(entry.agent_identity.clone(), plan);
}
}
}
if destroy_input_needed && let Err(error) = self.record_destroying_event().await {
report.push_error(format!("destroy marker append failed: {error}"));
return Err(MobDestroyError::Incomplete { report });
}
self.fail_all_pending_spawns("mob is destroying")
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"pending spawn cleanup during destroy failed",
error,
)
})?;
self.cancel_pending_peer_deliveries("mob is destroying")
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"peer delivery cancellation during destroy failed",
error,
)
})?;
if destroy_input_needed && self.has_orchestrator {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::StopOrchestrator,
"stop_orchestrator_destroy",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"stop orchestrator during destroy failed",
error,
)
})?;
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::DestroyOrchestrator,
"destroy_orchestrator",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy orchestrator during destroy failed",
error,
)
})?;
// `MobDestroying` above is the durable lifecycle notification.
// Never also lower destroy into a member turn: that detached turn
// can win admission after the disposal pre-check observes Idle,
// acquire the session's turn-finalization boundary, and make this
// same destroy wait for work it just created. Retaining the
// boundary wait is required for exact teardown authority; the
// liveness fix is to stop teardown from opening new member work.
}
if destroy_input_needed {
for entry in &entries {
if let Err(error) = self.admit_member_retire_for_destroy(entry).await {
report.push_error(format!(
"{}: destroy retire admission failed: {error}",
entry.agent_identity
));
return Err(MobDestroyError::Incomplete { report });
}
}
}
self.cancel_all_flow_tasks().await.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"cancel flow tasks during destroy failed",
error,
)
})?;
if !self.pending_routed_effects.is_empty() {
self.flush_routed_effects().await.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy routed effect dispatch failed",
error,
)
})?;
}
let released_remote_authorities = entries
.iter()
.filter(|entry| Self::runtime_binding_for_entry(entry).is_some())
.map(|entry| {
(
entry.agent_identity.to_string(),
entry.generation.get(),
entry.fence_token.get(),
)
})
.collect::<BTreeSet<_>>();
let (remote_entries, local_entries): (Vec<_>, Vec<_>) = entries
.into_iter()
.partition(|entry| Self::runtime_binding_for_entry(entry).is_some());
for entry in local_entries {
let plan = trust_cleanup_plan_by_member
.remove(&entry.agent_identity)
.unwrap_or_else(RetireTrustCleanupPlan::empty);
self.dispose_local_member_after_destroy_admission(entry, plan, &mut report)
.await?;
}
self.destroy_remote_members_for_destroy(
remote_entries,
&mut trust_cleanup_plan_by_member,
&mut report,
)
.await;
if report.remote_cleanup_deadline_exceeded
|| !report.orphaned_remote_members.is_empty()
|| !report.errors.is_empty()
{
return Err(MobDestroyError::Incomplete { report });
}
// Every remote host release above has now confirmed pruning for the
// released generation/fence, and stale watchers are machine-dropped.
// Durably record that external terminal before closing controller
// custody; never silently clear rows while a host may still own them.
let remote_turn_obligations = self
.dsl_authority
.state()
.pending_remote_turn_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.committed_remote_turn_outcomes
.iter(),
)
.chain(
self.dsl_authority
.state()
.resolved_remote_turn_outcomes
.iter(),
)
.cloned()
.collect::<std::collections::BTreeSet<_>>();
if let Err(error) = Self::verify_destroy_remote_turn_custody_drained(
&remote_turn_obligations,
&released_remote_authorities,
) {
report.push_error(error);
return Err(MobDestroyError::Incomplete { report });
}
if !self
.dsl_authority
.state()
.pending_placed_kickoff_outcomes
.is_empty()
|| !self
.dsl_authority
.state()
.resolved_placed_kickoff_outcomes
.is_empty()
{
report.push_error(
"destroy left placed-kickoff host outcome custody after exact remote release"
.to_string(),
);
return Err(MobDestroyError::Incomplete { report });
}
// Host authority is an independently durable, remotely accepted
// binding. Drain every Requested/Bound host through the authenticated
// RevokeHost protocol before the terminal machine transition. This
// preserves the host's durable receipt as the retry anchor and deletes
// the controlling-side authority row under a transition-derived
// witness; a raw metadata scrub could otherwise revive a stale host
// binding when this mob id is recovered later.
if destroy_input_needed {
// A retained-placement replacement request intentionally has no
// `host_bind_phase` entry until its ACK commits atomically. It is
// still a live bind window that Destroy must close. Snapshot the
// deduped union so a certified pre-write bind rejection cannot
// strand Destroy behind the replacement-map-empty guard.
let tracked_hosts = Self::host_ids_requiring_destroy_revoke(self.dsl_authority.state());
for host_id in tracked_hosts {
if let Err(error) = self.handle_revoke_host(&host_id).await {
report.push_error(format!(
"host '{host_id}' revoke during destroy failed: {error}"
));
return Err(MobDestroyError::Incomplete { report });
}
}
}
if destroy_input_needed {
self.require_placed_spawn_carriers_empty("destroy admission fence")
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"placed carrier drain before destroy failed",
error,
)
})?;
self.apply_dsl_input(mob_dsl::MobMachineInput::Destroy, "destroy_input")
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy machine transition failed",
error,
)
})?;
}
self.ensure_pending_spawn_alignment("handle_destroy completion")
.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
self.ensure_flow_tracker_alignment("handle_destroy completion")
.await
.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
if let Err(error) = self.cleanup_namespace().await {
report.push_error(error.to_string());
return Err(MobDestroyError::Incomplete { report });
}
report.namespace_cleaned = true;
self.require_placed_spawn_carriers_empty("destroy storage-finalizing fence")
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"placed carrier drain before storage finalization failed",
error,
)
})?;
self.require_host_authority_anchors_clear_for_action(
"cross MobDestroyStorageFinalizing terminal boundary",
)
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"host authority drain before storage finalization failed",
error,
)
})?;
self.record_destroy_storage_finalizing_event()
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"record destroy storage finalizing event failed",
error,
)
})?;
let runtime_metadata_snapshot =
self.runtime_metadata_snapshot().await.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
if let Err(error) = self
.runtime_metadata
.delete_external_binding_overlays(&self.definition.id)
.await
{
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
// Operator-grant rows are principal-keyed (not member/host-lifecycle
// tied), so the destroy sweep is the ONLY cleaner (DEC-P5P-7 step 3;
// no per-row revoke witness exists at destroy — the whole family is
// scrubbed under the MobDestroyStorageFinalizing fence).
if let Err(error) = self
.runtime_metadata
.delete_mob_operator_grants(&self.definition.id)
.await
{
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
// Member-upcall request rows are generation-pinned and otherwise
// retained without capacity eviction. Mob destroy is their sole
// lifecycle scrub; the snapshot above makes a later destroy failure
// retry-safe.
if let Err(error) = self
.runtime_metadata
.delete_member_operator_requests(&self.definition.id)
.await
{
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
// Pump cursors are plain controller metadata keyed by mob/member.
// Individual pump shutdown attempts an eager delete, but that path is
// deliberately best-effort so task teardown cannot deadlock member
// retirement. The destroy storage-finalizing fence is therefore the
// authoritative whole-mob scrub. This deletion is monotonic after all
// members and pumps are terminal; a later destroy retry need not
// restore cursors merely because another metadata family failed.
if let Err(error) = self
.runtime_metadata
.delete_member_event_cursors(&self.definition.id)
.await
{
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
let supervisor_delete = match runtime_metadata_snapshot.supervisor.as_ref() {
Some(supervisor) => Some(
self.prepare_supervisor_authority_deletion(
supervisor,
"handle_destroy metadata scrub",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"supervisor authority deletion admission failed",
error,
)
})?,
),
None => None,
};
if let (Some(supervisor), Some(prepared)) = (
runtime_metadata_snapshot.supervisor.as_ref(),
supervisor_delete,
) {
if let Err(error) = self
.runtime_metadata
.delete_supervisor_authority(&self.definition.id, supervisor, &prepared.authority)
.await
.and_then(|deleted| {
if deleted {
Ok(())
} else {
Err(crate::store::MobStoreError::CasConflict(format!(
"supervisor authority changed while deleting metadata for mob '{}'",
self.definition.id
)))
}
})
{
return Err(self
.incomplete_after_metadata_scrub_error(
report,
&runtime_metadata_snapshot,
error,
)
.await);
}
self.commit_prepared_dsl_transition(prepared.transition)?;
}
report.metadata_scrubbed = true;
if let Err(error) = self.events.clear().await {
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
report.events_cleared = true;
self.edge_locks.clear().await;
if report.remote_cleanup_deadline_exceeded
|| !report.orphaned_remote_members.is_empty()
|| !report.errors.is_empty()
{
return Err(MobDestroyError::Incomplete { report });
}
Ok(report)
}
/// Snapshot every controller-side host bind window Destroy must close.
/// Replacement requests deliberately live outside `host_bind_phase`
/// until atomic commit, so phase keys alone are incomplete.
pub(super) fn host_ids_requiring_destroy_revoke(
state: &mob_dsl::MobMachineState,
) -> BTreeSet<String> {
state
.host_bind_phase
.keys()
.chain(state.replacement_host_bind_endpoints.keys())
.map(|host_id| host_id.as_str().to_string())
.collect()
}
// =====================================================================
// Multi-host mobs (§7.2 steps 2 & 4): controlling-side host bind ceremony
// =====================================================================
/// The wire protocol version's integer projection. `Display` is the
/// sanctioned integer form: the wire type keeps its raw integer private
/// and serializes as that same integer.
fn bridge_protocol_version_number(
version: super::bridge_protocol::BridgeProtocolVersion,
) -> Result<u64, MobError> {
version.to_string().parse::<u64>().map_err(|error| {
MobError::Internal(format!(
"bridge protocol version '{version}' has no integer projection: {error}"
))
})
}
/// Validate a host-advertised live endpoint at the wire boundary (§16
/// DL5: scheme-qualified `ws`/`wss` absolute base URL; presence IS the
/// live capability).
fn live_ws_endpoint_from_wire(url: &str) -> Result<mob_dsl::LiveWsEndpointUrl, MobError> {
// WHATWG parsing repairs `ws:///path` into a host named `path`.
// Require an authority in the wire spelling first so malformed host
// advertisements cannot be silently reinterpreted as another peer.
let wire = url.trim();
let raw_authority_valid = wire
.split_once(':')
.filter(|(scheme, _)| {
scheme.eq_ignore_ascii_case("ws") || scheme.eq_ignore_ascii_case("wss")
})
.and_then(|(_, remainder)| remainder.strip_prefix("//"))
.map(|remainder| {
let authority_end = remainder.find(['/', '?', '#']).unwrap_or(remainder.len());
let authority = &remainder[..authority_end];
!authority.is_empty() && !authority.contains('\\')
})
.unwrap_or(false);
if !raw_authority_valid {
return Err(MobError::WiringError(format!(
"host live endpoint must be an absolute ws:// or wss:// base URL, got '{url}'"
)));
}
let parsed = url::Url::parse(wire).map_err(|_| {
MobError::WiringError(format!(
"host live endpoint must be an absolute ws:// or wss:// base URL, got '{url}'"
))
})?;
if matches!(parsed.scheme(), "ws" | "wss")
&& !parsed.cannot_be_a_base()
&& parsed.host_str().is_some_and(|host| !host.is_empty())
&& parsed.username().is_empty()
&& parsed.password().is_none()
&& parsed.query().is_none()
&& parsed.fragment().is_none()
{
Ok(mob_dsl::LiveWsEndpointUrl::from(
parsed.as_str().trim_end_matches('/').to_string(),
))
} else {
Err(MobError::WiringError(format!(
"host live endpoint must be an absolute ws:// or wss:// base URL without credentials, query, or fragment, got '{url}'"
)))
}
}
/// Project a bind/rebind reply's `BridgeCapabilities` onto the flattened
/// §6.1 capability record (the shape of the MobMachine host capability
/// maps and the domain `HostCapabilityReport`). This is the first — and
/// only — controlling-side consumer of the wire capability record.
fn host_capability_report_from_bridge(
capabilities: &super::bridge_protocol::BridgeCapabilities,
live_endpoint: Option<String>,
) -> Result<super::handle::HostCapabilityReport, MobError> {
let mut versions = capabilities.supported_protocol_versions.iter().copied();
let Some(first) = versions.next() else {
return Err(MobError::WiringError(
"host capability record declares no supported bridge protocol versions".to_string(),
));
};
let (mut min, mut max) = (first, first);
for version in versions {
if version < min {
min = version;
}
if version > max {
max = version;
}
}
Ok(super::handle::HostCapabilityReport {
protocol_min: Self::bridge_protocol_version_number(min)?,
protocol_max: Self::bridge_protocol_version_number(max)?,
engine_version: capabilities.engine_version.clone(),
durable_sessions: capabilities.durable_sessions,
autonomous_members: capabilities.autonomous_members,
hard_cancel_member: capabilities.hard_cancel_member,
tracked_input_cancel: capabilities.tracked_input_cancel,
memory_store: capabilities.memory_store,
mcp: capabilities.mcp,
resolvable_providers: capabilities
.resolvable_providers
.iter()
.map(|provider| provider.as_str().to_string())
.collect(),
approval_forwarding: capabilities.approval_forwarding,
live_endpoint,
})
}
fn required_host_capabilities(
state: &mob_dsl::MobMachineState,
host_id: &mob_dsl::HostId,
) -> super::bridge_protocol::BridgeHostCapabilityRequirements {
let has_remote_turn_custody = state
.pending_remote_turn_outcomes
.iter()
.chain(&state.committed_remote_turn_outcomes)
.chain(&state.resolved_remote_turn_outcomes)
.any(|row| &row.host_id == host_id);
let has_completion_custody = state
.pending_placed_completion_outcomes
.iter()
.chain(&state.resolved_placed_completion_outcomes)
.any(|row| &row.host_id == host_id);
let has_kickoff_custody = state
.pending_placed_kickoff_outcomes
.iter()
.chain(&state.resolved_placed_kickoff_outcomes)
.any(|row| &row.host_id == host_id);
let has_committed_autonomous_member =
state.member_placement.iter().any(|(identity, host)| {
host == host_id
&& state.member_runtime_modes.get(identity)
== Some(&mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost)
});
let has_pending_autonomous_member = state
.pending_autonomous_placed_spawns
.iter()
.any(|identity| state.pending_placed_spawn_hosts.get(identity) == Some(host_id));
let has_autonomous_member =
has_committed_autonomous_member || has_pending_autonomous_member;
let needs_tracked_turn_contract = has_remote_turn_custody
|| has_completion_custody
|| has_kickoff_custody
|| has_autonomous_member;
super::bridge_protocol::BridgeHostCapabilityRequirements {
durable_sessions: needs_tracked_turn_contract,
autonomous_members: has_autonomous_member,
tracked_input_cancel: needs_tracked_turn_contract,
protocol_v4: needs_tracked_turn_contract,
}
}
fn enforce_host_capability_contract(
host_id: &mob_dsl::HostId,
required: super::bridge_protocol::BridgeHostCapabilityRequirements,
capabilities: &super::handle::HostCapabilityReport,
) -> Result<(), MobError> {
let mut missing = Vec::new();
if required.durable_sessions && !capabilities.durable_sessions {
missing.push("durable_sessions".to_string());
}
if required.autonomous_members && !capabilities.autonomous_members {
missing.push("autonomous_members".to_string());
}
if required.tracked_input_cancel && !capabilities.tracked_input_cancel {
missing.push("tracked_input_cancel".to_string());
}
if required.protocol_v4
&& !(capabilities.protocol_min <= 4 && capabilities.protocol_max >= 4)
{
missing.push("protocol_v4".to_string());
}
if missing.is_empty() {
Ok(())
} else {
Err(MobError::HostCapabilityContractViolation {
host_id: host_id.as_str().to_string(),
missing,
})
}
}
/// Build the durable controlling-side host authority record (FLAG-3(a))
/// from committed machine facts. Single construction seam for the store
/// record family so the field mapping can never drift per call site.
fn host_authority_record(
host_id: &mob_dsl::HostId,
pubkey: mob_dsl::PeerSigningKey,
endpoint: &mob_dsl::PeerAddress,
epoch: u64,
binding_generation: u64,
capabilities: &super::handle::HostCapabilityReport,
) -> crate::store::MobHostAuthorityRecord {
crate::store::MobHostAuthorityRecord {
host_id: host_id.as_str().to_string(),
peer_id: host_id.as_str().to_string(),
signing_key: pubkey.0,
endpoint: endpoint.0.clone(),
authority_epoch: epoch,
binding_generation,
bind_phase: crate::store::MobHostBindPhaseRecord::Bound,
capabilities: crate::store::MobHostCapabilityRecord {
protocol_min: capabilities.protocol_min,
protocol_max: capabilities.protocol_max,
engine_version: capabilities.engine_version.clone(),
durable_sessions: capabilities.durable_sessions,
autonomous_members: capabilities.autonomous_members,
hard_cancel_member: capabilities.hard_cancel_member,
tracked_input_cancel: capabilities.tracked_input_cancel,
memory_store: capabilities.memory_store,
mcp: capabilities.mcp,
resolvable_providers: capabilities.resolvable_providers.clone(),
approval_forwarding: capabilities.approval_forwarding,
},
live_endpoint: capabilities.live_endpoint.clone(),
}
}
fn machine_host_facts_match_record(
state: &mob_dsl::MobMachineState,
host_id: &mob_dsl::HostId,
record: &crate::store::MobHostAuthorityRecord,
) -> bool {
state.mob_hosts.contains(host_id)
&& state.host_bind_phase.get(host_id) == Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_public_keys.get(host_id).map(|key| key.0) == Some(record.signing_key)
&& state
.host_endpoints
.get(host_id)
.map(|endpoint| endpoint.0.as_str())
== Some(record.endpoint.as_str())
&& state.host_authority_epochs.get(host_id).copied() == Some(record.authority_epoch)
&& state.host_binding_generations.get(host_id).copied()
== Some(record.binding_generation)
&& state.host_protocol_min.get(host_id).copied()
== Some(record.capabilities.protocol_min)
&& state.host_protocol_max.get(host_id).copied()
== Some(record.capabilities.protocol_max)
&& state.host_engine_versions.get(host_id).map(String::as_str)
== Some(record.capabilities.engine_version.as_str())
&& state.host_durable_sessions.get(host_id).copied()
== Some(record.capabilities.durable_sessions)
&& state.host_autonomous_members.get(host_id).copied()
== Some(record.capabilities.autonomous_members)
&& state.host_hard_cancel_member.get(host_id).copied()
== Some(record.capabilities.hard_cancel_member)
&& state.host_tracked_input_cancel.get(host_id).copied()
== Some(record.capabilities.tracked_input_cancel)
&& state.host_memory_store.get(host_id).copied()
== Some(record.capabilities.memory_store)
&& state.host_mcp.get(host_id).copied() == Some(record.capabilities.mcp)
&& state.host_resolvable_providers.get(host_id)
== Some(&record.capabilities.resolvable_providers)
&& state.host_approval_forwarding.get(host_id).copied()
== Some(record.capabilities.approval_forwarding)
&& state
.host_live_endpoints
.get(host_id)
.map(|endpoint| endpoint.0.as_str())
== record.live_endpoint.as_deref()
}
/// Apply one authenticated same-binding host fact declaration under a
/// generated persistence witness. Exact machine+store replay is pure;
/// drift writes durable truth by CAS/reread before publishing the prepared
/// machine transition.
async fn refresh_current_host_facts(
&mut self,
host_id: &mob_dsl::HostId,
capabilities: &super::handle::HostCapabilityReport,
context: &'static str,
) -> Result<bool, MobError> {
match self
.refresh_current_host_facts_inner(host_id, capabilities, context)
.await
{
Ok(changed) => Ok(changed),
Err(error) => {
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"{context}: authenticated host facts for '{}' could not converge durably: {error}; actor is fail-stopping for cold recovery",
host_id.as_str()
)))
}
}
}
async fn refresh_current_host_facts_inner(
&mut self,
host_id: &mob_dsl::HostId,
capabilities: &super::handle::HostCapabilityReport,
context: &'static str,
) -> Result<bool, MobError> {
let (pubkey, endpoint, epoch, binding_generation) = {
let state = self.dsl_authority.state();
(
state.host_public_keys.get(host_id).copied(),
state.host_endpoints.get(host_id).cloned(),
state.host_authority_epochs.get(host_id).copied(),
state.host_binding_generations.get(host_id).copied(),
)
};
let pubkey = pubkey.ok_or_else(|| {
MobError::Internal(format!(
"{context}: host '{}' has no signing key",
host_id.as_str()
))
})?;
let endpoint = endpoint.ok_or_else(|| {
MobError::Internal(format!(
"{context}: host '{}' has no endpoint",
host_id.as_str()
))
})?;
let epoch = epoch.ok_or_else(|| {
MobError::Internal(format!(
"{context}: host '{}' has no authority epoch",
host_id.as_str()
))
})?;
let binding_generation = binding_generation.ok_or_else(|| {
MobError::Internal(format!(
"{context}: host '{}' has no binding generation",
host_id.as_str()
))
})?;
let record = Self::host_authority_record(
host_id,
pubkey,
&endpoint,
epoch,
binding_generation,
capabilities,
);
let durable = self
.runtime_metadata
.load_mob_host_authority(&self.definition.id, host_id.as_str())
.await?;
if durable.as_ref() == Some(&record)
&& Self::machine_host_facts_match_record(self.dsl_authority.state(), host_id, &record)
{
return Ok(false);
}
let live_endpoint = capabilities
.live_endpoint
.as_deref()
.map(Self::live_ws_endpoint_from_wire)
.transpose()?;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RefreshHostCapabilities {
host_id: host_id.clone(),
epoch,
binding_generation,
protocol_min: capabilities.protocol_min,
protocol_max: capabilities.protocol_max,
engine_version: capabilities.engine_version.clone(),
durable_sessions: capabilities.durable_sessions,
autonomous_members: capabilities.autonomous_members,
hard_cancel_member: capabilities.hard_cancel_member,
tracked_input_cancel: capabilities.tracked_input_cancel,
memory_store: capabilities.memory_store,
mcp: capabilities.mcp,
resolvable_providers: capabilities.resolvable_providers.clone(),
approval_forwarding: capabilities.approval_forwarding,
live_endpoint,
},
context,
)?;
let witnessed = prepared.transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::HostCapabilitiesRefreshed {
host_id: effect_host,
epoch: effect_epoch,
binding_generation: effect_generation,
} if effect_host == host_id
&& *effect_epoch == epoch
&& *effect_generation == binding_generation
)
});
if !witnessed {
return Err(MobError::Internal(format!(
"{context}: RefreshHostCapabilities emitted no exact persistence witness"
)));
}
let authority = crate::store::MobHostAuthorityPersistenceAuthority::from_transition(
&record,
&prepared.transition,
)?;
let written = match durable.as_ref() {
Some(expected) => {
self.runtime_metadata
.compare_and_put_mob_host_authority(
&self.definition.id,
expected,
&record,
&authority,
)
.await?
}
None => {
self.runtime_metadata
.put_mob_host_authority_if_absent(&self.definition.id, &record, &authority)
.await?
}
};
if !written {
let reread = self
.runtime_metadata
.load_mob_host_authority(&self.definition.id, host_id.as_str())
.await?;
if reread.as_ref() != Some(&record) {
return Err(MobError::Internal(format!(
"{context}: host '{}' durable refresh CAS lost to non-equivalent authority",
host_id.as_str()
)));
}
}
self.commit_prepared_dsl_transition(prepared)?;
Ok(true)
}
/// §7.2 step 2: bind a member-host daemon to this mob.
///
/// Machine-owned ceremony: `BeginHostBind` opens the bind window and
/// emits the `RequestHostBind` handoff (fail-closed if absent — never the
/// `_ => {}` catch-all), this handler realizes the handoff as the
/// `OwnerRealizationOnly` owner by sending `BindHost` over the supervisor
/// bridge inside a recipient-trust obligation window, and `CommitHostBind`
/// records identity, endpoint, epoch (DEC-P2-9: the supervisor authority
/// epoch), and the declared capability record. The durable host authority
/// record is written BEFORE the in-memory commit (in-memory never past
/// durable). Certified pre-send failures/rejections leave `Requested` for
/// live retry; a sent-but-unconfirmed outcome retains trust plus the
/// pending obligation and fail-stops for cold exact replay (DEC-P2-10).
async fn handle_bind_host(
&mut self,
request: super::handle::HostBindRequest,
) -> Result<super::handle::HostBindReport, MobError> {
if self.state() == MobState::Destroyed {
return Err(self.invalid_transition_to(MobState::Destroyed));
}
// Identity-first (D1): the host id IS the canonical comms peer id
// derived from the descriptor's Ed25519 identity. The descriptor
// constructor re-validates peer-id ↔ pubkey consistency fail-closed.
let address = super::bridge_protocol::canonicalize_bridge_address(&request.address);
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
request.expected_peer_id.to_string(),
request.expected_peer_id.to_string(),
request.pubkey,
&address,
)
.map_err(|error| {
MobError::WiringError(format!("invalid host binding descriptor identity: {error}"))
})?;
let host_id = mob_dsl::HostId::from(peer.peer_id.to_string());
let expected_endpoint = mob_dsl::PeerAddress::from(address.clone());
// A prior revoke operation owns this host identity until its durable
// local terminal is closed. Never open/replay a bind across that
// anchor: an unconfirmed operation still needs remote convergence;
// a confirmed operation with an active authority row still needs the
// local revoke retry. Only the crash window after row deletion and
// machine revoke may be closed here before the next generation opens.
if let Some(anchor) = self
.current_pending_host_revoke_anchors()
.await?
.into_iter()
.find(|anchor| anchor.host_id == host_id.0)
{
if !anchor.confirmed {
return Err(MobError::Internal(format!(
"host '{}' has unconfirmed revoke operation '{}'; retry revoke before binding",
host_id.as_str(),
anchor.operation_id
)));
}
let durable_active = self
.runtime_metadata
.load_mob_host_authority(&self.definition.id, host_id.as_str())
.await?;
if durable_active.is_some() {
return Err(MobError::Internal(format!(
"host '{}' has confirmed revoke operation '{}' awaiting its local terminal; retry revoke before binding",
host_id.as_str(),
anchor.operation_id
)));
}
if self
.dsl_authority
.state()
.host_bind_phase
.contains_key(&host_id)
|| self
.dsl_authority
.state()
.replacement_host_bind_endpoints
.contains_key(&host_id)
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"host '{}' confirmed revoke operation '{}' lost its durable row while machine authority remains active",
host_id.as_str(),
anchor.operation_id
)));
}
if let Err(error) = self.complete_host_revoke_anchor(&anchor).await {
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
}
// FLAG-2 controlling half: `bind_host` of an ALREADY-BOUND host is an
// idempotent replay ceremony. The machine's bind window is terminal
// at Bound (BeginHostBind deliberately has no Bound arm), the host
// serves its bind-replay ladder at the recorded epoch, and the
// (re)bind is an orphan-reconciliation trigger (W-E trigger site 2).
// No machine facts move: identity/endpoint are verified against the
// recorded machine maps, the reply identity is verified like the
// fresh ceremony, and the sweep runs.
if self.dsl_authority.state().host_bind_phase.get(&host_id)
== Some(&mob_dsl::HostBindPhase::Bound)
{
let binding_generation = self.current_host_binding_generation(&host_id)?;
let recorded_endpoint = self
.dsl_authority
.state()
.host_endpoints
.get(&host_id)
.cloned();
if recorded_endpoint.as_ref() != Some(&expected_endpoint) {
return Err(MobError::WiringError(format!(
"host '{}' is bound at endpoint '{}' but the re-bind descriptor names '{}'",
host_id.as_str(),
recorded_endpoint
.map(|endpoint| endpoint.0)
.unwrap_or_default(),
expected_endpoint.0
)));
}
let authority = self.supervisor_bridge.authority().await;
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&peer)
.await?;
let required_capabilities =
Self::required_host_capabilities(self.dsl_authority.state(), &host_id);
let payload = super::bridge_protocol::BridgeHostBindPayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
expected_host_peer_id: peer.peer_id.to_string(),
expected_address: address.clone(),
// Filled with a request-bound HMAC immediately below;
// the raw descriptor token is never placed in a wire
// payload, even transiently.
bootstrap_proof: super::bridge_protocol::BridgeHostBootstrapProof::new(""),
mob_id: self.definition.id.to_string(),
required_capabilities,
};
let command = super::bridge_protocol::BridgeCommand::BindHost(
super::bridge_protocol::seal_host_bind_bootstrap_proof(
payload,
&request.bootstrap_token,
),
);
let _install = self.supervisor_bridge.trust_recipient(&peer).await?;
let value = self
.supervisor_bridge
.send_bridge_command(&peer, &command, std::time::Duration::from_secs(60))
.await?;
if let Some(rejection) =
Self::bridge_rejection_reply(command.protocol_version(), &value)
{
return Err(Self::bridge_rejection_error(rejection));
}
let bind: super::bridge_protocol::BridgeHostBindResponse =
super::bridge_protocol::decode_bridge_payload(&command, value, "bind host")?;
if bind.host_peer_id != peer.peer_id.to_string() {
return Err(MobError::WiringError(format!(
"host bind reply identity mismatch: expected '{}', got '{}'",
peer.peer_id, bind.host_peer_id
)));
}
if bind.binding_generation != binding_generation {
return Err(MobError::WiringError(format!(
"host bind reply generation mismatch: expected {binding_generation}, got {}",
bind.binding_generation
)));
}
let capabilities = Self::host_capability_report_from_bridge(
&bind.capabilities,
bind.live_endpoint.clone(),
)?;
Self::enforce_host_capability_contract(&host_id, required_capabilities, &capabilities)?;
self.refresh_current_host_facts(
&host_id,
&capabilities,
"bound BindHost replay fact convergence",
)
.await?;
if let Err(sweep_error) = self.observe_host_status(&host_id).await {
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %sweep_error,
"post-rebind host member reconciliation sweep failed"
);
}
if self.durable_uncertainty_fail_stop {
return Err(MobError::Internal(format!(
"host '{}' replay reconciliation entered durable fail-stop; suppressing replay success",
host_id.as_str()
)));
}
return Ok(super::handle::HostBindReport {
host_id: host_id.as_str().to_string(),
epoch: authority.epoch,
capabilities,
});
}
// BeginHostBind → RequestHostBind handoff, mirrored fail-closed.
let binding_generation = {
let state = self.dsl_authority.state();
if let Some(binding_generation) = state
.replacement_host_binding_generations
.get(&host_id)
.copied()
{
binding_generation
} else if state.host_bind_phase.get(&host_id)
== Some(&mob_dsl::HostBindPhase::Requested)
{
*state
.host_binding_generations
.get(&host_id)
.ok_or_else(|| {
MobError::Internal(format!(
"requested host '{}' has no durable binding-generation reservation",
host_id.as_str()
))
})?
} else {
state
.host_binding_generation_highwater
.get(&host_id)
.copied()
.unwrap_or(0)
.checked_add(1)
.ok_or_else(|| {
MobError::Internal(format!(
"host '{}' exhausted its durable binding-generation counter",
host_id.as_str()
))
})?
}
};
let begin_transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::BeginHostBind {
host_id: host_id.clone(),
expected_endpoint: expected_endpoint.clone(),
binding_generation,
},
"bind_host",
)?;
let (handoff_host, handoff_endpoint, handoff_generation) = begin_transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RequestHostBind {
host_id,
endpoint,
binding_generation,
} => Some((host_id.clone(), endpoint.clone(), *binding_generation)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted BeginHostBind but emitted no RequestHostBind handoff"
.into(),
)
})?;
if handoff_host != host_id
|| handoff_endpoint != expected_endpoint
|| handoff_generation != binding_generation
{
return Err(MobError::Internal(format!(
"MobMachine RequestHostBind drift: input host='{}' endpoint='{}', effect host='{}' endpoint='{}'",
host_id.as_str(),
expected_endpoint.0,
handoff_host.as_str(),
handoff_endpoint.0
)));
}
// Realize the handoff (disposition: external seam OwnerRealizationOnly
// — this handler IS the owner realization). The epoch stamped on the
// wire is the supervisor authority epoch (DEC-P2-9; no new counter).
let authority = self.supervisor_bridge.authority().await;
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&peer)
.await?;
let required_capabilities =
Self::required_host_capabilities(self.dsl_authority.state(), &host_id);
let payload = super::bridge_protocol::BridgeHostBindPayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
// New-command constructors always stamp V4 explicitly
// (CURRENT=V4, DEFAULT=V3 preserved).
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
expected_host_peer_id: peer.peer_id.to_string(),
expected_address: address.clone(),
// Filled with a request-bound HMAC immediately below; the
// raw descriptor token remains out-of-band only.
bootstrap_proof: super::bridge_protocol::BridgeHostBootstrapProof::new(""),
mob_id: self.definition.id.to_string(),
required_capabilities,
};
let command = super::bridge_protocol::BridgeCommand::BindHost(
super::bridge_protocol::seal_host_bind_bootstrap_proof(
payload,
&request.bootstrap_token,
),
);
let replacement = self
.dsl_authority
.state()
.replacement_host_bind_endpoints
.contains_key(&host_id);
let (bind_anchor, bind_anchor_is_new) = self
.ensure_host_bind_started(crate::event::RemoteHostBindRequestEvent {
host_id: host_id.as_str().to_string(),
peer_id: peer.peer_id.to_string(),
signing_key: request.pubkey,
endpoint: expected_endpoint.0.clone(),
authority_epoch: authority.epoch,
binding_generation,
replacement,
})
.await?;
// Trust window: record obligation → install → send → decode → verify
// canonical identity → resolve. Started is durable before this window.
// Before-send failures and authenticated pre-write rejections append
// AbortedNoEffect before rollback; once sent, an absent/invalid
// terminal leaves Started open and the actor fail-stops.
if let Err(error) = self.record_pending_recipient_trust_obligation(&peer, "bind_host") {
if bind_anchor_is_new {
if let Err(anchor_error) = self.abort_host_bind_anchor_no_effect(&bind_anchor).await
{
self.durable_uncertainty_fail_stop = true;
return Err(anchor_error);
}
return Err(error);
}
return Err(self.quarantine_uncertain_host_bind(&peer, error));
}
let install = match self.supervisor_bridge.trust_recipient(&peer).await {
Ok(install) => install,
Err(trust_error) => {
if bind_anchor_is_new {
if let Err(anchor_error) =
self.abort_host_bind_anchor_no_effect(&bind_anchor).await
{
self.durable_uncertainty_fail_stop = true;
return Err(anchor_error);
}
}
return Err(self.quarantine_uncertain_recipient_trust_install(
&peer,
"bind_host",
trust_error,
));
}
};
let value = match self
.supervisor_bridge
.send_bridge_command_classified(&peer, &command, std::time::Duration::from_secs(60))
.await
{
Ok(value) => value,
Err(super::supervisor_bridge::BridgeRequestFailure::BeforeSend(send_error)) => {
if bind_anchor_is_new {
if let Err(anchor_error) =
self.abort_host_bind_anchor_no_effect(&bind_anchor).await
{
self.durable_uncertainty_fail_stop = true;
return Err(anchor_error);
}
return Err(self
.rollback_supervisor_recipient_trust(&peer, install, send_error)
.await);
}
return Err(self.quarantine_uncertain_host_bind(&peer, send_error));
}
Err(super::supervisor_bridge::BridgeRequestFailure::AfterSend(send_error)) => {
return Err(self.quarantine_uncertain_host_bind(&peer, send_error));
}
};
if let Some(rejection) = Self::bridge_rejection_reply(command.protocol_version(), &value) {
let rejection_error = Self::bridge_rejection_error(rejection.clone());
if Self::bind_rejection_certifies_no_remote_effect(&rejection) {
if bind_anchor_is_new {
if let Err(anchor_error) =
self.abort_host_bind_anchor_no_effect(&bind_anchor).await
{
self.durable_uncertainty_fail_stop = true;
return Err(anchor_error);
}
return Err(self
.rollback_supervisor_recipient_trust(&peer, install, rejection_error)
.await);
}
return Err(self.quarantine_uncertain_host_bind(&peer, rejection_error));
}
return Err(self.quarantine_uncertain_host_bind(&peer, rejection_error));
}
let bind: super::bridge_protocol::BridgeHostBindResponse =
match super::bridge_protocol::decode_bridge_payload(&command, value, "bind host") {
Ok(bind) => bind,
Err(decode_error) => {
return Err(self.quarantine_uncertain_host_bind(&peer, decode_error));
}
};
if bind.host_peer_id != peer.peer_id.to_string() {
let mismatch = MobError::WiringError(format!(
"host bind reply identity mismatch: expected '{}', got '{}'",
peer.peer_id, bind.host_peer_id
));
return Err(self.quarantine_uncertain_host_bind(&peer, mismatch));
}
if bind.binding_generation != binding_generation {
return Err(self.quarantine_uncertain_host_bind(
&peer,
MobError::WiringError(format!(
"host bind reply generation mismatch: expected {binding_generation}, got {}",
bind.binding_generation
)),
));
}
let capabilities = match Self::host_capability_report_from_bridge(
&bind.capabilities,
bind.live_endpoint.clone(),
) {
Ok(capabilities) => capabilities,
Err(capability_error) => {
return Err(self.quarantine_uncertain_host_bind(&peer, capability_error));
}
};
if let Err(capability_error) =
Self::enforce_host_capability_contract(&host_id, required_capabilities, &capabilities)
{
return Err(self.quarantine_uncertain_host_bind(&peer, capability_error));
}
let live_endpoint = match bind
.live_endpoint
.as_deref()
.map(Self::live_ws_endpoint_from_wire)
.transpose()
{
Ok(live_endpoint) => live_endpoint,
Err(live_error) => {
return Err(self.quarantine_uncertain_host_bind(&peer, live_error));
}
};
let record = Self::host_authority_record(
&host_id,
mob_dsl::PeerSigningKey::from(request.pubkey),
&expected_endpoint,
authority.epoch,
binding_generation,
&capabilities,
);
if let Err(error) = self.confirm_host_bind_anchor(&bind_anchor, &record).await {
return Err(self.quarantine_uncertain_host_bind(&peer, error));
}
macro_rules! post_ack_try {
($result:expr) => {
match $result {
Ok(value) => value,
Err(error) => {
return Err(self.quarantine_uncertain_host_bind(&peer, error.into()));
}
}
};
}
// CommitHostBind: prepare → durable record (FLAG-3(a) writer, keyed on
// the prepared transition's witness) → commit in-memory. Every
// failure after the authenticated ACK is quarantined and
// fail-stops this actor. Cold recovery either replays the exact
// requested generation or restores Bound from the durable row.
let commit_input = mob_dsl::MobMachineInput::CommitHostBind {
host_id: host_id.clone(),
pubkey: mob_dsl::PeerSigningKey::from(request.pubkey),
endpoint: expected_endpoint.clone(),
epoch: authority.epoch,
binding_generation,
protocol_min: capabilities.protocol_min,
protocol_max: capabilities.protocol_max,
engine_version: capabilities.engine_version.clone(),
durable_sessions: capabilities.durable_sessions,
autonomous_members: capabilities.autonomous_members,
hard_cancel_member: capabilities.hard_cancel_member,
tracked_input_cancel: capabilities.tracked_input_cancel,
memory_store: capabilities.memory_store,
mcp: capabilities.mcp,
resolvable_providers: capabilities.resolvable_providers.clone(),
approval_forwarding: capabilities.approval_forwarding,
live_endpoint: live_endpoint.clone(),
};
let prepared =
post_ack_try!(self.prepare_dsl_input_transition(commit_input, "bind_host commit"));
let registered = prepared
.transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::HostRegistered {
host_id,
epoch,
binding_generation,
} => Some((host_id.clone(), *epoch, *binding_generation)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted CommitHostBind but emitted no HostRegistered effect"
.into(),
)
});
let (registered_host, registered_epoch, registered_generation) = post_ack_try!(registered);
if registered_host != host_id
|| registered_epoch != authority.epoch
|| registered_generation != binding_generation
{
return Err(self.quarantine_uncertain_host_bind(
&peer,
MobError::Internal(format!(
"MobMachine HostRegistered drift: input host='{}' epoch={}, effect host='{}' epoch={}",
host_id.as_str(),
authority.epoch,
registered_host.as_str(),
registered_epoch
)),
));
}
let persistence = post_ack_try!(
crate::store::MobHostAuthorityPersistenceAuthority::from_transition(
&record,
&prepared.transition,
)
);
post_ack_try!(
self.runtime_metadata
.put_mob_host_authority(&self.definition.id, &record, &persistence)
.await
);
post_ack_try!(self.commit_prepared_dsl_transition(prepared));
post_ack_try!(self.complete_host_bind_anchor(&bind_anchor).await);
post_ack_try!(Self::advance_host_binding_incarnation(
&mut self.host_binding_incarnations,
&mut self.orphan_release_reservations,
&host_id,
));
// Resolve last: every earlier fallible post-ACK step leaves this
// obligation available for quarantine diagnostics/cold convergence.
post_ack_try!(self.resolve_pending_recipient_trust_obligation(
&peer,
"bind_host confirmed and durably committed",
));
// §21.2 / W-E: (re)bind is an orphan-reconciliation trigger — a
// sweep failure never fails the bind (the sweep is a convergence
// driver, not a ceremony step).
if let Err(sweep_error) = self.observe_host_status(&host_id).await {
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %sweep_error,
"post-bind host member reconciliation sweep failed"
);
}
if self.durable_uncertainty_fail_stop {
return Err(MobError::Internal(format!(
"host '{}' bind reconciliation entered durable fail-stop; suppressing later route effects and success",
host_id.as_str()
)));
}
Ok(super::handle::HostBindReport {
host_id: host_id.as_str().to_string(),
epoch: registered_epoch,
capabilities,
})
}
async fn observe_host_status(&mut self, host_id: &mob_dsl::HostId) -> Result<(), MobError> {
let mut result = self.reconcile_host_members(host_id).await.map(|_| ());
if result
.as_ref()
.is_err_and(Self::host_status_rejection_requires_fail_stop)
{
self.durable_uncertainty_fail_stop = true;
}
// Bind/rebind is an explicit route-recovery boundary even if the
// follow-up inventory probe itself is temporarily unavailable. Keep
// the old pre-return convergence guarantee, but never emit route
// side effects after a durable fail-stop classification.
if !self.durable_uncertainty_fail_stop {
if let Err(route_error) = self.drain_route_installs_for_host(host_id).await {
if result.is_ok() {
result = Err(route_error);
} else {
tracing::error!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %route_error,
"host observation also found an invalid pending route-install ledger"
);
}
}
}
match &result {
Ok(()) => self
.reachability_observations
.mark_host_success(host_id.as_str()),
Err(error) => self
.reachability_observations
.mark_host_failure(host_id.as_str(), error),
}
result
}
async fn observe_host_status_as_authority(
&mut self,
host_id: &mob_dsl::HostId,
peer: &TrustedPeerDescriptor,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<(), MobError> {
let binding_generation = self.current_host_binding_generation(host_id)?;
let mut result = match Self::poll_bound_host_status_once(
Arc::clone(&self.supervisor_bridge),
authority,
&self.definition.id,
peer,
binding_generation,
)
.await
{
Ok(status) => self
.reconcile_host_status_response(host_id, status)
.await
.map(|_| ()),
Err(error) => Err(error),
};
if result
.as_ref()
.is_err_and(Self::host_status_rejection_requires_fail_stop)
{
self.durable_uncertainty_fail_stop = true;
}
if !self.durable_uncertainty_fail_stop {
if let Err(route_error) = self.drain_route_installs_for_host(host_id).await {
if result.is_ok() {
result = Err(route_error);
} else {
tracing::error!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %route_error,
"authority-scoped host observation also found an invalid pending route-install ledger"
);
}
}
}
match &result {
Ok(()) => self
.reachability_observations
.mark_host_success(host_id.as_str()),
Err(error) => self
.reachability_observations
.mark_host_failure(host_id.as_str(), error),
}
result
}
fn host_status_rejection_requires_fail_stop(error: &MobError) -> bool {
matches!(
error,
MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unsupported,
..
}
)
}
/// Start at most one detached status request per currently bound host.
///
/// Host I/O must not occupy the serialized actor: a partition can consume
/// the full request deadline and multiple hosts must be observed in
/// parallel. Completion re-enters through `HostStatusPollCompleted`,
/// where the binding epoch is rechecked before any result is applied.
async fn spawn_periodic_host_status_polls(
&mut self,
in_flight: &mut BTreeSet<mob_dsl::HostId>,
) {
let authority = self.supervisor_bridge.authority().await;
let targets = {
let state = self.dsl_authority.state();
let incarnations = &self.host_binding_incarnations;
state
.host_bind_phase
.iter()
.filter(|(host, phase)| {
if **phase != mob_dsl::HostBindPhase::Bound || in_flight.contains(*host) {
return false;
}
true
})
.map(|(host, _)| {
let result = (|| {
let missing = |fact: &str| {
MobError::Internal(format!(
"bound host '{}' has no {fact} for periodic status polling",
host.as_str()
))
};
let endpoint = state
.host_endpoints
.get(host)
.ok_or_else(|| missing("endpoint"))?;
let pubkey = state
.host_public_keys
.get(host)
.ok_or_else(|| missing("public key"))?;
let epoch = *state
.host_authority_epochs
.get(host)
.ok_or_else(|| missing("authority epoch"))?;
let generation = *state
.host_binding_generations
.get(host)
.ok_or_else(|| missing("binding generation"))?;
let incarnation = *incarnations
.get(host)
.ok_or_else(|| missing("binding incarnation"))?;
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
host.as_str(),
host.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::Internal(format!(
"bound host '{}' has invalid periodic status poll descriptor: {error}",
host.as_str()
))
})?;
Ok((epoch, generation, incarnation, peer))
})();
(host.clone(), result)
})
.collect::<Vec<_>>()
};
for (host_id, target) in targets {
let (binding_epoch, binding_generation, binding_incarnation, peer) = match target {
Ok(target) => target,
Err(error) => {
self.reachability_observations
.mark_host_failure(host_id.as_str(), &error);
tracing::error!(
host_id = %host_id.as_str(),
error = %error,
"bound host status target is malformed; leaving it retry-visible"
);
continue;
}
};
if !in_flight.insert(host_id.clone()) {
continue;
}
let bridge = Arc::clone(&self.supervisor_bridge);
let command_tx = self.command_tx.clone();
let mob_id = self.definition.id.clone();
let authority = authority.clone();
// Stable, small per-host jitter keeps many recovered mobs from
// synchronizing all host probes on the same timer edge.
let jitter_ms = self
.definition
.id
.as_str()
.bytes()
.chain(host_id.as_str().bytes())
.fold(0_u64, |acc, byte| {
acc.wrapping_mul(33).wrapping_add(byte as u64)
})
% 750;
self.actor_io_tasks.spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(jitter_ms)).await;
let result = Self::poll_bound_host_status_once(
bridge,
&authority,
&mob_id,
&peer,
binding_generation,
)
.await;
let _ = command_tx
.send(RoutedMobCommand::internal(
MobCommand::HostStatusPollCompleted {
host_id: host_id.as_str().to_string(),
binding_epoch,
binding_generation,
binding_incarnation,
result,
},
))
.await;
});
}
}
/// Poll an already-bound host without acquiring actor-owned recipient
/// trust. Bind/rebind installed that durable trust edge; a missing edge
/// is therefore an observation failure for the next ceremony to repair,
/// not permission for a detached task to mutate machine obligations.
async fn poll_bound_host_status_once(
bridge: Arc<super::MobSupervisorBridge>,
authority: &crate::store::SupervisorAuthorityRecord,
mob_id: &crate::MobId,
peer: &TrustedPeerDescriptor,
binding_generation: u64,
) -> Result<super::bridge_protocol::BridgeHostStatusResponse, MobError> {
let supervisor_spec = bridge
.supervisor_spec_for_authority_and_recipient(authority, peer)
.await?;
let command = super::bridge_protocol::BridgeCommand::HostStatus(
super::bridge_protocol::BridgeHostStatusPayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
mob_id: mob_id.to_string(),
},
);
let value = bridge
.send_bridge_command_as_authority(
authority,
peer,
&command,
std::time::Duration::from_secs(10),
)
.await?;
if let Some(rejection) = Self::bridge_rejection_reply(command.protocol_version(), &value) {
return Err(Self::bridge_rejection_error(rejection));
}
super::bridge_protocol::decode_bridge_payload(&command, value, "periodic host status")
}
/// W-E orphan reconciliation sweep: read the host's `HostStatus`
/// inventory and release, AT THE HOST-REPORTED TUPLE, every row the
/// machine holds no live facts for. A confirmed current member is never
/// released; a row the machine still has in `MaterializePending` at the
/// same tuple is skipped (its spawn ladder owns it — the sweep runs on
/// the actor task, serialized with spawn finalization). Orphan releases
/// feed NO machine signal — the machine has no facts for an orphan
/// (roster never showed it, §9 row 2).
async fn reconcile_host_members(
&mut self,
host_id: &mob_dsl::HostId,
) -> Result<bool, MobError> {
let binding_generation = self.current_host_binding_generation(host_id)?;
let (endpoint, pubkey) = {
let state = self.dsl_authority.state();
(
state.host_endpoints.get(host_id).cloned(),
state.host_public_keys.get(host_id).copied(),
)
};
let (Some(endpoint), Some(pubkey)) = (endpoint, pubkey) else {
return Err(MobError::Internal(format!(
"host '{}' has no recorded binding facts for the reconciliation sweep",
host_id.as_str()
)));
};
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
host_id.as_str(),
host_id.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::Internal(format!(
"bound host '{}' facts do not form a canonical peer descriptor: {error}",
host_id.as_str()
))
})?;
let authority = self.supervisor_bridge.authority().await;
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&peer)
.await?;
let command = super::bridge_protocol::BridgeCommand::HostStatus(
super::bridge_protocol::BridgeHostStatusPayload {
supervisor: supervisor_spec.into(),
epoch: authority.epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
mob_id: self.definition.id.to_string(),
},
);
let status: super::bridge_protocol::BridgeHostStatusResponse = self
.send_bridge_command_typed(&peer, &command, std::time::Duration::from_secs(10))
.await?;
self.reconcile_host_status_response(host_id, status).await
}
/// Apply one current host inventory. Network releases are themselves
/// detached: the actor only classifies the immutable response against
/// current machine facts and schedules idempotent convergence work.
fn unreported_placement_requires_revival(
placed_host: &mob_dsl::HostId,
polled_host: &mob_dsl::HostId,
host_reported_member: bool,
spawn_exec_phase: Option<mob_dsl::SpawnExecPhase>,
) -> bool {
placed_host == polled_host
&& !host_reported_member
&& !matches!(
spawn_exec_phase,
Some(mob_dsl::SpawnExecPhase::MaterializePending)
)
}
fn host_status_expected_member_tuple(
state: &mob_dsl::MobMachineState,
identity: &mob_dsl::AgentIdentity,
) -> (
Option<mob_dsl::Generation>,
Option<mob_dsl::FenceToken>,
bool,
) {
let materialize_pending = matches!(
state.spawn_exec_phase.get(identity),
Some(mob_dsl::SpawnExecPhase::MaterializePending)
);
if materialize_pending {
(
state
.pending_placed_spawn_generations
.get(identity)
.copied(),
state
.pending_placed_spawn_fence_tokens
.get(identity)
.copied(),
true,
)
} else {
(
state.identity_runtime_generations.get(identity).copied(),
state.identity_runtime_fence_tokens.get(identity).copied(),
false,
)
}
}
/// Advance the volatile incarnation for one host binding and invalidate
/// every detached release reservation minted by its predecessor.
///
/// This counter is intentionally actor-local. No detached task survives
/// actor restart, while a supervisor authority epoch can survive (and be
/// reused across) revoke + fresh bind. Keeping the two concepts separate
/// closes the old-completion/new-route ABA window without inventing a
/// second durable host authority.
fn advance_host_binding_incarnation(
incarnations: &mut BTreeMap<mob_dsl::HostId, u64>,
reservations: &mut BTreeMap<
super::state::HostOrphanReleaseKey,
HostOrphanReleaseReservation,
>,
host_id: &mob_dsl::HostId,
) -> Result<u64, MobError> {
let next = incarnations
.get(host_id)
.copied()
.unwrap_or(0)
.checked_add(1)
.ok_or_else(|| {
MobError::Internal(format!(
"host '{}' exhausted its volatile binding-incarnation counter",
host_id.as_str()
))
})?;
reservations.retain(|key, _| key.host_id != *host_id);
incarnations.insert(host_id.clone(), next);
Ok(next)
}
fn current_host_binding_incarnation(&self, host_id: &mob_dsl::HostId) -> Result<u64, MobError> {
if self.dsl_authority.state().host_bind_phase.get(host_id)
!= Some(&mob_dsl::HostBindPhase::Bound)
{
return Err(MobError::Internal(format!(
"host '{}' has no current bound incarnation",
host_id.as_str()
)));
}
self.host_binding_incarnations
.get(host_id)
.copied()
.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no volatile binding-incarnation fence",
host_id.as_str()
))
})
}
fn current_host_binding_generation(&self, host_id: &mob_dsl::HostId) -> Result<u64, MobError> {
if self.dsl_authority.state().host_bind_phase.get(host_id)
!= Some(&mob_dsl::HostBindPhase::Bound)
{
return Err(MobError::Internal(format!(
"host '{}' has no current durable binding generation",
host_id.as_str()
)));
}
self.dsl_authority
.state()
.host_binding_generations
.get(host_id)
.copied()
.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no durable binding-generation fence",
host_id.as_str()
))
})
}
fn reserve_host_orphan_release(
reservations: &mut BTreeMap<
super::state::HostOrphanReleaseKey,
HostOrphanReleaseReservation,
>,
key: &super::state::HostOrphanReleaseKey,
) -> bool {
match reservations.entry(key.clone()) {
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(HostOrphanReleaseReservation::InFlight);
true
}
std::collections::btree_map::Entry::Occupied(_) => false,
}
}
fn absorb_host_orphan_release_completion(
reservations: &mut BTreeMap<
super::state::HostOrphanReleaseKey,
HostOrphanReleaseReservation,
>,
key: &super::state::HostOrphanReleaseKey,
succeeded: bool,
) -> bool {
let std::collections::btree_map::Entry::Occupied(mut entry) =
reservations.entry(key.clone())
else {
return false;
};
if *entry.get() != HostOrphanReleaseReservation::InFlight {
return false;
}
if succeeded {
entry.insert(HostOrphanReleaseReservation::ReleasedAwaitingFreshOmission);
} else {
entry.remove();
}
true
}
fn absorb_fresh_host_inventory_for_orphan_releases(
reservations: &mut BTreeMap<
super::state::HostOrphanReleaseKey,
HostOrphanReleaseReservation,
>,
host_id: &mob_dsl::HostId,
reported: &BTreeSet<super::state::HostOrphanReleaseKey>,
) {
reservations.retain(|key, phase| {
key.host_id != *host_id
|| *phase == HostOrphanReleaseReservation::InFlight
|| reported.contains(key)
});
}
/// Revalidate the full member residency carried by an authenticated
/// events page against current MobMachine facts. The event pump also
/// checks its local lease before and after this command; this actor-side
/// check closes the serialized state-transition interval in between.
fn validate_member_events_runtime_observation(
&self,
expected: &super::bridge_protocol::BridgeMemberIncarnation,
) -> Result<mob_dsl::HostId, MobError> {
if self.definition.id != expected.mob_id {
return Err(MobError::Internal(format!(
"member-events runtime observation names mob '{}' while actor owns '{}'",
expected.mob_id, self.definition.id
)));
}
let host_id = mob_dsl::HostId::from(expected.host_id.as_str());
let identity = mob_dsl::AgentIdentity::from(expected.agent_identity.as_str());
let state = self.dsl_authority.state();
let current_matches = state.host_bind_phase.get(&host_id)
== Some(&mob_dsl::HostBindPhase::Bound)
&& state.host_binding_generations.get(&host_id).copied()
== Some(expected.binding_generation)
&& state.member_placement.get(&identity) == Some(&host_id)
&& state
.current_placed_spawn_host_binding_generations
.get(&identity)
.copied()
== Some(expected.binding_generation)
&& state
.identity_runtime_generations
.get(&identity)
.is_some_and(|generation| generation.0 == expected.generation)
&& state
.identity_runtime_fence_tokens
.get(&identity)
.is_some_and(|fence| fence.0 == expected.fence_token)
&& state
.member_session_bindings
.get(&identity)
.is_some_and(|session| session.0 == expected.member_session_id);
if !current_matches {
return Err(MobError::Internal(format!(
"member-events runtime observation for '{}' is stale against the current host/member residency",
expected.agent_identity
)));
}
Ok(host_id)
}
fn record_host_runtime_incarnation(
&mut self,
host_id: &mob_dsl::HostId,
runtime_incarnation: super::bridge_protocol::BridgeHostRuntimeIncarnation,
source: &'static str,
) -> bool {
let previous = self
.host_runtime_incarnations
.insert(host_id.clone(), runtime_incarnation);
let changed = previous != Some(runtime_incarnation);
if changed {
tracing::info!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
previous_runtime_incarnation = ?previous,
runtime_incarnation = %runtime_incarnation,
source,
"authenticated host runtime incarnation changed; route intent will be re-realized"
);
}
changed
}
async fn converge_routes_after_host_runtime_observation(
&mut self,
host_id: &mob_dsl::HostId,
runtime_incarnation_changed: bool,
) -> Result<(), MobError> {
if runtime_incarnation_changed {
self.drain_route_installs_for_host(host_id).await
} else {
self.realize_pending_route_installs(Some(host_id)).await
}
}
async fn reconcile_host_status_response(
&mut self,
host_id: &mob_dsl::HostId,
status: super::bridge_protocol::BridgeHostStatusResponse,
) -> Result<bool, MobError> {
let recorded_live_endpoint = self
.dsl_authority
.state()
.host_live_endpoints
.get(host_id)
.map(|endpoint| endpoint.0.clone());
// Live endpoint has one carrier and changes only on bind/rebind.
// Periodic status refreshes the non-live capability record while
// preserving that machine-owned endpoint verbatim.
let capabilities =
Self::host_capability_report_from_bridge(&status.capabilities, recorded_live_endpoint)?;
if let Err(error) = Self::enforce_host_capability_contract(
host_id,
Self::required_host_capabilities(self.dsl_authority.state(), host_id),
&capabilities,
) {
// The host answered authoritatively but can no longer serve
// retained residency/custody. Do not rewrite machine capability
// facts or keep operating on a degraded substrate; cold recovery
// is the only boundary allowed to re-establish the contract.
self.durable_uncertainty_fail_stop = true;
return Err(error);
}
self.refresh_current_host_facts(host_id, &capabilities, "periodic host capability refresh")
.await?;
let peer = {
let state = self.dsl_authority.state();
let (Some(endpoint), Some(pubkey)) = (
state.host_endpoints.get(host_id),
state.host_public_keys.get(host_id),
) else {
return Err(MobError::Internal(format!(
"host '{}' lost binding facts during status reconciliation",
host_id.as_str()
)));
};
let Ok(peer) = TrustedPeerDescriptor::unsigned_with_pubkey(
host_id.as_str(),
host_id.as_str(),
pubkey.0,
endpoint.0.as_str(),
) else {
return Err(MobError::Internal(format!(
"host '{}' binding facts do not form a peer descriptor",
host_id.as_str()
)));
};
peer
};
let binding_incarnation = self.current_host_binding_incarnation(host_id)?;
let binding_generation = self.current_host_binding_generation(host_id)?;
let supervisor_authority = self.supervisor_bridge.authority().await;
let supervisor = self
.supervisor_bridge
.supervisor_spec_for_authority_and_recipient(&supervisor_authority, &peer)
.await?;
let reported_release_keys = status
.members
.iter()
.map(|record| super::state::HostOrphanReleaseKey {
host_id: host_id.clone(),
binding_incarnation,
agent_identity: mob_dsl::AgentIdentity::from(record.agent_identity.clone()),
generation: mob_dsl::Generation(record.generation),
fence_token: mob_dsl::FenceToken(record.fence_token),
})
.collect::<BTreeSet<_>>();
Self::absorb_fresh_host_inventory_for_orphan_releases(
&mut self.orphan_release_reservations,
host_id,
&reported_release_keys,
);
let mut current_reported = std::collections::BTreeSet::new();
for record in &status.members {
let dsl_identity = mob_dsl::AgentIdentity::from(record.agent_identity.clone());
let (placed_here, machine_generation, machine_fence, materialize_pending) = {
let state = self.dsl_authority.state();
let (machine_generation, machine_fence, materialize_pending) =
Self::host_status_expected_member_tuple(state, &dsl_identity);
(
state.member_placement.get(&dsl_identity) == Some(host_id),
machine_generation,
machine_fence,
materialize_pending,
)
};
let tuple_matches = machine_generation.as_ref().map(|generation| generation.0)
== Some(record.generation)
&& machine_fence.as_ref().map(|fence| fence.0) == Some(record.fence_token);
if placed_here && tuple_matches {
if materialize_pending {
// The spawn ladder owns this row: its ack or failure is
// in flight; releasing here would race the commit.
continue;
}
let carrier = match self
.runtime_metadata
.load_placed_spawn(&self.definition.id, &record.agent_identity)
.await
{
Ok(Some(carrier)) => carrier,
Ok(None) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"authenticated HostStatus reports placed member '{}' but its committed carrier is absent; actor is fail-stopping for cold recovery",
record.agent_identity
)));
}
Err(error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"authenticated HostStatus reports placed member '{}' but its carrier read failed; actor is fail-stopping for cold recovery: {error}",
record.agent_identity
)));
}
};
let crate::store::PlacedSpawnCarrierPhase::Committed(committed) = &carrier.phase
else {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"authenticated HostStatus reports placed member '{}' while its carrier is Pending; actor is fail-stopping for cold recovery",
record.agent_identity
)));
};
if carrier.host_id.to_string() != host_id.as_str()
|| carrier.generation != record.generation
|| carrier.fence_token != record.fence_token
|| carrier.spec_digest != record.spec_digest
|| committed.member_session_id.to_string() != record.session_id
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"authenticated HostStatus member '{}' conflicts with its exact committed carrier tuple; actor is fail-stopping for cold recovery",
record.agent_identity
)));
}
self.promote_committed_placed_carrier_binding_generation(
&carrier,
host_id,
binding_generation,
"promote placed carrier from exact host status presence",
)
.await?;
current_reported.insert(record.agent_identity.clone());
if !record.healthy {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %record.agent_identity,
host = %host_id.as_str(),
"host reports current member unhealthy (revival trigger)"
);
// W-D.2 trigger (2): deferred through the command
// channel so the sweep (which runs inside bind/rebind
// handling) never blocks on a bridge round trip.
self.try_fire_placed_revival_trigger(
Some(AgentIdentity::from(record.agent_identity.as_str())),
&MobError::Internal(format!(
"host '{}' reports current member unhealthy",
host_id.as_str()
)),
);
}
continue;
}
if placed_here && materialize_pending && tuple_matches {
continue;
}
let release_key = super::state::HostOrphanReleaseKey {
host_id: host_id.clone(),
binding_incarnation,
agent_identity: dsl_identity,
generation: mob_dsl::Generation(record.generation),
fence_token: mob_dsl::FenceToken(record.fence_token),
};
if !Self::reserve_host_orphan_release(
&mut self.orphan_release_reservations,
&release_key,
) {
tracing::debug!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
binding_incarnation,
agent_identity = %record.agent_identity,
generation = record.generation,
fence_token = record.fence_token,
session_id = %record.session_id,
"exact orphan release is already in flight"
);
continue;
}
// Stale/orphan: release at the HOST-REPORTED tuple; the host's
// release admission admits exactly its recorded materialized
// tuple and its dedup replays converge.
let release = super::provisioner::HostMemberReleaseRequest {
mob_id: self.definition.id.clone(),
agent_identity: record.agent_identity.clone(),
generation: record.generation,
fence_token: record.fence_token,
supervisor_authority: supervisor_authority.clone(),
supervisor: supervisor.clone().into(),
binding_generation,
host: peer.clone(),
};
let provisioner = Arc::clone(&self.provisioner);
let command_tx = self.command_tx.clone();
self.actor_io_tasks.spawn(async move {
let result = provisioner.release_host_member(release).await;
let _ = command_tx
.send(RoutedMobCommand::internal(
MobCommand::HostOrphanReleaseCompleted {
key: release_key,
result,
},
))
.await;
});
}
// W-D.2 trigger (3): members the MACHINE places on this host that
// the host does NOT report — the revival classification adjudicates
// Broken vs revive from the machine facts.
let unreported: Vec<AgentIdentity> = {
let state = self.dsl_authority.state();
state
.member_placement
.iter()
.filter(|(identity, placed_host)| {
Self::unreported_placement_requires_revival(
placed_host,
host_id,
current_reported.contains(identity.0.as_str()),
state.spawn_exec_phase.get(*identity).copied(),
)
})
.map(|(identity, _)| AgentIdentity::from(identity.0.as_str()))
.collect()
};
for identity in unreported {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %identity,
host = %host_id.as_str(),
"machine places member on host but the host reports no row (revival trigger)"
);
self.try_fire_placed_revival_trigger(
Some(identity),
&MobError::Internal(format!(
"host '{}' reports no materialized row for a machine-placed member",
host_id.as_str()
)),
);
}
// Advance this observer-local cache only after the complete inventory
// reconciliation succeeds. If an earlier step fails, the next
// authenticated reply must still be able to trigger route recovery.
Ok(
self.record_host_runtime_incarnation(
host_id,
status.runtime_incarnation,
"host status",
),
)
}
async fn current_pending_host_bind_anchors(
&self,
) -> Result<Vec<PendingHostBindAnchor>, MobError> {
let all_events = self.events.replay_all().await?;
let mob_events = all_events
.into_iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |position| position + 1);
pending_host_bind_anchors(&mob_events[epoch_start..])
}
async fn ensure_host_bind_started(
&mut self,
request: crate::event::RemoteHostBindRequestEvent,
) -> Result<(PendingHostBindAnchor, bool), MobError> {
if let Some(existing) = self
.current_pending_host_bind_anchors()
.await?
.into_iter()
.find(|anchor| anchor.request.host_id == request.host_id)
{
if existing.request != request {
return Err(MobError::Internal(format!(
"unfinished host bind '{}' for host '{}' does not match the requested descriptor authority",
existing.operation_id, request.host_id
)));
}
if existing.confirmed_authority.is_some() {
return Err(MobError::Internal(format!(
"confirmed host bind '{}' for host '{}' awaits local recovery completion",
existing.operation_id, request.host_id
)));
}
return Ok((existing, false));
}
let operation_id = meerkat_core::time_compat::new_uuid_v7().to_string();
self.ensure_exact_structural_event(MobEventKind::RemoteHostBindStarted {
operation_id: operation_id.clone(),
request: request.clone(),
})
.await?;
Ok((
PendingHostBindAnchor {
operation_id,
request,
confirmed_authority: None,
},
true,
))
}
async fn confirm_host_bind_anchor(
&mut self,
anchor: &PendingHostBindAnchor,
authority: &crate::store::MobHostAuthorityRecord,
) -> Result<(), MobError> {
if !host_bind_request_matches_confirmed_authority(&anchor.request, authority) {
return Err(MobError::Internal(format!(
"host bind confirmation for operation '{}' drifts from Started authority",
anchor.operation_id
)));
}
self.ensure_exact_structural_event(MobEventKind::RemoteHostBindConfirmed {
operation_id: anchor.operation_id.clone(),
authority: authority.clone(),
})
.await
}
async fn complete_host_bind_anchor(
&mut self,
anchor: &PendingHostBindAnchor,
) -> Result<(), MobError> {
self.ensure_exact_structural_event(MobEventKind::RemoteHostBindCompleted {
operation_id: anchor.operation_id.clone(),
host_id: anchor.request.host_id.clone(),
authority_epoch: anchor.request.authority_epoch,
binding_generation: anchor.request.binding_generation,
})
.await
}
async fn abort_host_bind_anchor_no_effect(
&mut self,
anchor: &PendingHostBindAnchor,
) -> Result<(), MobError> {
self.ensure_exact_structural_event(MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id: anchor.operation_id.clone(),
host_id: anchor.request.host_id.clone(),
authority_epoch: anchor.request.authority_epoch,
binding_generation: anchor.request.binding_generation,
})
.await
}
async fn current_pending_host_revoke_anchors(
&self,
) -> Result<Vec<PendingHostRevokeAnchor>, MobError> {
let all_events = self.events.replay_all().await?;
let mob_events = all_events
.into_iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |position| position + 1);
pending_host_revoke_anchors(&mob_events[epoch_start..])
}
async fn require_host_authority_anchors_clear_for_action(
&self,
action: &str,
) -> Result<(), MobError> {
let pending_binds = self.current_pending_host_bind_anchors().await?;
let pending_revokes = self.current_pending_host_revoke_anchors().await?;
require_host_authority_anchors_clear(action, &pending_binds, &pending_revokes)
}
async fn ensure_host_revoke_started(
&mut self,
host_id: &mob_dsl::HostId,
epoch: u64,
binding_generation: u64,
) -> Result<PendingHostRevokeAnchor, MobError> {
if let Some(existing) = self
.current_pending_host_revoke_anchors()
.await?
.into_iter()
.find(|anchor| anchor.host_id == host_id.0)
{
if existing.epoch != epoch || existing.binding_generation != binding_generation {
return Err(MobError::Internal(format!(
"unfinished host revoke '{}' targets host '{}' epoch {}, current epoch is {epoch}",
existing.operation_id, host_id.0, existing.epoch,
)));
}
return Ok(existing);
}
let operation_id = meerkat_core::time_compat::new_uuid_v7().to_string();
self.ensure_exact_structural_event(MobEventKind::RemoteHostRevokeStarted {
operation_id: operation_id.clone(),
host_id: host_id.0.clone(),
epoch,
binding_generation,
})
.await?;
Ok(PendingHostRevokeAnchor {
operation_id,
host_id: host_id.0.clone(),
epoch,
binding_generation,
confirmed: false,
})
}
async fn complete_host_revoke_anchor(
&mut self,
anchor: &PendingHostRevokeAnchor,
) -> Result<(), MobError> {
self.ensure_exact_structural_event(MobEventKind::RemoteHostRevokeCompleted {
operation_id: anchor.operation_id.clone(),
host_id: anchor.host_id.clone(),
epoch: anchor.epoch,
binding_generation: anchor.binding_generation,
})
.await
}
/// Revoke a bound (or bind-requested) host. Bound hosts execute the
/// authenticated HOST-side terminal first; only a validated durable
/// `HostRevoked` receipt permits this controlling actor to clear its
/// local machine/record. Reply loss therefore leaves the local binding as
/// the retry anchor, while the host replays its durable receipt.
/// Requested-only windows never reached a host binding and clear locally.
/// Placed members keep `member_placement`; the §9 revival ladder owns
/// re-placement.
async fn handle_revoke_host(
&mut self,
host_id_raw: &str,
) -> Result<super::handle::HostRevokeReport, MobError> {
let host_id = mob_dsl::HostId::from(host_id_raw.to_string());
if let Some(anchor) = self
.current_pending_host_bind_anchors()
.await?
.into_iter()
.find(|anchor| anchor.request.host_id == host_id.0)
{
return Err(MobError::Internal(format!(
"host '{}' bind operation '{}' is unfinished (confirmed={}); exact bind convergence is required before revoke",
host_id.as_str(),
anchor.operation_id,
anchor.confirmed_authority.is_some(),
)));
}
// Capture every remote addressing/auth fact before either authority
// clears it. The actor is serialized, so no host command can race
// this terminal choreography.
let (
phase,
endpoint,
pubkey,
binding_epoch,
binding_generation,
released_members,
committed_released_members,
) = {
let state = self.dsl_authority.state();
let released_members: Vec<AgentIdentity> = state
.member_placement
.iter()
.filter(|(_, placed_host)| **placed_host == host_id)
.map(|(identity, _)| AgentIdentity::from(identity.0.as_str()))
.collect();
// A placement is opened before the remote materialization ACK.
// Only identities with a committed host session belong in the
// receipt's required proof set; a truthful host may legitimately
// omit a MaterializePending/failed window that never became a
// durable host row. Any extra host-reported row remains allowed
// and is released by revoke.
let committed_released_members =
Self::committed_host_member_release_proof_set(state, &host_id);
(
state.host_bind_phase.get(&host_id).copied().or_else(|| {
state
.replacement_host_bind_endpoints
.contains_key(&host_id)
.then_some(mob_dsl::HostBindPhase::Requested)
}),
state.host_endpoints.get(&host_id).cloned(),
state.host_public_keys.get(&host_id).copied(),
state.host_authority_epochs.get(&host_id).copied(),
state
.host_binding_generations
.get(&host_id)
.copied()
.or_else(|| {
state
.replacement_host_binding_generations
.get(&host_id)
.copied()
}),
released_members,
committed_released_members,
)
};
let pending_anchor = self
.current_pending_host_revoke_anchors()
.await?
.into_iter()
.find(|anchor| anchor.host_id == host_id.0);
if phase.is_none()
&& let Some(anchor) = pending_anchor.as_ref()
&& self
.runtime_metadata
.load_mob_host_authority(&self.definition.id, host_id.as_str())
.await?
.is_none()
{
if !anchor.confirmed {
return Err(MobError::Internal(format!(
"host revoke '{}' lost local authority before durable remote confirmation",
anchor.operation_id,
)));
}
// Crash after durable authority deletion/local machine commit but
// before the final event. The absent record is the restart fact
// that the local terminal already holds; close only this exact op.
self.complete_host_revoke_anchor(anchor).await?;
return Ok(super::handle::HostRevokeReport {
host_id: host_id.as_str().to_string(),
released_members,
});
}
let binding_generation = binding_generation.ok_or_else(|| {
MobError::Internal(format!(
"host '{}' has no binding-generation fact for revoke",
host_id.as_str()
))
})?;
if phase == Some(mob_dsl::HostBindPhase::Bound) && pending_anchor.is_none() {
// The first durable revoke carrier also becomes the mutation
// admission fence above. Establish the live-channel terminal
// while the old host authority is still addressable; retries of
// an existing carrier rely on the cleanup-backed HostRevoked
// receipt instead of trying to use already-revoked authority.
self.quiesce_member_live_for_identities(
committed_released_members.clone(),
"host revoke live-channel barrier",
)
.await?;
}
let revoke_anchor = match phase {
Some(mob_dsl::HostBindPhase::Bound) => {
let epoch = binding_epoch.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no authority epoch for revoke intent",
host_id.as_str()
))
})?;
Some(
self.ensure_host_revoke_started(&host_id, epoch, binding_generation)
.await?,
)
}
_ => None,
};
let bound_peer = match phase {
Some(mob_dsl::HostBindPhase::Bound) => {
let (Some(endpoint), Some(pubkey), Some(binding_epoch)) =
(endpoint.as_ref(), pubkey, binding_epoch)
else {
return Err(MobError::Internal(format!(
"bound host '{}' lacks endpoint, signing key, or authority epoch",
host_id.as_str()
)));
};
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
host_id.as_str(),
host_id.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::WiringError(format!(
"bound host '{}' facts do not form a canonical revoke target: {error}",
host_id.as_str()
))
})?;
let authority = self.supervisor_bridge.authority().await;
if authority.epoch != binding_epoch {
return Err(MobError::Internal(format!(
"bound host '{}' authority epoch drift: bridge={}, host_record={binding_epoch}",
host_id.as_str(),
authority.epoch
)));
}
let supervisor = self
.supervisor_bridge
.supervisor_spec_for_recipient(&peer)
.await?;
let command = super::bridge_protocol::BridgeCommand::RevokeHost(
super::bridge_protocol::BridgeHostRevokePayload {
supervisor: supervisor.into(),
epoch: binding_epoch,
binding_generation,
protocol_version: super::bridge_protocol::BridgeProtocolVersion::V4,
mob_id: self.definition.id.to_string(),
},
);
// Recipient trust is volatile transport state, while this
// exact host binding is durable MobMachine authority. Rebuild
// the route after a cold controlling restart and retain it on
// every pre-terminal failure because the host remains bound
// and retryable. The confirmed-revoke path below removes it.
self.supervisor_bridge.trust_recipient(&peer).await?;
let value = match self
.supervisor_bridge
.send_bridge_command_classified(&peer, &command, HOST_REVOKE_BRIDGE_TIMEOUT)
.await
{
Ok(value) => value,
Err(super::supervisor_bridge::BridgeRequestFailure::BeforeSend(error)) => {
return Err(error);
}
Err(super::supervisor_bridge::BridgeRequestFailure::AfterSend(error)) => {
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
};
if let Some(rejection) =
Self::bridge_rejection_reply(command.protocol_version(), &value)
{
let error = Self::bridge_rejection_error(rejection.clone());
if Self::revoke_rejection_certifies_no_remote_effect(&rejection) {
return Err(error);
}
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
let receipt = match super::bridge_protocol::decode_bridge_payload::<
super::bridge_protocol::BridgeHostRevokedResponse,
>(&command, value, "revoke host")
{
Ok(receipt) => receipt,
Err(error) => {
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
};
if receipt.host_peer_id != host_id.as_str()
|| receipt.mob_id != self.definition.id.as_str()
|| receipt.epoch != binding_epoch
|| receipt.binding_generation != binding_generation
{
let error = MobError::Internal(format!(
"host revoke receipt mismatch: expected host='{}' mob='{}' epoch={} generation={}, got host='{}' mob='{}' epoch={} generation={}",
host_id.as_str(),
self.definition.id,
binding_epoch,
binding_generation,
receipt.host_peer_id,
receipt.mob_id,
receipt.epoch,
receipt.binding_generation,
));
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
let receipt_members = receipt
.released_members
.iter()
.cloned()
.collect::<BTreeSet<_>>();
let expected_members = committed_released_members
.iter()
.map(ToString::to_string)
.collect::<BTreeSet<_>>();
if receipt_members.len() != receipt.released_members.len()
|| !expected_members.is_subset(&receipt_members)
{
let error = MobError::Internal(format!(
"host revoke receipt released-member proof is incomplete or non-canonical: expected at least {expected_members:?}, got {:?}",
receipt.released_members
));
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
Some(peer)
}
Some(mob_dsl::HostBindPhase::Requested) => None,
None => None,
};
let remote_terminal_accepted = bound_peer.is_some();
macro_rules! revoke_terminal_try {
($result:expr) => {
match $result {
Ok(value) => value,
Err(error) => {
let error: MobError = error.into();
if remote_terminal_accepted {
return Err(self.quarantine_uncertain_host_revoke(&host_id, error));
}
return Err(error);
}
}
};
}
macro_rules! revoke_terminal_error {
($error:expr) => {{
let error: MobError = $error;
if remote_terminal_accepted {
self.quarantine_uncertain_host_revoke(&host_id, error)
} else {
error
}
}};
}
if bound_peer.is_some() {
let epoch = revoke_terminal_try!(binding_epoch.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' lost its authority epoch before revoke confirmation",
host_id.as_str()
))
}));
let anchor = revoke_terminal_try!(revoke_anchor.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' revoke has no durable Started anchor",
host_id.as_str()
))
}));
revoke_terminal_try!(
self.ensure_exact_structural_event(MobEventKind::RemoteHostRevokeConfirmed {
operation_id: anchor.operation_id.clone(),
host_id: host_id.0.clone(),
epoch,
binding_generation,
})
.await
);
let obligations = self
.dsl_authority
.state()
.pending_remote_turn_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.committed_remote_turn_outcomes
.iter(),
)
.chain(
self.dsl_authority
.state()
.resolved_remote_turn_outcomes
.iter(),
)
.filter(|obligation| obligation.host_id == host_id)
.cloned()
.collect::<BTreeSet<_>>();
for obligation in obligations {
let carrier =
revoke_terminal_try!(super::remote_flow_ticket::obligation_event(&obligation));
if self
.dsl_authority
.state()
.pending_remote_turn_outcomes
.contains(&obligation)
{
// The authenticated HostRevoked receipt and its exact
// structural carrier are already durable. Before waking
// a live flow task (or relying on recovery when no ticket
// exists), record one canonical public step failure and
// its private receipt while the exact intent still owns
// custody. The queued commit then converges through the
// exact terminal+Disposed replay seam after cleanup.
let reason =
super::remote_flow_ticket::HOST_REVOKED_STEP_FAILURE_REASON.to_string();
revoke_terminal_try!(
self.commit_remote_turn_receipt_in_actor(
crate::run::MobRunRemoteTurnReceipt {
obligation: carrier.clone(),
outcome: crate::run::MobRunRemoteTurnReceiptOutcome::Failed {
reason: reason.clone(),
no_effect_proof: None,
},
},
)
.await
);
self.remote_flow_tickets.fail_armed(
&carrier.agent_identity,
&carrier.input_id,
reason,
);
}
revoke_terminal_try!(
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnOutcomeDisposed {
obligation: carrier.clone(),
})
.await
);
revoke_terminal_try!(
self.run_store
.delete_remote_turn_receipt(&carrier.run_id, carrier.dispatch_sequence)
.await
);
revoke_terminal_try!(
self.run_store
.delete_remote_turn_intent(&carrier.run_id, carrier.dispatch_sequence)
.await
);
revoke_terminal_try!(self.apply_dsl_input(
mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { obligation },
"dispose_remote_turn_obligation_after_host_revoke",
));
}
let completion_obligations = self
.dsl_authority
.state()
.pending_placed_completion_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.resolved_placed_completion_outcomes
.iter(),
)
.filter(|obligation| obligation.host_id == host_id)
.cloned()
.collect::<BTreeSet<_>>();
for obligation in completion_obligations {
let identity = AgentIdentity::from(obligation.agent_identity.0.as_str());
revoke_terminal_try!(
self.dispose_placed_completion_custody_for_exact_residency(
ExactRemoteTurnResidency {
agent_identity: &identity,
host_id: &obligation.host_id,
host_binding_generation: obligation.host_binding_generation,
member_session_id: &obligation.member_session_id,
generation: obligation.generation.0,
fence_token: obligation.fence_token.0,
},
)
.await
);
}
let kickoff_obligations = self
.dsl_authority
.state()
.pending_placed_kickoff_outcomes
.iter()
.chain(
self.dsl_authority
.state()
.resolved_placed_kickoff_outcomes
.iter(),
)
.filter(|obligation| obligation.host_id == host_id)
.cloned()
.collect::<BTreeSet<_>>();
for obligation in kickoff_obligations {
let identity = AgentIdentity::from(obligation.agent_identity.0.as_str());
revoke_terminal_try!(
self.dispose_placed_kickoff_custody_for_exact_residency(
ExactRemoteTurnResidency {
agent_identity: &identity,
host_id: &obligation.host_id,
host_binding_generation: obligation.host_binding_generation,
member_session_id: &obligation.member_session_id,
generation: obligation.generation.0,
fence_token: obligation.fence_token.0,
},
)
.await
);
}
}
// If local recipient-trust cleanup fails, retain the local binding
// and return failure: the next call replays the host's durable
// receipt and `send_bridge_command_typed` repairs recipient trust.
if let Some(peer) = bound_peer.as_ref() {
revoke_terminal_try!(
self.supervisor_bridge
.untrust_recipient(peer)
.await
.map_err(|error| {
MobError::WiringError(format!(
"host '{}' durably revoked but controlling recipient trust could not be removed: {error}",
host_id.as_str()
))
})
);
}
// A Requested-only host has no durable record (records exist only
// for committed binds); its revoke clears the machine window only.
let durable_record = revoke_terminal_try!(
self.runtime_metadata
.load_mob_host_authority(&self.definition.id, host_id.as_str())
.await
);
match (phase, durable_record.as_ref()) {
(Some(mob_dsl::HostBindPhase::Bound), None) => {
return Err(revoke_terminal_error!(MobError::Internal(format!(
"bound host '{}' has no durable authority record for revoke",
host_id.as_str()
))));
}
(Some(mob_dsl::HostBindPhase::Requested), Some(_)) => {
return Err(revoke_terminal_error!(MobError::Internal(format!(
"requested host '{}' unexpectedly has a durable authority record",
host_id.as_str()
))));
}
_ => {}
}
let prepared = revoke_terminal_try!(self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RevokeHost {
host_id: host_id.clone(),
binding_generation,
},
"revoke_host",
));
let (revoked_host, revoked_generation) = revoke_terminal_try!(
prepared
.transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::HostRevoked {
host_id,
binding_generation,
} => {
Some((host_id.clone(), *binding_generation))
}
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted RevokeHost but emitted no HostRevoked effect".into(),
)
})
);
if revoked_host != host_id || revoked_generation != binding_generation {
return Err(revoke_terminal_error!(MobError::Internal(format!(
"MobMachine HostRevoked drift: input host='{}', effect host='{}'",
host_id.as_str(),
revoked_host.as_str()
))));
}
if let Some(durable_record) = durable_record.as_ref() {
let deletion = revoke_terminal_try!(
crate::store::MobHostAuthorityDeletionAuthority::from_transition(
durable_record,
&prepared.transition,
)
);
if durable_record.binding_generation > 0 {
revoke_terminal_try!(
self.runtime_metadata
.put_mob_host_binding_generation_highwater(
&self.definition.id,
durable_record,
&deletion,
)
.await
);
}
let removed = revoke_terminal_try!(
self.runtime_metadata
.delete_mob_host_authority(&self.definition.id, durable_record, &deletion)
.await
);
if !removed {
return Err(revoke_terminal_error!(MobError::Internal(format!(
"durable host authority delete CAS missed for host '{}'",
host_id.as_str()
))));
}
}
revoke_terminal_try!(self.commit_prepared_dsl_transition(prepared));
revoke_terminal_try!(Self::advance_host_binding_incarnation(
&mut self.host_binding_incarnations,
&mut self.orphan_release_reservations,
&host_id,
));
self.reachability_observations.clear_host(host_id.as_str());
self.host_runtime_incarnations.remove(&host_id);
if let Some(anchor) = revoke_anchor.as_ref() {
revoke_terminal_try!(self.complete_host_revoke_anchor(anchor).await);
}
Ok(super::handle::HostRevokeReport {
host_id: host_id.as_str().to_string(),
released_members,
})
}
fn committed_host_member_release_proof_set(
state: &mob_dsl::MobMachineState,
host_id: &mob_dsl::HostId,
) -> Vec<AgentIdentity> {
state
.member_placement
.iter()
.filter(|(identity, placed_host)| {
*placed_host == host_id
&& state.member_session_bindings.contains_key(identity)
&& !matches!(
state.spawn_exec_phase.get(identity),
Some(mob_dsl::SpawnExecPhase::MaterializePending)
)
})
.map(|(identity, _)| AgentIdentity::from(identity.0.as_str()))
.collect()
}
/// Enumerate the bound hosts that join the rotation recipient set (§7.2
/// step 4): `host_bind_phase == Bound`, peer descriptor rebuilt from the
/// machine's recorded endpoint + signing key, failing closed when a bound
/// host's facts are incomplete.
fn bound_host_rotation_targets(
&self,
) -> Result<Vec<(mob_dsl::HostId, TrustedPeerDescriptor)>, MobError> {
let state = self.dsl_authority.state();
let mut targets = Vec::new();
for (host_id, phase) in &state.host_bind_phase {
if *phase != mob_dsl::HostBindPhase::Bound {
continue;
}
let endpoint = state.host_endpoints.get(host_id).ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no recorded endpoint",
host_id.as_str()
))
})?;
let pubkey = state
.host_public_keys
.get(host_id)
.copied()
.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no recorded signing key",
host_id.as_str()
))
})?;
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
host_id.as_str(),
host_id.as_str(),
pubkey.0,
endpoint.0.as_str(),
)
.map_err(|error| {
MobError::Internal(format!(
"bound host '{}' facts do not form a canonical peer descriptor: {error}",
host_id.as_str()
))
})?;
targets.push((host_id.clone(), peer));
}
Ok(targets)
}
/// Absorb a bound host's typed acceptance of the attempted supervisor
/// authority — a fresh `HostRebound` accept or the idempotent
/// rebind-replay ack (FLAG-2(ii)). On a fresh accept the machine records
/// the new epoch + re-declared capability record and the durable host
/// authority record is rewritten under the transition witness (durable
/// before in-memory). A replay ack whose facts are already recorded is a
/// pure convergence probe: no machine input, no durable write.
async fn absorb_host_rebind_acceptance(
&mut self,
host_id: &mob_dsl::HostId,
peer: &TrustedPeerDescriptor,
next: &crate::store::SupervisorAuthorityRecord,
rebound: &super::bridge_protocol::BridgeHostReboundResponse,
) -> Result<(), MobError> {
if let Err(error) = self
.absorb_host_rebind_acceptance_inner(host_id, peer, next, rebound)
.await
{
self.durable_uncertainty_fail_stop = true;
tracing::error!(
host = %host_id.as_str(),
epoch = next.epoch,
error = %error,
"authenticated HostRebound could not be absorbed exactly; fail-stopping for cold recovery"
);
return Err(MobError::Internal(format!(
"authenticated HostRebound for host '{}' epoch {} could not be absorbed exactly: {error}; actor is fail-stopping for cold recovery",
host_id.as_str(),
next.epoch,
)));
}
Ok(())
}
async fn absorb_host_rebind_acceptance_inner(
&mut self,
host_id: &mob_dsl::HostId,
peer: &TrustedPeerDescriptor,
next: &crate::store::SupervisorAuthorityRecord,
rebound: &super::bridge_protocol::BridgeHostReboundResponse,
) -> Result<(), MobError> {
let binding_generation = self.current_host_binding_generation(host_id)?;
if rebound.host_peer_id != peer.peer_id.to_string() {
return Err(MobError::WiringError(format!(
"host rebind reply identity mismatch for host '{}': got '{}'",
host_id.as_str(),
rebound.host_peer_id
)));
}
if rebound.binding_generation != binding_generation {
return Err(MobError::WiringError(format!(
"host rebind reply generation mismatch for host '{}': expected {binding_generation}, got {}",
host_id.as_str(),
rebound.binding_generation
)));
}
let (recorded_epoch, pubkey, endpoint) = {
let state = self.dsl_authority.state();
(
state.host_authority_epochs.get(host_id).copied(),
state.host_public_keys.get(host_id).copied(),
state.host_endpoints.get(host_id).cloned(),
)
};
// Every ACK re-declares the complete host fact set. Same-epoch replay
// is pure only when both machine and durable record already match;
// otherwise it uses the exact refresh persistence transition.
let capabilities = Self::host_capability_report_from_bridge(
&rebound.capabilities,
rebound.live_endpoint.clone(),
)?;
Self::enforce_host_capability_contract(
host_id,
Self::required_host_capabilities(self.dsl_authority.state(), host_id),
&capabilities,
)?;
if recorded_epoch == Some(next.epoch) {
// Replay-converged: an earlier accepted attempt already recorded
// the attempted epoch in the machine and the durable record. This
// is fact-preserving, so it must not advance the actor-local
// incarnation or invalidate legitimate in-flight orphan releases.
self.refresh_current_host_facts(
host_id,
&capabilities,
"same-epoch HostRebound replay fact convergence",
)
.await?;
if let Err(sweep_error) = self
.observe_host_status_as_authority(host_id, peer, next)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %sweep_error,
"post-replay HostRebound capability/inventory sweep failed"
);
}
if self.durable_uncertainty_fail_stop {
return Err(MobError::Internal(format!(
"host '{}' replayed HostRebound revealed an incompatible retained capability contract; suppressing rotation success",
host_id.as_str()
)));
}
return Ok(());
}
let pubkey = pubkey.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no recorded signing key",
host_id.as_str()
))
})?;
let endpoint = endpoint.ok_or_else(|| {
MobError::Internal(format!(
"bound host '{}' has no recorded endpoint",
host_id.as_str()
))
})?;
// Restart truthfulness (D5): the rebind reply re-declares the whole
// capability record; an absent live endpoint CLEARS the entry.
let live_endpoint = rebound
.live_endpoint
.as_deref()
.map(Self::live_ws_endpoint_from_wire)
.transpose()?;
let input = mob_dsl::MobMachineInput::HostRebound {
host_id: host_id.clone(),
epoch: next.epoch,
binding_generation,
protocol_min: capabilities.protocol_min,
protocol_max: capabilities.protocol_max,
engine_version: capabilities.engine_version.clone(),
durable_sessions: capabilities.durable_sessions,
autonomous_members: capabilities.autonomous_members,
hard_cancel_member: capabilities.hard_cancel_member,
tracked_input_cancel: capabilities.tracked_input_cancel,
memory_store: capabilities.memory_store,
mcp: capabilities.mcp,
resolvable_providers: capabilities.resolvable_providers.clone(),
approval_forwarding: capabilities.approval_forwarding,
live_endpoint,
};
let prepared = self.prepare_dsl_input_transition(input, "rotate_supervisor host rebind")?;
let (rebound_host, rebound_epoch, rebound_generation) = prepared
.transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::HostReboundRecorded {
host_id,
epoch,
binding_generation,
} => Some((host_id.clone(), *epoch, *binding_generation)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted HostRebound but emitted no HostReboundRecorded effect"
.into(),
)
})?;
if rebound_host != *host_id
|| rebound_epoch != next.epoch
|| rebound_generation != binding_generation
{
return Err(MobError::Internal(format!(
"MobMachine HostReboundRecorded drift: input host='{}' epoch={}, effect host='{}' epoch={}",
host_id.as_str(),
next.epoch,
rebound_host.as_str(),
rebound_epoch
)));
}
let record = Self::host_authority_record(
host_id,
pubkey,
&endpoint,
next.epoch,
binding_generation,
&capabilities,
);
let persistence = crate::store::MobHostAuthorityPersistenceAuthority::from_transition(
&record,
&prepared.transition,
)?;
self.runtime_metadata
.put_mob_host_authority(&self.definition.id, &record, &persistence)
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Self::advance_host_binding_incarnation(
&mut self.host_binding_incarnations,
&mut self.orphan_release_reservations,
host_id,
)?;
// Rebind absorption is an orphan-reconciliation trigger (W-E);
// sweep failures never fail the rotation absorption.
if let Err(sweep_error) = self
.observe_host_status_as_authority(host_id, peer, next)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
host = %host_id.as_str(),
error = %sweep_error,
"post-rebind host member reconciliation sweep failed"
);
}
if self.durable_uncertainty_fail_stop {
return Err(MobError::Internal(format!(
"host '{}' rebind reconciliation entered durable fail-stop; suppressing later route and rotation effects",
host_id.as_str()
)));
}
Ok(())
}
async fn observe_supervisor_rotation_as_authority(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
peer: &TrustedPeerDescriptor,
command: &super::bridge_protocol::BridgeCommand,
timeout: std::time::Duration,
context: &'static str,
) -> Result<super::bridge_protocol::BridgeSupervisorRotationObservation, MobError> {
let value = self
.supervisor_bridge
.send_bridge_command_as_authority(authority, peer, command, timeout)
.await?;
super::bridge_protocol::decode_bridge_payload(command, value, context)
}
async fn handle_rotate_supervisor(
&mut self,
) -> Result<super::handle::SupervisorRotationReport, MobError> {
use super::bridge_protocol::{
BridgeCommand, BridgeSupervisorDelivery, BridgeSupervisorPayload,
BridgeSupervisorRotationObservation, BridgeSupervisorRotationObserve,
BridgeSupervisorRotationState, BridgeSupervisorRotationSubmit,
SupervisorRotationOperationId,
};
if self.state() == MobState::Destroyed {
return Err(self.invalid_transition_to(MobState::Destroyed));
}
let pending_binds = self.current_pending_host_bind_anchors().await?;
let pending_revokes = self.current_pending_host_revoke_anchors().await?;
require_host_authority_anchors_clear_for_supervisor_rotation(
&pending_binds,
&pending_revokes,
)?;
let remote_peers = {
let roster = self.roster.read().await;
roster
.list_all()
.filter_map(Self::runtime_binding_for_entry)
.map(|binding| {
Self::peer_only_spec_for_binding(&binding, "handle_rotate_supervisor")
})
.collect::<Result<Vec<_>, _>>()?
};
let bound_hosts = self.bound_host_rotation_targets()?;
for peer in remote_peers
.iter()
.chain(bound_hosts.iter().map(|(_, peer)| peer))
{
self.supervisor_bridge
.require_supported_rotation_endpoint(peer.address.transport())?;
}
let prepared_rotation_admission = self
.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::AdmitSupervisorRotation,
"handle_rotate_supervisor_admission",
)
.map_err(|error| MobError::MobMachineRejected {
context: "handle_rotate_supervisor_admission",
reason: error.to_string(),
})?;
let loaded = self
.load_supervisor_authority_snapshot()
.await?
.ok_or_else(|| {
MobError::Internal(format!(
"cannot rotate supervisor for mob '{}': missing supervisor runtime metadata",
self.definition.id
))
})?;
let current = loaded.durable;
let stable_current = current.without_pending_rotation();
let existing_pending = current.pending_rotation.clone();
if existing_pending.is_none() {
let affected = {
let state = self.dsl_authority.state();
state
.member_placement
.keys()
.filter(|identity| state.placed_carrier_binding_active_for_identity(identity))
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<Vec<_>>()
};
// A fresh RebindHost rotates the authority that member-side live
// responders authenticate. Drain and close under the old
// authority before publishing the rotation admission/pending
// record. An existing durable pending rotation is the retry
// anchor proving this barrier already ran; reusing the old route
// after a remote accepted rebind would be invalid.
self.quiesce_member_live_for_identities(
affected,
"supervisor rebind live-channel barrier",
)
.await?;
}
self.commit_prepared_dsl_transition(prepared_rotation_admission)?;
// Epoch is a durable authority fence. Refuse exhaustion before
// generating an operation/candidate, enumerating targets, writing the
// pending anchor, or sending any bridge command. Wrapping to zero would
// let authority older than the durable high-water look current again.
let next_epoch =
stable_current
.epoch
.checked_add(1)
.ok_or(MobError::SupervisorEpochExhausted {
current_epoch: stable_current.epoch,
})?;
let operation_id = existing_pending
.as_ref()
.and_then(|pending| pending.operation_id)
.unwrap_or_else(SupervisorRotationOperationId::new);
let mut next = existing_pending
.as_ref()
.map(|pending| pending.authority_record())
.unwrap_or_else(|| {
let mut generated = crate::store::SupervisorAuthorityRecord::generate(
super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
);
generated.epoch = next_epoch;
generated
});
if next.epoch != next_epoch || next.public_peer_id == stable_current.public_peer_id {
return Err(MobError::Internal(format!(
"durable pending supervisor authority is not the exact successor of peer={} epoch={}: pending peer={} epoch={}",
stable_current.public_peer_id,
stable_current.epoch,
next.public_peer_id,
next.epoch,
)));
}
// Supervisor rotation operations are a V4 protocol. A pre-operation
// pending record may retain a V2/V3 target; the CAS migration below
// upgrades that same key/epoch target before any operation delivery.
next.protocol_version = super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION;
next.pending_rotation = None;
// A carried legacy accepted id is a durable fact that the member is
// already fenced onto `next`. Until it has an operation receipt it is
// retried under `next`; operation-era accepted ids follow the same
// safe path and are re-observed on every activation attempt.
let mut accepted_peer_ids: BTreeSet<String> = existing_pending
.as_ref()
.map(|pending| pending.accepted_peer_ids.iter().cloned().collect())
.unwrap_or_default();
let mut active_peer_ids: BTreeSet<String> = remote_peers
.iter()
.map(|peer| peer.peer_id.to_string())
.collect();
for (_, host_peer) in &bound_hosts {
let host_peer_id = host_peer.peer_id.to_string();
active_peer_ids.insert(host_peer_id);
}
if existing_pending
.as_ref()
.is_some_and(|pending| pending.operation_id.is_none())
{
// V2/V3 records have accepted evidence but no exact operation or
// target map. Reconcile that legacy evidence to the current active
// roster before installing the first V4 operation id. The generated
// RecordSupervisorPendingRotation transition verifies this is
// exactly old-accepted intersect active; the shell cannot prune an
// active accepted peer or retain a retired/missing one.
accepted_peer_ids.retain(|peer_id| active_peer_ids.contains(peer_id));
}
let mut member_targets = existing_pending
.as_ref()
.map(|pending| pending.member_targets.clone())
.unwrap_or_default();
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
let missing_peer_id = match member_targets.entry(peer_id) {
std::collections::btree_map::Entry::Occupied(_) => None,
std::collections::btree_map::Entry::Vacant(entry) => Some(entry.into_key()),
};
if let Some(peer_id) = missing_peer_id {
// Do not retain the vacant-entry borrow across the async target
// projection. This map is actor-local, so the second entry is
// still vacant when the projection completes.
let target = self
.bridge_supervisor_payload_for_authority_and_recipient(&next, peer)
.await?
.supervisor;
member_targets.entry(peer_id).or_insert(target);
}
}
for (_, host_peer) in &bound_hosts {
let peer_id = host_peer.peer_id.to_string();
let missing_peer_id = match member_targets.entry(peer_id) {
std::collections::btree_map::Entry::Occupied(_) => None,
std::collections::btree_map::Entry::Vacant(entry) => Some(entry.into_key()),
};
if let Some(peer_id) = missing_peer_id {
let target = self
.bridge_supervisor_payload_for_authority_and_recipient(&next, host_peer)
.await?
.supervisor;
member_targets.entry(peer_id).or_insert(target);
}
}
for (peer_id, target) in &member_targets {
if target.peer_id != next.public_peer_id || target.pubkey != next.public_signing_key() {
return Err(MobError::Internal(format!(
"persisted supervisor rotation target for member '{peer_id}' does not match operation authority {}",
next.public_peer_id
)));
}
}
// The mob-side operation/target record is committed before any remote
// delivery. This is the stable retry anchor even when no peer has yet
// completed the member-owned operation.
let mut durable_write_expected = current.clone();
if existing_pending.as_ref().is_none_or(|pending| {
pending.operation_id.is_none()
|| pending.member_targets != member_targets
|| pending
.accepted_peer_ids
.iter()
.cloned()
.collect::<BTreeSet<_>>()
!= accepted_peer_ids
|| !pending
.protocol_version
.same_protocol_as(next.protocol_version)
}) {
let persisted = self
.persist_pending_supervisor_rotation(PendingSupervisorRotationPersistenceRequest {
current: &stable_current,
expected_durable: &durable_write_expected,
pending: &next,
operation_id,
accepted_peer_ids: &accepted_peer_ids,
active_peer_ids: &active_peer_ids,
member_targets: &member_targets,
})
.await?;
let Some(record) = persisted.persisted_record else {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: 0,
rollback_succeeded: false,
pending_authority_recorded: false,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} was not durably recorded before submission"
),
});
};
durable_write_expected = record;
}
// The generated initial-operation guard deliberately installs an
// empty acceptance set. Advance bound hosts into the exact operation
// in a second monotonic CAS before any RebindHost delivery: host
// epochs cannot roll back, so a rejection or lost reply must retain
// the same operation/target tuple for retry.
let mut host_write_ahead_changed = false;
for (_, host_peer) in &bound_hosts {
host_write_ahead_changed |= accepted_peer_ids.insert(host_peer.peer_id.to_string());
}
if host_write_ahead_changed {
let persisted = self
.persist_pending_supervisor_rotation(PendingSupervisorRotationPersistenceRequest {
current: &stable_current,
expected_durable: &durable_write_expected,
pending: &next,
operation_id,
accepted_peer_ids: &accepted_peer_ids,
active_peer_ids: &active_peer_ids,
member_targets: &member_targets,
})
.await?;
let Some(record) = persisted.persisted_record else {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: persisted.pending_authority_recorded,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} did not durably record bound-host write-ahead membership before rebind"
),
});
};
durable_write_expected = record;
}
self.rotate_supervisor_bridge_to(&stable_current).await?;
// Bound hosts rotate before member operations, so no member can
// advance beyond a host that still serves the previous supervisor.
// The exact V4 operation/target record above is the write-ahead
// authority for this host leg too; host acceptance itself is recorded
// in the durable host-authority row by `absorb_host_rebind_acceptance`.
// Host epochs are monotonic, so retry first signs with the stable
// authority and then retries the exact tuple as `next` when the host
// already committed but the prior reply was lost.
for (host_id, host_peer) in &bound_hosts {
let binding_generation = self.current_host_binding_generation(host_id)?;
let required_capabilities =
Self::required_host_capabilities(self.dsl_authority.state(), host_id);
let supervisor_spec = self
.supervisor_bridge
.supervisor_spec_for_authority_and_recipient(&next, host_peer)
.await?;
let command = super::bridge_protocol::BridgeCommand::RebindHost(
super::bridge_protocol::BridgeHostRebindPayload {
supervisor: supervisor_spec.into(),
epoch: next.epoch,
binding_generation,
protocol_version: super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
mob_id: self.definition.id.to_string(),
required_capabilities,
},
);
let first_rebind_result = self
.supervisor_bridge
.send_bridge_command_as_authority_classified(
&stable_current,
host_peer,
&command,
std::time::Duration::from_secs(5),
)
.await;
let first_send_ambiguous = matches!(
&first_rebind_result,
Err(super::supervisor_bridge::BridgeRequestFailure::AfterSend(_))
);
let retry_as_next = match &first_rebind_result {
Err(super::supervisor_bridge::BridgeRequestFailure::AfterSend(_)) => true,
Err(super::supervisor_bridge::BridgeRequestFailure::BeforeSend(_)) => false,
Ok(value) => Self::bridge_rejection_reply(
super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
value,
)
.and_then(|rejection| rejection.typed_cause())
.is_some_and(|cause| {
matches!(
cause,
super::bridge_protocol::BridgeRejectionCause::SenderMismatch
| super::bridge_protocol::BridgeRejectionCause::StaleSupervisor
)
}),
};
let rebind_result = if retry_as_next {
self.supervisor_bridge
.send_bridge_command_as_authority_classified(
&next,
host_peer,
&command,
std::time::Duration::from_secs(5),
)
.await
} else {
first_rebind_result
};
let rebind_error = match rebind_result {
Ok(value) => {
if let Some(rejection) = Self::bridge_rejection_reply(
super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
&value,
) {
let error = Self::bridge_rejection_error(rejection.clone());
if Self::rebind_rejection_certifies_no_remote_effect(&rejection) {
Some(error)
} else {
Some(self.quarantine_uncertain_host_rebind(host_id, next.epoch, error))
}
} else {
match super::bridge_protocol::decode_bridge_payload::<
super::bridge_protocol::BridgeHostReboundResponse,
>(&command, value, "rebind host")
{
Ok(rebound) => self
.absorb_host_rebind_acceptance(host_id, host_peer, &next, &rebound)
.await
.err(),
Err(decode_error) => Some(self.quarantine_uncertain_host_rebind(
host_id,
next.epoch,
decode_error,
)),
}
}
}
Err(super::supervisor_bridge::BridgeRequestFailure::BeforeSend(error)) => {
if first_send_ambiguous {
Some(self.quarantine_uncertain_host_rebind(host_id, next.epoch, error))
} else {
Some(error)
}
}
Err(super::supervisor_bridge::BridgeRequestFailure::AfterSend(error)) => {
Some(self.quarantine_uncertain_host_rebind(host_id, next.epoch, error))
}
};
if let Some(error) = rebind_error {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"failed to rotate bound host '{}': {error}",
host_id.as_str()
),
});
}
}
// Submission is genuine one-way delivery. A comms send failure means
// only "not yet observed"; it never rolls back or cancels the durable
// member operation, and retry reuses this exact operation id.
let mut submission_diagnostics = BTreeMap::new();
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
if accepted_peer_ids.contains(&peer_id) {
// Accepted evidence is observed first below. A terminal receipt
// must never be route-refreshed or resubmitted after restart.
continue;
}
let target = member_targets.get(&peer_id).cloned().ok_or_else(|| {
MobError::Internal(format!(
"supervisor rotation operation {operation_id} has no durable target for peer '{peer_id}'"
))
})?;
let delivery = BridgeSupervisorDelivery::SubmitSupervisorRotation(
BridgeSupervisorRotationSubmit {
operation_id,
target: target.clone(),
target_epoch: next.epoch,
protocol_version: next.protocol_version,
},
);
if let Err(error) = self.supervisor_bridge.trust_recipient(peer).await {
submission_diagnostics.insert(peer_id, error.to_string());
tracing::debug!(%operation_id, peer_id = %peer.peer_id, %error, "supervisor rotation delivery route is not yet available");
continue;
}
let send_result = self
.supervisor_bridge
.send_supervisor_delivery(peer, &delivery)
.await;
if let Err(error) = send_result {
submission_diagnostics.insert(peer_id, error.to_string());
tracing::debug!(%operation_id, peer_id = %peer.peer_id, %error, "one-way supervisor rotation submission remains pending");
}
}
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
let accepted_before_observation = accepted_peer_ids.contains(&peer_id);
let expected_target = member_targets.get(&peer_id).cloned().ok_or_else(|| {
MobError::Internal(format!(
"supervisor rotation operation {operation_id} has no durable target for peer '{peer_id}'"
))
})?;
let next_observe_command =
BridgeCommand::ObserveSupervisorRotation(BridgeSupervisorRotationObserve {
operation_id,
observer: expected_target.clone(),
observer_epoch: next.epoch,
protocol_version: next.protocol_version,
});
let retained_observer = self
.supervisor_bridge
.supervisor_spec_for_authority_and_recipient(&stable_current, peer)
.await?;
let retained_observe_command =
BridgeCommand::ObserveSupervisorRotation(BridgeSupervisorRotationObserve {
operation_id,
observer: retained_observer.into(),
observer_epoch: stable_current.epoch,
protocol_version: next.protocol_version,
});
let observation_window = if cfg!(test) {
std::time::Duration::from_secs(1)
} else {
std::time::Duration::from_secs(5)
};
let deadline = Instant::now() + observation_window;
let mut last_observation = submission_diagnostics.remove(&peer_id).map_or_else(
|| "operation not yet observable".to_string(),
|error| format!("one-way submission was not delivered: {error}"),
);
let mut legacy_adoption_submitted = false;
let completed = loop {
let now = Instant::now();
if now >= deadline {
break false;
}
// Reserve half of the remaining observation window for the
// retained-authority read. A rejected attempted authority can
// consume its whole request timeout without returning a typed
// rejection because it has no reply route.
let request_timeout = std::cmp::min(
deadline.saturating_duration_since(now) / 2,
std::time::Duration::from_millis(500),
);
// Completed rotations are observable under `next`, while a
// terminal rejection deliberately retains the previous/current
// supervisor. Try the attempted authority first, then fall back
// to that retained authority whenever the attempted path is not
// authoritative (NotFound, unroutable, or rejected). Without
// this cross-path read, a durable rejection looks forever
// pending because the rejected target is forbidden to inspect
// its receipt.
let next_observation = self
.observe_supervisor_rotation_as_authority(
&next,
peer,
&next_observe_command,
request_timeout,
"observe supervisor rotation as attempted authority",
)
.await;
let observation = match next_observation {
Ok(observation @ BridgeSupervisorRotationObservation::Found { .. }) => {
observation
}
Ok(BridgeSupervisorRotationObservation::NotFound { .. }) => {
let retained_timeout = std::cmp::min(
deadline.saturating_duration_since(Instant::now()),
std::time::Duration::from_millis(500),
);
match self
.observe_supervisor_rotation_as_authority(
&stable_current,
peer,
&retained_observe_command,
retained_timeout,
"observe supervisor rotation as retained authority",
)
.await
{
Ok(observation) => observation,
Err(error) => {
last_observation = format!(
"{last_observation}; attempted authority returned not-found; retained-authority observation failed: {error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
}
}
Ok(observation) => observation,
Err(
next_error @ MobError::BridgeCommandRejected {
cause:
super::bridge_protocol::BridgeRejectionCause::SenderMismatch
| super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
..
},
) => {
let retained_timeout = std::cmp::min(
deadline.saturating_duration_since(Instant::now()),
std::time::Duration::from_millis(500),
);
match self
.observe_supervisor_rotation_as_authority(
&stable_current,
peer,
&retained_observe_command,
retained_timeout,
"observe supervisor rotation as retained authority",
)
.await
{
Ok(observation) => observation,
Err(retained_error) => {
last_observation = format!(
"{last_observation}; attempted-authority observation failed: {next_error}; retained-authority observation failed: {retained_error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
}
}
Err(next_error) => {
// Observation is read-only and never mints remote
// trust. If `next` committed, the member has revoked
// `stable_current` and this retained probe simply
// remains unroutable. If `next` was rejected, the
// retained route is the only authority able to read
// the durable rejection receipt.
let retained_timeout = std::cmp::min(
deadline.saturating_duration_since(Instant::now()),
std::time::Duration::from_millis(500),
);
match self
.observe_supervisor_rotation_as_authority(
&stable_current,
peer,
&retained_observe_command,
retained_timeout,
"observe supervisor rotation as retained authority after attempted-authority transport failure",
)
.await
{
Ok(observation) => observation,
Err(retained_error) => {
last_observation = format!(
"{last_observation}; attempted-authority observation failed: {next_error}; retained-authority observation failed: {retained_error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
}
}
};
match observation {
BridgeSupervisorRotationObservation::NotFound {
operation_id: observed_operation_id,
} => {
if observed_operation_id != operation_id {
return Err(MobError::Internal(format!(
"supervisor rotation observation for peer '{peer_id}' returned mismatched not-found operation {observed_operation_id} (expected {operation_id})"
)));
}
if accepted_before_observation && !legacy_adoption_submitted {
// Only NotFound distinguishes a legacy accepted
// member from an operation-era terminal receipt.
// Repair mutable route metadata under the exact next
// key/epoch and require a decoded ACK before adopting
// the operation id. A timeout leaves it pending and
// never submits a target that could be rejected
// durably because refresh did not happen.
let route_refresh =
BridgeCommand::AuthorizeSupervisor(BridgeSupervisorPayload {
supervisor: expected_target.clone(),
epoch: next.epoch,
protocol_version: next.protocol_version,
});
// A lost refresh reply is ambiguous. Retry the
// exact idempotent command here, before returning
// to observation, so later reads cannot consume
// the budget reserved for that retry. The first
// attempt receives at most half of the remaining
// convergence window; the retry receives what is
// left, without extending the deadline.
let mut refresh_value = None;
for attempt in 0..2 {
let remaining = deadline.saturating_duration_since(Instant::now());
let attempt_budget = if attempt == 0 {
remaining / 2
} else {
remaining
};
let refresh_timeout = std::cmp::min(
attempt_budget,
std::time::Duration::from_millis(500),
);
// Never dispatch a request whose diagnostic
// timeout rounds down to 0ms at the end of the
// convergence window.
if refresh_timeout < std::time::Duration::from_millis(1) {
break;
}
match self
.supervisor_bridge
.send_bridge_command_as_authority(
&next,
peer,
&route_refresh,
refresh_timeout,
)
.await
{
Ok(value) => {
refresh_value = Some(value);
break;
}
Err(error) => {
last_observation = format!(
"legacy next-authority route refresh remains unconfirmed: {error}"
);
}
}
}
let Some(refresh_value) = refresh_value else {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
};
if let Err(error) = super::bridge_protocol::decode_bridge_ack(
&route_refresh,
refresh_value,
"legacy supervisor route refresh",
) {
last_observation = format!(
"legacy next-authority route refresh was not acknowledged: {error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
let delivery = BridgeSupervisorDelivery::SubmitSupervisorRotation(
BridgeSupervisorRotationSubmit {
operation_id,
target: expected_target.clone(),
target_epoch: next.epoch,
protocol_version: next.protocol_version,
},
);
match self
.supervisor_bridge
.send_supervisor_delivery_as_authority(&next, peer, &delivery)
.await
{
Ok(()) => {
legacy_adoption_submitted = true;
last_observation =
"legacy operation adoption submitted".to_string();
}
Err(error) => {
last_observation = format!(
"legacy operation adoption remains pending: {error}"
);
}
}
} else {
last_observation = "operation not found yet".to_string();
}
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Pending { operation, phase },
} => {
if operation.operation_id != operation_id
|| operation.target.target_epoch != next.epoch
|| operation.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched pending receipt for peer '{peer_id}'"
)));
}
last_observation = format!("operation pending in phase {phase:?}");
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Completed { receipt },
} => {
if receipt.operation_id != operation_id
|| receipt.target.target_epoch != next.epoch
|| receipt.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched terminal receipt for peer '{peer_id}'"
)));
}
break true;
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Rejected { receipt },
} => {
if receipt.operation.operation_id != operation_id
|| receipt.operation.target.target_epoch != next.epoch
|| receipt.operation.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched rejection receipt for peer '{peer_id}'"
)));
}
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} was rejected by peer '{peer_id}' ({:?}): {}",
receipt.cause, receipt.reason
),
});
}
BridgeSupervisorRotationObservation::Found { state } => {
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned unsupported state {state:?} for peer '{peer_id}'"
)));
}
_ => {
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned an unsupported observation for peer '{peer_id}'"
)));
}
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
};
if !completed {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} remains pending for peer '{peer_id}': {last_observation}"
),
});
}
if accepted_peer_ids.insert(peer_id) {
let persisted = self
.persist_pending_supervisor_rotation(
PendingSupervisorRotationPersistenceRequest {
current: &stable_current,
expected_durable: &durable_write_expected,
pending: &next,
operation_id,
accepted_peer_ids: &accepted_peer_ids,
active_peer_ids: &active_peer_ids,
member_targets: &member_targets,
},
)
.await?;
let Some(record) = persisted.persisted_record else {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: persisted.pending_authority_recorded,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} completed remotely but its observation checkpoint was not persisted"
),
});
};
durable_write_expected = record;
}
}
let public_peer_id = next.public_peer_id.clone();
match self
.activate_supervisor_authority(
&stable_current,
&durable_write_expected,
operation_id,
&next,
)
.await
{
Ok(()) => Ok(super::handle::SupervisorRotationReport {
previous_epoch: stable_current.epoch,
current_epoch: next.epoch,
public_peer_id,
}),
Err(error) => Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: error.rollback_succeeded,
pending_authority_recorded: error.pending_authority_recorded,
rollback_error: error.rollback_error,
reason: format!(
"supervisor rotation operation {operation_id} completed remotely but local authority activation failed: {}",
error.error
),
}),
}
}
async fn rotate_supervisor_bridge_to(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<(), MobError> {
if !self.supervisor_authority_record_is_machine_authorized(authority) {
return Err(MobError::Internal(format!(
"refusing to rotate supervisor bridge to peer={} epoch={} without generated MobMachine authority",
authority.public_peer_id, authority.epoch
)));
}
let active = self.supervisor_bridge.authority().await;
if active.public_peer_id != authority.public_peer_id
|| active.epoch != authority.epoch
|| active.protocol_version != authority.protocol_version
{
let bridge_authority = self.supervisor_bridge_authority_for_record(authority)?;
let prepared = self
.supervisor_bridge
.prepare_rotation(authority.clone(), &bridge_authority)
.await?;
self.supervisor_bridge
.commit_prepared_rotation(prepared)
.await?;
}
Ok(())
}
fn commit_durable_supervisor_authority_transition(
&mut self,
prepared: PreparedDslTransition,
durable: &crate::store::SupervisorAuthorityRecord,
context: &'static str,
) -> Result<(), MobError> {
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
// The store is already authoritative at `durable`. Continuing
// with an older in-memory MobMachine would let a retry mint a new
// epoch or sign with a stale bridge authority. Only cold recovery
// may rebuild the machine from the exact durable record.
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"{context}: durable supervisor authority peer={} epoch={} could not publish its prepared MobMachine transition; actor is fail-stopping for cold recovery: {error}",
durable.public_peer_id, durable.epoch,
)));
}
Ok(())
}
async fn persist_pending_supervisor_rotation(
&mut self,
request: PendingSupervisorRotationPersistenceRequest<'_>,
) -> Result<SupervisorPendingRotationPersistence, MobError> {
let PendingSupervisorRotationPersistenceRequest {
current,
expected_durable,
pending,
operation_id,
accepted_peer_ids,
active_peer_ids,
member_targets,
} = request;
let mut record = current.without_pending_rotation();
let pending_rotation = crate::store::SupervisorPendingRotationRecord::from_authority(
pending,
operation_id,
accepted_peer_ids.iter().cloned().collect(),
member_targets.clone(),
);
// The operation and target are durable before the first remote send,
// even while the completed-peer set is empty. This record is the
// caller-side retry anchor; bridge delivery/response timing is not.
let record_pending_input =
record.dsl_record_pending_rotation_input(&pending_rotation, active_peer_ids);
record.pending_rotation = Some(pending_rotation);
let prepared = self.prepare_supervisor_authority_persistence(
record_pending_input,
&record,
"persist_pending_supervisor_rotation",
)?;
let cas_result = self
.runtime_metadata
.compare_and_put_supervisor_authority(
&self.definition.id,
expected_durable,
&record,
&prepared.authority,
)
.await;
let original_error = match cas_result {
Ok(true) => {
self.commit_durable_supervisor_authority_transition(
prepared.transition,
&record,
"persist_pending_supervisor_rotation",
)?;
return Ok(SupervisorPendingRotationPersistence {
pending_authority_recorded: record.pending_rotation.is_some(),
persisted_record: Some(record),
});
}
Ok(false) => MobError::from(crate::store::MobStoreError::CasConflict(format!(
"supervisor authority changed while persisting pending rotation for mob '{}'",
self.definition.id
))),
Err(error) => MobError::from(error),
};
let original_detail = original_error.to_string();
match self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await
{
Ok(Some(durable)) if durable == record => {
// The CAS committed and only its completion was lost. Publish
// the exact prepared machine transition once; this is success,
// not a fresh rotation attempt.
self.commit_durable_supervisor_authority_transition(
prepared.transition,
&record,
"persist_pending_supervisor_rotation reread convergence",
)?;
Ok(SupervisorPendingRotationPersistence {
pending_authority_recorded: record.pending_rotation.is_some(),
persisted_record: Some(record),
})
}
Ok(Some(durable)) if durable == *expected_durable => {
// Exact expected proves the attempted write had no effect.
Err(original_error)
}
Ok(Some(durable)) => {
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"persist_pending_supervisor_rotation CAS failed ({original_detail}) and exact reread found divergent supervisor authority peer={} epoch={}; actor is fail-stopping for cold recovery",
durable.public_peer_id, durable.epoch,
)))
}
Ok(None) => {
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"persist_pending_supervisor_rotation CAS failed ({original_detail}) and exact reread found the supervisor authority missing; actor is fail-stopping for cold recovery"
)))
}
Err(read_error) => {
self.durable_uncertainty_fail_stop = true;
Err(MobError::Internal(format!(
"persist_pending_supervisor_rotation CAS failed ({original_detail}) and exact reread remained unreadable; actor is fail-stopping for cold recovery: {read_error}"
)))
}
}
}
async fn activate_supervisor_authority(
&mut self,
current: &crate::store::SupervisorAuthorityRecord,
expected_durable: &crate::store::SupervisorAuthorityRecord,
operation_id: super::bridge_protocol::SupervisorRotationOperationId,
next: &crate::store::SupervisorAuthorityRecord,
) -> Result<(), SupervisorAuthorityActivationError> {
let prepared_commit = self
.prepare_supervisor_authority_persistence(
current.dsl_commit_rotation_input(&operation_id.to_string(), next),
next,
"activate_supervisor_authority",
)
.map_err(|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let prepared_bridge_authority =
crate::store::SupervisorAuthorityBridgeAuthority::from_persistence_authority(
next,
&prepared_commit.authority,
)
.map_err(|error| SupervisorAuthorityActivationError {
error: MobError::from(error),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let prepared_bridge_rotation = self
.supervisor_bridge
.prepare_rotation(next.clone(), &prepared_bridge_authority)
.await
.map_err(|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let previous_private_trust_removal_key = current.public_peer_id.clone();
let session_member_refs = {
let roster = self.roster.read().await;
roster
.list_all()
.filter_map(|entry| match &entry.member_ref {
MemberRef::Session { .. } => Some(entry.member_ref.clone()),
MemberRef::BackendPeer { .. } => None,
})
.collect::<Vec<_>>()
};
for member_ref in session_member_refs {
if let (Some(session_id), Some(comms)) = (
member_ref.bridge_session_id().cloned(),
self.provisioner_comms(&member_ref).await,
) {
let supervisor_spec =
Self::supervisor_spec_for_authority(&self.definition.id, next).map_err(
|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
},
)?;
match self
.install_supervisor_private_trust_for_session_authority(
&session_id,
&comms,
next,
supervisor_spec,
Some(current),
Some(&previous_private_trust_removal_key),
)
.await
{
Ok(_) => {}
Err(SupervisorPrivateTrustInstallError {
error,
new_trust_cleanup_failed,
}) => {
return Err(Self::supervisor_activation_error(
error,
new_trust_cleanup_failed,
true,
));
}
}
}
}
// The durable current+pending record is the remote-complete/local-
// activation-pending retry anchor. Keep it intact across the only
// fallible bridge-activation step (notably a fixed-port listener
// rebuild). Once the live bridge is ready under `next`, the CAS below
// atomically publishes the final durable authority and removes that
// anchor. A rebuild failure therefore retries this exact operation;
// it can never surface Incomplete after deleting the operation id and
// exact member targets.
if let Err(error) = self
.supervisor_bridge
.commit_prepared_rotation(prepared_bridge_rotation)
.await
{
return Err(Self::supervisor_activation_error(error, false, true));
}
let cas_result = self
.runtime_metadata
.compare_and_put_supervisor_authority(
&self.definition.id,
expected_durable,
next,
&prepared_commit.authority,
)
.await;
let original_error = match cas_result {
Ok(true) => {
return self
.commit_durable_supervisor_authority_transition(
prepared_commit.transition,
next,
"activate_supervisor_authority",
)
.map_err(|error| Self::supervisor_activation_error(error, false, false));
}
Ok(false) => MobError::from(crate::store::MobStoreError::CasConflict(format!(
"supervisor authority changed before final commit for mob '{}'",
self.definition.id
))),
Err(error) => MobError::from(error),
};
let original_detail = original_error.to_string();
match self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await
{
Ok(Some(durable)) if durable == *next => self
.commit_durable_supervisor_authority_transition(
prepared_commit.transition,
next,
"activate_supervisor_authority reread convergence",
)
.map_err(|error| Self::supervisor_activation_error(error, false, false)),
Ok(Some(durable)) if durable == *expected_durable => {
Err(Self::supervisor_activation_error(
original_error,
false,
expected_durable.pending_rotation.is_some(),
))
}
Ok(Some(durable)) => {
self.durable_uncertainty_fail_stop = true;
Err(Self::supervisor_activation_error(
MobError::Internal(format!(
"final supervisor authority CAS failed ({original_detail}) and exact reread found divergent authority peer={} epoch={}; actor is fail-stopping for cold recovery",
durable.public_peer_id, durable.epoch,
)),
false,
durable.pending_rotation.is_some(),
))
}
Ok(None) => {
self.durable_uncertainty_fail_stop = true;
Err(Self::supervisor_activation_error(
MobError::Internal(format!(
"final supervisor authority CAS failed ({original_detail}) and exact reread found the authority missing; actor is fail-stopping for cold recovery"
)),
false,
false,
))
}
Err(read_error) => {
self.durable_uncertainty_fail_stop = true;
Err(Self::supervisor_activation_error(
MobError::Internal(format!(
"final supervisor authority CAS failed ({original_detail}) and exact reread remained unreadable; actor is fail-stopping for cold recovery: {read_error}"
)),
false,
true,
))
}
}
}
fn supervisor_activation_error(
error: MobError,
new_trust_cleanup_failed: bool,
pending_authority_recorded: bool,
) -> SupervisorAuthorityActivationError {
// Once member-owned operations complete, activation recovery must not
// reconstruct old-supervisor trust on member runtimes. Leave the local
// activation retryable in place and only retain the typed cleanup fact.
SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded,
rollback_error: new_trust_cleanup_failed
.then(|| "supervisor private trust cleanup failed".to_string()),
}
}
/// Cancel checkpointers and transition to Stopped. Used by `handle_reset`
/// error paths after destructive steps have already been taken.
async fn fail_reset_to_stopped(&mut self) {
self.provisioner.cancel_all_checkpointers().await;
let result = self
.drive_placed_completion_lifecycle_cleanup(
None,
true,
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop),
)
.await;
let result = match result {
Ok(()) => self.commit_stopped_lifecycle_after_cleanup().await,
Err(error) => Err(error),
};
if let Err(e) = result {
tracing::warn!(error = %e, "authority rejected Stop in fail_reset_to_stopped");
}
}
async fn handle_reset(&mut self, prior_state: MobState) -> Result<(), MobError> {
self.require_host_authority_anchors_clear_for_action("reset mob")
.await?;
self.ensure_pending_spawn_alignment("handle_reset preflight")?;
self.ensure_flow_tracker_alignment("handle_reset preflight")
.await?;
let was_stopped = prior_state == MobState::Stopped;
self.cancel_all_flow_tasks().await?;
// Rearm checkpointers temporarily so retire can checkpoint if needed.
if was_stopped {
self.provisioner.rearm_all_checkpointers().await;
}
// --- Destructive phase: retire members and stop MCP servers. ---
// After this point the mob is effectively stopped regardless of what
// the prior state field says.
if let Err(error) = self.retire_all_members("reset").await {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
// ResetToRunning owns active/pending run counters and coordinator
// binding. The durable epoch marker is appended only after the
// generated authority has accepted the prepared transition.
let prepared = match self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Reset, "reset_to_running")
.map_err(|error| {
tracing::debug!(
context = "reset_to_running",
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(MobState::Running)
}) {
Ok(prepared) => prepared,
Err(error) => {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
};
if let Err(error) = Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Reset,
"reset_to_running",
) {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
// --- Event rewrite phase: append the new epoch marker. ---
// Append-only epoch model: projections clear on MobReset; the original
// MobCreated definition remains the durable resume authority for this
// mob. No clear() needed -- crash-safe.
let events = self.events.clone();
let mob_id = self.definition.id.clone();
if let Err(error) = self
.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobReset,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await
{
self.fail_reset_to_stopped().await;
return Err(error);
}
// Clear in-memory projections.
self.edge_locks.clear().await;
self.retired_event_index.write().await.clear();
self.retirement_started_event_index.write().await.clear();
self.preserved_respawn_topology_event_index
.write()
.await
.clear();
self.ensure_pending_spawn_alignment("handle_reset completion")?;
self.ensure_flow_tracker_alignment("handle_reset completion")
.await?;
Ok(())
}
/// Retire all roster members in parallel (sliding window of
/// `MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS`). handle_retire only returns Err on
/// event-append failures (pre-cleanup); cleanup errors are best-effort.
/// If any member fails to retire the operation is aborted — the caller
/// can retry since already-retired members are idempotent.
async fn retire_all_members(&mut self, context: &str) -> Result<(), MobError> {
let prepared_retire_all = self.prepare_command_admission(
mob_dsl::MobMachineInput::RetireAll,
MobState::Running,
context,
)?;
self.commit_prepared_dsl_input(prepared_retire_all)?;
self.ensure_pending_spawn_alignment("retire_all_members preflight")?;
let ids = {
let roster = self.roster.read().await;
roster
.list_all()
.map(|entry| entry.agent_identity.clone())
.collect::<Vec<_>>()
};
let mut retire_failures: Vec<String> = Vec::new();
let mut retirement_start_failures: Vec<String> = Vec::new();
for id in ids {
let result = self.retire_one(id).await;
if let Err((id, error)) = result {
let retained_generation = {
let roster = self.roster.read().await;
roster.get(&id).map(|entry| entry.generation)
};
let Some(generation) = retained_generation else {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %id,
error = %error,
"{context}: retire reported cleanup failure after removing member; continuing"
);
continue;
};
match self.retirement_started_event_exists(&id, generation).await {
Ok(true) => {}
Ok(false) => retirement_start_failures.push(format!(
"{id}: retirement failed before MemberRetirementStarted became durable"
)),
Err(start_check_error) => retirement_start_failures.push(format!(
"{id}: could not prove durable MemberRetirementStarted: {start_check_error}"
)),
}
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %id,
error = %error,
"{context}: retire failed for member"
);
retire_failures.push(format!("{id}: {error}"));
}
}
// Global pending-spawn cancellation can affect identities whose
// member retirement has not yet acquired a durable retry anchor. Do
// not expose that cancellation unless every retained failed member is
// proven to have persisted MemberRetirementStarted.
if !retirement_start_failures.is_empty() {
return Err(MobError::Internal(format!(
"{context} aborted before pending-spawn drain: {} member retirement start(s) were not durable: {}",
retirement_start_failures.len(),
retirement_start_failures.join("; ")
)));
}
// Existing member incarnations have each persisted their own
// MemberRetirementStarted carrier before global pending-spawn
// cancellation becomes externally visible.
let pending_reason =
format!("{context}: draining pending spawns after member retirement starts");
self.fail_all_pending_spawns(&pending_reason).await?;
self.ensure_pending_spawn_alignment("retire_all_members after pending drain")?;
if !retire_failures.is_empty() {
return Err(MobError::Internal(format!(
"{context} aborted: {} member(s) could not be retired: {}",
retire_failures.len(),
retire_failures.join("; ")
)));
}
self.ensure_pending_spawn_alignment("retire_all_members completion")?;
Ok(())
}
async fn retire_one(&mut self, id: AgentIdentity) -> Result<(), (AgentIdentity, MobError)> {
self.handle_retire_inner(&id, true, false, false, false, None)
.await
.map_err(|error| (id, error))
}
async fn reserve_placed_completion_before_submit_work(
&mut self,
entry: &RosterEntry,
interaction_id: meerkat_core::interaction::InteractionId,
) -> Result<crate::event::PlacedCompletionObligationEvent, MobError> {
let canonical_input_id = interaction_id.0.to_string();
let parsed = uuid::Uuid::parse_str(&canonical_input_id).map_err(|_| {
MobError::InvalidPlacedInteractionId {
interaction_id: canonical_input_id.clone(),
}
})?;
if parsed.is_nil() || parsed.to_string() != canonical_input_id {
return Err(MobError::InvalidPlacedInteractionId {
interaction_id: canonical_input_id,
});
}
let incarnation = self.placed_member_incarnation(entry)?;
// The append-only Record is also the permanent used-id ledger for
// every host-tracked placed delivery family. ACK/close removes live
// custody, never history; the host dedup/tombstone namespace is
// shared by ordinary completion, flow, and kickoff rows.
let used_key = PlacedTrackedInputKey {
agent_identity: entry.agent_identity.to_string(),
host_id: incarnation.host_id.clone(),
generation: entry.generation.get(),
fence_token: entry.fence_token.get(),
input_id: canonical_input_id.clone(),
};
let state = self.dsl_authority.state();
let dispatch_sequence = state
.placed_completion_dispatch_sequence
.checked_add(1)
.ok_or_else(|| {
MobError::Internal("placed completion dispatch sequence exhausted".to_string())
})?;
let obligation_event = crate::event::PlacedCompletionObligationEvent {
agent_identity: entry.agent_identity.clone(),
host_id: incarnation.host_id,
host_binding_generation: incarnation.binding_generation,
member_session_id: incarnation.member_session_id,
generation: entry.generation,
fence_token: entry.fence_token,
dispatch_sequence,
input_id: canonical_input_id,
};
let obligation =
super::placed_completion_reconciler::obligation_from_event(&obligation_event)?;
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordPlacedCompletionObligation { obligation },
"record_placed_completion_before_submit_work",
)?;
self.placed_completion_durable_index
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.reserve_fresh_tracked_input(
used_key,
PlacedTrackedInputOwner::Completion(obligation_event.clone()),
)?;
self.ensure_placed_completion_carrier(MobEventKind::PlacedCompletionObligationRecorded {
obligation: obligation_event.clone(),
})
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"placed completion Record was durable but machine commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(obligation_event)
}
/// Unified work-lane entry.
///
/// The `MobMachine` DSL owns work-origin legality: whether this runtime is
/// live, which origin is admissible (External vs Internal), and whether
/// external callers may address this runtime. The shell no longer
/// re-decides any of those facts — it forwards the caller-declared
/// [`WorkOrigin`] to the DSL and lets the guards accept or reject.
///
/// Shell-owned pre-work (shell is the only place that can do these):
/// * Auto-spawn when the target member is absent and MobMachine accepts
/// typed spawn-policy resolution feedback. Only meaningful for
/// externally-originated work — internal origins never auto-spawn.
/// * Post-authorization dispatch — reading the machine's
/// generated `RequestRuntimeIngress` / `RequestPeerRuntimeIngress`
/// effect and materializing it as actual runtime ingress (event
/// injector or `StartTurnRequest`). The effect payload is the
/// authority token for the dispatch shape; the shell verifies it
/// before touching session or peer transport.
async fn handle_submit_work(
&mut self,
payload: Box<super::state::SubmitWorkPayload>,
) -> Result<SubmitWorkDispatchCompletion, MobError> {
let super::state::SubmitWorkPayload {
runtime_id,
fence_token,
work_ref,
content,
origin,
system_prompt,
injected_context,
mut interaction_id,
objective_id,
handling_mode,
external_delivery_identity,
turn_metadata,
event_tx,
completion_tx,
llm_identity_applied_tx,
ack_mode,
} = *payload;
tracing::debug!(
agent_identity = %runtime_id.identity,
runtime_id = %runtime_id,
work_ref = %work_ref,
origin = ?origin,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
"handle_submit_work started"
);
self.ensure_pending_spawn_alignment("handle_submit_work preflight")?;
let agent_identity = runtime_id.identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let declared_dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&runtime_id);
let declared_dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
// SubmitWork admission belongs to MobMachine even when the shell may
// need to auto-spawn an absent external target. Probe the declared
// command before policy resolution so stopped/completed mobs reject
// without staging spawn side effects.
let declared_submit_work_admitted = match self.probe_command_admission(
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: declared_dsl_runtime_id.clone(),
fence_token: declared_dsl_fence_token,
work_id: mob_dsl::WorkId::from_work_ref(&work_ref),
origin: mob_dsl::WorkOrigin::from(origin),
},
MobState::Running,
"submit_work_command_admission",
) {
Ok(()) => true,
Err(error) => {
if self.state() != MobState::Running {
return Err(error);
}
false
}
};
// A non-Running lifecycle is rejected above by the canonical
// SubmitWork transition. Only after that generated admission probe
// may the shell report an in-progress Running-phase quiesce; doing
// this first would shadow the machine's Stopped/Completed guard.
if lifecycle_origin_fenced(self.dsl_authority.state()) {
return Err(MobError::LifecycleOperationPending {
intent: self
.dsl_authority
.state()
.placed_completion_lifecycle_intent
.map(|intent| format!("{intent:?}").to_ascii_lowercase())
.unwrap_or_else(|| "unknown".to_string()),
});
}
// Auto-spawn is an external-only policy seam that runs when the target
// member is absent and MobMachine accepts typed spawn-policy feedback.
// For existing members, the caller's runtime/fence pair is forwarded
// into MobMachine so generated authority owns stale-incarnation
// rejection.
let initial_entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
let initial_entry_present = initial_entry.is_some();
let entry = match initial_entry {
Some(e) => {
self.ensure_member_not_broken(&e.agent_identity).await?;
e
}
None => {
if matches!(origin, WorkOrigin::Internal) {
let current_state = self.state();
return Err(Self::resolve_submit_work_projection_missing_or_rejection(
&mut self.dsl_authority,
declared_submit_work_admitted,
&dsl_identity,
&declared_dsl_runtime_id,
declared_dsl_fence_token,
&runtime_id,
origin,
&agent_identity,
current_state,
));
}
let identity = AgentIdentity::from(agent_identity.as_str());
if let Some(spec) = self.resolve_spawn_policy_via_machine(&identity).await? {
Box::pin(self.spawn_from_policy_inline(&identity, spec, &work_ref, origin))
.await?;
{
let roster = self.roster.read().await;
roster.get(&identity).cloned()
}
.ok_or_else(|| {
MobError::Internal(format!(
"auto-spawned member '{identity}' missing from roster after completion"
))
})?
} else {
let current_state = self.state();
return Err(Self::resolve_submit_work_projection_missing_or_rejection(
&mut self.dsl_authority,
declared_submit_work_admitted,
&dsl_identity,
&declared_dsl_runtime_id,
declared_dsl_fence_token,
&runtime_id,
origin,
&agent_identity,
current_state,
));
}
}
};
let admission_runtime_id = if initial_entry_present {
runtime_id.clone()
} else {
entry.agent_runtime_id.clone()
};
let admission_fence_token = if initial_entry_present {
fence_token
} else {
entry.fence_token
};
if let Some(identity) = &external_delivery_identity {
identity.validate()?;
let correlation = uuid::Uuid::parse_str(&identity.correlation_id).map_err(|_| {
MobError::Internal(
"external-delivery correlation identity is not a UUID".to_string(),
)
})?;
let correlation = meerkat_core::interaction::InteractionId(correlation);
if interaction_id.is_some_and(|existing| existing != correlation) {
return Err(MobError::Internal(
"external-delivery correlation conflicts with supplied transcript identity"
.to_string(),
));
}
interaction_id = Some(correlation);
}
// Per-turn LLM identity fields require a queued executor boundary.
// Reject Steer explicitly before the more general carrier and host
// capability checks below.
let requests_llm_reconfigure = turn_metadata.as_ref().is_some_and(|metadata| {
metadata.model.is_some()
|| metadata.provider.is_some()
|| metadata.self_hosted_server_id.is_some()
|| metadata.provider_params.is_some()
|| metadata.auth_binding.is_some()
});
if requests_llm_reconfigure && handling_mode == meerkat_core::types::HandlingMode::Steer {
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "per-turn LLM identity overrides require a queued executor turn; running Steer has no executor application boundary"
.to_string(),
});
}
// Merge actor-owned WorkSpec causality into the caller's typed carrier
// before admission. Any conflict or carrier that cannot be realized
// on this exact placement/backend is rejected before MobMachine emits
// an ingress effect.
let turn_metadata = submit_work_turn_metadata(turn_metadata, interaction_id, objective_id)?;
let remotely_hosted =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity);
if remotely_hosted
&& !injected_context.is_empty()
&& turn_metadata
.as_ref()
.is_some_and(|metadata| metadata.transient_turn_context.is_some())
{
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "remote member delivery cannot combine injected_context with transient_turn_context"
.to_string(),
});
}
validate_member_turn_carriers(
&entry,
remotely_hosted,
handling_mode,
external_delivery_identity.is_some(),
turn_metadata.as_ref(),
event_tx.is_some(),
completion_tx.is_some(),
)?;
#[cfg(feature = "runtime-adapter")]
let local_external_identity_supported = self.runtime_adapter.is_some();
#[cfg(not(feature = "runtime-adapter"))]
let local_external_identity_supported = false;
if external_delivery_identity.is_some()
&& !remotely_hosted
&& !local_external_identity_supported
{
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "stable external input identity requires a runtime-backed member"
.to_string(),
});
}
// Only representable carriers reach the concrete executor capability
// check. Actor-global adapter presence is not evidence that a direct,
// peer-only, or remotely hosted backend can realize an override, but
// a missing host capability must not shadow a path-level
// UnsupportedForMode rejection.
if requests_llm_reconfigure
&& !self
.provisioner
.supports_member_turn_llm_reconfigure(&entry.member_ref)
{
return Err(MobError::MissingMemberCapability {
member_id: AgentIdentity::from(runtime_id.identity.as_str()),
capability: crate::error::MobMemberCapability::SessionLlmReconfigure,
context: "member turn LLM identity override",
});
}
let interaction_id = turn_metadata
.as_ref()
.and_then(|metadata| metadata.transcript_identity.interaction_id);
let objective_id = turn_metadata
.as_ref()
.and_then(|metadata| metadata.transcript_identity.objective_id);
// Injected-context deliverability is validated BEFORE the MobMachine
// SubmitWork input is applied: refusing realization after admission
// would abandon the machine-emitted ingress effect outside the
// machine's typed rejection vocabulary. Steer dispatch realizes as
// live system-context appends and the autonomous inbox path flows
// through comms plain events (classified external event ->
// SystemNotice transcript append) — neither carries a user-channel
// transcript boundary for typed injected-context messages to precede,
// so both fail closed here: one canonical terminal path for the same
// semantic condition, and no admitted effect is ever left
// unrealized over it.
if !injected_context.is_empty() {
if handling_mode == meerkat_core::types::HandlingMode::Steer {
return Err(MobError::InjectedContextUndeliverable {
member_id: AgentIdentity::from(agent_identity.as_str()),
reason: "steer dispatch carries no transcript boundary for injected context",
});
}
if entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
return Err(MobError::InjectedContextUndeliverable {
member_id: AgentIdentity::from(agent_identity.as_str()),
reason: "autonomous inbox delivery carries no user-channel work boundary",
});
}
}
if system_prompt.is_some()
&& !remotely_hosted
&& entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost
{
return Err(MobError::UnsupportedForMode {
mode: entry.runtime_mode,
reason: "autonomous inbox delivery carries no admitted turn boundary for ordinary System content"
.to_string(),
});
}
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity)
&& interaction_id.is_some_and(|interaction_id| interaction_id.0.is_nil())
{
return Err(MobError::InvalidPlacedInteractionId {
interaction_id: uuid::Uuid::nil().to_string(),
});
}
// Project the caller's identifiers into DSL bridging types. Existing
// members use the caller-supplied runtime/fence so MobMachine can
// reject stale generations; auto-spawned members use the generated
// runtime/fence created by the spawn authority path.
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&admission_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(admission_fence_token);
let dsl_work_id = mob_dsl::WorkId::from_work_ref(&work_ref);
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let submit_work_input = mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: dsl_work_id.clone(),
origin: dsl_origin,
};
// Completion-specific admission is a prerequisite of authoritative
// work ingress. Probe the exact SubmitWork first, then persist and
// commit Record while the actor owns the no-interleaving window. The
// Record transition changes no SubmitWork guard, so the subsequent
// real input cannot reject for history/capability/quota after an
// ingress effect has been emitted.
if self
.probe_command_admission(
submit_work_input.clone(),
MobState::Running,
"submit_work_exact_pre_record_probe",
)
.is_err()
{
let current_state = self.state();
return Err(Self::resolve_submit_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&admission_runtime_id,
origin,
&agent_identity,
current_state,
));
}
let mut effective_interaction_id = interaction_id;
let mut placed_completion_obligation = None;
let mut placed_completion_context = None;
if ack_mode == crate::mob_machine::SubmitWorkAckMode::TurnCompleted
&& super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&entry.agent_identity,
)
{
let completion_id = placed_submit_interaction_id(interaction_id)?;
// Pump construction and exact route validation are shell
// preflight, not post-ingress fallibility. A later pump death is
// covered by the durable cancellation reconciler.
self.ensure_member_event_pump(&entry.agent_identity).await?;
let incarnation = self.placed_member_incarnation(&entry)?;
let member_ref =
self.machine_member_ref_for_behavior(&entry, "placed completion preflight")?;
let route_exact = matches!(
&member_ref,
MemberRef::BackendPeer { session_id, .. }
if optional_route_session_matches_machine(
session_id.as_ref(),
&incarnation.member_session_id,
)
);
let context = self
.member_event_pumps
.remote_completion_context(&entry.agent_identity, &incarnation);
if !route_exact || context.is_none() {
return Err(MobError::Internal(format!(
"placed completion preflight has no exact active route/pump for '{}'",
entry.agent_identity
)));
}
placed_completion_context = context;
placed_completion_obligation = Some(
self.reserve_placed_completion_before_submit_work(&entry, completion_id)
.await?,
);
effective_interaction_id = Some(completion_id);
}
// Apply the DSL SubmitWork input. The MobMachine owns work-origin and
// fence-token legality: `SubmitWorkRunningExternal` /
// `SubmitWorkRunningInternal` encode origin, addressability,
// live-runtime, fence-token, and phase guards.
let transition = match mob_dsl::MobMachineMutator::apply(
&mut self.dsl_authority,
submit_work_input,
) {
Ok(transition) => transition,
Err(_) => {
let current_state = self.state();
let rejection = Self::resolve_submit_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&admission_runtime_id,
origin,
&agent_identity,
current_state,
);
if let Some(obligation) = placed_completion_obligation.take()
&& let Err(close_error) = self
.request_placed_completion_cancellation_in_actor(obligation)
.await
{
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"SubmitWork rejected after durable completion Record ({rejection}); pre-send closure also failed ({close_error})"
)));
}
return Err(rejection);
}
};
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
// The generated ingress effect is the authority token for realization.
// Its payload must match the admitted work before the shell dispatches
// to a session-bound runtime or a peer-only runtime.
let ingress_authority = match SubmitWorkIngressAuthority::from_transition(
&transition,
&dsl_runtime_id,
dsl_fence_token,
mob_dsl::Generation::from_domain(admission_runtime_id.generation),
&dsl_work_id,
dsl_origin,
) {
Ok(authority) => authority,
Err(error) => {
if let Some(obligation) = placed_completion_obligation.take() {
return Err(self
.unwind_placed_completion_prepare_failure(obligation, error)
.await);
}
return Err(error);
}
};
drop(transition);
let completion = match self
.dispatch_member_turn_after_machine_admission(
&entry,
ingress_authority,
SubmitWorkDispatchRequest {
content,
system_prompt,
injected_context,
interaction_id: effective_interaction_id,
objective_id,
handling_mode,
external_delivery_identity,
turn_metadata,
event_tx,
completion_tx,
llm_identity_applied_tx,
ack_mode,
operation_id: None,
placed_completion_obligation: placed_completion_obligation.clone(),
placed_completion_context,
},
)
.await
{
Ok(completion) => completion,
Err(error) => {
if let Some(obligation) = placed_completion_obligation.take() {
return Err(self
.unwind_placed_completion_prepare_failure(obligation, error)
.await);
}
return Err(error);
}
};
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
work_ref = %work_ref,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
completion = completion.kind(),
"handle_submit_work dispatched after machine admission"
);
Ok(completion)
}
/// W-D.2 trigger 1: a typed delivery failure against a PLACED member's
/// peer fires the revival classification back onto the actor (the remote
/// analog of the local `has_live_session == false` dispatch trigger).
/// The observation is raw — the machine's Classify ladder decides; a
/// spurious trigger for a live member converges through the host's
/// idempotent ensure (`ResumedLive`).
async fn fire_placed_revival_trigger(
revival: Option<(mpsc::Sender<RoutedMobCommand>, AgentIdentity)>,
error: &MobError,
) {
let Some((command_tx, agent_identity)) = revival else {
return;
};
let reason = format!("placed member delivery failed: {error}");
if command_tx
.send(RoutedMobCommand::internal(MobCommand::RevivePlacedMember {
agent_identity: agent_identity.clone(),
reason,
}))
.await
.is_err()
{
tracing::warn!(
agent_identity = %agent_identity,
"placed member revival trigger dropped (actor gone)"
);
}
}
async fn persist_placed_completion_cancellation_until_durable(
handle: &MobHandle,
identity: &AgentIdentity,
obligation: &crate::event::PlacedCompletionObligationEvent,
reason: &'static str,
) {
let machine_obligation =
match super::placed_completion_reconciler::obligation_from_event(obligation) {
Ok(obligation) => obligation,
Err(error) => {
tracing::error!(
agent_identity = %identity,
input_id = %obligation.input_id,
error = %error,
reason,
"placed completion aftercare could not decode durable obligation"
);
return;
}
};
let mut delay = std::time::Duration::from_millis(100);
loop {
// Natural terminal, certified closure, ACK, or lifecycle Dispose
// may win while aftercare retries. Once Pending is gone there is
// no cancellation fact left to persist.
if !handle
.machine_state_watch_rx
.borrow()
.pending_placed_completion_outcomes
.contains(&machine_obligation)
{
return;
}
match handle
.request_placed_completion_cancellation(obligation.clone())
.await
{
Ok(()) => {
if let Err(error) = handle.ensure_pump_for_obligation(identity).await {
tracing::warn!(
agent_identity = %identity,
input_id = %obligation.input_id,
error = %error,
reason,
"placed completion cancellation is durable but immediate pump ensure failed"
);
}
return;
}
Err(MobError::ActorCommandChannelClosed | MobError::ActorReplyChannelClosed) => {
// Actor restart replays the Record and adopts custody.
return;
}
Err(error) => {
tracing::warn!(
agent_identity = %identity,
input_id = %obligation.input_id,
error = %error,
reason,
retry_delay_ms = delay.as_millis() as u64,
"retrying placed completion cancellation until durable"
);
tokio::time::sleep(delay).await;
delay = (delay * 2).min(std::time::Duration::from_secs(5));
}
}
}
}
fn prepare_placed_completion_wait(
&self,
identity: &AgentIdentity,
context: super::event_pump::RemoteCompletionContext,
expected_member: super::bridge_protocol::BridgeMemberIncarnation,
input_id: String,
obligation: crate::event::PlacedCompletionObligationEvent,
) -> Result<PreparedPlacedCompletionWait, MobError> {
let parsed =
uuid::Uuid::parse_str(&input_id).map_err(|_| MobError::InvalidPlacedInteractionId {
interaction_id: input_id.clone(),
})?;
if parsed.is_nil() || parsed.to_string() != input_id || obligation.input_id != input_id {
return Err(MobError::InvalidPlacedInteractionId {
interaction_id: input_id,
});
}
if obligation.agent_identity != *identity
|| obligation.host_id != expected_member.host_id
|| obligation.host_binding_generation != expected_member.binding_generation
|| obligation.member_session_id != expected_member.member_session_id
|| obligation.generation.get() != expected_member.generation
|| obligation.fence_token.get() != expected_member.fence_token
|| expected_member.agent_identity != identity.as_str()
|| self.definition.id != expected_member.mob_id
{
return Err(MobError::Internal(format!(
"placed completion '{}' drifted from its durable exact residency",
obligation.input_id
)));
}
let interaction_id = meerkat_core::interaction::InteractionId(parsed);
// Registration is deliberately the final fallible step before the
// detached sender is spawned: no terminal can beat the observer.
let waiter = context
.register(interaction_id)
.map_err(|reason| MobError::Internal(reason.to_string()))?;
Ok(PreparedPlacedCompletionWait {
expected_member,
input_id: obligation.input_id.clone(),
waiter,
obligation,
handle: self.mob_handle_for_tools(),
identity: identity.clone(),
})
}
async fn unwind_placed_completion_prepare_failure(
&mut self,
obligation: crate::event::PlacedCompletionObligationEvent,
error: MobError,
) -> MobError {
let identity = obligation.agent_identity.clone();
if let Err(cancel_error) = self
.request_placed_completion_cancellation_in_actor(obligation)
.await
{
self.durable_uncertainty_fail_stop = true;
return MobError::Internal(format!(
"placed completion preparation failed ({error}) and its durable cancellation request failed ({cancel_error}); actor is fail-stopping for cold recovery"
));
}
if let Err(ensure_error) = self.ensure_member_event_pump(&identity).await {
// Cancellation is already durable; the owned reconciler retries
// the exact pump/cancel path. Preserve the initiating error while
// making the immediate lease failure visible.
tracing::warn!(
agent_identity = %identity,
error = %ensure_error,
"placed completion cancellation is durable but immediate pump ensure failed"
);
}
error
}
fn spawn_turn_completed_reply(
&mut self,
provisioner: Arc<dyn MobProvisioner>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
mut reply_tx: oneshot::Sender<Result<(), MobError>>,
revival: Option<(mpsc::Sender<RoutedMobCommand>, AgentIdentity)>,
remote: Option<PreparedPlacedCompletionWait>,
) {
self.actor_io_tasks.spawn(async move {
// Placed completion arrives with Record committed and its exact
// waiter already registered by the actor. The detached task may
// realize that authority, but never derive/mint custody itself.
let (placed, remote_wait) = match remote {
Some(PreparedPlacedCompletionWait {
expected_member,
input_id,
waiter,
obligation,
handle,
identity,
}) => {
(
Some(super::provisioner::PlacedTurnDeliveryContext {
input_id: input_id.clone(),
transcript_interaction_id: Some(input_id.clone()),
expected_member,
// The machine Record is the controller-side
// custody paired with this retained host sidecar.
outcome_tracking: Some(
super::bridge_protocol::BridgeOutcomeTracking::Interaction,
),
}),
Some((
handle,
identity,
obligation,
input_id,
waiter,
)),
)
}
None => (None, None),
};
let delivery = provisioner
.start_turn_with_correlation(&member_ref, *req, placed)
.await;
let result = match remote_wait {
// Local sessions complete inside start_turn. The explicit
// legacy peer-only lane returns Some(envelope id) but has
// pre-phase-6 dispatch-ack semantics.
None => match delivery {
Ok(_) => Ok(()),
Err(error) => {
Self::fire_placed_revival_trigger(revival, &error).await;
Err(error)
}
},
Some((handle, identity, obligation, input_id, waiter)) => {
let (delivery_uncertainty, receipt_mismatch) = match delivery {
Err(error @ MobError::BridgeDeliveryRejected { .. }) => {
// This is the one positively certified no-effect
// result. Close durable machine custody before the
// user can observe the rejection; Close performs
// the distinct exact waiter fanout.
if let Err(close_error) = handle
.close_placed_completion_outcome(
obligation.clone(),
crate::event::PlacedCompletionClosureEvent::DeliveryRejected,
)
.await
{
let close_error = MobError::Internal(format!(
"placed delivery was rejected ({error}) but durable completion closure failed: {close_error}"
));
Self::fire_placed_revival_trigger(revival, &close_error).await;
let _ = reply_tx.send(Err(close_error));
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"delivery_rejection_close_failed",
)
.await;
return;
}
Self::fire_placed_revival_trigger(revival, &error).await;
let _ = reply_tx.send(Err(error));
return;
}
Err(error) => (Some(error), None),
Ok(receipt) if receipt.as_deref() != Some(input_id.as_str()) => (
None,
Some(MobError::Internal(format!(
"placed turn returned correlation receipt {receipt:?}, expected '{input_id}'"
))),
),
Ok(_) => (None, None),
};
if let Some(error) = receipt_mismatch {
// A changed authenticated receipt makes the send
// result untrustworthy. Report promptly, then keep the
// actor-owned task alive until cancellation is durable.
Self::fire_placed_revival_trigger(revival, &error).await;
let _ = reply_tx.send(Err(error));
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"receipt_identity_mismatch",
)
.await;
return;
}
if delivery_uncertainty.is_some() {
// Ambiguous transport failure may mean the host never
// saw the work, in which case no sidecar can ever wake
// the waiter. Durably switch this exact key to the
// cancellation lane immediately; retain the waiter so
// authenticated NoEffect/Cancelled/Terminal authority
// remains the only semantic answer.
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"ambiguous_delivery_result",
)
.await;
}
// A timeout/Unavailable/Internal delivery result is not
// no-effect evidence: both byte-identical replies may have
// been lost after host acceptance. Keep exact waiter and
// pump custody until the authenticated sidecar wins. There
// is deliberately no local-duration effect decision; an
// upstream caller may abandon its volatile reply while the
// durable cleanup obligation continues independently.
let terminal = tokio::select! {
terminal = waiter => terminal,
() = reply_tx.closed() => {
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"caller_dropped",
).await;
return;
}
};
match terminal {
Err(_) => {
let error = delivery_uncertainty.unwrap_or_else(|| {
MobError::Internal(
super::event_pump::PUMP_STOPPED_WAITER_REASON.to_string(),
)
});
Self::fire_placed_revival_trigger(revival, &error).await;
let _ = reply_tx.send(Err(error));
Self::persist_placed_completion_cancellation_until_durable(
&handle,
&identity,
&obligation,
"waiter_infrastructure_loss",
)
.await;
return;
}
Ok(terminal) => {
match terminal {
super::event_pump::RemoteInteractionTerminal::Complete
| super::event_pump::RemoteInteractionTerminal::CallbackPending {
..
} => Ok(()),
super::event_pump::RemoteInteractionTerminal::Failed {
reason,
} => Err(MobError::Internal(format!(
"remote turn failed: {reason}"
))),
super::event_pump::RemoteInteractionTerminal::Closed {
closure,
} => Err(match closure {
crate::event::PlacedCompletionClosureEvent::DeliveryRejected => {
MobError::PlacedCompletionDeliveryRejected
}
crate::event::PlacedCompletionClosureEvent::HostNoEffect => {
MobError::PlacedCompletionHostNoEffect
}
crate::event::PlacedCompletionClosureEvent::HostCancelled => {
MobError::PlacedCompletionHostCancelled
}
}),
super::event_pump::RemoteInteractionTerminal::Disposed => {
Err(MobError::PlacedCompletionDisposed)
}
}
}
}
}
};
let _ = reply_tx.send(result);
});
}
// The detached reply task receives the complete admitted turn tuple; keep
// those independent carriers visible at the spawn boundary.
#[allow(clippy::too_many_arguments)]
fn spawn_turn_admission_reply(
&mut self,
provisioner: Arc<dyn MobProvisioner>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
completion_tx: Option<oneshot::Sender<Result<(), MobError>>>,
llm_identity_applied_tx: Option<super::handle::MemberTurnLlmIdentityAppliedSender>,
reply_tx: oneshot::Sender<Result<(), MobError>>,
revival: Option<(mpsc::Sender<RoutedMobCommand>, AgentIdentity)>,
placed_incarnation: Option<super::bridge_protocol::BridgeMemberIncarnation>,
placed_input_id: Option<String>,
) {
debug_assert_eq!(revival.is_some(), placed_incarnation.is_some());
self.actor_io_tasks.spawn(async move {
let result = match (placed_incarnation, placed_input_id) {
(Some(expected_member), Some(input_id)) => {
if completion_tx.is_some() || llm_identity_applied_tx.is_some() {
Err(MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
reason: "tracked completion is not supported for remotely hosted members"
.to_string(),
})
} else {
match placed_turn_supplied_interaction_id(&req) {
Ok(transcript_interaction_id) => {
let expected_receipt = input_id.clone();
match provisioner
.start_turn_with_correlation(
&member_ref,
*req,
Some(super::provisioner::PlacedTurnDeliveryContext {
input_id,
transcript_interaction_id: transcript_interaction_id
.map(|interaction_id| interaction_id.0.to_string()),
expected_member,
// IngressAccepted carries no retained
// terminal-publication custody.
outcome_tracking: None,
}),
)
.await
{
Ok(receipt)
if receipt.as_deref() == Some(expected_receipt.as_str()) =>
{
Ok(())
}
Ok(receipt) => Err(MobError::Internal(format!(
"placed admission returned transport receipt {receipt:?}, expected '{expected_receipt}'"
))),
Err(error) => Err(error),
}
}
Err(error) => Err(error),
}
}
}
(None, None) => {
if let Some(completion_tx) = completion_tx {
provisioner
.admit_tracked_turn(
&member_ref,
*req,
completion_tx,
llm_identity_applied_tx,
)
.await
} else {
provisioner.admit_turn(&member_ref, *req).await
}
}
_ => Err(MobError::Internal(
"turn admission placement/transport fields drifted".to_string(),
)),
};
if let Err(error) = &result {
Self::fire_placed_revival_trigger(revival, error).await;
}
let _ = reply_tx.send(result);
});
}
async fn dispatch_turn_driven_spawn_initial_turn(
&mut self,
agent_identity: &AgentIdentity,
agent_runtime_id: &AgentRuntimeId,
fence_token: FenceToken,
operation_id: &meerkat_core::ops::OperationId,
content: ContentInput,
inherited_objective_id: Option<meerkat_core::interaction::ObjectiveId>,
) -> Result<(), MobError> {
let entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
}
.ok_or_else(|| {
MobError::Internal(format!(
"turn-driven spawn initial SubmitWork for '{agent_identity}' had no roster projection after Spawn admission"
))
})?;
let work_ref = WorkRef::new();
let origin = WorkOrigin::Internal;
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(agent_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
let dsl_work_id = mob_dsl::WorkId::from_work_ref(&work_ref);
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let transition = match mob_dsl::MobMachineMutator::apply(
&mut self.dsl_authority,
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: dsl_work_id.clone(),
origin: dsl_origin,
},
) {
Ok(transition) => transition,
Err(_) => {
let current_state = self.state();
return Err(Self::resolve_submit_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
agent_runtime_id,
origin,
agent_identity,
current_state,
));
}
};
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
let ingress_authority = SubmitWorkIngressAuthority::from_transition(
&transition,
&dsl_runtime_id,
dsl_fence_token,
mob_dsl::Generation::from_domain(agent_runtime_id.generation),
&dsl_work_id,
dsl_origin,
)?;
drop(transition);
let completion = self
.dispatch_member_turn_after_machine_admission(
&entry,
ingress_authority,
SubmitWorkDispatchRequest {
content,
system_prompt: None,
// Spawn kickoff is mob-internal coordination content; the
// injected-context slot belongs to the submit-work lane.
injected_context: Vec::new(),
// Mob-internal kickoff carries no host interaction id.
interaction_id: None,
objective_id: inherited_objective_id.or(machine_kickoff_objective_id(
self.dsl_authority.state(),
agent_identity,
)?),
handling_mode: meerkat_core::types::HandlingMode::Queue,
external_delivery_identity: None,
turn_metadata: None,
event_tx: None,
completion_tx: None,
llm_identity_applied_tx: None,
ack_mode: crate::mob_machine::SubmitWorkAckMode::IngressAccepted,
operation_id: Some(operation_id.clone()),
placed_completion_obligation: None,
placed_completion_context: None,
},
)
.await?;
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
completion = completion.kind(),
"dispatch_turn_driven_spawn_initial_turn dispatched after machine admission"
);
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
"dispatch_turn_driven_spawn_initial_turn finishing dispatch"
);
self.finish_submit_work_dispatch(completion).await
}
/// Unified work-lane cancel entry.
///
/// The MobMachine DSL `CancelAllWork` transition owns live-runtime
/// membership, fence-token freshness, and phase legality. Once the machine
/// accepts, the shell dispatches `interrupt_member` on the current bridge
/// session.
async fn handle_cancel_all_work(
&mut self,
runtime_id: AgentRuntimeId,
fence_token: FenceToken,
) -> Result<(), MobError> {
let agent_identity = runtime_id.identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
let prepared = self
.prepare_dsl_input(
mob_dsl::MobMachineInput::CancelAllWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
},
"handle_cancel_all_work",
)
.map_err(|_| {
let current_state = self.state();
Self::resolve_cancel_all_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&runtime_id,
&agent_identity,
current_state,
)
})?;
let entry = entry.ok_or_else(|| {
MobError::Internal(format!(
"MobMachine accepted CancelAllWork for '{agent_identity}' but the roster projection has no member entry"
))
})?;
let expected_member = self
.dsl_authority
.state()
.member_placement
.contains_key(&dsl_identity)
.then(|| self.placed_member_incarnation(&entry))
.transpose()?;
// Feed the DSL CancelAllWork input. Guards have already accepted the
// runtime binding, fence token, and phase.
self.commit_prepared_dsl_input(prepared)?;
// Dispatch the interrupt now that the machine has authorized.
let machine_member_ref =
self.machine_member_ref_for_behavior(&entry, "cancel all work interrupt")?;
self.provisioner
.interrupt_member(&machine_member_ref, expected_member.as_ref())
.await
}
async fn finish_submit_work_dispatch(
&self,
completion: SubmitWorkDispatchCompletion,
) -> Result<(), MobError> {
match completion {
SubmitWorkDispatchCompletion::Completed => {
tracing::debug!("finish_submit_work_dispatch completed without runtime call");
Ok(())
}
SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref,
req,
completion_tx,
llm_identity_applied_tx,
placed_identity,
placed_incarnation,
placed_input_id,
} => {
tracing::debug!(
member_ref = ?member_ref,
operation_id = ?operation_id,
"finish_submit_work_dispatch admitting turn"
);
debug_assert_eq!(placed_identity.is_some(), placed_incarnation.is_some());
let result = if let Some(expected_member) = placed_incarnation {
if completion_tx.is_some() || llm_identity_applied_tx.is_some() {
return Err(MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
reason:
"tracked completion is not supported for remotely hosted members"
.to_string(),
});
}
let transcript_interaction_id = placed_turn_supplied_interaction_id(&req)?
.map(|interaction_id| interaction_id.0.to_string());
let input_id = placed_input_id.ok_or_else(|| {
MobError::Internal(
"placed admission lost its transport input id".to_string(),
)
})?;
// The peer provisioner's operation-specific arm already
// ignored the local operation id. Preserve that behavior
// while carrying exact placed incarnation authority.
self.provisioner
.start_turn_with_correlation(
&member_ref,
*req,
Some(super::provisioner::PlacedTurnDeliveryContext {
input_id,
transcript_interaction_id,
expected_member,
outcome_tracking: None,
}),
)
.await
.map(|_| ())
} else if let Some(completion_tx) = completion_tx {
self.provisioner
.admit_tracked_turn(
&member_ref,
*req,
completion_tx,
llm_identity_applied_tx,
)
.await
} else if let Some(operation_id) = operation_id.as_ref() {
self.provisioner
.admit_turn_for_operation(&member_ref, operation_id, *req)
.await
} else {
self.provisioner.admit_turn(&member_ref, *req).await
};
tracing::debug!(
member_ref = ?member_ref,
ok = result.is_ok(),
"finish_submit_work_dispatch admitted turn"
);
if let Err(error) = &result {
self.try_fire_placed_revival_trigger(placed_identity, error);
}
result
}
SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref,
req,
placed_identity,
placed_incarnation,
placed_input_id: _,
placed_completion_obligation: _,
placed_completion_context: _,
} => {
tracing::debug!(
member_ref = ?member_ref,
"finish_submit_work_dispatch starting turn"
);
let result = match (placed_identity.as_ref(), placed_incarnation) {
(Some(_), Some(_)) => Err(MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
reason: "inline placed TurnCompleted dispatch has no detached actor-owned event-pump waiter"
.to_string(),
}),
(None, None) => self.provisioner.start_turn(&member_ref, *req).await,
_ => Err(MobError::Internal(
"turn completion placement identity/incarnation drifted".to_string(),
)),
};
tracing::debug!(
member_ref = ?member_ref,
ok = result.is_ok(),
"finish_submit_work_dispatch started turn"
);
if let Err(error) = &result {
self.try_fire_placed_revival_trigger(placed_identity, error);
}
result
}
}
}
/// Inline (actor-task) flavor of the revival trigger: `try_send` because
/// awaiting the actor's own bounded command channel from the actor task
/// could deadlock when full; a dropped trigger is re-fired by the next
/// delivery attempt or the `HostStatus` sweep.
fn try_fire_placed_revival_trigger(
&self,
placed_identity: Option<AgentIdentity>,
error: &MobError,
) {
let Some(agent_identity) = placed_identity else {
return;
};
let reason = format!("placed member delivery failed: {error}");
if let Err(send_error) =
self.command_tx
.try_send(RoutedMobCommand::internal(MobCommand::RevivePlacedMember {
agent_identity: agent_identity.clone(),
reason,
}))
{
tracing::warn!(
agent_identity = %agent_identity,
error = %send_error,
"placed member revival trigger dropped (channel full or actor gone)"
);
}
}
async fn dispatch_member_turn_after_machine_admission(
&mut self,
entry: &RosterEntry,
ingress_authority: SubmitWorkIngressAuthority,
request: SubmitWorkDispatchRequest,
) -> Result<SubmitWorkDispatchCompletion, MobError> {
let SubmitWorkDispatchRequest {
content,
system_prompt,
injected_context,
interaction_id,
objective_id,
handling_mode,
external_delivery_identity,
turn_metadata,
event_tx,
completion_tx,
llm_identity_applied_tx,
ack_mode,
operation_id,
placed_completion_obligation,
placed_completion_context,
} = request;
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
runtime_mode = ?entry.runtime_mode,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
ingress_authority = ingress_authority.variant(),
"dispatch_member_turn_after_machine_admission started"
);
// Injected-context deliverability (steer / autonomous-inbox) was
// validated in `handle_submit_work` BEFORE the MobMachine admission,
// so no admitted ingress effect can be abandoned over it here.
let live_steer_admission = handling_mode == meerkat_core::types::HandlingMode::Steer
&& ack_mode == crate::mob_machine::SubmitWorkAckMode::IngressAccepted;
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission resolving member ref"
);
let machine_member_ref =
self.machine_member_ref_for_behavior(entry, "direct turn delivery")?;
tracing::debug!(
agent_identity = %entry.agent_identity,
member_ref = ?machine_member_ref,
"dispatch_member_turn_after_machine_admission resolved member ref"
);
ingress_authority.verify_member_ref(&machine_member_ref, "direct turn delivery")?;
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission verified ingress authority"
);
// W-D.2 trigger material: machine placement fact, read once at
// dispatch. A typed bridge-delivery failure for a placed member
// fires the revival classification back onto the actor.
let placed_identity =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity)
.then(|| entry.agent_identity.clone());
let placed_incarnation = placed_identity
.as_ref()
.map(|_| self.placed_member_incarnation(entry))
.transpose()?;
let placed_input_id = match (&placed_identity, ack_mode) {
(Some(_), crate::mob_machine::SubmitWorkAckMode::IngressAccepted) => {
// A caller-stable external delivery derives the remote
// protocol's UUID-shaped dedup token from its exact
// idempotency key. Ordinary callers retain one fresh key per
// admitted SubmitWork.
Some(external_delivery_identity.as_ref().map_or_else(
|| meerkat_core::time_compat::new_uuid_v7().to_string(),
|identity| {
WorkRef::for_external_delivery(
&self.definition.id,
&entry.agent_identity,
&identity.idempotency_key,
)
.to_string()
},
))
}
(Some(_), crate::mob_machine::SubmitWorkAckMode::TurnCompleted) => Some(
interaction_id
.ok_or_else(|| {
MobError::Internal(
"placed TurnCompleted admission lost its reserved interaction id"
.to_string(),
)
})?
.0
.to_string(),
),
(None, _) => None,
};
if let Some(obligation) = placed_completion_obligation.as_ref()
&& placed_input_id.as_deref() != Some(obligation.input_id.as_str())
{
return Err(MobError::Internal(
"placed completion dispatch id drifted from its durable Record".to_string(),
));
}
let effective_interaction_id = interaction_id;
// §19.L5/W-D.2: the member HOST owns a placed member's liveness — the
// controlling realm never holds its session, so the local ensure
// below would mint a dishonest DurableSnapshotMissing. Placed
// revival is delivery-failure-triggered (fire_placed_revival_trigger)
// and host-observed instead.
if placed_identity.is_none()
&& !live_steer_admission
&& let Some(bridge_session_id) = machine_member_ref.bridge_session_id()
{
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission checking live session actor"
);
match self
.session_service
.live_session_actor_registered(bridge_session_id)
.await
{
Ok(true) => {
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission live session actor exists"
);
}
Ok(false) | Err(meerkat_core::service::SessionError::NotFound { .. }) => {
// #37: the live materialization is gone while MobMachine
// still owns the member as Active. Machine-authorized
// revival rebuilds the live session from the durable
// snapshot through the existing resume materialization
// path; an unrecoverable or failed revival resolves into
// the typed terminal `MemberRestoreFailed`.
self.revive_member_live_materialization(
entry,
&machine_member_ref,
bridge_session_id,
false,
true,
)
.await?;
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission revived live session"
);
}
Err(error) => return Err(MobError::SessionError(error)),
}
}
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_mode = ?entry.runtime_mode,
"dispatch_member_turn_after_machine_admission dispatching runtime mode"
);
// Placement selects the physical owner before runtime mode. Both
// TurnDriven and AutonomousHost members can live on a remote member
// host; in either case the controller realizes RequestRuntimeIngress
// as DeliverMemberInput with the exact machine incarnation. Sending
// the remote session id to a local injector/runtime adapter would
// target the wrong owner.
if placed_identity.is_some() {
let turn_metadata =
submit_work_turn_metadata(turn_metadata, effective_interaction_id, objective_id)?;
let req = meerkat_core::service::StartTurnRequest {
injected_context,
prompt: content,
system_prompt,
event_tx: None,
runtime: submit_work_runtime_semantics(
handling_mode,
turn_metadata,
external_delivery_identity.as_ref(),
),
};
return match ack_mode {
crate::mob_machine::SubmitWorkAckMode::IngressAccepted => {
Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref: machine_member_ref,
req: Box::new(req),
completion_tx,
llm_identity_applied_tx,
placed_identity,
placed_incarnation,
placed_input_id,
})
}
crate::mob_machine::SubmitWorkAckMode::TurnCompleted => {
Ok(SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref: machine_member_ref,
req: Box::new(req),
placed_identity,
placed_incarnation,
placed_input_id,
placed_completion_obligation,
placed_completion_context,
})
}
};
}
match entry.runtime_mode {
crate::MobRuntimeMode::AutonomousHost => {
if ingress_authority.is_peer_runtime() {
return Err(MobError::Internal(format!(
"autonomous direct turn delivery requires generated RequestRuntimeIngress authority for '{}'",
entry.agent_identity
)));
}
let bridge_session_id = machine_member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"autonomous direct turn delivery requires MobMachine session binding for '{}'",
entry.agent_identity
))
})?;
self.ensure_autonomous_runtime_ready(&entry.agent_identity, &machine_member_ref)
.await?;
let render_metadata = turn_metadata
.as_ref()
.and_then(|metadata| metadata.render_metadata.clone());
if self
.autonomous_steer_requires_admission_barrier(
entry,
&machine_member_ref,
handling_mode,
ack_mode,
)
.await?
{
let req = meerkat_core::service::StartTurnRequest {
// The admission barrier is steer-only; steer dispatch
// with injected context was rejected before the mode
// fork, so this carrier is invariantly empty here.
injected_context: Vec::new(),
prompt: content,
system_prompt,
event_tx,
runtime: submit_work_runtime_semantics(
handling_mode,
turn_metadata,
external_delivery_identity.as_ref(),
),
};
return Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref: machine_member_ref,
req: Box::new(req),
completion_tx,
llm_identity_applied_tx,
placed_identity,
placed_incarnation,
placed_input_id,
});
}
// Injected context on the autonomous inbox path was rejected
// pre-admission in `handle_submit_work` (the plain-event path
// has no user-channel work boundary); the carrier is
// invariantly empty here.
let injector = self
.provisioner
.interaction_event_injector(&bridge_session_id)
.await
.ok_or_else(|| MobError::MissingMemberCapability {
member_id: crate::ids::AgentIdentity::from(entry.agent_identity.as_str()),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous direct turn delivery",
})?;
// A host-supplied interaction id rides the injected inbox
// event so the comms classification (and therefore the
// runtime transcript identity) carries the SAME id as the
// host's live interaction frames instead of minting a fresh
// unrelated one.
let inject_result = injector.inject_with_turn_identity(
interaction_id,
objective_id,
content,
meerkat_core::PlainEventSource::Rpc,
handling_mode,
render_metadata,
);
inject_result.map_err(|error| {
MobError::Internal(format!(
"autonomous dispatch inject failed for '{}': {}",
entry.agent_identity, error
))
})?;
Ok(SubmitWorkDispatchCompletion::Completed)
}
crate::MobRuntimeMode::TurnDriven => {
tracing::debug!(
agent_identity = %entry.agent_identity,
ingress_is_peer_runtime = ingress_authority.is_peer_runtime(),
"dispatch_member_turn_after_machine_admission entering turn-driven dispatch"
);
let machine_member_ref = if ingress_authority.is_peer_runtime() {
self.authorize_peer_only_member_ref_for_behavior(
&machine_member_ref,
"turn-driven direct turn delivery",
)
.await?
} else if placed_identity.is_some() {
// §19.L5/§15.3: the machine authorized runtime ingress at
// the MEMBER HOST's session; the phase-3 transport to that
// session is the member peer (`DeliverMemberInput` over
// comms — the provisioner's peer arm authorizes the
// supervisor itself). Projecting the remote session into
// the delivery ref would route the LOCAL session arms at
// a session this realm does not hold.
Self::project_member_ref_session_binding(&entry.member_ref, None).ok_or_else(
|| {
MobError::Internal(format!(
"direct turn delivery requires a peer-shaped ref for placed member '{}'",
entry.agent_identity
))
},
)?
} else {
machine_member_ref
};
tracing::debug!(
agent_identity = %entry.agent_identity,
member_ref = ?machine_member_ref,
"dispatch_member_turn_after_machine_admission building turn request"
);
let req = meerkat_core::service::StartTurnRequest {
// Turn-driven work requests carry no typed_turn_appends;
// the injected-context field is the single lowering
// carrier here. Runtime-backed members re-lower it into
// the prompt input's typed slot
// (`runtime_input_from_turn_request`); direct
// session-service members materialize it in the runner.
injected_context,
prompt: content,
system_prompt,
event_tx,
runtime: submit_work_runtime_semantics(
handling_mode,
turn_metadata,
external_delivery_identity.as_ref(),
),
};
tracing::debug!(
agent_identity = %entry.agent_identity,
ack_mode = ?ack_mode,
"dispatch_member_turn_after_machine_admission built turn request"
);
match ack_mode {
crate::mob_machine::SubmitWorkAckMode::IngressAccepted => {
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxing turn admission request"
);
let req = Box::new(req);
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxed turn admission request"
);
Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref: machine_member_ref,
req,
completion_tx,
llm_identity_applied_tx,
placed_identity,
placed_incarnation,
placed_input_id,
})
}
crate::mob_machine::SubmitWorkAckMode::TurnCompleted => {
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxing turn completion request"
);
let req = Box::new(req);
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxed turn completion request"
);
Ok(SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref: machine_member_ref,
req,
placed_identity,
placed_incarnation,
placed_input_id,
placed_completion_obligation,
placed_completion_context,
})
}
}
}
}
}
async fn autonomous_steer_requires_admission_barrier(
&self,
entry: &RosterEntry,
member_ref: &MemberRef,
handling_mode: meerkat_core::types::HandlingMode,
ack_mode: crate::mob_machine::SubmitWorkAckMode,
) -> Result<bool, MobError> {
if handling_mode != meerkat_core::types::HandlingMode::Steer
|| ack_mode != crate::mob_machine::SubmitWorkAckMode::IngressAccepted
{
return Ok(false);
}
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity) {
// A placed autonomous runtime has no controller-local state to
// probe. If this helper is reached independently, require the
// admission path; start_turn_with_correlation owns the remote send.
return Ok(true);
}
#[cfg(feature = "runtime-adapter")]
if let (Some(adapter), Some(session_id)) =
(&self.runtime_adapter, member_ref.bridge_session_id())
{
use meerkat_runtime::service_ext::SessionServiceRuntimeExt as _;
match adapter.runtime_state(session_id).await {
Ok(meerkat_runtime::RuntimeState::Running) => {
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
"active steer admission barrier enabled by running runtime state"
);
return Ok(true);
}
Ok(state) => {
let session_active = self
.provisioner
.is_member_active(member_ref)
.await?
.unwrap_or(false);
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
runtime_state = ?state,
session_active,
"active steer admission barrier checked non-running runtime state"
);
if session_active {
return Ok(true);
}
return Ok(false);
}
Err(error) => {
// Fail closed: an indeterminate runtime state must REQUIRE the
// admission barrier so the steer routes through real machine
// admission instead of bypassing it with a direct event inject.
// Bypassing on an unknown state would ack Completed before the
// machine admits the turn, so we demand admission whenever the
// runtime state cannot be determined.
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
error = %error,
"runtime state unavailable; requiring autonomous steer admission barrier (fail closed)"
);
return Ok(true);
}
}
}
Ok(false)
}
async fn commit_remote_turn_receipt_in_actor(
&mut self,
receipt: crate::run::MobRunRemoteTurnReceipt,
) -> Result<(), MobError> {
let matching_intent = self
.run_store
.list_remote_turn_intents(&receipt.obligation.run_id)
.await?
.into_iter()
.find(|intent| intent.obligation == receipt.obligation);
let Some(matching_intent) = matching_intent else {
// Release/rematerialization may have actor-serialized a typed
// failure + Disposed carrier and then privacy-cleaned the private
// rows before the awakened flow task's queued receipt command can
// run. Accept only that exact public terminal class + exact
// obligation + exact Disposed proof; this lets the flow reducer
// continue to its recorded StepTargetFailure without recreating
// private custody or executing anything twice.
let events = self.events.replay_all().await?;
if Self::remote_turn_disposed_replay_matches(&events, &self.definition.id, &receipt) {
return Ok(());
}
return Err(MobError::Internal(format!(
"remote-turn receipt has no exact durable intent for sequence {}",
receipt.obligation.dispatch_sequence
)));
};
matching_intent
.validate_for(&receipt.obligation.run_id, &self.definition.id)
.map_err(MobError::Internal)?;
// A reservation may have proved its first Record append absent and
// allowed the reconciler to repair it later. A fast host terminal can
// beat that scan, so every receipt commit self-heals the public chain
// before writing private receipt or terminal/finalizer carriers.
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnObligationRecorded {
obligation: receipt.obligation.clone(),
})
.await?;
let obligation = super::remote_flow_ticket::obligation_from_event(&receipt.obligation);
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::CommitRemoteTurnOutcome {
obligation: obligation.clone(),
},
"commit_remote_turn_receipt",
)?;
let terminal_kind = Self::remote_turn_receipt_terminal_kind(&receipt);
self.run_store
.put_remote_turn_receipt(&receipt.obligation.run_id, &receipt)
.await?;
self.ensure_remote_turn_carrier(terminal_kind).await?;
self.commit_prepared_dsl_transition(prepared)?;
let dispose_context = match &receipt.outcome {
crate::run::MobRunRemoteTurnReceiptOutcome::Failed {
no_effect_proof: Some(_),
..
} => Some("dispose_remote_turn_after_certified_no_effect"),
crate::run::MobRunRemoteTurnReceiptOutcome::TrackedInputCancel { .. } => {
Some("dispose_remote_turn_after_tracked_cancel")
}
_ => None,
};
if let Some(dispose_context) = dispose_context {
let carrier = receipt.obligation.clone();
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnOutcomeDisposed {
obligation: carrier.clone(),
})
.await?;
self.run_store
.delete_remote_turn_receipt(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.run_store
.delete_remote_turn_intent(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.apply_dsl_input(
mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { obligation },
dispose_context,
)?;
}
Ok(())
}
async fn finalize_remote_turn_privacy_cleanup_in_actor(
&mut self,
cleanup: super::remote_turn_reconciler::FinalizedRemoteTurnPrivacyCleanup,
) -> Result<(), MobError> {
let events = self.events.replay_all().await?;
let mob_events = events
.into_iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |position| position + 1);
let validated = super::remote_turn_reconciler::validate_finalized_remote_turn_public_chain(
&mob_events[epoch_start..],
cleanup.obligation.dispatch_sequence,
)?;
if validated != cleanup {
return Err(MobError::Internal(format!(
"remote-turn finalized cleanup sequence {} changed after scan validation",
cleanup.obligation.dispatch_sequence
)));
}
let obligation = super::remote_flow_ticket::obligation_from_event(&cleanup.obligation);
let (input, context) = match cleanup.finalizer {
super::remote_turn_reconciler::RemoteTurnPublicFinalizer::Acknowledged => (
mob_dsl::MobMachineInput::AcknowledgeRemoteTurnOutcome { obligation },
"reconcile_finalized_remote_turn_ack_cleanup",
),
super::remote_turn_reconciler::RemoteTurnPublicFinalizer::Disposed => (
mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { obligation },
"reconcile_finalized_remote_turn_dispose_cleanup",
),
};
let prepared = self.prepare_dsl_input_transition(input, context)?;
self.run_store
.delete_remote_turn_receipt(
&cleanup.obligation.run_id,
cleanup.obligation.dispatch_sequence,
)
.await?;
self.run_store
.delete_remote_turn_intent(
&cleanup.obligation.run_id,
cleanup.obligation.dispatch_sequence,
)
.await?;
if let Err(error) = self.commit_prepared_dsl_transition(prepared) {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"remote-turn finalized privacy cleanup is durable but machine finalizer commit failed; actor is fail-stopping for cold recovery: {error}"
)));
}
Ok(())
}
async fn close_remote_turn_after_tracked_cancel_in_actor(
&mut self,
receipt: crate::run::MobRunRemoteTurnReceipt,
) -> Result<(), MobError> {
let carrier = receipt.obligation.clone();
let obligation = super::remote_flow_ticket::obligation_from_event(&carrier);
self.commit_remote_turn_receipt_in_actor(receipt).await?;
// The generic commit path closes typed tracked-cancel receipts itself
// so a crash-replayed caller cannot accidentally leave Committed
// custody behind. Keep this exact fallback for an older in-flight
// phase that still owns the obligation after an idempotent commit.
let still_owned = {
let state = self.dsl_authority.state();
state.pending_remote_turn_outcomes.contains(&obligation)
|| state.committed_remote_turn_outcomes.contains(&obligation)
|| state.resolved_remote_turn_outcomes.contains(&obligation)
};
if !still_owned {
return Ok(());
}
self.ensure_remote_turn_carrier(MobEventKind::RemoteTurnOutcomeDisposed {
obligation: carrier.clone(),
})
.await?;
self.run_store
.delete_remote_turn_receipt(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.run_store
.delete_remote_turn_intent(&carrier.run_id, carrier.dispatch_sequence)
.await?;
self.apply_dsl_input(
mob_dsl::MobMachineInput::DisposeRemoteTurnObligation { obligation },
"dispose_remote_turn_after_tracked_cancel",
)?;
Ok(())
}
fn remote_turn_receipt_terminal_kind(
receipt: &crate::run::MobRunRemoteTurnReceipt,
) -> MobEventKind {
let target = crate::ids::AgentRuntimeId::new(
receipt.obligation.agent_identity.clone(),
receipt.obligation.generation,
);
match &receipt.outcome {
crate::run::MobRunRemoteTurnReceiptOutcome::Completed { value } => {
MobEventKind::StepTargetCompleted {
run_id: receipt.obligation.run_id.clone(),
step_id: receipt.obligation.step_id.clone(),
target,
output: Some(value.clone()),
remote_turn_obligation: Some(receipt.obligation.clone()),
}
}
crate::run::MobRunRemoteTurnReceiptOutcome::Failed { reason, .. } => {
MobEventKind::StepTargetFailed {
run_id: receipt.obligation.run_id.clone(),
step_id: receipt.obligation.step_id.clone(),
target,
reason: reason.clone(),
remote_turn_obligation: Some(receipt.obligation.clone()),
error_report: None,
error: None,
}
}
crate::run::MobRunRemoteTurnReceiptOutcome::TrackedInputCancel { reason, .. } => {
MobEventKind::StepTargetFailed {
run_id: receipt.obligation.run_id.clone(),
step_id: receipt.obligation.step_id.clone(),
target,
reason: reason.clone(),
remote_turn_obligation: Some(receipt.obligation.clone()),
error_report: None,
error: None,
}
}
}
}
fn remote_turn_disposed_replay_matches(
events: &[crate::event::MobEvent],
mob_id: &MobId,
receipt: &crate::run::MobRunRemoteTurnReceipt,
) -> bool {
let mob_events = events
.iter()
.filter(|event| &event.mob_id == mob_id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
let expected_terminal = Self::remote_turn_receipt_terminal_kind(receipt);
let terminal_matches = mob_events[epoch_start..]
.iter()
.any(|event| event.kind == expected_terminal);
let disposed = mob_events[epoch_start..].iter().any(|event| {
matches!(
&event.kind,
MobEventKind::RemoteTurnOutcomeDisposed { obligation }
if obligation == &receipt.obligation
)
});
terminal_matches && disposed
}
async fn converge_recovered_flow_run_in_actor(
&mut self,
run_id: RunId,
) -> Result<(), MobError> {
if self.run_tasks.contains_key(&run_id) || self.run_cancel_tokens.contains_key(&run_id) {
return Err(MobError::Internal(format!(
"recovered flow convergence refused for live run task '{run_id}'"
)));
}
if self
.dsl_authority
.state()
.pending_remote_turn_outcomes
.iter()
.any(|obligation| obligation.run_id.0 == run_id.to_string())
{
// Fan-out recovery: every adopter calls this seam; only the last
// Pending terminal may run the single cancellation convergence.
return Ok(());
}
let run = self
.run_store
.get_run(&run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
if crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status)? {
// Custody recovery may finish after a lifecycle cancellation has
// already terminalized the run. Terminality wins: replay cleanup
// must not attempt to project a recovered failure over Canceled.
return Ok(());
}
if let Some((failed_step_id, reason)) = self.recovered_remote_step_failure(&run_id).await? {
// Recovery has no live FlowEngine future left to consume the
// ticket, but the authenticated terminal receipt and its exact
// StepTargetFailed carrier are durable. Project that failure
// through the same machine-authorized FailStep + Failed seams as
// live execution. Falling back to Canceled here would erase the
// semantic result that already closed remote custody. Unresolved
// siblings are canceled rather than assigned a failure reason
// that belongs only to this authenticated carrier.
self.fail_recovered_step_in_actor(&run_id, &failed_step_id, &reason)
.await?;
self.flow_engine
.repair_persisted_fail_step_projections(&run_id, &failed_step_id, &reason)
.await?;
self.cancel_unfinished_steps_in_actor(&run_id).await?;
self.terminalize_failed_in_actor(
run_id,
run.flow_id,
FlowFailureCause::from_step_error(&MobError::FlowFailed {
run_id: run.run_id,
reason,
}),
"recovered_remote_failure_terminalize_failed",
)
.await?;
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::CompleteFlow,
"recovered_remote_failure_complete",
)?;
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::FinishRun,
"recovered_remote_failure_finish",
)?;
self.publish_machine_state_projection();
return Ok(());
}
// A recovered run's frame reducer still contains the pre-crash
// Dispatched state. Re-entering FlowEngine here could redispatch that
// step with a fresh input id. Preserve the established controlling
// restart contract instead: once every same-run Pending custody row
// has terminalized, converge the unfinished run to Canceled through
// the exact same durable helper used during startup recovery.
let terminalization = super::terminalization::FlowTerminalizationAuthority::new(
self.run_store.clone(),
self.events.clone(),
self.definition.id.clone(),
);
super::builder::converge_recovered_active_flow_run(
&mut self.dsl_authority,
self.run_store.clone(),
&terminalization,
run_id,
)
.await?;
self.publish_machine_state_projection();
Ok(())
}
async fn recovered_remote_step_failure(
&self,
run_id: &RunId,
) -> Result<Option<(StepId, String)>, MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
let failure_candidate_steps = run
.flow_state
.step_status
.iter()
.filter_map(|(step_id, status)| {
status
.is_none_or(|status| {
matches!(
status,
flow_run::StepRunStatus::Dispatched | flow_run::StepRunStatus::Failed
)
})
.then_some(step_id.clone())
})
.collect::<BTreeSet<_>>();
let events = self.events.replay_all().await?;
let mob_events = events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = mob_events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
// A single terminal carrier for a still-unresolved or already-Failed
// step is the OneToOne/single-target case and can be projected
// exactly. Multiple target carriers need the live collection-policy
// reducer (Any, Quorum, etc.); recovery deliberately retains the prior
// conservative Canceled fallback rather than guessing Failed from one
// target.
let mut terminals = BTreeMap::<(StepId, u64), Option<String>>::new();
for event in &mob_events[epoch_start..] {
let (event_run_id, event_step_id, obligation, outcome) = match &event.kind {
MobEventKind::StepTargetFailed {
run_id,
step_id,
reason,
remote_turn_obligation: Some(obligation),
..
} => (run_id, step_id, obligation, Some(reason.clone())),
MobEventKind::StepTargetCompleted {
run_id,
step_id,
remote_turn_obligation: Some(obligation),
..
} => (run_id, step_id, obligation, None),
_ => continue,
};
if event_run_id != run_id
|| &obligation.run_id != run_id
|| event_step_id != &obligation.step_id
|| !failure_candidate_steps.contains(event_step_id)
{
continue;
}
let key = (event_step_id.clone(), obligation.dispatch_sequence);
match terminals.insert(key, outcome.clone()) {
Some(existing) if existing != outcome => {
return Err(MobError::Internal(format!(
"recovered remote-turn sequence {} has conflicting terminal carriers",
obligation.dispatch_sequence
)));
}
_ => {}
}
}
Ok(recovered_single_target_failure(
&terminals,
&run.flow_state.step_target_counts,
run.flow_state.max_step_retries,
))
}
async fn handle_run_flow(
&mut self,
requested_run_id: Option<RunId>,
flow_id: FlowId,
activation_params: serde_json::Value,
scoped_event_tx: Option<mpsc::Sender<meerkat_core::ScopedAgentEvent>>,
) -> Result<RunId, MobError> {
self.ensure_pending_spawn_alignment("handle_run_flow preflight")?;
self.ensure_flow_tracker_alignment("handle_run_flow preflight")
.await?;
let run_id = requested_run_id.unwrap_or_default();
if let Some(existing) = self.run_store.get_run(&run_id).await? {
if existing.mob_id != self.definition.id
|| existing.flow_id != flow_id
|| existing.activation_params != activation_params
{
return Err(MobError::Internal(format!(
"stable external flow run '{run_id}' was reused for different mob, flow, or activation parameters"
)));
}
let terminal =
crate::run::mob_machine_run_status_is_terminal(&existing.run_id, &existing.status)?;
if !terminal
&& !self.run_tasks.contains_key(&run_id)
&& !self.run_cancel_tokens.contains_key(&run_id)
{
return Err(MobError::Internal(format!(
"stable external flow run '{run_id}' is durable but has no live execution tracker; it is held for recovery instead of being reported as resumable"
)));
}
return Ok(run_id);
}
self.preview_run_flow_command_admission(&run_id)?;
let config = FlowRunConfig::from_definition(flow_id, &self.definition)?;
let run_flow = MobRun::run_flow_input(&run_id, &config)?;
debug_assert!(matches!(run_flow, mob_dsl::MobMachineInput::RunFlow { .. }));
let prepared_run_flow = self
.prepare_dsl_input(run_flow.clone(), "run_flow")
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
self.create_pending_run(
run_id.clone(),
&config,
prepared_run_flow.authority.state(),
activation_params.clone(),
vec![run_flow],
)
.await
.inspect_err(|error| {
tracing::warn!(
run_id = %run_id,
flow_id = %config.flow_id,
error = %error,
"flow admission run-store create failed before committing MobMachine RunFlow"
);
})?;
self.commit_prepared_dsl_input(prepared_run_flow)?;
if let Err(error) = self.apply_dsl_signal(mob_dsl::MobMachineSignal::StartRun, "start_run")
{
let mut details = Vec::new();
if let Err(rollback_error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::CompleteFlow,
"complete_flow_rollback",
) {
details.push(format!(
"RunFlow CompleteFlow rollback failed: {rollback_error}"
));
}
let terminalize_cause = FlowFailureCause::AdmissionFailed {
detail: format!(
"lifecycle StartRun transition failed during flow admission: {error}"
),
};
if let Err(terminalize_error) = self
.terminalize_failed_in_actor(
run_id.clone(),
config.flow_id.clone(),
terminalize_cause,
"run_flow_start_signal_terminalize_failed",
)
.await
{
details.push(format!(
"terminalizing pending run failed: {terminalize_error}"
));
}
let detail_suffix = if details.is_empty() {
String::new()
} else {
format!("; {}", details.join("; "))
};
return Err(MobError::Internal(format!(
"lifecycle StartRun transition failed during flow admission: {error}{detail_suffix}"
)));
}
let cancel_token = tokio_util::sync::CancellationToken::new();
self.run_cancel_tokens.insert(
run_id.clone(),
(cancel_token.clone(), config.flow_id.clone()),
);
if let Some(scoped_event_tx) = scoped_event_tx {
self.flow_streams
.lock()
.await
.insert(run_id.clone(), scoped_event_tx);
}
let engine = self.flow_engine.clone();
let cleanup_tx = self.command_tx.clone();
let flow_engine = self.flow_engine.clone();
let flow_run_id = run_id.clone();
let flow_id_for_task = config.flow_id.clone();
let cleanup_run_id = run_id.clone();
let handle = tokio::spawn(async move {
let run_id_for_execute = flow_run_id.clone();
let execution_cancel = cancel_token.clone();
if let Err(error) = engine
.execute_flow(
run_id_for_execute,
config,
activation_params,
execution_cancel,
)
.await
{
tracing::error!(
run_id = %flow_run_id,
flow_id = %flow_id_for_task,
error = %error,
"flow task execution failed; delegating terminalization to flow-run kernel"
);
if cancel_token.is_cancelled() {
if let Err(finalize_error) = flow_engine
.terminalize_canceled(flow_run_id.clone(), flow_id_for_task)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize canceled run after flow task cancellation"
);
}
} else {
match error {
MobError::RunCanceled(_) => {
if let Err(finalize_error) = flow_engine
.terminalize_canceled(flow_run_id.clone(), flow_id_for_task)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize canceled run after flow task cancellation"
);
}
}
other => {
if let Err(finalize_error) = flow_engine
.terminalize_failed(
flow_run_id.clone(),
flow_id_for_task,
FlowFailureCause::from_step_error(&other),
)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize run after flow task error"
);
}
}
}
}
}
if cleanup_tx
.send(RoutedMobCommand::internal(MobCommand::FlowFinished {
run_id: cleanup_run_id,
}))
.await
.is_err()
{
tracing::warn!(
run_id = %flow_run_id,
"failed to send FlowFinished cleanup command"
);
}
});
self.run_tasks.insert(run_id.clone(), handle);
self.ensure_flow_tracker_alignment("handle_run_flow completion")
.await?;
Ok(run_id)
}
async fn create_pending_run(
&self,
run_id: RunId,
config: &FlowRunConfig,
machine_state: &mob_dsl::MobMachineState,
activation_params: serde_json::Value,
authority_inputs: Vec<mob_dsl::MobMachineInput>,
) -> Result<RunId, MobError> {
let flow_state =
MobRun::flow_state_for_config_with_authority(&run_id, config, machine_state)?;
let mut run = MobRun::pending_with_run_id(
run_id.clone(),
self.definition.id.clone(),
config.flow_id.clone(),
flow_state,
activation_params,
);
run.append_flow_authority_inputs(authority_inputs)?;
self.run_store.create_run(run).await?;
Ok(run_id)
}
async fn handle_flow_cleanup(
&mut self,
run_id: RunId,
context: &'static str,
) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_flow_cleanup preflight")?;
self.ensure_flow_tracker_alignment("handle_flow_cleanup preflight")
.await?;
let has_task = self.run_tasks.contains_key(&run_id);
let has_token = self.run_cancel_tokens.contains_key(&run_id);
let has_stream = self.flow_streams.lock().await.contains_key(&run_id);
let run_terminal = self
.run_store
.get_run(&run_id)
.await?
.as_ref()
.map(|run| crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status))
.transpose()?
.unwrap_or(false);
// Tracker presence is not execution ownership: the JoinHandle that
// sent this command has already completed, and cancellation cleanup
// replaces it with a finite helper task. Never discard those last
// volatile ownership markers until durable run terminality is proven.
if !run_terminal {
return Err(MobError::Internal(format!(
"{context}: run {run_id} cleanup arrived before persisted terminalization"
)));
}
if !has_task && !has_token && !has_stream {
tracing::debug!(
run_id = %run_id,
context = context,
"flow cleanup command had no local run-tracker entries"
);
return Ok(());
}
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "flow_cleanup")?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "flow_cleanup")?;
let _ = self.run_tasks.remove(&run_id);
let _ = self.run_cancel_tokens.remove(&run_id);
let _ = self.flow_streams.lock().await.remove(&run_id);
self.ensure_flow_tracker_alignment("handle_flow_cleanup completion")
.await?;
Ok(())
}
async fn handle_cancel_flow(&mut self, run_id: RunId) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_cancel_flow preflight")?;
self.apply_command_admission(
mob_dsl::MobMachineInput::CancelFlow {
run_id: mob_dsl::RunId::from(run_id.to_string()),
},
MobState::Running,
"cancel_flow",
)?;
let Some((cancel_token, flow_id)) = self
.run_cancel_tokens
.get(&run_id)
.map(|(token, flow_id)| (token.clone(), flow_id.clone()))
else {
if self.run_tasks.contains_key(&run_id)
|| self.flow_streams.lock().await.contains_key(&run_id)
{
return Err(MobError::Internal(format!(
"handle_cancel_flow: run {run_id} missing cancel token despite live task/stream trackers"
)));
}
self.ensure_flow_tracker_alignment("handle_cancel_flow no-op completion")
.await?;
return Ok(());
};
self.flow_streams.lock().await.remove(&run_id);
cancel_token.cancel();
let Some(mut handle) = self.run_tasks.remove(&run_id) else {
self.flow_engine.cancel_unfinished_steps(&run_id).await?;
self.terminalize_canceled_in_actor(
run_id.clone(),
flow_id,
"cancel_flow_no_handle_terminalize_canceled",
)
.await?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "cancel_flow_no_handle")
.map_err(|error| {
MobError::Internal(format!(
"flow canceled cleanup (no task handle): lifecycle CompleteFlow transition failed for run {run_id}: {error}"
))
})?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "cancel_flow_no_handle")
.map_err(|error| {
MobError::Internal(format!(
"flow canceled cleanup (no task handle): lifecycle FinishRun transition failed for run {run_id}: {error}"
))
})?;
let _ = self.run_cancel_tokens.remove(&run_id);
self.ensure_flow_tracker_alignment("handle_cancel_flow no-task cleanup")
.await?;
return Ok(());
};
let flow_engine = self.flow_engine.clone();
let cleanup_tx = self.command_tx.clone();
let cancel_grace_timeout = self
.definition
.limits
.as_ref()
.and_then(|limits| limits.cancel_grace_timeout_ms)
.map_or_else(
|| std::time::Duration::from_secs(5),
std::time::Duration::from_millis,
);
let cleanup_run_tracker = run_id.clone();
let cleanup_run_id_for_error = run_id.clone();
let cleanup_handle = tokio::spawn(async move {
let cleanup_run_id = run_id.clone();
let completed = tokio::select! {
_ = &mut handle => true,
() = tokio::time::sleep(cancel_grace_timeout) => false,
};
if completed {
let mut terminalized = true;
if let Err(error) = flow_engine.cancel_unfinished_steps(&run_id).await {
terminalized = false;
tracing::error!(
error = %error,
"failed to settle dispatched steps after flow task completion during cancellation"
);
}
if let Err(error) = flow_engine
.terminalize_canceled(run_id.clone(), flow_id)
.await
{
terminalized = false;
tracing::error!(
error = %error,
"failed to apply canceled terminalization after flow task completion"
);
}
if cleanup_tx
.send(RoutedMobCommand::internal(
MobCommand::FlowCanceledCleanup {
run_id: cleanup_run_id,
terminalized,
},
))
.await
.is_err()
{
tracing::warn!(
"failed to send FlowCanceledCleanup command after task completion"
);
}
return;
}
handle.abort();
let mut terminalized = true;
if let Err(error) = flow_engine.cancel_unfinished_steps(&run_id).await {
terminalized = false;
tracing::error!(
error = %error,
"failed to settle dispatched steps before flow cancellation terminalization"
);
}
if let Err(error) = flow_engine
.terminalize_canceled(run_id.clone(), flow_id)
.await
{
terminalized = false;
tracing::error!(
error = %error,
"failed flow-run kernel cancellation terminalization"
);
}
if cleanup_tx
.send(RoutedMobCommand::internal(
MobCommand::FlowCanceledCleanup {
run_id: cleanup_run_id,
terminalized,
},
))
.await
.is_err()
{
tracing::warn!("failed to send FlowCanceledCleanup command");
}
});
if let Some(replaced) = self.run_tasks.insert(cleanup_run_tracker, cleanup_handle) {
replaced.abort();
return Err(MobError::Internal(format!(
"handle_cancel_flow: duplicate flow cleanup task registration for run {cleanup_run_id_for_error}"
)));
}
self.ensure_flow_tracker_alignment("handle_cancel_flow completion")
.await?;
Ok(())
}
async fn apply_flow_run_command_in_actor(
&mut self,
run_id: &RunId,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<Option<Vec<flow_run::Effect>>, MobError> {
let authority_input = command.authority_input(run_id);
let prepared = self.prepare_dsl_input(authority_input.clone(), context)?;
let machine_state = prepared.authority.state().clone();
let machine_effects = prepared.effects.clone();
let authority =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&authority_input)?;
let effects = if matches!(command, MobMachineFlowRunCommand::StartRun(_)) {
self.flow_engine
.start_run_state_with_machine_state(
run_id,
machine_state,
authority,
machine_effects,
context,
)
.await?
} else {
Some(
self.flow_engine
.apply_command_with_machine_state(
run_id,
command,
machine_state,
authority,
machine_effects,
context,
)
.await?,
)
};
if effects.is_some() {
self.commit_prepared_dsl_input(prepared)?;
}
Ok(effects)
}
async fn commit_flow_run_command_in_actor(
&mut self,
run_id: &RunId,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<Option<Vec<flow_run::Effect>>, MobError> {
self.apply_flow_run_command_in_actor(run_id, command, context)
.await
}
async fn commit_flow_frame_store_plan_in_actor(
&mut self,
run_id: &RunId,
plan: FlowFrameLoopStorePlan,
) -> Result<bool, MobError> {
if !self
.flow_frame_store_plan_expected_matches(run_id, &plan)
.await?
{
return Ok(false);
}
let prepared =
self.prepare_dsl_inputs(plan.machine_inputs(), "flow_frame_loop_store_plan")?;
let authority_inputs = plan.machine_inputs().to_vec();
let won = match &plan {
FlowFrameLoopStorePlan::InsertFrame {
frame_id,
initial_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
None,
initial_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::FrameState {
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
Some(expected_frame),
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteStepAndRecordOutput {
frame_id,
expected_frame,
next_frame,
step_output_key,
step_output,
loop_context,
..
} => self
.run_store
.cas_complete_step_and_record_output_with_authority(
run_id,
frame_id,
expected_frame,
next_frame.clone(),
step_output_key.clone(),
step_output.clone(),
loop_context
.as_ref()
.map(|(loop_id, iteration)| (loop_id, *iteration)),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::GrantNodeSlot {
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_grant_node_slot_with_authority(
run_id,
expected_run_state,
next_run_state.clone(),
frame_id,
expected_frame,
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::StartLoop {
loop_instance_id,
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
initial_loop,
..
} => self
.run_store
.cas_start_loop_with_authority(
run_id,
loop_instance_id,
expected_run_state,
next_run_state.clone(),
frame_id,
expected_frame,
next_frame.clone(),
initial_loop.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::GrantBodyFrameStart {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
initial_frame,
ledger_entry,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_grant_body_frame_start_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
initial_frame.clone(),
ledger_entry.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::RunStateOnly {
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_flow_state_with_authority(
run_id,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::SealFrame {
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
Some(expected_frame),
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteBodyFrame {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_complete_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
expected_frame,
next_frame.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::LoopRequestBodyFrame {
loop_instance_id,
expected_loop,
next_loop,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_loop_request_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteLoop {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_complete_loop_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
expected_frame,
next_frame.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
};
if won {
self.commit_prepared_dsl_input(prepared)?;
}
Ok(won)
}
async fn flow_frame_store_plan_expected_matches(
&self,
run_id: &RunId,
plan: &FlowFrameLoopStorePlan,
) -> Result<bool, MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
Ok(match plan {
FlowFrameLoopStorePlan::InsertFrame { frame_id, .. } => {
!run.frames.contains_key(frame_id)
}
FlowFrameLoopStorePlan::FrameState {
frame_id,
expected_frame,
..
}
| FlowFrameLoopStorePlan::CompleteStepAndRecordOutput {
frame_id,
expected_frame,
..
}
| FlowFrameLoopStorePlan::SealFrame {
frame_id,
expected_frame,
..
} => run.frames.get(frame_id) == Some(expected_frame),
FlowFrameLoopStorePlan::GrantNodeSlot {
expected_run_state,
frame_id,
expected_frame,
..
} => {
&run.flow_state == expected_run_state
&& run.frames.get(frame_id) == Some(expected_frame)
}
FlowFrameLoopStorePlan::StartLoop {
loop_instance_id,
expected_run_state,
frame_id,
expected_frame,
..
} => {
&run.flow_state == expected_run_state
&& run.frames.get(frame_id) == Some(expected_frame)
&& !run.loops.contains_key(loop_instance_id)
}
FlowFrameLoopStorePlan::GrantBodyFrameStart {
loop_instance_id,
expected_loop,
frame_id,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
&& !run.frames.contains_key(frame_id)
}
FlowFrameLoopStorePlan::RunStateOnly {
expected_run_state, ..
} => &run.flow_state == expected_run_state,
FlowFrameLoopStorePlan::CompleteBodyFrame {
loop_instance_id,
expected_loop,
frame_id,
expected_frame,
expected_run_state,
..
}
| FlowFrameLoopStorePlan::CompleteLoop {
loop_instance_id,
expected_loop,
frame_id,
expected_frame,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
&& run.frames.get(frame_id) == Some(expected_frame)
}
FlowFrameLoopStorePlan::LoopRequestBodyFrame {
loop_instance_id,
expected_loop,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
}
})
}
async fn commit_flow_terminalization_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
target: TerminalizationTarget,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<TerminalizationOutcome, MobError> {
let run = self
.run_store
.get_run(&run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
if crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status)?
&& !matches!(
(&target, &run.status),
(TerminalizationTarget::Canceled, MobRunStatus::Failed)
)
{
let repaired = self
.flow_engine
.repair_persisted_terminalization(run_id, flow_id, target)
.await;
if repaired.is_err() {
// A terminal run snapshot is already durable, but its public
// terminal carrier could not be repaired. Continuing to serve
// commands would let the live MobMachine project from a state
// whose durable event truth is incomplete. Cold recovery owns
// the only safe retry boundary for this ambiguous commit.
self.durable_uncertainty_fail_stop = true;
}
return repaired;
}
let authority_input = command.authority_input(&run_id);
let prepared = self.prepare_dsl_input(authority_input.clone(), context)?;
let machine_state = prepared.authority.state().clone();
let machine_effects = prepared.effects.clone();
let authority =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&authority_input)?;
let outcome = self
.flow_engine
.terminalize_with_machine_state(
run_id.clone(),
flow_id,
target.clone(),
command,
machine_state,
authority,
machine_effects,
)
.await;
match outcome {
Ok(TerminalizationOutcome::Transitioned) => {
self.commit_prepared_dsl_input(prepared)?;
Ok(TerminalizationOutcome::Transitioned)
}
Ok(TerminalizationOutcome::Noop) => Ok(TerminalizationOutcome::Noop),
Err(error) => {
let persisted_target = match self
.persisted_terminal_status_matches_target(&run_id, &target)
.await
{
Ok(persisted_target) => persisted_target,
Err(reconcile_error) => {
// Once the combined terminalization seam errors, an
// unreadable run snapshot leaves the commit outcome
// unknowable. Do not keep serving from volatile state.
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"terminalization failed for run '{run_id}' ({error}) and durable status reconciliation failed; actor is fail-stopping for cold recovery: {reconcile_error}"
)));
}
};
if persisted_target {
// The store committed the terminal snapshot before the
// terminal carrier failed. Keep the in-memory authority
// aligned for this command, then fail-stop so only cold
// recovery can repair the missing carrier and reopen the
// actor from durable truth. Set the fence before the
// volatile authority commit too: if that alignment step
// rejects, the actor must still leave the serving set.
self.durable_uncertainty_fail_stop = true;
self.commit_prepared_dsl_input(prepared)?;
}
Err(error)
}
}
}
async fn persisted_terminal_status_matches_target(
&self,
run_id: &RunId,
target: &TerminalizationTarget,
) -> Result<bool, MobError> {
Ok(self
.run_store
.get_run(run_id)
.await?
.is_some_and(|run| run.status == target.status()))
}
async fn cancel_unfinished_steps_in_actor(&mut self, run_id: &RunId) -> Result<(), MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
for step_id in run.ordered_steps()? {
if run
.flow_state
.step_status
.get(&step_id)
.and_then(|status| *status)
.is_some_and(|status| !matches!(status, flow_run::StepRunStatus::Dispatched))
{
continue;
}
self.apply_flow_run_command_in_actor(
run_id,
MobMachineFlowRunCommand::CancelStep(flow_run::inputs::CancelStep { step_id }),
"actor_cancel_unfinished_step",
)
.await?;
}
Ok(())
}
async fn fail_recovered_step_in_actor(
&mut self,
run_id: &RunId,
failed_step_id: &StepId,
reason: &str,
) -> Result<(), MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
let status = run
.flow_state
.step_status
.get(failed_step_id)
.and_then(|status| *status);
if status == Some(flow_run::StepRunStatus::Failed) {
return Ok(());
}
if status.is_some_and(|status| status != flow_run::StepRunStatus::Dispatched) {
return Err(MobError::Internal(format!(
"recovered exact failure step '{failed_step_id}' in run '{run_id}' has incompatible status {status:?}"
)));
}
let effects = self
.apply_flow_run_command_in_actor(
run_id,
MobMachineFlowRunCommand::FailStep(flow_run::inputs::FailStep {
step_id: failed_step_id.clone(),
}),
"actor_fail_recovered_exact_step",
)
.await?;
let _ = self
.flow_engine
.apply_fail_step_projection(effects, run_id, failed_step_id, reason)
.await?;
Ok(())
}
async fn terminalize_canceled_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
context: &'static str,
) -> Result<(), MobError> {
let _ = self
.commit_flow_terminalization_in_actor(
run_id,
flow_id,
TerminalizationTarget::Canceled,
MobMachineFlowRunCommand::TerminalizeCanceled(
flow_run::inputs::TerminalizeCanceled {},
),
context,
)
.await?;
Ok(())
}
async fn terminalize_failed_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
cause: FlowFailureCause,
context: &'static str,
) -> Result<(), MobError> {
let _ = self
.commit_flow_terminalization_in_actor(
run_id,
flow_id,
TerminalizationTarget::Failed { cause },
MobMachineFlowRunCommand::TerminalizeFailed(flow_run::inputs::TerminalizeFailed {}),
context,
)
.await?;
Ok(())
}
async fn cancel_all_flow_tasks(&mut self) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("cancel_all_flow_tasks preflight")?;
let tracked_run_ids = self.run_cancel_tokens.keys().cloned().collect::<Vec<_>>();
for run_id in tracked_run_ids {
let Some((token, flow_id)) = self
.run_cancel_tokens
.get(&run_id)
.map(|(token, flow_id)| (token.clone(), flow_id.clone()))
else {
continue;
};
token.cancel();
if let Some(handle) = self.run_tasks.remove(&run_id) {
handle.abort();
}
self.flow_streams.lock().await.remove(&run_id);
self.cancel_unfinished_steps_in_actor(&run_id).await?;
self.terminalize_canceled_in_actor(
run_id.clone(),
flow_id.clone(),
"cancel_all_flow_terminalize_canceled",
)
.await?;
// CompleteFlow / FinishRun accept `active_run_count == 0` as
// a legitimate terminal convergence (see
// `CompleteFlowRunningZero` and `FinishRunRunningZero` in
// the mob_machine DSL). The natural `FlowFinished` cleanup
// races with this destroy-driven cancel; both paths drive
// the authority toward the same terminal state.
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "cancel_all_flow")?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "cancel_all_flow")?;
let _ = self.run_cancel_tokens.remove(&run_id);
}
self.ensure_flow_tracker_alignment("cancel_all_flow_tasks completion")
.await?;
Ok(())
}
// -----------------------------------------------------------------------
// Internal helpers
// -----------------------------------------------------------------------
/// Compensate a failed spawn wiring path to avoid partial state.
async fn rollback_failed_spawn(
&mut self,
agent_identity: &AgentIdentity,
rollback_context: FailedSpawnRollback<'_>,
) -> Result<(), MobError> {
let FailedSpawnRollback {
generation,
profile_name,
member_ref,
operation_id,
session_origin,
successful_wiring_targets,
planned_wiring_targets,
} = rollback_context;
let spawned_entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let retire_event_already_present =
self.retire_event_exists(agent_identity, generation).await?;
let retirement_started_already_present = match spawned_entry.as_ref() {
Some(entry) => {
self.retirement_started_event_exists(agent_identity, entry.generation)
.await?
}
None => false,
};
if spawned_entry.is_none() && !retire_event_already_present {
return Err(MobError::WiringError(format!(
"spawn rollback requires roster entry for '{agent_identity}' before retiring durable member state"
)));
}
// Prepare and validate the generated retirement authority, then make
// its crash/retry carrier durable before peer notices, trust removal,
// or machine unwiring. Rollback is itself a lifecycle transaction:
// without this ordering, a crash can leave externally mutated
// topology with no replayable cleanup intent.
let rollback_retire_input = if let Some(entry) = spawned_entry.as_ref() {
let releasing = member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain);
let session_id_for_route = releasing.clone();
let retirement_started_journal_kind = if session_id_for_route.is_some() {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing
} else {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
};
let input = mob_dsl::MobMachineInput::Retire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
generation: mob_dsl::Generation::from_domain(entry.generation),
releasing,
session_id: session_id_for_route.clone(),
};
let prepared = self.prepare_dsl_input_transition(
input.clone(),
"rollback_failed_spawn_prepare_retiring_before_cleanup",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
retirement_started_journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"rollback_failed_spawn_prepare_retiring_before_cleanup",
)?;
if !retirement_started_already_present {
self.append_retirement_started_event_for_entry(
entry,
retirement_started_journal_kind,
session_id_for_route,
false,
)
.await?;
}
Some(input)
} else {
None
};
let mut wired_peers = successful_wiring_targets.to_vec();
wired_peers.sort();
wired_peers.dedup();
let mut cleanup_peers = wired_peers.clone();
for peer_id in planned_wiring_targets {
if peer_id != agent_identity && !cleanup_peers.contains(peer_id) {
cleanup_peers.push(peer_id.clone());
}
}
let mut cleanup_handoffs = BTreeMap::new();
if spawned_entry.is_some() {
for peer_member_identity in &cleanup_peers {
let peer_entry = {
let roster = self.roster.read().await;
roster.get(peer_member_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let cleanup_edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(agent_identity),
mob_dsl::AgentIdentity::from_domain(&peer_entry.agent_identity),
);
match self.authorize_member_trust_cleanup(
&cleanup_edge,
"spawn_rollback_trust_cleanup_authority",
) {
Ok(handoff) => {
let retry_handoff = self
.authorize_member_trust_cleanup(
&cleanup_edge,
"spawn_rollback_trust_cleanup_retry_authority",
)
.ok();
cleanup_handoffs
.insert(peer_entry.agent_identity.clone(), (handoff, retry_handoff));
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback trust cleanup skipped without generated unwiring authority"
);
}
}
}
}
let spawned_comms =
if super::member_runtime_is_host_owned(self.dsl_authority.state(), agent_identity) {
None
} else {
self.provisioner_comms(member_ref).await
};
let mut rollback = LifecycleRollback::new("spawn rollback");
if !wired_peers.is_empty() {
let spawned_entry = spawned_entry.as_ref().ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires roster entry for '{agent_identity}'"
))
})?;
let spawned_sender = self
.sender_runtime_for_entry(spawned_entry)
.await
.ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires sender runtime for '{agent_identity}'"
))
})?;
let spawned_spec = match self
.resolve_wiring_endpoint(spawned_entry, "spawn rollback spawned member")
.await?
{
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. }
| WiringEndpoint::Placed { spec, .. } => spec,
};
let spawned_peer_description = self
.definition
.resolve_profile(&spawned_entry.role, self.realm_profile_store.as_ref())
.await
.map(|p| p.peer_description)
.unwrap_or_default();
for peer_member_identity in &wired_peers {
let peer_spec = {
let roster = self.roster.read().await;
let peer_entry = roster.get(peer_member_identity).cloned().ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires roster entry for wired peer '{peer_member_identity}'"
))
})?;
drop(roster);
match self
.resolve_wiring_endpoint(&peer_entry, "spawn rollback")
.await?
{
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. }
| WiringEndpoint::Placed { spec, .. } => spec,
}
};
if let Err(error) = self
.notify_peer_retired(
&peer_spec,
agent_identity,
spawned_entry,
&spawned_spec,
&spawned_sender,
)
.await
{
return Err(rollback.fail(error).await);
}
rollback.defer(
format!(
"compensating mob.peer_added '{agent_identity}' -> '{peer_member_identity}'"
),
{
let spawned_sender = spawned_sender.clone();
let peer_spec = peer_spec.clone();
let agent_identity = agent_identity.clone();
let role = spawned_entry.role.clone();
let peer_description = spawned_peer_description.clone();
let spawned_spec = spawned_spec.clone();
move || async move {
let peer_route = PeerRoute::with_display_name(
peer_spec.peer_id,
peer_spec.name.clone(),
);
// K15: lifecycle params are the typed wire
// contract (`CommsPeerLifecycleParams`); no
// shadow `peer_name`/`peer_id`/`address` mirrors.
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: agent_identity.as_str().to_string(),
role: Some(role.as_str().to_string()),
description: Some(peer_description),
peer_spec: Some(
super::bridge_protocol::BridgePeerSpec::from(spawned_spec),
),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{agent_identity}': {error}"
))
})?;
let cmd = CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerAdded,
params,
};
spawned_sender.send(cmd).await?;
Ok(())
}
},
);
}
}
if let Some(spawned_entry) = spawned_entry.as_ref()
&& let Ok(spawned_endpoint) = self
.resolve_wiring_endpoint(spawned_entry, "spawn rollback trust cleanup spawned")
.await
{
let (spawned_spec, spawned_comms, spawned_binding) = match spawned_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (spec, Some(comms), None),
WiringEndpoint::PeerOnly { spec, binding } => (spec, None, Some(binding)),
// Remote runtime: no local comms handle and no V3 binding —
// the placed side's trust cleanup rides the obligation lane.
WiringEndpoint::Placed { spec, .. } => (spec, None, None),
};
for peer_member_identity in &cleanup_peers {
let peer_entry = {
let roster = self.roster.read().await;
roster.get(peer_member_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let Ok(peer_endpoint) = self
.resolve_wiring_endpoint(&peer_entry, "spawn rollback trust cleanup peer")
.await
else {
continue;
};
let (peer_spec, peer_comms, peer_binding) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (spec, Some(comms), None),
WiringEndpoint::PeerOnly { spec, binding } => (spec, None, Some(binding)),
// Remote runtime: cleanup of the placed side rides the
// obligation lane, not a local comms handle.
WiringEndpoint::Placed { spec, .. } => (spec, None, None),
};
let cleanup_handoff = cleanup_handoffs.get(&peer_entry.agent_identity);
if let Some(spawned_comms) = spawned_comms.as_ref() {
let peer_key = Self::trusted_peer_removal_key(&peer_spec);
if let Some((handoff, retry_handoff)) = cleanup_handoff.as_ref() {
let authority =
handoff.unwiring_authority_for(&peer_entry.agent_identity, &peer_key);
if let Ok(authority) = authority {
let removed = self
.apply_trusted_peer_remove(
spawned_comms.as_ref(),
peer_key.clone(),
authority,
)
.await;
if removed.is_err()
&& let Some(retry_handoff) = retry_handoff.as_ref()
&& let Ok(retry_authority) = retry_handoff
.unwiring_authority_for(&peer_entry.agent_identity, &peer_key)
{
let _ = self
.apply_trusted_peer_remove(
spawned_comms.as_ref(),
peer_key.clone(),
retry_authority,
)
.await;
}
}
}
}
if let Some(peer_comms) = peer_comms {
let spawned_key = Self::trusted_peer_removal_key(&spawned_spec);
if let Some((handoff, retry_handoff)) = cleanup_handoff.as_ref() {
let authority =
handoff.unwiring_authority_for(agent_identity, &spawned_key);
if let Ok(authority) = authority {
let removed = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
spawned_key.clone(),
authority,
)
.await;
if removed.is_err()
&& let Some(retry_handoff) = retry_handoff.as_ref()
&& let Ok(retry_authority) = retry_handoff
.unwiring_authority_for(agent_identity, &spawned_key)
{
let _ = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
spawned_key.clone(),
retry_authority,
)
.await;
}
}
}
}
if spawned_binding.is_some() || peer_binding.is_some() {
if let Err(error) = self.cleanup_member_machine_wiring_edge(
agent_identity,
&peer_entry.agent_identity,
"spawn_rollback_peer_only_machine_wiring_cleanup",
) {
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback could not clean generated peer-only wiring graph"
);
continue;
}
}
if let Some(spawned_binding) = spawned_binding.as_ref()
&& let Err(error) = self
.unwire_peer_only_recipient(
&spawned_spec,
Some(spawned_binding),
&peer_spec,
std::time::Duration::from_secs(2),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback failed to unwire spawned peer-only trust"
);
}
if let Some(peer_binding) = peer_binding.as_ref()
&& let Err(error) = self
.unwire_peer_only_recipient(
&peer_spec,
Some(peer_binding),
&spawned_spec,
std::time::Duration::from_secs(2),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback failed to unwire peer-only trust for spawned member"
);
}
}
}
if !cleanup_peers.is_empty() {
let rollback_inputs = cleanup_peers
.iter()
.filter(|peer_id| *peer_id != agent_identity)
.map(|peer_id| mob_dsl::MobMachineInput::UnwireMembers {
edge: mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(agent_identity),
mob_dsl::AgentIdentity::from_domain(peer_id),
),
})
.collect::<Vec<_>>();
if !rollback_inputs.is_empty() {
match self.prepare_dsl_inputs(&rollback_inputs, "spawn_rollback_wiring_cleanup") {
Ok(prepared) => {
if let Err(error) = self.commit_prepared_dsl_input(prepared) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"spawn rollback could not commit generated wiring graph cleanup"
);
}
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"spawn rollback could not clean generated wiring graph"
);
}
}
}
}
if matches!(
session_origin,
super::provisioner::ProvisionSessionOrigin::ResumedDurable
| super::provisioner::ProvisionSessionOrigin::RevivedRetired
) {
let entry = spawned_entry.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"resumed spawn rollback lost roster incarnation for '{agent_identity}'"
))
})?;
if let Some(session_id) = member_ref.bridge_session_id() {
// The failed incarnation must not consume a late routed bind
// after its durable session has been returned to idle.
self.discard_pending_routed_effects_for_session(session_id);
}
// Persist the membership rollback obligation before touching the
// only durable session. A crash or restore failure therefore
// leaves a replayable Retiring incarnation correlated with the
// exact preserved session instead of a live roster row pointing
// at a session that has already been returned to idle.
let session_id = member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain)
.ok_or_else(|| {
MobError::Internal(format!(
"resumed spawn rollback for '{agent_identity}' lost its durable session binding"
))
})?;
let prepared_retire = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::Retire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
generation: mob_dsl::Generation::from_domain(entry.generation),
releasing: None,
session_id: Some(session_id.clone()),
},
"rollback_resumed_spawn_mark_retiring",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared_retire.transition,
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
Some(session_id.clone()),
"rollback_resumed_spawn_mark_retiring",
)?;
if !retire_event_already_present {
self.append_retirement_started_event_for_entry(
entry,
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding,
Some(session_id),
false,
)
.await?;
}
self.commit_prepared_dsl_transition(prepared_retire)?;
let rollback_authority = match self
.provisioner
.capture_resumed_member_rollback_authority(member_ref)
.await
{
Ok(authority) => authority,
Err(error) => return Err(rollback.fail(error).await),
};
if let Err(error) = self
.provisioner
.restore_resumed_member(
member_ref,
operation_id,
session_origin,
&rollback_authority,
)
.await
{
return Err(rollback.fail(error).await);
}
if !retire_event_already_present {
self.append_retire_event_for_entry(entry).await?;
}
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::RecoverRosterMemberRetired {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
generation: mob_dsl::Generation::from_domain(entry.generation),
preserve_machine_topology: false,
preservation_started: false,
},
"rollback_resumed_spawn_membership",
)?;
self.delete_external_binding_overlay_for_member(agent_identity, generation)
.await?;
self.roster.write().await.remove_member(agent_identity);
self.per_spawn_external_tools
.write()
.await
.remove(agent_identity);
self.restore_diagnostics
.write()
.await
.remove(agent_identity);
return Ok(());
}
if let (Some(entry), Some(retire_input)) = (spawned_entry.as_ref(), rollback_retire_input) {
let session_id_for_route = member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain);
let retirement_started_journal_kind = if session_id_for_route.is_some() {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing
} else {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
};
// Cleanup authorization and unwiring advance topology_epoch. The
// pre-cleanup preparation proved admission but is intentionally
// not committed; prepare again against the current authority.
let prepared_retire = match self.prepare_dsl_input_transition(
retire_input,
"rollback_failed_spawn_mark_retiring_after_cleanup",
) {
Ok(prepared_retire) => prepared_retire,
Err(error) => {
tracing::warn!(
agent_identity = %agent_identity,
%error,
"spawn rollback could not mark runtime retired in DSL"
);
return Err(rollback.fail(error).await);
}
};
if let Err(error) = Self::require_member_lifecycle_journal_effect(
&prepared_retire.transition,
retirement_started_journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route,
"rollback_failed_spawn_mark_retiring_after_cleanup",
) {
return Err(rollback.fail(error).await);
}
let detach_obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared_retire.transition,
);
self.commit_prepared_dsl_transition(prepared_retire)?;
if let Err(error) = self
.realize_member_retire_ingress_detach(
entry,
detach_obligations,
"rollback_failed_spawn_request_pending_session_ingress_detach",
)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self.flush_routed_effects().await {
return Err(rollback.fail(error).await);
}
}
// Reuse disposal pipeline methods for session archive + roster removal.
let rollback_ctx = DisposalContext {
agent_identity: agent_identity.clone(),
entry: spawned_entry.clone().unwrap_or_else(|| {
let identity = AgentIdentity::from(agent_identity.as_str());
RosterEntry {
agent_identity: identity.clone(),
generation: crate::ids::Generation::INITIAL,
fence_token: crate::ids::FenceToken::new(0),
agent_runtime_id: crate::ids::AgentRuntimeId::initial(identity),
role: profile_name.clone(),
member_ref: member_ref.clone(),
runtime_mode: crate::MobRuntimeMode::TurnDriven,
peer_id: spawned_comms.as_ref().and_then(|c| c.peer_id()),
transport_public_key: spawned_comms.as_ref().and_then(|c| c.public_key()),
wired_to: std::collections::BTreeSet::new(),
external_peer_specs: std::collections::BTreeMap::new(),
labels: std::collections::BTreeMap::new(),
kickoff: None,
effective_profile_override: None,
effective_model_override: None,
}
}),
retiring_key: spawned_comms.as_ref().and_then(|c| c.public_key()),
retiring_comms: None,
retiring_spec: None,
preserve_machine_topology: false,
machine_wired_peer_identities: BTreeSet::new(),
trust_unwire_authority_by_peer: BTreeMap::new(),
historical_trust_unwire_authorities_by_peer: BTreeMap::new(),
};
let disposal = match self.dispose_archive_session(&rollback_ctx).await {
Ok(disposal) => disposal,
Err(error) => return Err(rollback.fail(error).await),
};
let placed = super::member_runtime_is_host_owned(
self.dsl_authority.state(),
&rollback_ctx.entry.agent_identity,
);
if !placed && let Some(binding) = Self::runtime_binding_for_entry(&rollback_ctx.entry) {
if let Err(error) = self
.record_remote_member_runtime_retired(&rollback_ctx.entry)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self
.revoke_supervisor_for_retiring_entry(
&rollback_ctx.entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
{
return Err(rollback
.fail(MobError::RetirementTopologyIncomplete(format!(
"failed-spawn rollback could not revoke peer-only supervisor authority: {error}"
)))
.await);
}
}
if let Err(error) = self
.delete_external_binding_overlay_for_member(
&rollback_ctx.entry.agent_identity,
rollback_ctx.entry.generation,
)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self
.observe_member_retirement_archived(&rollback_ctx, disposal)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self
.delete_retired_placed_member_carriers(&rollback_ctx.agent_identity)
.await
{
return Err(rollback.fail(error).await);
}
self.dispose_remove_from_roster(&rollback_ctx, true).await;
Ok(())
}
/// Resolve profile-declared rust tool bundles to a dispatcher.
fn external_tools_for_profile(
&self,
profile: &crate::profile::Profile,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
) -> Result<Option<Arc<dyn AgentToolDispatcher>>, MobError> {
let default_tools = self
.default_external_tools_provider
.as_ref()
.and_then(|p| p());
compose_external_tools_for_profile(
profile,
&self.tool_bundles,
self.mob_handle_for_tools(),
default_tools,
per_spawn_external_tools,
None,
)
}
async fn retirement_started_event_exists(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<bool, MobError> {
let key = format!("{}:{}", agent_identity, generation.get());
Ok(self
.retirement_started_event_index
.read()
.await
.contains(&key))
}
async fn preserved_respawn_topology_event_exists(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> bool {
let key = format!("{}:{}", agent_identity, generation.get());
self.preserved_respawn_topology_event_index
.read()
.await
.contains(&key)
}
async fn append_retirement_started_event_for_entry(
&self,
entry: &RosterEntry,
journal_kind: mob_dsl::MobLifecycleJournalKind,
session_id: Option<mob_dsl::SessionId>,
preserve_machine_topology: bool,
) -> Result<(), MobError> {
let parse_session = |session_id: mob_dsl::SessionId| {
SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"generated retirement-start journal carried invalid session id '{}': {error}",
session_id.0
))
})
};
let session_id = session_id.map(parse_session).transpose()?;
let releasing = match journal_kind {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing => {
Some(session_id.clone().ok_or_else(|| {
MobError::Internal(
"generated releasing retirement-start journal omitted session_id"
.to_string(),
)
})?)
}
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding => {
if session_id.is_none() {
return Err(MobError::Internal(
"generated preserving retirement-start journal omitted session_id"
.to_string(),
));
}
None
}
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly => {
if session_id.is_some() {
return Err(MobError::Internal(
"generated peer-only retirement-start journal carried session_id"
.to_string(),
));
}
None
}
other => {
return Err(MobError::Internal(format!(
"generated retirement-start append received non-start journal kind {other:?}"
)));
}
};
let machine_wired_peer_identities =
self.machine_member_wired_peer_identities_for(&entry.agent_identity);
let retiring_peer_endpoint = match self.machine_member_peer_spec_for(
&entry.agent_identity,
"append retirement-start peer endpoint",
)? {
Some(endpoint) => Some(endpoint),
None => match self
.roster_member_peer_spec_for(entry, "append retirement-start peer endpoint")?
{
Some(endpoint) => Some(endpoint),
None => {
self.retained_member_peer_spec_from_wired_peer_trust(
entry,
&machine_wired_peer_identities,
"append retirement-start peer endpoint",
)
.await?
}
},
};
let has_retained_trust_topology = !machine_wired_peer_identities.is_empty()
|| !self
.machine_external_peer_edges_for(&entry.agent_identity)
.is_empty();
if has_retained_trust_topology && retiring_peer_endpoint.is_none() {
return Err(MobError::RetirementTopologyIncomplete(format!(
"retirement-start for '{}' cannot persist wired cleanup authority without its exact peer endpoint",
entry.agent_identity
)));
}
let desired = MobEventKind::MemberRetirementStarted {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
generation: entry.generation,
role: entry.role.clone(),
releasing,
session_id,
retiring_peer_endpoint,
preserve_machine_topology,
};
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await;
if let Err(error) = append {
// Resolve wrote-then-error only against this mob's current epoch;
// another mob may use the same identity/generation in the shared
// event store.
let all_events = self.events.replay_all().await?;
let events = all_events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
if !events[epoch_start..]
.iter()
.any(|event| event.kind == desired)
{
return Err(MobError::from(error));
}
}
let key = format!("{}:{}", entry.agent_identity, entry.generation.get());
self.retirement_started_event_index
.write()
.await
.insert(key.clone());
if preserve_machine_topology {
self.preserved_respawn_topology_event_index
.write()
.await
.insert(key);
}
Ok(())
}
async fn retire_event_exists(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<bool, MobError> {
let key = Self::retire_event_key(agent_identity, generation);
let index = self.retired_event_index.read().await;
Ok(index.contains(&key))
}
async fn append_retire_event_for_entry(&mut self, entry: &RosterEntry) -> Result<(), MobError> {
let desired = MobEventKind::MemberRetired {
agent_identity: entry.agent_identity.clone(),
generation: entry.generation,
role: entry.role.clone(),
};
let append = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await;
if let Err(error) = append {
// The store may durably append and then lose the acknowledgement.
// Reconcile only the exact terminal carrier in this mob's current
// reset epoch; a same-identity event from another mob or an older
// epoch cannot authorize this retirement completion.
let all_events = match self.events.replay_all().await {
Ok(events) => events,
Err(reconcile_error) => {
self.durable_uncertainty_fail_stop = true;
return Err(MobError::Internal(format!(
"MemberRetired append for '{}' failed ({error}) and durable reconciliation remained unreadable; actor is fail-stopping for cold recovery: {reconcile_error}",
entry.agent_identity
)));
}
};
let events = all_events
.iter()
.filter(|event| event.mob_id == self.definition.id)
.collect::<Vec<_>>();
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |index| index + 1);
if !events[epoch_start..]
.iter()
.any(|event| event.kind == desired)
{
return Err(MobError::from(error));
}
}
let key = Self::retire_event_key(&entry.agent_identity, entry.generation);
self.retired_event_index.write().await.insert(key);
Ok(())
}
/// Get the comms runtime for a session, if available.
async fn provisioner_comms(&self, member_ref: &MemberRef) -> Option<Arc<dyn CoreCommsRuntime>> {
self.provisioner.comms_runtime(member_ref).await
}
fn trusted_peer_descriptor_from_machine_endpoint(
endpoint: &mob_dsl::ExternalPeerEndpoint,
) -> Result<TrustedPeerDescriptor, MobError> {
TrustedPeerDescriptor::unsigned_with_pubkey(
endpoint.name.0.clone(),
&endpoint.peer_id.0,
endpoint.signing_key.0,
&endpoint.address.0,
)
.map_err(|error| {
MobError::WiringError(format!(
"MobMachine external peer edge has invalid descriptor '{}': {error}",
endpoint.name.0
))
})
}
fn machine_restore_wiring_plan(
&self,
agent_identity: &AgentIdentity,
) -> Result<RestoreWiringPlan, MobError> {
let local =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let mut local_peers = Vec::new();
for edge in &self.dsl_authority.state().wiring_edges {
let peer = if edge.a == local {
Some(&edge.b)
} else if edge.b == local {
Some(&edge.a)
} else {
None
};
if let Some(peer) = peer {
local_peers.push(AgentIdentity::from(peer.0.as_str()));
}
}
local_peers.sort();
local_peers.dedup();
let mut external_peers = Vec::new();
for edge in self
.dsl_authority
.state()
.external_peer_edges_by_key
.values()
{
if edge.local == local {
external_peers.push(Self::trusted_peer_descriptor_from_machine_endpoint(
&edge.endpoint,
)?);
}
}
external_peers.sort_by(|a, b| {
a.name
.as_str()
.cmp(b.name.as_str())
.then_with(|| a.peer_id.to_string().cmp(&b.peer_id.to_string()))
});
external_peers.dedup_by(|a, b| a.name == b.name && a.peer_id == b.peer_id);
Ok(RestoreWiringPlan {
local_peers,
external_peers,
})
}
async fn sender_runtime_for_entry(
&self,
entry: &RosterEntry,
) -> Option<Arc<dyn CoreCommsRuntime>> {
if super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity) {
// A placed member has no controller-local sender runtime. Its
// trust rows are realized through host route obligations; using
// the supervisor runtime here would impersonate the member, while
// following a projected session id would target the wrong host.
return None;
}
if let Some(comms) = self.provisioner_comms(&entry.member_ref).await {
return Some(comms);
}
if matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
) {
return Some(self.supervisor_bridge.runtime_core().await);
}
None
}
/// Generate the comms name for a roster entry.
fn comms_name_for(&self, entry: &RosterEntry) -> Result<String, MobError> {
render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
entry.agent_identity.as_str(),
)
}
async fn local_wiring_spec(
&self,
entry: &RosterEntry,
member_ref: &MemberRef,
comms: &Arc<dyn CoreCommsRuntime>,
comms_name: &str,
public_key: &str,
) -> Result<TrustedPeerDescriptor, MobError> {
// Build the trusted-peer spec from the MACHINE-derived member ref (not
// the roster's stored copy), keeping `entry` only for diagnostics.
let mut spec = self
.provisioner
.trusted_peer_spec(member_ref, comms_name, public_key)
.await?;
if let Some(address) = comms.advertised_address() {
spec.address = PeerAddress::parse(&address).map_err(|error| {
MobError::WiringError(format!(
"invalid advertised comms address for '{}': {error}",
entry.agent_identity
))
})?;
}
Ok(spec)
}
async fn resolve_wiring_endpoint(
&self,
entry: &RosterEntry,
context: &'static str,
) -> Result<WiringEndpoint, MobError> {
let active_placement =
self.ensure_placed_carrier_binding_active(&entry.agent_identity, context)?;
let comms_name = self.comms_name_for(entry)?;
let member_ref = self.machine_member_ref_for_behavior(entry, context)?;
// Placement owns the physical lane before the transport ref is
// inspected. `machine_member_ref_for_behavior` intentionally projects
// the machine's remote session id for fencing, so asking the composite
// provisioner for comms first would submit a foreign id to the
// controller-local session backend and could misclassify this endpoint
// as Local.
if let Some(host) = active_placement {
let spec = self
.machine_member_peer_spec_for(&entry.agent_identity, context)?
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: placed member '{}' has no machine-recorded peer endpoint",
entry.agent_identity
))
})?;
return Ok(WiringEndpoint::Placed {
identity: entry.agent_identity.clone(),
host,
spec,
});
}
if let Some(comms) = self.provisioner_comms(&member_ref).await {
let public_key = comms.public_key().ok_or_else(|| {
MobError::WiringError(format!(
"{context} requires public key for '{}'",
entry.agent_identity
))
})?;
let spec = self
.local_wiring_spec(entry, &member_ref, &comms, &comms_name, &public_key)
.await?;
return Ok(WiringEndpoint::Local {
entry: Box::new(entry.clone()),
comms,
spec,
comms_name,
});
}
match &member_ref {
MemberRef::BackendPeer { .. } => {
let binding =
Self::runtime_binding_for_member_ref(&member_ref).ok_or_else(|| {
MobError::WiringError(format!(
"{context} requires external runtime binding for '{}'",
entry.agent_identity
))
})?;
let spec = Self::peer_only_spec_for_binding(&binding, context)?;
Ok(WiringEndpoint::PeerOnly { spec, binding })
}
MemberRef::Session { .. } => Err(MobError::WiringError(format!(
"{context} requires comms runtime for '{}'",
entry.agent_identity
))),
}
}
/// Retirement-only endpoint resolution. An exact confirmed revoke means
/// the dormant member runtime/trust store is gone, but its retained
/// machine endpoint remains the generated authority for cleaning surviving
/// local or active-host trust rows. No ordinary behavior may use this lane.
async fn resolve_wiring_endpoint_for_retirement(
&self,
entry: &RosterEntry,
) -> Result<WiringEndpoint, MobError> {
if let Some(host) = self.confirmed_revoked_placed_host(&entry.agent_identity) {
let spec = self
.machine_member_peer_spec_for(
&entry.agent_identity,
"confirmed-revoked retirement endpoint",
)?
.ok_or_else(|| {
MobError::RetirementTopologyIncomplete(format!(
"confirmed-revoked placed member '{}' has no retained machine peer endpoint",
entry.agent_identity
))
})?;
return Ok(WiringEndpoint::Placed {
identity: entry.agent_identity.clone(),
host,
spec,
});
}
self.resolve_wiring_endpoint(entry, "unwire retirement")
.await
}
/// Notify a peer that a new peer was added.
///
/// Sends a `PeerRequest` with intent `mob.peer_added` FROM `sender_comms`
/// TO the peer identified by `recipient_comms_name`. The params contain
/// the new peer's identity and role.
///
/// REQ-MOB-010/011: Notification is required for successful wiring.
async fn notify_peer_added(
&self,
sender_comms: &Arc<dyn CoreCommsRuntime>,
recipient_spec: &TrustedPeerDescriptor,
new_peer_id: &AgentIdentity,
new_peer_entry: &RosterEntry,
) -> Result<(), MobError> {
let peer_description = self
.definition
.resolve_profile(&new_peer_entry.role, self.realm_profile_store.as_ref())
.await
.map(|p| p.peer_description)
.unwrap_or_default();
let new_peer_spec = match self
.resolve_wiring_endpoint(new_peer_entry, "notify_peer_added")
.await?
{
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. }
| WiringEndpoint::Placed { spec, .. } => spec,
};
let peer_route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
// K15: lifecycle params are the typed wire contract
// (`CommsPeerLifecycleParams`) — `peer_spec` carries the canonical
// typed peer identity; no shadow `peer_name`/`peer_id`/`address`
// mirror fields.
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: new_peer_id.as_str().to_string(),
role: Some(new_peer_entry.role.as_str().to_string()),
description: Some(peer_description),
peer_spec: Some(super::bridge_protocol::BridgePeerSpec::from(new_peer_spec)),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{new_peer_id}': {error}"
))
})?;
let cmd = CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerAdded,
params,
};
sender_comms.send(cmd).await?;
Ok(())
}
async fn notify_peer_event(
&self,
intent: &'static str,
recipient_spec: &TrustedPeerDescriptor,
other_peer_id: &AgentIdentity,
other_peer_entry: &RosterEntry,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
let other_peer_spec = match self
.resolve_wiring_endpoint(other_peer_entry, "notify_peer_event")
.await?
{
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. }
| WiringEndpoint::Placed { spec, .. } => spec,
};
self.notify_peer_event_with_spec(
intent,
recipient_spec,
other_peer_id,
other_peer_entry,
&other_peer_spec,
sender_comms,
)
.await
}
async fn notify_peer_event_with_spec(
&self,
intent: &'static str,
recipient_spec: &TrustedPeerDescriptor,
other_peer_id: &AgentIdentity,
other_peer_entry: &RosterEntry,
other_peer_spec: &TrustedPeerDescriptor,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
let peer_route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
// K15: lifecycle params are the typed wire contract
// (`CommsPeerLifecycleParams`) — `peer_spec` carries the canonical
// typed peer identity; no shadow `peer_name`/`peer_id`/`address`
// mirror fields.
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: other_peer_id.as_str().to_string(),
role: Some(other_peer_entry.role.as_str().to_string()),
description: None,
peer_spec: Some(super::bridge_protocol::BridgePeerSpec::from(
other_peer_spec.clone(),
)),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{other_peer_id}': {error}"
))
})?;
let cmd = match intent {
"mob.peer_retired" => CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerRetired,
params,
},
"mob.peer_unwired" => CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerUnwired,
params,
},
_ => CommsCommand::PeerRequest {
objective_id: None,
to: peer_route,
intent: intent.to_string(),
params,
blocks: None,
content_taint: None,
handling_mode: meerkat_core::types::HandlingMode::Queue,
stream: meerkat_core::comms::InputStreamMode::None,
},
};
sender_comms.send(cmd).await?;
Ok(())
}
async fn notify_kickoff_event(
&self,
agent_identity: &AgentIdentity,
intent: &'static str,
) -> Result<(), MobError> {
let (entry, wired_peers) = {
let machine_peer_identities = self
.machine_wired_peer_identities_for(&AgentIdentity::from(agent_identity.as_str()));
let roster = self.roster.read().await;
let Some(entry) = roster.get(agent_identity).cloned() else {
return Ok(());
};
let wired_peers: Vec<AgentIdentity> = machine_peer_identities
.iter()
.filter_map(|id| roster.get_by_identity(id).map(|e| e.agent_identity.clone()))
.collect();
(entry, wired_peers)
};
let placed =
super::member_runtime_is_host_owned(self.dsl_authority.state(), &entry.agent_identity);
if placed {
return if wired_peers.is_empty() {
Ok(())
} else {
Err(MobError::WiringError(format!(
"kickoff lifecycle notice from placed member '{}' has no remote-sender realization",
entry.agent_identity
)))
};
}
let sender_comms = self.provisioner_comms(&entry.member_ref).await;
let effects = MobRuntimeBridgeAuthority::plan_lifecycle_notice(
sender_comms.is_some()
|| matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
),
&wired_peers,
intent,
);
let Some(sender_comms) = self.sender_runtime_for_entry(&entry).await else {
return Ok(());
};
for effect in effects {
let MobRuntimeBridgeEffect::DeliverLifecycleNotice { peer_id, intent } = effect;
let recipient_entry = {
let roster = self.roster.read().await;
roster.get(&peer_id).cloned()
};
let Some(recipient_entry) = recipient_entry else {
continue;
};
let recipient_spec = match self
.resolve_wiring_endpoint(&recipient_entry, "notify_kickoff_event")
.await?
{
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
WiringEndpoint::Placed { .. } => {
return Err(MobError::WiringError(format!(
"kickoff lifecycle notice to placed member '{}' has no remote-recipient realization",
recipient_entry.agent_identity
)));
}
};
self.notify_peer_event(
intent,
&recipient_spec,
agent_identity,
&entry,
&sender_comms,
)
.await?;
}
Ok(())
}
/// Notify a peer that another peer was retired from the mob.
async fn notify_peer_retired(
&self,
recipient_spec: &TrustedPeerDescriptor,
retired_id: &AgentIdentity,
retired_entry: &RosterEntry,
retired_spec: &TrustedPeerDescriptor,
retiring_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
self.notify_peer_event_with_spec(
"mob.peer_retired",
recipient_spec,
retired_id,
retired_entry,
retired_spec,
retiring_comms,
)
.await
}
/// Notify a peer that another peer was unwired (trust link removed).
async fn notify_peer_unwired(
&self,
recipient_spec: &TrustedPeerDescriptor,
unwired_id: &AgentIdentity,
unwired_entry: &RosterEntry,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
self.notify_peer_event(
"mob.peer_unwired",
recipient_spec,
unwired_id,
unwired_entry,
sender_comms,
)
.await
}
}
struct ClosedRuntimeEffectRefusal {
kind: crate::error::RuntimeEffectKind,
session_id: SessionId,
refusal_code: String,
reason: String,
broken_member: Option<AgentIdentity>,
}
impl ClosedRuntimeEffectRefusal {
fn into_mob_error(self) -> MobError {
MobError::RuntimeEffectRefused {
kind: self.kind,
session_id: self.session_id,
refusal_code: self.refusal_code,
reason: self.reason,
}
}
}
/// Extract the one machine-owned terminal classification emitted by a
/// generated refusal-feedback input. Absence or multiplicity is a composition
/// contract break; the shell never infers a fallback class from the original
/// dispatch error.
fn closed_runtime_effect_refusal_from_transition(
transition: &mob_dsl::MobMachineTransition,
) -> Result<ClosedRuntimeEffectRefusal, MobError> {
let mut closed = None;
for effect in transition.effects() {
let candidate = match effect {
mob_dsl::MobMachineEffect::RuntimeBindingRefusalClassified {
agent_identity,
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeBinding,
session_id,
refusal_code,
reason,
Some(AgentIdentity::from(agent_identity.0.as_str())),
)),
mob_dsl::MobMachineEffect::RuntimeIngressRefusalClassified {
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeIngress,
session_id,
refusal_code,
reason,
None,
)),
mob_dsl::MobMachineEffect::RuntimeRetireRefusalClassified {
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeRetire,
session_id,
refusal_code,
reason,
None,
)),
_ => None,
};
let Some((kind, session_id, refusal_code, reason, broken_member)) = candidate else {
continue;
};
if closed.is_some() {
return Err(MobError::Internal(
"MobMachine refusal feedback emitted multiple runtime refusal classifications"
.to_owned(),
));
}
let session_id = SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"MobMachine refusal classification carried invalid session id `{}`: {error}",
session_id.0
))
})?;
closed = Some(ClosedRuntimeEffectRefusal {
kind,
session_id,
refusal_code: refusal_code.clone(),
reason: reason.clone(),
broken_member,
});
}
closed.ok_or_else(|| {
MobError::Internal(
"MobMachine refusal feedback emitted no runtime refusal classification".to_owned(),
)
})
}
/// The bridge-session scope of a routed composition effect, in DSL form.
/// Every effect variant the MobMachine routes to MeerkatMachine consumers
/// carries its target session. Refusal closure is selected by generated route
/// metadata, not by reverse-looking up this session in shell state.
fn routed_effect_session_scope_dsl(
effect: &mob_dsl::MobMachineEffect,
) -> Option<&mob_dsl::SessionId> {
match effect {
mob_dsl::MobMachineEffect::RequestRuntimeBinding { session_id, .. }
| mob_dsl::MobMachineEffect::RequestRuntimeRetire { session_id, .. }
| mob_dsl::MobMachineEffect::RequestRuntimeDestroy { session_id }
| mob_dsl::MobMachineEffect::RequestRuntimeIngress { session_id, .. } => Some(session_id),
_ => None,
}
}
/// Domain-typed bridge-session scope of a routed composition effect.
fn routed_effect_session_scope(effect: &mob_dsl::MobMachineEffect) -> Option<SessionId> {
routed_effect_session_scope_dsl(effect).and_then(|id| SessionId::parse(&id.0).ok())
}
fn revival_error_means_session_already_live(
error: &MobError,
bridge_session_id: &SessionId,
) -> bool {
matches!(
error,
MobError::SessionError(meerkat_core::service::SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
)) if session_id == bridge_session_id
)
}
fn recovered_single_target_failure(
terminals: &BTreeMap<(StepId, u64), Option<String>>,
step_target_counts: &BTreeMap<StepId, u32>,
max_step_retries: u32,
) -> Option<(StepId, String)> {
if terminals.len() != 1 || max_step_retries != 0 {
return None;
}
let ((step_id, _), outcome) = terminals.iter().next()?;
if step_target_counts.get(step_id).copied() != Some(1) {
return None;
}
outcome.clone().map(|reason| (step_id.clone(), reason))
}
#[cfg(test)]
mod autonomous_stop_planning_tests {
use super::{
AutonomousStopPhase, MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS, advance_rotating_cursor,
autonomous_stop_phase, disposal_uses_host_release_authority, lifecycle_origin_fenced,
mob_dsl,
};
#[test]
fn bounded_rotation_advances_past_each_full_window() {
assert_eq!(MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS, 8);
let host_count = 100;
let first = advance_rotating_cursor(
host_count,
0,
Some(MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS - 1),
);
let second = advance_rotating_cursor(
host_count,
first,
Some(MAX_CONCURRENT_AUTONOMOUS_STOP_INTERRUPTS - 1),
);
assert_eq!(first, 8, "the next window must not overlap 7/8 hosts");
assert_eq!(second, 16);
assert_eq!(advance_rotating_cursor(host_count, 16, Some(11)), 28);
}
#[test]
fn placed_kickoff_custody_precedes_interrupt_phase() {
assert_eq!(
autonomous_stop_phase(1, 0),
AutonomousStopPhase::WaitForPlacedKickoffCleanup
);
assert_eq!(
autonomous_stop_phase(0, 1),
AutonomousStopPhase::WaitForPlacedKickoffCleanup
);
assert_eq!(
autonomous_stop_phase(0, 0),
AutonomousStopPhase::DriveInterrupts
);
}
#[test]
fn placed_disposal_uses_release_instead_of_interrupt_barrier() {
assert!(disposal_uses_host_release_authority(true));
assert!(!disposal_uses_host_release_authority(false));
}
#[test]
fn typed_lifecycle_intent_fences_actor_origin_while_public_phase_is_running() {
let mut state = mob_dsl::MobMachineState::default();
assert!(!lifecycle_origin_fenced(&state));
state.placed_completion_lifecycle_quiescing = true;
state.placed_completion_lifecycle_intent =
Some(mob_dsl::PlacedCompletionLifecycleIntentKind::Stop);
assert!(lifecycle_origin_fenced(&state));
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod fence_token_allocator_tests {
use super::*;
#[test]
fn final_fence_tokens_are_issued_once_then_preview_and_allocation_fail() {
let counter = std::sync::atomic::AtomicU64::new(u64::MAX - 1);
assert_eq!(
MobActor::preview_fence_token(&counter).unwrap().get(),
u64::MAX - 1
);
assert_eq!(
MobActor::allocate_fence_token(&counter).unwrap().get(),
u64::MAX - 1
);
assert_eq!(
MobActor::preview_fence_token(&counter).unwrap().get(),
u64::MAX
);
assert_eq!(
MobActor::allocate_fence_token(&counter).unwrap().get(),
u64::MAX
);
assert_eq!(
counter.load(std::sync::atomic::Ordering::Relaxed),
0,
"allocator stores the runtime-only exhaustion sentinel"
);
assert!(MobActor::preview_fence_token(&counter).is_err());
assert!(MobActor::allocate_fence_token(&counter).is_err());
}
}
#[cfg(test)]
mod member_live_cleanup_tests {
use super::super::bridge_protocol::BridgeRejectionCause;
use super::MobActor;
use crate::MobError;
use crate::event::MemberRef;
use meerkat_core::types::SessionId;
fn rejection(cause: BridgeRejectionCause, reason: &str) -> MobError {
MobError::BridgeCommandRejected {
cause,
reason: reason.to_string(),
}
}
#[test]
fn typed_absent_live_substrate_is_lifecycle_absence_proof() {
for cause in [
BridgeRejectionCause::LiveChannelNotFound,
BridgeRejectionCause::LiveTransportUnavailable,
] {
let error = rejection(cause, "authenticated member-host reply");
assert!(MobActor::member_live_cleanup_proves_absent(&error));
assert!(MobActor::member_live_status_proves_absent(&error));
}
}
#[test]
fn transient_or_stringly_live_failures_are_not_absence_proof() {
for error in [
rejection(
BridgeRejectionCause::Unavailable,
"member host serves no live substrate",
),
rejection(
BridgeRejectionCause::Internal,
"member host serves no live substrate",
),
MobError::BridgeRequestTimedOut {
request_envelope_id: "member-live-status".to_string(),
timeout_ms: 15_000,
},
] {
assert!(!MobActor::member_live_cleanup_proves_absent(&error));
assert!(!MobActor::member_live_status_proves_absent(&error));
}
}
#[test]
fn shutdown_live_cleanup_selects_only_controller_owned_session_members() {
assert!(MobActor::member_live_ref_is_controller_local(
&MemberRef::Session {
session_id: SessionId::new(),
}
));
for session_id in [None, Some(SessionId::new())] {
assert!(!MobActor::member_live_ref_is_controller_local(
&MemberRef::BackendPeer {
peer_id: "external-peer".to_string(),
address: "tcp://127.0.0.1:1".to_string(),
pubkey: [0; 32],
bootstrap_token: None,
session_id,
}
));
}
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod runtime_observation_tests {
use super::super::remote_flow_ticket::{
HOST_REVOKED_STEP_FAILURE_REASON, obligation_from_event,
};
use super::{
ExactRemoteTurnResidency, MobActor, RemoteTurnCustodyPhase, foreign_runtime_observation,
mob_dsl, pending_host_bind_anchors, pending_host_revoke_anchors,
recovered_single_target_failure,
require_host_authority_anchors_clear_for_supervisor_rotation,
validate_host_authority_anchor_disjointness,
validate_host_authority_anchor_epoch_boundaries,
};
use crate::event::{
MobEvent, MobEventKind, RemoteHostBindRequestEvent, RemoteTurnObligationEvent,
};
use crate::ids::{AgentIdentity, FenceToken, Generation, MobId, RunId, StepId};
use crate::run::{
MobRunRemoteTurnReceipt, MobRunRemoteTurnReceiptOutcome,
MobRunRemoteTurnTrackedCancelTerminal,
};
use chrono::Utc;
use std::collections::{BTreeMap, BTreeSet};
#[test]
fn promotion_cleanup_plan_terminalizes_all_old_phases_and_leaves_g2_private_rows() {
let identity = AgentIdentity::from("worker");
let host = mob_dsl::HostId("host".to_string());
let session = mob_dsl::SessionId("session".to_string());
let old_event = RemoteTurnObligationEvent {
agent_identity: identity.clone(),
host_id: host.0.clone(),
host_binding_generation: 1,
member_session_id: session.0.clone(),
generation: Generation::new(1),
fence_token: FenceToken::new(7),
dispatch_sequence: 1,
input_id: "old-input".to_string(),
run_id: RunId::new(),
step_id: StepId::from("step"),
};
let pending_old = obligation_from_event(&old_event);
let committed_old = mob_dsl::RemoteTurnObligation {
dispatch_sequence: 2,
input_id: mob_dsl::InputId("committed-old".to_string()),
..pending_old.clone()
};
let resolved_old = mob_dsl::RemoteTurnObligation {
dispatch_sequence: 3,
input_id: mob_dsl::InputId("resolved-old".to_string()),
..pending_old.clone()
};
let g2 = mob_dsl::RemoteTurnObligation {
host_binding_generation: 2,
input_id: mob_dsl::InputId("g2-input".to_string()),
dispatch_sequence: 4,
..pending_old.clone()
};
let pending = BTreeSet::from([pending_old.clone(), g2.clone()]);
let committed = BTreeSet::from([committed_old.clone()]);
let resolved = BTreeSet::from([resolved_old.clone()]);
let plan = MobActor::remote_turn_custody_cleanup_plan(
&pending,
&committed,
&resolved,
ExactRemoteTurnResidency {
agent_identity: &identity,
host_id: &host,
host_binding_generation: 1,
member_session_id: &session,
generation: 1,
fence_token: 7,
},
);
let selected = plan
.iter()
.map(|cleanup| (cleanup.obligation.input_id.0.as_str(), cleanup.phase))
.collect::<Vec<_>>();
assert_eq!(
selected,
vec![
("old-input", RemoteTurnCustodyPhase::Pending),
("committed-old", RemoteTurnCustodyPhase::Committed),
("resolved-old", RemoteTurnCustodyPhase::Resolved),
]
);
let pending_terminals = plan
.iter()
.filter(|cleanup| cleanup.phase == RemoteTurnCustodyPhase::Pending)
.map(|cleanup| {
(
cleanup.obligation.input_id.0.clone(),
super::super::remote_flow_ticket::REMATERIALIZED_STEP_FAILURE_REASON,
)
})
.collect::<Vec<_>>();
assert!(
pending_terminals
== vec![(
"old-input".to_string(),
super::super::remote_flow_ticket::REMATERIALIZED_STEP_FAILURE_REASON,
)],
"only Pending G1 custody receives the synthetic rematerialization terminal"
);
// Fake private run-store rows exercise the production plan's privacy
// keys: every G1 phase is removed and the same-member G2 sibling is
// untouched.
let mut intents = BTreeSet::from([
"old-input".to_string(),
"committed-old".to_string(),
"resolved-old".to_string(),
"g2-input".to_string(),
]);
let mut receipts = intents.clone();
for cleanup in &plan {
intents.remove(&cleanup.obligation.input_id.0);
receipts.remove(&cleanup.obligation.input_id.0);
}
assert_eq!(intents, BTreeSet::from(["g2-input".to_string()]));
assert_eq!(receipts, BTreeSet::from(["g2-input".to_string()]));
}
#[test]
fn recovered_failure_promotion_does_not_bypass_fanout_collection_policy() {
let step = StepId::from("fanout");
let single_failure = std::collections::BTreeMap::from([(
(step.clone(), 1),
Some("typed single-target failure".to_string()),
)]);
let single_target_count = BTreeMap::from([(step.clone(), 1)]);
assert_eq!(
recovered_single_target_failure(&single_failure, &single_target_count, 0),
Some((step.clone(), "typed single-target failure".to_string())),
"one terminal target has unambiguous failure semantics"
);
assert_eq!(
recovered_single_target_failure(&single_failure, &single_target_count, 1),
None,
"a persisted failed attempt does not prove an unconsumed retry budget was exhausted"
);
let incomplete_fanout = std::collections::BTreeMap::from([(
(step.clone(), 1),
Some("first target failed before sibling terminal".to_string()),
)]);
let fanout_target_count = BTreeMap::from([(step.clone(), 2)]);
assert_eq!(
recovered_single_target_failure(&incomplete_fanout, &fanout_target_count, 0),
None,
"one observed carrier does not prove a one-target dispatch"
);
let fanout_any = std::collections::BTreeMap::from([
((step.clone(), 1), Some("one target failed".to_string())),
((step.clone(), 2), None),
]);
assert_eq!(
recovered_single_target_failure(&fanout_any, &fanout_target_count, 0),
None,
"FanOut Any may succeed from the completed target"
);
let fanout_quorum = std::collections::BTreeMap::from([
((step.clone(), 1), Some("first target failed".to_string())),
((step, 2), Some("second target failed".to_string())),
]);
assert_eq!(
recovered_single_target_failure(&fanout_quorum, &fanout_target_count, 0),
None,
"FanOut Quorum requires the collection-policy reducer, not one target verdict"
);
}
#[test]
fn remote_turn_carriers_are_scoped_to_their_mob_before_epoch_and_conflict_checks() {
let mob_a = MobId::from("carrier-mob-a");
let mob_b = MobId::from("carrier-mob-b");
let obligation = RemoteTurnObligationEvent {
agent_identity: AgentIdentity::from("worker"),
host_id: "host".to_string(),
host_binding_generation: 1,
member_session_id: "session".to_string(),
generation: Generation::new(1),
fence_token: FenceToken::new(7),
dispatch_sequence: 1,
input_id: "input-a".to_string(),
run_id: RunId::new(),
step_id: StepId::from("step"),
};
let events = vec![MobEvent {
cursor: 1,
timestamp: Utc::now(),
mob_id: mob_a,
kind: MobEventKind::RemoteTurnObligationRecorded {
obligation: obligation.clone(),
},
}];
let identical_other_mob = MobEventKind::RemoteTurnObligationRecorded {
obligation: obligation.clone(),
};
assert!(
!MobActor::remote_turn_carrier_present(&events, &mob_b, &identical_other_mob)
.expect("other mob's identical carrier is not a match")
);
let mut conflicting = obligation;
conflicting.input_id = "input-b".to_string();
let conflicting_other_mob = MobEventKind::RemoteTurnObligationRecorded {
obligation: conflicting,
};
assert!(
!MobActor::remote_turn_carrier_present(&events, &mob_b, &conflicting_other_mob)
.expect("other mob's same-sequence carrier is not a conflict")
);
}
#[test]
fn disposed_remote_turn_replay_requires_exact_terminal_payload_and_current_epoch() {
let mob_id = MobId::from("disposed-replay-mob");
let obligation = RemoteTurnObligationEvent {
agent_identity: AgentIdentity::from("worker"),
host_id: "host".to_string(),
host_binding_generation: 1,
member_session_id: "session".to_string(),
generation: Generation::new(1),
fence_token: FenceToken::new(7),
dispatch_sequence: 1,
input_id: "019f0000-0000-7000-8000-000000000001".to_string(),
run_id: RunId::new(),
step_id: StepId::from("step"),
};
let receipt = MobRunRemoteTurnReceipt {
obligation: obligation.clone(),
outcome: MobRunRemoteTurnReceiptOutcome::Failed {
reason: HOST_REVOKED_STEP_FAILURE_REASON.to_string(),
no_effect_proof: None,
},
};
let terminal = MobActor::remote_turn_receipt_terminal_kind(&receipt);
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let disposed = MobEventKind::RemoteTurnOutcomeDisposed {
obligation: obligation.clone(),
};
assert!(
!MobActor::remote_turn_disposed_replay_matches(
&[event(1, terminal.clone())],
&mob_id,
&receipt,
),
"a crash after the public failure but before Dispose must retain custody for replay"
);
let exact_events = vec![event(1, terminal.clone()), event(2, disposed.clone())];
assert!(MobActor::remote_turn_disposed_replay_matches(
&exact_events,
&mob_id,
&receipt,
));
let tracked_cancel = MobRunRemoteTurnReceipt {
obligation: obligation.clone(),
outcome: MobRunRemoteTurnReceiptOutcome::TrackedInputCancel {
reason: "host cancelled recovered turn".to_string(),
terminal: MobRunRemoteTurnTrackedCancelTerminal::Cancelled,
},
};
tracked_cancel
.validate_shape_for(&obligation.run_id)
.expect("tracked-cancel receipt is durable replay material");
let tracked_terminal = MobActor::remote_turn_receipt_terminal_kind(&tracked_cancel);
assert!(MobActor::remote_turn_disposed_replay_matches(
&[event(1, tracked_terminal), event(2, disposed.clone())],
&mob_id,
&tracked_cancel,
));
let replayed_events = vec![
event(1, terminal.clone()),
event(2, terminal.clone()),
event(3, disposed.clone()),
event(4, disposed.clone()),
];
assert!(
MobActor::remote_turn_disposed_replay_matches(&replayed_events, &mob_id, &receipt,),
"replayed revoke convergence remains exact and idempotent even when no live ticket owns the receipt"
);
let mut wrong_reason = receipt.clone();
wrong_reason.outcome = MobRunRemoteTurnReceiptOutcome::Failed {
reason: "different failure".to_string(),
no_effect_proof: None,
};
assert!(
!MobActor::remote_turn_disposed_replay_matches(&exact_events, &mob_id, &wrong_reason,),
"a terminal for the same obligation but a different reason is not replay authority"
);
let mut wrong_outer_run = terminal.clone();
if let MobEventKind::StepTargetFailed { run_id, .. } = &mut wrong_outer_run {
*run_id = RunId::new();
} else {
panic!("failed receipt must produce StepTargetFailed");
}
assert!(
!MobActor::remote_turn_disposed_replay_matches(
&[event(1, wrong_outer_run), event(2, disposed.clone())],
&mob_id,
&receipt,
),
"inner obligation equality cannot excuse a corrupt outer run correlation"
);
assert!(
!MobActor::remote_turn_disposed_replay_matches(
&[
event(1, terminal),
event(2, disposed.clone()),
event(3, MobEventKind::MobReset),
event(4, disposed),
],
&mob_id,
&receipt,
),
"a terminal carrier from a prior reset epoch cannot authorize current cleanup"
);
}
#[test]
fn revoke_receipt_proof_excludes_materialize_pending_placement() {
let authority = mob_dsl::MobMachineAuthority::new();
let mut state = authority.state().clone();
let identity = mob_dsl::AgentIdentity("pending-member".to_string());
let host = mob_dsl::HostId("host-b".to_string());
state
.member_placement
.insert(identity.clone(), host.clone());
state.spawn_exec_phase.insert(
identity.clone(),
mob_dsl::SpawnExecPhase::MaterializePending,
);
assert!(
MobActor::committed_host_member_release_proof_set(&state, &host).is_empty(),
"an opened placement without a committed host session is not required in the revoke receipt"
);
state.member_session_bindings.insert(
identity.clone(),
mob_dsl::SessionId("member-session".to_string()),
);
state
.spawn_exec_phase
.insert(identity, mob_dsl::SpawnExecPhase::MembershipCommitted);
assert_eq!(
MobActor::committed_host_member_release_proof_set(&state, &host),
vec![AgentIdentity::from("pending-member")],
"the member joins the required proof set only after host session commit"
);
}
#[test]
fn unreported_materialize_pending_placement_does_not_trigger_revival() {
let host = mob_dsl::HostId("host-b".to_string());
assert!(
!MobActor::unreported_placement_requires_revival(
&host,
&host,
false,
Some(mob_dsl::SpawnExecPhase::MaterializePending),
),
"a status poll may not revive an in-flight placed spawn before its session binding exists"
);
assert!(
MobActor::unreported_placement_requires_revival(
&host,
&host,
false,
Some(mob_dsl::SpawnExecPhase::MembershipCommitted),
),
"a committed machine placement missing from the host inventory remains a revival trigger"
);
assert!(
!MobActor::unreported_placement_requires_revival(
&host,
&host,
true,
Some(mob_dsl::SpawnExecPhase::MembershipCommitted),
),
"a host-reported current member is not an unreported-placement revival candidate"
);
}
#[test]
fn host_status_uses_pending_placed_tuple_before_membership_commit() {
let authority = mob_dsl::MobMachineAuthority::new();
let mut state = authority.state().clone();
let identity = mob_dsl::AgentIdentity("pending-member".to_string());
state.spawn_exec_phase.insert(
identity.clone(),
mob_dsl::SpawnExecPhase::MaterializePending,
);
state
.pending_placed_spawn_generations
.insert(identity.clone(), mob_dsl::Generation(3));
state
.pending_placed_spawn_fence_tokens
.insert(identity.clone(), mob_dsl::FenceToken(7));
// A prior incarnation may still have a lower live tuple; it must not
// classify the in-flight host row as stale.
state
.identity_runtime_generations
.insert(identity.clone(), mob_dsl::Generation(2));
state
.identity_runtime_fence_tokens
.insert(identity.clone(), mob_dsl::FenceToken(6));
assert_eq!(
MobActor::host_status_expected_member_tuple(&state, &identity),
(
Some(mob_dsl::Generation(3)),
Some(mob_dsl::FenceToken(7)),
true,
)
);
}
#[test]
fn orphan_release_reservation_deduplicates_only_the_exact_authority_tuple() {
let exact = crate::runtime::state::HostOrphanReleaseKey {
host_id: mob_dsl::HostId("host-b".to_string()),
binding_incarnation: 1,
agent_identity: mob_dsl::AgentIdentity("worker".to_string()),
generation: mob_dsl::Generation(3),
fence_token: mob_dsl::FenceToken(7),
};
let successor_fence = crate::runtime::state::HostOrphanReleaseKey {
fence_token: mob_dsl::FenceToken(8),
..exact.clone()
};
let mut reservations = std::collections::BTreeMap::new();
assert!(MobActor::reserve_host_orphan_release(
&mut reservations,
&exact
));
assert!(
!MobActor::reserve_host_orphan_release(&mut reservations, &exact),
"overlapping observations must not schedule the same release twice"
);
assert!(
MobActor::reserve_host_orphan_release(&mut reservations, &successor_fence),
"a successor fence is a distinct release authority tuple"
);
assert!(MobActor::absorb_host_orphan_release_completion(
&mut reservations,
&exact,
true,
));
assert!(reservations.contains_key(&successor_fence));
assert!(
!MobActor::absorb_host_orphan_release_completion(&mut reservations, &exact, true,),
"a duplicate completion cannot clear another in-flight operation"
);
assert!(
!MobActor::reserve_host_orphan_release(&mut reservations, &exact),
"a successful release remains reserved while stale inventory still reports it"
);
MobActor::absorb_fresh_host_inventory_for_orphan_releases(
&mut reservations,
&exact.host_id,
&std::collections::BTreeSet::from([exact.clone(), successor_fence.clone()]),
);
assert!(!MobActor::reserve_host_orphan_release(
&mut reservations,
&exact
));
MobActor::absorb_fresh_host_inventory_for_orphan_releases(
&mut reservations,
&exact.host_id,
&std::collections::BTreeSet::from([successor_fence.clone()]),
);
assert!(
MobActor::reserve_host_orphan_release(&mut reservations, &exact),
"a fresh host inventory omission reopens the exact tuple"
);
assert!(MobActor::absorb_host_orphan_release_completion(
&mut reservations,
&successor_fence,
false,
));
assert!(
MobActor::reserve_host_orphan_release(&mut reservations, &successor_fence),
"a failed typed completion reopens the exact tuple for retry"
);
}
#[test]
fn revoke_then_fresh_bind_reopens_identical_orphan_tuple_after_late_completion() {
let host_id = mob_dsl::HostId("host-b".to_string());
let mut incarnations = std::collections::BTreeMap::new();
let mut reservations = std::collections::BTreeMap::new();
let old_incarnation = MobActor::advance_host_binding_incarnation(
&mut incarnations,
&mut reservations,
&host_id,
)
.expect("initial bind incarnation");
let old_key = crate::runtime::state::HostOrphanReleaseKey {
host_id: host_id.clone(),
binding_incarnation: old_incarnation,
agent_identity: mob_dsl::AgentIdentity("ghost".to_string()),
generation: mob_dsl::Generation(3),
fence_token: mob_dsl::FenceToken(7),
};
assert!(MobActor::reserve_host_orphan_release(
&mut reservations,
&old_key,
));
// Revoke is a binding-incarnation boundary. It must invalidate the
// in-flight key before an old endpoint's successful completion is
// absorbed.
let revoked_incarnation = MobActor::advance_host_binding_incarnation(
&mut incarnations,
&mut reservations,
&host_id,
)
.expect("revoke incarnation");
assert!(revoked_incarnation > old_incarnation);
assert!(reservations.is_empty());
assert!(
!MobActor::absorb_host_orphan_release_completion(&mut reservations, &old_key, true,),
"late success from the revoked route must not mint a tombstone"
);
// A fresh bind may reuse the host id, supervisor epoch, and exact
// release tuple while pointing at a replacement/recovered host. Its
// own inventory must be able to schedule a new ReleaseMember.
let fresh_incarnation = MobActor::advance_host_binding_incarnation(
&mut incarnations,
&mut reservations,
&host_id,
)
.expect("fresh bind incarnation");
let fresh_key = crate::runtime::state::HostOrphanReleaseKey {
binding_incarnation: fresh_incarnation,
..old_key.clone()
};
MobActor::absorb_fresh_host_inventory_for_orphan_releases(
&mut reservations,
&host_id,
&std::collections::BTreeSet::from([fresh_key.clone()]),
);
assert!(
MobActor::reserve_host_orphan_release(&mut reservations, &fresh_key),
"the replacement host's identical tuple must issue a new release"
);
}
#[test]
fn destroy_refuses_to_delete_residual_pending_custody_after_release() {
let obligation = mob_dsl::RemoteTurnObligation {
agent_identity: mob_dsl::AgentIdentity("worker".to_string()),
host_id: mob_dsl::HostId("host".to_string()),
host_binding_generation: 1,
member_session_id: mob_dsl::SessionId("session".to_string()),
generation: mob_dsl::Generation(1),
fence_token: mob_dsl::FenceToken(7),
dispatch_sequence: 3,
input_id: mob_dsl::InputId("input".to_string()),
run_id: mob_dsl::RunId("run".to_string()),
step_id: mob_dsl::StepId("step".to_string()),
};
let obligations = std::collections::BTreeSet::from([obligation]);
let exact_release_witness =
std::collections::BTreeSet::from([("worker".to_string(), 1, 7)]);
let error = MobActor::verify_destroy_remote_turn_custody_drained(
&obligations,
&exact_release_witness,
)
.expect_err("release proof alone cannot authorize residual Pending deletion");
assert!(error.contains("refusing to fabricate Dispose"));
assert!(
MobActor::verify_destroy_remote_turn_custody_drained(
&std::collections::BTreeSet::new(),
&exact_release_witness,
)
.is_ok()
);
}
#[test]
fn host_revoke_retry_anchors_distinguish_same_epoch_rebind_cycles() {
let mob_id = MobId::from("host-revoke-anchors");
let first = meerkat_core::time_compat::new_uuid_v7().to_string();
let second = meerkat_core::time_compat::new_uuid_v7().to_string();
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let events = vec![
event(
1,
MobEventKind::RemoteHostRevokeStarted {
operation_id: first.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 1,
},
),
event(
2,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: first.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 1,
},
),
event(
3,
MobEventKind::RemoteHostRevokeCompleted {
operation_id: first,
host_id: "host".to_string(),
epoch: 7,
binding_generation: 1,
},
),
event(
4,
MobEventKind::RemoteHostRevokeStarted {
operation_id: second.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 2,
},
),
];
assert_eq!(
pending_host_revoke_anchors(&events).expect("valid anchored revoke history"),
vec![super::PendingHostRevokeAnchor {
operation_id: second,
host_id: "host".to_string(),
epoch: 7,
binding_generation: 2,
confirmed: false,
}],
"a completed old operation cannot target a later same-epoch rebind"
);
}
#[test]
fn host_revoke_anchor_reducer_rejects_skipped_regressed_and_drifted_phases() {
let mob_id = MobId::from("host-revoke-anchor-order");
let operation_id = meerkat_core::time_compat::new_uuid_v7().to_string();
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let started = MobEventKind::RemoteHostRevokeStarted {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 2,
};
let confirmed = MobEventKind::RemoteHostRevokeConfirmed {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 2,
};
let completed = MobEventKind::RemoteHostRevokeCompleted {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 2,
};
assert!(
pending_host_revoke_anchors(&[
event(1, started.clone()),
event(2, started.clone()),
event(3, confirmed.clone()),
event(4, confirmed.clone()),
event(5, completed.clone()),
event(6, completed.clone()),
])
.expect("exact same-phase duplicates are idempotent")
.is_empty()
);
for malformed in [
vec![event(1, confirmed.clone()), event(2, started.clone())],
vec![event(1, started.clone()), event(2, completed.clone())],
vec![
event(1, started.clone()),
event(2, confirmed.clone()),
event(3, completed.clone()),
event(4, confirmed.clone()),
],
vec![
event(1, started.clone()),
event(
2,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
epoch: 7,
binding_generation: 3,
},
),
],
vec![event(
1,
MobEventKind::RemoteHostRevokeStarted {
operation_id: operation_id.clone(),
host_id: String::new(),
epoch: 7,
binding_generation: 0,
},
)],
] {
assert!(
pending_host_revoke_anchors(&malformed).is_err(),
"malformed revoke phase history must fail closed"
);
}
}
#[test]
fn supervisor_rotation_is_blocked_by_cold_unfinished_host_authority_anchors() {
let mob_id = MobId::from("host-anchor-rotation-fence");
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let bind_operation = meerkat_core::time_compat::new_uuid_v7().to_string();
let bind_request = bind_anchor_request();
let pending_binds = pending_host_bind_anchors(&[event(
1,
MobEventKind::RemoteHostBindStarted {
operation_id: bind_operation,
request: bind_request,
},
)])
.expect("cold Started-only bind anchor");
let bind_error =
require_host_authority_anchors_clear_for_supervisor_rotation(&pending_binds, &[])
.expect_err("unfinished bind must fence authority rotation");
assert!(bind_error.to_string().contains("bind operation"));
let complete_bind_error =
super::require_host_authority_anchors_clear("complete mob", &pending_binds, &[])
.expect_err("unfinished bind must fence completion before side effects");
assert!(
complete_bind_error
.to_string()
.contains("cannot complete mob")
);
let revoke_operation = meerkat_core::time_compat::new_uuid_v7().to_string();
let pending_revokes = pending_host_revoke_anchors(&[event(
2,
MobEventKind::RemoteHostRevokeStarted {
operation_id: revoke_operation,
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
},
)])
.expect("cold Started-only revoke anchor");
let revoke_error =
require_host_authority_anchors_clear_for_supervisor_rotation(&[], &pending_revokes)
.expect_err("unfinished revoke must fence authority rotation");
assert!(revoke_error.to_string().contains("revoke operation"));
let complete_revoke_error =
super::require_host_authority_anchors_clear("complete mob", &[], &pending_revokes)
.expect_err("unfinished revoke must fence completion before side effects");
assert!(
complete_revoke_error
.to_string()
.contains("cannot complete mob")
);
let overlap_error =
validate_host_authority_anchor_disjointness(&pending_binds, &pending_revokes)
.expect_err("one host cannot own unfinished bind and revoke operations together");
assert!(overlap_error.to_string().contains("overlapping unfinished"));
}
#[test]
fn unfinished_host_authority_operations_cannot_cross_lifecycle_boundaries() {
let mob_id = MobId::from("host-anchor-lifecycle-boundary");
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
for (name, boundary) in [
("MobReset", MobEventKind::MobReset),
("MobCompleted", MobEventKind::MobCompleted),
("MobDestroying", MobEventKind::MobDestroying),
(
"MobDestroyStorageFinalizing",
MobEventKind::MobDestroyStorageFinalizing,
),
] {
let bind_history = vec![
event(
1,
MobEventKind::RemoteHostBindStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
request: bind_anchor_request(),
},
),
event(2, boundary.clone()),
];
assert!(
validate_host_authority_anchor_epoch_boundaries(&bind_history)
.expect_err("Started bind cannot cross lifecycle boundary")
.to_string()
.contains(&format!("cross {name}"))
);
let revoke_history = vec![
event(
1,
MobEventKind::RemoteHostRevokeStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
},
),
event(2, boundary),
];
assert!(
validate_host_authority_anchor_epoch_boundaries(&revoke_history)
.expect_err("Started revoke cannot cross lifecycle boundary")
.to_string()
.contains(&format!("cross {name}"))
);
}
for terminal in [
MobEventKind::MobCompleted,
MobEventKind::MobDestroyStorageFinalizing,
] {
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, terminal.clone()),
event(
2,
MobEventKind::RemoteHostBindStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
request: bind_anchor_request(),
},
),
])
.expect_err("terminal history cannot open a later bind before reset")
.to_string()
.contains("after terminal")
);
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, terminal),
event(
2,
MobEventKind::RemoteHostRevokeStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
},
),
])
.expect_err("terminal history cannot open a later revoke before reset")
.to_string()
.contains("after terminal")
);
}
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, MobEventKind::MobDestroying),
event(
2,
MobEventKind::RemoteHostBindStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
request: bind_anchor_request(),
},
),
])
.expect_err("destroy cleanup may never open a host bind")
.to_string()
.contains("only new revoke")
);
let destroy_revoke = meerkat_core::time_compat::new_uuid_v7().to_string();
let destroy_started = MobEventKind::RemoteHostRevokeStarted {
operation_id: destroy_revoke.clone(),
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
};
validate_host_authority_anchor_epoch_boundaries(&[
event(1, MobEventKind::MobDestroying),
event(2, destroy_started.clone()),
])
.expect("destroy-owned revoke may remain recoverable before finalization");
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, MobEventKind::MobDestroying),
event(2, destroy_started.clone()),
event(3, MobEventKind::MobDestroyStorageFinalizing),
])
.expect_err("storage finalization requires destroy-owned revokes to close")
.to_string()
.contains("cross MobDestroyStorageFinalizing")
);
validate_host_authority_anchor_epoch_boundaries(&[
event(1, MobEventKind::MobDestroying),
event(2, destroy_started),
event(
3,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: destroy_revoke.clone(),
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
},
),
event(
4,
MobEventKind::RemoteHostRevokeCompleted {
operation_id: destroy_revoke,
host_id: "host".to_string(),
epoch: 9,
binding_generation: 3,
},
),
event(5, MobEventKind::MobDestroyStorageFinalizing),
])
.expect("completed destroy-owned revokes may cross storage finalization");
}
#[test]
fn host_revoke_history_targets_only_the_current_bound_generation() {
let mob_id = MobId::from("host-revoke-generation-order");
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let mut bind_g1 = bind_anchor_request();
bind_g1.binding_generation = 1;
let mut authority_g1 = bind_anchor_authority();
authority_g1.binding_generation = 1;
let bind_one = meerkat_core::time_compat::new_uuid_v7().to_string();
let revoke_one = meerkat_core::time_compat::new_uuid_v7().to_string();
let base = vec![
event(
1,
MobEventKind::RemoteHostBindStarted {
operation_id: bind_one.clone(),
request: bind_g1.clone(),
},
),
event(
2,
MobEventKind::RemoteHostBindConfirmed {
operation_id: bind_one.clone(),
authority: authority_g1,
},
),
event(
3,
MobEventKind::RemoteHostBindCompleted {
operation_id: bind_one,
host_id: bind_g1.host_id.clone(),
authority_epoch: bind_g1.authority_epoch,
binding_generation: 1,
},
),
event(
4,
MobEventKind::RemoteHostRevokeStarted {
operation_id: revoke_one.clone(),
host_id: bind_g1.host_id.clone(),
epoch: bind_g1.authority_epoch,
binding_generation: 1,
},
),
event(
5,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: revoke_one.clone(),
host_id: bind_g1.host_id.clone(),
epoch: bind_g1.authority_epoch,
binding_generation: 1,
},
),
event(
6,
MobEventKind::RemoteHostRevokeCompleted {
operation_id: revoke_one,
host_id: bind_g1.host_id.clone(),
epoch: bind_g1.authority_epoch,
binding_generation: 1,
},
),
];
let mut duplicate = base.clone();
duplicate.push(event(
7,
MobEventKind::RemoteHostRevokeStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
host_id: bind_g1.host_id.clone(),
epoch: bind_g1.authority_epoch,
binding_generation: 1,
},
));
assert!(
validate_host_authority_anchor_epoch_boundaries(&duplicate)
.expect_err("completed generation cannot be revoked twice")
.to_string()
.contains("current bound authority generation")
);
let mut bind_g2 = bind_g1.clone();
bind_g2.binding_generation = 2;
let mut authority_g2 = bind_anchor_authority();
authority_g2.binding_generation = 2;
let bind_two = meerkat_core::time_compat::new_uuid_v7().to_string();
let mut stale = base;
stale.extend([
event(
7,
MobEventKind::RemoteHostBindStarted {
operation_id: bind_two.clone(),
request: bind_g2.clone(),
},
),
event(
8,
MobEventKind::RemoteHostBindConfirmed {
operation_id: bind_two.clone(),
authority: authority_g2,
},
),
event(
9,
MobEventKind::RemoteHostBindCompleted {
operation_id: bind_two,
host_id: bind_g2.host_id.clone(),
authority_epoch: bind_g2.authority_epoch,
binding_generation: 2,
},
),
event(
10,
MobEventKind::RemoteHostRevokeStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
host_id: bind_g2.host_id,
epoch: bind_g2.authority_epoch,
binding_generation: 1,
},
),
]);
assert!(
validate_host_authority_anchor_epoch_boundaries(&stale)
.expect_err("a delayed G1 revoke cannot cross the G2 bind")
.to_string()
.contains("current bound authority generation 2")
);
}
#[test]
fn cross_family_host_authority_history_forbids_overlap_and_nonadvancing_rebind() {
let mob_id = MobId::from("host-anchor-cross-family");
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let revoke_id = meerkat_core::time_compat::new_uuid_v7().to_string();
let mut bind_request = bind_anchor_request();
bind_request.binding_generation = 1;
let bind_started = MobEventKind::RemoteHostBindStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
request: bind_request.clone(),
};
let revoke_started = MobEventKind::RemoteHostRevokeStarted {
operation_id: revoke_id.clone(),
host_id: bind_request.host_id.clone(),
epoch: 9,
binding_generation: 1,
};
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, bind_started.clone()),
event(2, revoke_started.clone()),
])
.expect_err("revoke cannot overlap an unfinished bind")
.to_string()
.contains("overlap")
);
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, revoke_started.clone()),
event(2, bind_started.clone()),
])
.expect_err("bind cannot overlap an unfinished revoke")
.to_string()
.contains("overlap")
);
assert!(
validate_host_authority_anchor_epoch_boundaries(&[
event(1, revoke_started),
event(
2,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: revoke_id.clone(),
host_id: bind_request.host_id.clone(),
epoch: 9,
binding_generation: 1,
},
),
event(
3,
MobEventKind::RemoteHostRevokeCompleted {
operation_id: revoke_id,
host_id: bind_request.host_id.clone(),
epoch: 9,
binding_generation: 1,
},
),
event(4, bind_started),
])
.expect_err("post-revoke bind must advance the consumed generation")
.to_string()
.contains("does not advance")
);
}
#[test]
fn proven_no_effect_bind_abort_allows_new_operation_at_same_generation() {
let mob_id = MobId::from("host-bind-no-effect-retry");
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let first = meerkat_core::time_compat::new_uuid_v7().to_string();
let retry = meerkat_core::time_compat::new_uuid_v7().to_string();
let mut request = bind_anchor_request();
request.binding_generation = 1;
let mut authority = bind_anchor_authority();
authority.binding_generation = 1;
let history = vec![
event(
1,
MobEventKind::RemoteHostBindStarted {
operation_id: first.clone(),
request: request.clone(),
},
),
event(
2,
MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id: first.clone(),
host_id: request.host_id.clone(),
authority_epoch: request.authority_epoch,
binding_generation: request.binding_generation,
},
),
event(
3,
MobEventKind::RemoteHostBindStarted {
operation_id: retry.clone(),
request: request.clone(),
},
),
event(
4,
MobEventKind::RemoteHostBindConfirmed {
operation_id: retry.clone(),
authority,
},
),
event(
5,
MobEventKind::RemoteHostBindCompleted {
operation_id: retry,
host_id: request.host_id.clone(),
authority_epoch: request.authority_epoch,
binding_generation: request.binding_generation,
},
),
];
validate_host_authority_anchor_epoch_boundaries(&history)
.expect("provable pre-write abort may retry the Requested generation with a new op id");
let revoke = meerkat_core::time_compat::new_uuid_v7().to_string();
let late_retry = meerkat_core::time_compat::new_uuid_v7().to_string();
let mut late = history;
late.extend([
event(
6,
MobEventKind::RemoteHostRevokeStarted {
operation_id: revoke.clone(),
host_id: request.host_id.clone(),
epoch: request.authority_epoch,
binding_generation: 1,
},
),
event(
7,
MobEventKind::RemoteHostRevokeConfirmed {
operation_id: revoke.clone(),
host_id: request.host_id.clone(),
epoch: request.authority_epoch,
binding_generation: 1,
},
),
event(
8,
MobEventKind::RemoteHostRevokeCompleted {
operation_id: revoke,
host_id: request.host_id.clone(),
epoch: request.authority_epoch,
binding_generation: 1,
},
),
event(
9,
MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id: first,
host_id: request.host_id.clone(),
authority_epoch: request.authority_epoch,
binding_generation: 1,
},
),
event(
10,
MobEventKind::RemoteHostBindStarted {
operation_id: late_retry,
request,
},
),
]);
assert!(
validate_host_authority_anchor_epoch_boundaries(&late)
.expect_err("late duplicate abort cannot reopen a remotely consumed generation")
.to_string()
.contains("does not advance")
);
}
fn bind_anchor_request() -> RemoteHostBindRequestEvent {
RemoteHostBindRequestEvent {
host_id: "host".to_string(),
peer_id: "host".to_string(),
signing_key: [7; 32],
endpoint: "tcp://host".to_string(),
authority_epoch: 9,
binding_generation: 3,
replacement: true,
}
}
fn bind_anchor_authority() -> crate::store::MobHostAuthorityRecord {
crate::store::MobHostAuthorityRecord {
host_id: "host".to_string(),
peer_id: "host".to_string(),
signing_key: [7; 32],
endpoint: "tcp://host".to_string(),
authority_epoch: 9,
binding_generation: 3,
bind_phase: crate::store::MobHostBindPhaseRecord::Bound,
capabilities: crate::store::MobHostCapabilityRecord {
protocol_min: 4,
protocol_max: 4,
engine_version: "test".to_string(),
durable_sessions: true,
autonomous_members: true,
hard_cancel_member: true,
tracked_input_cancel: true,
memory_store: false,
mcp: true,
resolvable_providers: BTreeSet::from(["anthropic".to_string()]),
approval_forwarding: false,
},
live_endpoint: None,
}
}
#[test]
fn host_live_endpoint_requires_an_absolute_base_url() {
assert_eq!(
MobActor::live_ws_endpoint_from_wire("wss://host.example/prefix/")
.expect("an absolute wss base URL is valid")
.0,
"wss://host.example/prefix",
"the stored base must join the fixed live path with one slash"
);
assert_eq!(
MobActor::live_ws_endpoint_from_wire(" \twSs://HOST.EXAMPLE/prefix/\r\n")
.expect("WHATWG-normalizable whitespace and case are accepted")
.0,
"wss://host.example/prefix",
"wire validation must persist the parsed URL, never the raw input bytes"
);
for invalid in [
"ws://",
"ws:///live",
"wss://host.example/live?token=secret",
"wss://host.example/live#fragment",
"wss://user:secret@host.example/live",
] {
assert!(
MobActor::live_ws_endpoint_from_wire(invalid).is_err(),
"'{invalid}' must not enter durable host capability state"
);
}
}
#[test]
fn host_bind_anchor_reducer_accepts_only_exact_monotone_phases() {
let mob_id = MobId::from("host-bind-anchors");
let operation_id = meerkat_core::time_compat::new_uuid_v7().to_string();
let request = bind_anchor_request();
let authority = bind_anchor_authority();
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
let started = MobEventKind::RemoteHostBindStarted {
operation_id: operation_id.clone(),
request: request.clone(),
};
let confirmed = MobEventKind::RemoteHostBindConfirmed {
operation_id: operation_id.clone(),
authority: authority.clone(),
};
let completed = MobEventKind::RemoteHostBindCompleted {
operation_id: operation_id.clone(),
host_id: request.host_id.clone(),
authority_epoch: request.authority_epoch,
binding_generation: request.binding_generation,
};
let pending = pending_host_bind_anchors(&[
event(1, started.clone()),
event(2, started.clone()),
event(3, confirmed.clone()),
event(4, confirmed.clone()),
])
.expect("exact same-phase duplicates remain idempotent");
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].operation_id, operation_id);
assert_eq!(pending[0].request, request);
assert_eq!(pending[0].confirmed_authority.as_ref(), Some(&authority));
assert!(
pending_host_bind_anchors(&[
event(1, started.clone()),
event(2, confirmed.clone()),
event(3, completed.clone()),
event(4, completed),
])
.expect("exact terminal replay is idempotent")
.is_empty()
);
for regressed in [
vec![
event(1, started.clone()),
event(2, confirmed.clone()),
event(
3,
MobEventKind::RemoteHostBindCompleted {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
authority_epoch: 9,
binding_generation: 3,
},
),
event(4, confirmed.clone()),
],
vec![
event(1, started.clone()),
event(2, confirmed.clone()),
event(3, started.clone()),
],
vec![
event(1, started.clone()),
event(
2,
MobEventKind::RemoteHostBindAbortedNoEffect {
operation_id: operation_id.clone(),
host_id: "host".to_string(),
authority_epoch: 9,
binding_generation: 3,
},
),
event(3, started.clone()),
],
] {
assert!(
pending_host_bind_anchors(®ressed).is_err(),
"phase regression must be durable corruption"
);
}
}
#[test]
fn host_bind_anchor_reducer_rejects_missing_or_drifted_started_authority() {
let mob_id = MobId::from("host-bind-anchor-drift");
let operation_id = meerkat_core::time_compat::new_uuid_v7().to_string();
let request = bind_anchor_request();
let event = |cursor, kind| MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: mob_id.clone(),
kind,
};
assert!(
pending_host_bind_anchors(&[event(
1,
MobEventKind::RemoteHostBindConfirmed {
operation_id: operation_id.clone(),
authority: bind_anchor_authority(),
},
)])
.is_err()
);
let mut drifted = request.clone();
drifted.binding_generation += 1;
assert!(
pending_host_bind_anchors(&[
event(
1,
MobEventKind::RemoteHostBindStarted {
operation_id: operation_id.clone(),
request,
},
),
event(
2,
MobEventKind::RemoteHostBindStarted {
operation_id,
request: drifted,
},
),
])
.is_err()
);
}
#[test]
fn foreign_runtime_observations_are_identified_for_log_downgrade() {
let authority = mob_dsl::MobMachineAuthority::new();
let signal = mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id: mob_dsl::AgentRuntimeId("rt:session:parent".to_string()),
fence_token: mob_dsl::FenceToken(0),
};
assert!(foreign_runtime_observation(authority.state(), &signal).is_some());
}
#[test]
fn live_member_runtime_observations_remain_machine_owned() {
let mut authority = mob_dsl::MobMachineAuthority::new();
let identity = mob_dsl::AgentIdentity("member".to_string());
let runtime_id = mob_dsl::AgentRuntimeId("member:0".to_string());
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: identity.clone(),
profile_name: "test".to_string(),
model: "test-model".to_string(),
profile_material_digest: "test-profile-digest".to_string(),
tool_config_digest: "test-tool-config-digest".to_string(),
skills_digest: "test-skills-digest".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: true,
resolved_spec_digest: None,
},
)
.expect("AuthorizeSpawnProfile should seed live runtime ownership");
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: identity.clone(),
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: mob_dsl::Generation(0),
profile_material_digest: "test-profile-digest".to_string(),
external_addressable: true,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(mob_dsl::SessionId("member-session".to_string())),
replacing: None,
placement: None,
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: false,
default_llm_client_override_present: false,
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
},
)
.expect("BeginSpawnExec should open the spawn-exec phase");
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: identity,
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: mob_dsl::Generation(0),
profile_material_digest: "test-profile-digest".to_string(),
external_addressable: true,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(mob_dsl::SessionId("member-session".to_string())),
replacing: None,
member_peer_endpoint: None,
spec_digest_echo: None,
ack_engine_version: None,
placed_spawn_id: None,
provision_operation_id: None,
},
)
.expect("CommitSpawnMembership should seed live runtime ownership");
let signal = mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id: runtime_id,
fence_token: mob_dsl::FenceToken(1),
};
assert!(foreign_runtime_observation(authority.state(), &signal).is_none());
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod routed_effect_containment_tests {
use super::*;
#[test]
fn runtime_retire_route_is_deferred_until_correlated_detach_closes() {
let identity = mob_dsl::AgentIdentity::from("worker");
let runtime_id = mob_dsl::AgentRuntimeId::from("worker:1");
let session_id = mob_dsl::SessionId::from("session-worker");
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
state
.runtime_retire_pending_sessions
.insert(runtime_id.clone(), session_id.clone());
state
.pending_session_ingress_detach_runtime_ids
.insert(runtime_id.clone());
let effect = super::super::composition::MobSeamEffect::routed(
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: identity,
agent_runtime_id: runtime_id.clone(),
session_id,
},
)
.expect("runtime retire is a generated routed effect");
assert!(MobActor::runtime_retire_blocked_on_pending_detach(
&state, &effect
));
state
.pending_session_ingress_detach_runtime_ids
.remove(&runtime_id);
assert!(!MobActor::runtime_retire_blocked_on_pending_detach(
&state, &effect
));
}
#[test]
fn actor_boundary_retries_closed_refusals_until_independent_tail_drains() {
let session_id = SessionId::parse("019f0000-0000-7000-8000-000000000001")
.expect("valid test session id");
let closed = |kind| {
Err(MobError::RuntimeEffectRefused {
kind,
session_id: session_id.clone(),
refusal_code: "runtime_refused".to_string(),
reason: "test refusal".to_string(),
})
};
let mut outcomes = VecDeque::from([
closed(crate::error::RuntimeEffectKind::RuntimeBinding),
closed(crate::error::RuntimeEffectKind::RuntimeIngress),
Ok(()),
]);
let mut boundary_attempts = 0;
let mut closed_refusals = 0;
loop {
boundary_attempts += 1;
let outcome = outcomes
.pop_front()
.expect("boundary test must terminate on the drained outcome");
match MobActor::classify_actor_boundary_flush(outcome) {
ActorBoundaryFlushDisposition::Drained => break,
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(_) => {
closed_refusals += 1;
continue;
}
ActorBoundaryFlushDisposition::Fatal(error) => {
panic!("machine-closed tail drain became fatal: {error}")
}
}
}
assert_eq!(boundary_attempts, 3);
assert_eq!(closed_refusals, 2);
assert!(outcomes.is_empty());
assert!(matches!(
MobActor::classify_actor_boundary_flush(Err(MobError::WiringError(
"structural refusal".to_string()
))),
ActorBoundaryFlushDisposition::Fatal(MobError::WiringError(_))
));
}
/// Containment precondition: every effect variant the MobMachine routes
/// to MeerkatMachine consumers carries its target bridge session, so a
/// dispatch failure always resolves to ONE member instead of terminating
/// the mob actor task (field: one revived member's PrepareBindings guard
/// rejection killed every healthy sibling).
#[test]
fn every_routed_effect_variant_is_session_scoped() {
let session = mob_dsl::SessionId::from("019f0000-0000-7000-8000-000000000001");
let runtime_id = mob_dsl::AgentRuntimeId("identity:1".to_string());
let routed = [
mob_dsl::MobMachineEffect::RequestRuntimeBinding {
session_id: session.clone(),
agent_identity: mob_dsl::AgentIdentity::from("worker-1"),
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: None,
},
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: mob_dsl::AgentIdentity::from("worker-1"),
agent_runtime_id: runtime_id.clone(),
session_id: session.clone(),
},
mob_dsl::MobMachineEffect::RequestRuntimeDestroy {
session_id: session.clone(),
},
mob_dsl::MobMachineEffect::RequestRuntimeIngress {
session_id: session.clone(),
agent_runtime_id: runtime_id,
fence_token: mob_dsl::FenceToken(1),
generation: None,
work_id: mob_dsl::WorkId::from("work-1"),
origin: mob_dsl::WorkOrigin::External,
},
];
for effect in &routed {
assert_eq!(
routed_effect_session_scope_dsl(effect),
Some(&session),
"routed effect must expose its session scope: {effect:?}"
);
assert!(
routed_effect_session_scope(effect).is_some(),
"session scope must parse to a domain session id"
);
}
}
}
#[cfg(test)]
mod placed_tracked_input_index_tests {
use super::*;
fn private_remote_turn() -> crate::event::RemoteTurnObligationEvent {
crate::event::RemoteTurnObligationEvent {
agent_identity: AgentIdentity::from("placed-worker"),
host_id: "host-a".to_string(),
host_binding_generation: 3,
member_session_id: "session-a".to_string(),
generation: crate::ids::Generation::new(2),
fence_token: crate::ids::FenceToken::new(7),
dispatch_sequence: 41,
input_id: uuid::Uuid::new_v4().to_string(),
run_id: crate::ids::RunId::new(),
step_id: crate::ids::StepId::from("step"),
}
}
#[test]
fn private_remote_turn_reserves_missing_public_turn_key_across_rebuild() {
let mob_id = MobId::from("private-remote-turn-index");
let remote = private_remote_turn();
let mut index = PlacedCompletionDurableIndex::recover_with_private_remote_turns(
&[],
&mob_id,
std::slice::from_ref(&remote),
)
.expect("seed validated private remote-turn owner");
index
.rebuild_from_events(&[], &mob_id)
.expect("ambiguous-append rebuild preserves private owners");
let completion = crate::event::PlacedCompletionObligationEvent {
agent_identity: remote.agent_identity.clone(),
host_id: remote.host_id.clone(),
host_binding_generation: remote.host_binding_generation,
member_session_id: remote.member_session_id.clone(),
generation: remote.generation,
fence_token: remote.fence_token,
dispatch_sequence: 1,
input_id: remote.input_id.clone(),
};
assert!(matches!(
index.reserve_fresh_tracked_input(
PlacedTrackedInputKey::completion(&completion),
PlacedTrackedInputOwner::Completion(completion),
),
Err(MobError::PlacedInteractionIdAlreadyUsed { .. })
));
}
#[test]
fn finalized_remote_turn_carrier_permanently_reserves_its_turn_key() {
let mob_id = MobId::from("finalized-remote-turn-index");
let remote = private_remote_turn();
let events = vec![MobEvent {
cursor: 1,
timestamp: chrono::Utc::now(),
mob_id: mob_id.clone(),
kind: MobEventKind::RemoteTurnOutcomeDisposed {
obligation: remote.clone(),
},
}];
let mut index = PlacedCompletionDurableIndex::recover(&events, &mob_id)
.expect("any exact final carrier burns the remote TurnKey");
let completion = crate::event::PlacedCompletionObligationEvent {
agent_identity: remote.agent_identity.clone(),
host_id: remote.host_id.clone(),
host_binding_generation: remote.host_binding_generation,
member_session_id: remote.member_session_id.clone(),
generation: remote.generation,
fence_token: remote.fence_token,
dispatch_sequence: 1,
input_id: remote.input_id.clone(),
};
assert!(matches!(
index.reserve_fresh_tracked_input(
PlacedTrackedInputKey::completion(&completion),
PlacedTrackedInputOwner::Completion(completion),
),
Err(MobError::PlacedInteractionIdAlreadyUsed { .. })
));
}
}
#[cfg(test)]
mod kickoff_tests {
use super::*;
fn placed_obligation(identity: &AgentIdentity) -> crate::event::PlacedKickoffObligationEvent {
crate::event::PlacedKickoffObligationEvent {
agent_identity: identity.clone(),
host_id: "host-a".to_string(),
host_binding_generation: 3,
member_session_id: "session-a".to_string(),
generation: crate::ids::Generation::new(2),
fence_token: crate::ids::FenceToken::new(7),
input_id: uuid::Uuid::new_v4().to_string(),
objective_id: meerkat_core::interaction::ObjectiveId::new(),
}
}
fn kickoff_event(mob_id: &MobId, cursor: u64, kind: MobEventKind) -> MobEvent {
MobEvent {
cursor,
timestamp: chrono::Utc::now(),
mob_id: mob_id.clone(),
kind,
}
}
fn kickoff_snapshot(
obligation: &crate::event::PlacedKickoffObligationEvent,
phase: crate::roster::MobMemberKickoffPhase,
error: Option<&str>,
seconds: u64,
) -> crate::roster::MobMemberKickoffSnapshot {
crate::roster::MobMemberKickoffSnapshot {
objective_id: Some(obligation.objective_id),
phase,
error: error.map(ToOwned::to_owned),
updated_at: std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds),
}
}
#[test]
fn structural_snapshot_reuses_only_an_exact_projection_timestamp() {
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
let mut state = mob_dsl::MobMachineState::default();
state
.member_kickoff_objective_ids
.insert(dsl_identity.clone(), obligation.objective_id.to_string());
state.member_kickoff_starting.insert(dsl_identity.clone());
let starting = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Starting,
None,
11,
);
let later = std::time::UNIX_EPOCH + std::time::Duration::from_secs(99);
let replayed =
placed_kickoff_snapshot_for_structural_event(&state, &identity, Some(&starting), later)
.expect("exact Starting replay snapshot");
assert_eq!(replayed, starting);
state.member_kickoff_starting.remove(&dsl_identity);
state.member_kickoff_cancelled.insert(dsl_identity);
let changed =
placed_kickoff_snapshot_for_structural_event(&state, &identity, Some(&starting), later)
.expect("fresh Cancelled snapshot");
assert_eq!(
changed.phase,
crate::roster::MobMemberKickoffPhase::Cancelled
);
assert_eq!(changed.updated_at, later);
}
#[test]
fn fresh_start_retry_reuses_append_committed_machine_uncommitted_pair() {
let mob_id = MobId::from("placed-start-write-before-commit");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let starting = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Starting,
None,
17,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: starting.clone(),
},
),
];
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Record {
obligation: &obligation,
},
)
.expect("fresh Start retry durable preflight"),
Some(starting),
);
}
#[test]
fn kickoff_structural_lookup_ignores_same_uuid_in_another_turn_scope() {
let mob_id = MobId::from("placed-start-scoped-input");
let expected_identity = AgentIdentity::from("placed-worker");
let expected = placed_obligation(&expected_identity);
let historical_identity = AgentIdentity::from("other-worker");
let mut historical = placed_obligation(&historical_identity);
historical.input_id = expected.input_id.clone();
let historical_starting = kickoff_snapshot(
&historical,
crate::roster::MobMemberKickoffPhase::Starting,
None,
19,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: historical,
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: historical_identity,
kickoff: historical_starting,
},
),
];
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Record {
obligation: &expected,
},
)
.expect("another scoped TurnKey is unrelated"),
None
);
}
#[test]
fn kickoff_structural_lookup_rejects_conflict_within_same_turn_scope() {
let mob_id = MobId::from("placed-start-scoped-conflict");
let identity = AgentIdentity::from("placed-worker");
let expected = placed_obligation(&identity);
let mut conflicting = expected.clone();
conflicting.objective_id = meerkat_core::interaction::ObjectiveId::new();
let starting = kickoff_snapshot(
&conflicting,
crate::roster::MobMemberKickoffPhase::Starting,
None,
21,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: conflicting,
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: starting,
},
),
];
assert!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Record {
obligation: &expected,
},
)
.expect_err("same scoped TurnKey with different custody must fail closed")
.to_string()
.contains("conflicts with its durable carrier")
);
}
#[test]
fn fresh_resolve_retry_reuses_append_committed_machine_uncommitted_pair() {
let mob_id = MobId::from("placed-resolve-write-before-commit");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let outcome = crate::event::PlacedKickoffHostOutcomeEvent::InteractionComplete;
let started = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Started,
None,
23,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffOutcomeResolved {
obligation: obligation.clone(),
outcome: outcome.clone(),
kickoff: started.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: started.clone(),
},
),
];
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Resolved {
obligation: &obligation,
outcome: &outcome,
},
)
.expect("fresh Resolve retry durable preflight"),
Some(started),
);
}
#[test]
fn fresh_reject_retry_reuses_append_committed_machine_uncommitted_pair() {
let mob_id = MobId::from("placed-reject-write-before-commit");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let error = "authenticated no-effect";
let failed = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Failed,
Some(error),
29,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffRejectedNoEffect {
obligation: obligation.clone(),
error: error.to_string(),
kickoff: failed.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: failed.clone(),
},
),
];
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::RejectedNoEffect {
obligation: &obligation,
error,
},
)
.expect("fresh Reject retry durable preflight"),
Some(failed),
);
}
#[test]
fn fresh_retry_rejects_a_partial_structural_carrier() {
let mob_id = MobId::from("placed-partial-write-before-commit");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let events = vec![kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation.clone(),
},
)];
let error = existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Record {
obligation: &obligation,
},
)
.expect_err("a partial structural batch must fail closed");
assert!(error.to_string().contains("no paired kickoff projection"));
}
#[test]
fn start_replay_after_resolve_and_ack_reuses_original_record_pair() {
let mob_id = MobId::from("placed-start-replay");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let starting = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Starting,
None,
1,
);
let started = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Started,
None,
2,
);
let outcome = crate::event::PlacedKickoffHostOutcomeEvent::InteractionComplete;
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: starting.clone(),
},
),
kickoff_event(
&mob_id,
3,
MobEventKind::PlacedKickoffOutcomeResolved {
obligation: obligation.clone(),
outcome: outcome.clone(),
kickoff: started.clone(),
},
),
kickoff_event(
&mob_id,
4,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: started,
},
),
kickoff_event(
&mob_id,
5,
MobEventKind::PlacedKickoffOutcomeAcknowledged {
obligation: obligation.clone(),
},
),
];
let count = events.len();
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Record {
obligation: &obligation,
},
)
.expect("Start replay must validate the original Starting pair after ACK"),
Some(starting),
);
assert!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Resolved {
obligation: &obligation,
outcome: &outcome,
},
)
.expect("terminal replay must validate its existing pair after ACK")
.is_some()
);
assert_eq!(events.len(), count, "replay validation must append nothing");
}
#[test]
fn cancellation_winning_resolved_replay_reuses_exact_terminal_pair() {
let mob_id = MobId::from("placed-cancelled-resolve-replay");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let starting = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Starting,
None,
1,
);
let cancelled = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Cancelled,
None,
2,
);
let outcome = crate::event::PlacedKickoffHostOutcomeEvent::InteractionFailed {
error: "host failed after Stop".to_string(),
};
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: starting,
},
),
kickoff_event(
&mob_id,
3,
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: cancelled.clone(),
},
),
kickoff_event(
&mob_id,
4,
MobEventKind::PlacedKickoffOutcomeResolved {
obligation: obligation.clone(),
outcome: outcome.clone(),
kickoff: cancelled.clone(),
},
),
kickoff_event(
&mob_id,
5,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: cancelled.clone(),
},
),
];
let count = events.len();
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::Resolved {
obligation: &obligation,
outcome: &outcome,
},
)
.expect("cancellation-winning host terminal replay must be exact"),
Some(cancelled),
);
assert_eq!(events.len(), count, "replay validation must append nothing");
}
#[test]
fn structural_batch_allows_repeated_projection_tail_but_rejects_partial_head() {
let mob_id = MobId::from("placed-cancelled-batch-head");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let cancelled = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Cancelled,
None,
31,
);
let head = MobEventKind::PlacedKickoffOutcomeResolved {
obligation: obligation.clone(),
outcome: crate::event::PlacedKickoffHostOutcomeEvent::InteractionCancelled,
kickoff: cancelled.clone(),
};
let tail = MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: cancelled,
};
let desired = vec![head.clone(), tail.clone()];
let prior_cancel = vec![kickoff_event(&mob_id, 1, tail.clone())];
assert!(
!exact_structural_batch_present(&prior_cancel, &mob_id, &desired)
.expect("an earlier identical projection is not a partial structural batch")
);
let partial = vec![
kickoff_event(&mob_id, 1, tail.clone()),
kickoff_event(&mob_id, 2, head.clone()),
];
assert!(
exact_structural_batch_present(&partial, &mob_id, &desired)
.expect_err("a structural head without its exact tail must fail closed")
.to_string()
.contains("partial or out of order")
);
let complete = vec![
kickoff_event(&mob_id, 1, tail.clone()),
kickoff_event(&mob_id, 2, head),
kickoff_event(&mob_id, 3, tail),
];
assert!(
exact_structural_batch_present(&complete, &mob_id, &desired)
.expect("the exact head-tail batch remains idempotent")
);
}
#[test]
fn rejected_no_effect_replay_reuses_exact_failure_pair() {
let mob_id = MobId::from("placed-rejection-replay");
let identity = AgentIdentity::from("placed-worker");
let obligation = placed_obligation(&identity);
let starting = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Starting,
None,
1,
);
let error = "authenticated pre-admission rejection";
let failed = kickoff_snapshot(
&obligation,
crate::roster::MobMemberKickoffPhase::Failed,
Some(error),
2,
);
let events = vec![
kickoff_event(
&mob_id,
1,
MobEventKind::PlacedKickoffObligationRecorded {
obligation: obligation.clone(),
},
),
kickoff_event(
&mob_id,
2,
MobEventKind::MemberKickoffUpdated {
member: identity.clone(),
kickoff: starting,
},
),
kickoff_event(
&mob_id,
3,
MobEventKind::PlacedKickoffRejectedNoEffect {
obligation: obligation.clone(),
error: error.to_string(),
kickoff: failed.clone(),
},
),
kickoff_event(
&mob_id,
4,
MobEventKind::MemberKickoffUpdated {
member: identity,
kickoff: failed.clone(),
},
),
];
let count = events.len();
assert_eq!(
existing_placed_kickoff_structural_carrier(
&events,
&mob_id,
PlacedKickoffStructuralReplay::RejectedNoEffect {
obligation: &obligation,
error,
},
)
.expect("no-effect rejection replay must validate its existing pair"),
Some(failed),
);
assert_eq!(events.len(), count, "replay validation must append nothing");
}
#[cfg(feature = "runtime-adapter")]
#[test]
fn kickoff_finalization_failure_never_projects_started() {
let identity = AgentIdentity::from("worker-finalization-failure");
for (error, expected_error) in [
(
meerkat_core::TurnErrorMetadata::runtime_apply_failure(
"durable transcript commit failed",
),
"durable transcript commit failed",
),
(
meerkat_core::TurnErrorMetadata {
detail: None,
..meerkat_core::TurnErrorMetadata::runtime_apply_failure("discarded")
},
"turn finalization failed",
),
] {
let input = kickoff_resolution_input(
&identity,
meerkat_runtime::completion::CompletionOutcome::CompletedWithFinalizationFailure {
error,
},
);
match input {
mob_dsl::MobMachineInput::KickoffResolveFailed { member_id, error } => {
assert_eq!(member_id.0, identity.as_str());
assert_eq!(error, expected_error);
}
other => {
panic!("non-durable kickoff completion must resolve Failed, got {other:?}")
}
}
}
}
#[test]
fn callback_pending_is_settled_for_the_kickoff_barrier() {
let pending = mob_dsl::AgentIdentity::from("pending-member");
let starting = mob_dsl::AgentIdentity::from("starting-member");
let callback_pending = mob_dsl::AgentIdentity::from("callback-member");
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
state.member_kickoff_pending.insert(pending.clone());
state.member_kickoff_starting.insert(starting.clone());
state
.member_kickoff_callback_pending
.insert(callback_pending.clone());
let unresolved = unresolved_kickoff_member_ids(&state);
assert!(unresolved.contains(&pending.0));
assert!(unresolved.contains(&starting.0));
assert!(
!unresolved.contains(&callback_pending.0),
"callback-pending is a one-shot interaction terminal, not an unresolved kickoff"
);
assert_eq!(
state
.kickoff_material_for_member_id(&callback_pending.0)
.expect("callback phase remains projected")
.phase,
mob_dsl::KickoffPhase::CallbackPending,
"settling the barrier must not erase the external-fulfillment diagnostic"
);
}
#[test]
fn machine_kickoff_objective_is_typed_for_delivery() {
let identity = AgentIdentity::from("worker-1");
let objective_id = meerkat_core::interaction::ObjectiveId::new();
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
state.member_kickoff_objective_ids.insert(
mob_dsl::AgentIdentity::from_domain(&identity),
objective_id.to_string(),
);
let metadata = machine_kickoff_turn_metadata(&state, &identity)
.expect("machine objective should parse")
.expect("kickoff objective should produce turn metadata");
assert_eq!(
metadata.transcript_identity.objective_id,
Some(objective_id)
);
}
#[test]
fn malformed_machine_kickoff_objective_fails_closed() {
let identity = AgentIdentity::from("worker-1");
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
state.member_kickoff_objective_ids.insert(
mob_dsl::AgentIdentity::from_domain(&identity),
"not-a-uuid".to_string(),
);
let error = machine_kickoff_objective_id(&state, &identity)
.expect_err("corrupt machine objective must not become an unscoped turn");
assert!(error.to_string().contains("invalid kickoff objective id"));
}
#[test]
fn missing_machine_kickoff_objective_fails_closed() {
let identity = AgentIdentity::from("worker-1");
let state = mob_dsl::MobMachineAuthority::new().state().clone();
let error = machine_kickoff_turn_metadata(&state, &identity)
.expect_err("autonomous kickoff must not become an unscoped turn");
assert!(error.to_string().contains("no kickoff objective id"));
}
#[test]
fn kickoff_generated_rejection_maps_to_typed_mob_error() {
let mut authority = mob_dsl::MobMachineAuthority::new();
// `KickoffMarkStarting` requires the member to be in the pending kickoff
// set; on a fresh authority it is genuinely rejected by the generated
// machine (unlike the late-arrival `KickoffResolve*` / `KickoffCancel*`
// inputs, which the 0.7.2 L5 D2a totality work made into typed no-op
// self-loops). This keeps the test exercising the real
// generated-rejection → typed-error mapping.
let generated = mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::KickoffMarkStarting {
member_id: mob_dsl::AgentIdentity::from("worker-1"),
},
)
.expect_err("mark-starting without a pending kickoff must be generated rejection");
let error = MobActor::kickoff_rejection_error("test_kickoff", generated);
match error {
MobError::MobMachineRejected { context, reason } => {
assert_eq!(context, "test_kickoff");
assert!(reason.contains("KickoffMarkStarting"));
}
other => panic!("expected typed MobMachineRejected, got {other:?}"),
}
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod member_comms_name_tests {
use super::render_member_comms_name;
use crate::MobError;
#[test]
fn valid_slugs_render_via_single_owner_display() {
let name = render_member_comms_name("mob-1", "worker", "w-a")
.expect("valid identifier-safe slugs should render");
assert_eq!(name, "mob-1/worker/w-a");
}
#[test]
fn invalid_component_fails_closed_with_typed_error() {
// A component that violates the comms-name slug rule (leading digit /
// illegal `/` inside) must fail closed as the typed
// `MobError::MemberCommsName` rather than reconstructing the raw join.
let error = render_member_comms_name("mob-1", "wor/ker", "w-a")
.expect_err("a component containing '/' must fail closed");
assert!(
matches!(error, MobError::MemberCommsName(_)),
"comms-name failure must surface the typed owner error, got: {error}"
);
// No raw-join fallback: the rejected component must not leak back out as
// a rendered routing name.
assert!(
!error.to_string().contains("mob-1/wor/ker/w-a"),
"rejected component must not be laundered into a raw routing name: {error}"
);
}
#[test]
fn empty_component_fails_closed() {
let error = render_member_comms_name("mob-1", "", "w-a")
.expect_err("an empty component is not identifier-safe");
assert!(matches!(error, MobError::MemberCommsName(_)));
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod bridge_rejection_tests {
use super::{ExpectedRevokeCleanupFailure, MobActor};
use crate::MobError;
use crate::runtime::bridge_protocol::{
BridgeRejectionCause, SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
decode_legacy_v1_raw_string_rejection,
};
use meerkat_core::comms::{AdmissionDropReason, SendError};
use serde_json::json;
fn known_bridge_rejection_causes() -> &'static [(BridgeRejectionCause, &'static str)] {
&[
(BridgeRejectionCause::NotBound, "not_bound"),
(BridgeRejectionCause::StaleSupervisor, "stale_supervisor"),
(BridgeRejectionCause::SenderMismatch, "sender_mismatch"),
(BridgeRejectionCause::AlreadyBound, "already_bound"),
(
BridgeRejectionCause::InvalidBootstrapToken,
"invalid_bootstrap_token",
),
(
BridgeRejectionCause::UnsupportedProtocolVersion,
"unsupported_protocol_version",
),
(
BridgeRejectionCause::InvalidSupervisorSpec,
"invalid_supervisor_spec",
),
(BridgeRejectionCause::InvalidPeerSpec, "invalid_peer_spec"),
(BridgeRejectionCause::AddressMismatch, "address_mismatch"),
(BridgeRejectionCause::Unsupported, "unsupported"),
(BridgeRejectionCause::Internal, "internal"),
]
}
#[test]
fn actor_decodes_typed_protocol_v2_bridge_rejection() {
let value = json!({
"result": "rejected",
"cause": "stale_supervisor",
"reason": "epoch too low",
});
let rejection =
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value)
.expect("typed rejection should decode");
assert_eq!(
rejection.typed_cause(),
Some(BridgeRejectionCause::StaleSupervisor)
);
assert_eq!(rejection.reason(), "epoch too low");
}
#[test]
fn actor_does_not_promote_raw_string_as_protocol_v2_rejection() {
let value = json!("legacy rejection");
assert!(
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value).is_none()
);
let legacy = decode_legacy_v1_raw_string_rejection(&value)
.expect("legacy raw string should decode only through the explicit v1 helper");
assert_eq!(legacy.typed_cause(), None);
assert!(legacy.is_legacy_v1_raw_string());
}
#[test]
fn actor_bridge_rejection_error_preserves_each_known_typed_cause() {
for (cause, wire_name) in known_bridge_rejection_causes() {
let reason = format!("typed bridge rejection: {wire_name}");
let value = json!({
"result": "rejected",
"cause": wire_name,
"reason": reason,
});
let rejection =
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value)
.expect("typed rejection should decode");
let error = MobActor::bridge_rejection_error(rejection);
match error {
MobError::BridgeCommandRejected {
cause: actual,
reason: actual_reason,
} => {
assert_eq!(actual, *cause);
assert_eq!(actual_reason, reason);
}
other => panic!("expected typed bridge rejection error, got {other:?}"),
}
}
}
#[test]
fn actor_legacy_bridge_rejection_error_stays_untyped() {
let value = json!("legacy rejection");
let legacy = decode_legacy_v1_raw_string_rejection(&value)
.expect("legacy raw string should decode only through the explicit v1 helper");
let error = MobActor::bridge_rejection_error(legacy);
assert!(matches!(
error,
MobError::WiringError(reason) if reason == "legacy rejection"
));
}
#[test]
fn revoke_cleanup_accepts_only_typed_not_bound() {
let not_bound = MobError::BridgeCommandRejected {
cause: BridgeRejectionCause::NotBound,
reason: "no authorized supervisor registered".to_string(),
};
assert_eq!(
MobActor::expected_revoke_cleanup_failure(¬_bound),
Some(ExpectedRevokeCleanupFailure::BridgeRejected {
cause: BridgeRejectionCause::NotBound,
})
);
for unproven_absence in [
MobError::CommsError(SendError::PeerNotFound("peer-1".to_string())),
MobError::CommsError(SendError::PeerOffline),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::UntrustedSender,
}),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::SessionClosed,
}),
] {
assert_eq!(
MobActor::expected_revoke_cleanup_failure(&unproven_absence),
None,
"connectivity failure does not prove supervisor trust was revoked"
);
}
}
#[test]
fn revoke_cleanup_does_not_treat_error_text_as_a_semantic_cause() {
let cases = [
MobError::WiringError("peer offline".to_string()),
MobError::Internal("no authorized supervisor registered".to_string()),
MobError::CommsError(SendError::Validation("peer not found".to_string())),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::InboxFull,
}),
MobError::BridgeCommandRejected {
cause: BridgeRejectionCause::SenderMismatch,
reason: "no authorized supervisor registered".to_string(),
},
];
for error in cases {
assert_eq!(MobActor::expected_revoke_cleanup_failure(&error), None);
}
}
#[test]
fn revoke_cleanup_expected_failure_ratchet_rejects_string_parsing() {
let source = include_str!("actor.rs");
let start = source
.find("fn expected_revoke_cleanup_failure")
.expect("expected cleanup classifier exists");
let end = source[start..]
.find("\n async fn destroy_remote_member_for_destroy")
.expect("cleanup classifier precedes remote destroy cleanup");
let body = &source[start..start + end];
for disallowed in [
"to_ascii_lowercase",
"to_lowercase",
"error.to_string()",
".contains(",
] {
assert!(
!body.contains(disallowed),
"expected revoke cleanup classifier must not parse error text with {disallowed}"
);
}
}
// Row #53 (atomicity_window) / dogma row R044: the former TEXT-BASED
// ratchet for ensure_supervisor_authorized trust rollback was replaced by
// behavioral tests in runtime/tests.rs:
// - test_authorize_rejection_rolls_back_newly_installed_recipient_trust
// - test_authorize_rejection_preserves_preexisting_recipient_trust
// - test_bind_rejection_rolls_back_provisioner_recipient_trust
// - test_bind_send_failure_rolls_back_provisioner_recipient_trust
// - test_post_commit_bind_decode_failure_quarantines_and_fail_stops
// - test_pending_recipient_trust_obligation_resolves_on_success
// which assert the live trust surface (supervisor bridge runtime peers)
// and the MobMachine `pending_recipient_trust` obligation instead of
// grepping source text.
/// Row #155 (fault_laundered): supervisor-activation failure handling must
/// classify "did the attempted new-supervisor-trust cleanup also fail" by a
/// typed discriminant carried on the install error, never by
/// string-matching the formatted error message.
#[test]
fn supervisor_activation_failure_is_typed_and_never_reauthorizes_old_trust() {
let source = include_str!("actor.rs");
let start = source
.find("fn supervisor_activation_error")
.expect("supervisor_activation_error exists");
let end = source[start..]
.find("\n /// Cancel checkpointers and transition to Stopped.")
.expect("rollback classifier precedes fail_reset_to_stopped");
let body = &source[start..start + end];
assert!(
body.contains("rollback_error: new_trust_cleanup_failed"),
"failure handling must key on the typed new_trust_cleanup_failed discriminant"
);
assert!(
body.contains("reconstruct old-supervisor trust"),
"activation recovery must explicitly preserve the no-old-reauthorization boundary"
);
assert!(
!body.contains("install_supervisor_private_trust_for_session("),
"activation failure handling must not reinstall old supervisor trust"
);
for disallowed in [
"error_text",
"error.to_string()",
".contains(",
"to_lowercase",
] {
assert!(
!body.contains(disallowed),
"supervisor activation rollback classifier must not parse error text with {disallowed}"
);
}
}
/// Row #155: the install path must surface the cleanup verdict as a typed
/// outcome (`SupervisorPrivateTrustInstallError` carrying
/// `new_trust_cleanup_failed`) rather than burying it in a formatted string
/// that a downstream `.contains()` would have to recover.
#[test]
fn supervisor_private_trust_install_carries_typed_cleanup_verdict() {
let source = include_str!("actor.rs");
let start = source
.find("async fn install_supervisor_private_trust_for_session_authority")
.expect("install_supervisor_private_trust_for_session_authority exists");
let end = source[start..]
.find("\n async fn stage_supervisor_trust_published_for_private_trust")
.expect("install fn precedes its publish-feedback staging helper");
let body = &source[start..start + end];
assert!(
body.contains(
"Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError>"
),
"install must return the typed install error so callers read a structured cleanup verdict"
);
assert!(
body.contains(
"SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)"
),
"a failed attempted-trust cleanup must be recorded as a typed discriminant"
);
assert!(
body.contains(".is_err()"),
"the cleanup attempt result must be inspected, not discarded"
);
}
}