use super::disposal::{
AbortOnError, BulkBestEffort, DisposalContext, DisposalReport, DisposalStep, ErrorPolicy,
WarnAndContinue,
};
use super::flow_frame_engine::FlowFrameLoopStorePlan;
use super::mob_member_lifecycle_projection::{
CanonicalMemberSnapshotMaterial, MobMemberLifecycleInput, MobMemberLifecycleProjection,
kickoff_snapshot_from_machine_state,
};
use super::mob_runtime_bridge_authority::{MobRuntimeBridgeAuthority, MobRuntimeBridgeEffect};
use super::provision_guard::PendingProvision;
use super::terminalization::{FlowFailureCause, TerminalizationOutcome, TerminalizationTarget};
use super::transaction::LifecycleRollback;
use super::*;
use crate::generated::protocol_mob_destroying_session_ingress::MobDestroyingSessionIngressObligation;
use crate::ids::{AgentIdentity, AgentRuntimeId, RespawnTopologyPeerId};
use crate::machines::mob_machine as mob_dsl;
use crate::run::{MobMachineFlowAuthorityToken, MobMachineFlowRunCommand, MobRunStatus, flow_run};
#[cfg(target_arch = "wasm32")]
use crate::tokio;
use futures::FutureExt;
use futures::stream::{FuturesUnordered, StreamExt};
use meerkat_core::comms::{
CommsTrustMutation, CommsTrustMutationAuthority, CommsTrustMutationResult, PeerAddress,
PeerLifecycleKind, PeerName, PeerRoute, SendError, TrustedPeerDescriptor,
};
use meerkat_core::time_compat::{Instant, SystemTime};
use meerkat_machine_kernels::generated::mob::command_capabilities as generated_mob_command_capabilities;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
pub(super) struct InitialTurnHandle {
handle: tokio::task::JoinHandle<()>,
}
impl InitialTurnHandle {
async fn abort_and_join(self) {
self.handle.abort();
let _ = self.handle.await;
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct WiringSides {
local: bool,
peer: bool,
}
impl WiringSides {
fn empty() -> Self {
Self {
local: false,
peer: false,
}
}
fn local() -> Self {
Self {
local: true,
peer: false,
}
}
fn peer() -> Self {
Self {
local: false,
peer: true,
}
}
fn both() -> Self {
Self {
local: true,
peer: true,
}
}
fn has_local(self) -> bool {
self.local
}
fn has_peer(self) -> bool {
self.peer
}
}
enum SubmitWorkDispatchCompletion {
Completed,
AwaitTurnAdmission {
operation_id: Option<meerkat_core::ops::OperationId>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
},
AwaitTurnCompletion {
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
},
}
impl SubmitWorkDispatchCompletion {
fn kind(&self) -> &'static str {
match self {
Self::Completed => "Completed",
Self::AwaitTurnAdmission { .. } => "AwaitTurnAdmission",
Self::AwaitTurnCompletion { .. } => "AwaitTurnCompletion",
}
}
}
struct SubmitWorkDispatchRequest {
content: ContentInput,
injected_context: Vec<ContentInput>,
interaction_id: Option<meerkat_core::interaction::InteractionId>,
handling_mode: meerkat_core::types::HandlingMode,
render_metadata: Option<meerkat_core::types::RenderMetadata>,
ack_mode: crate::mob_machine::SubmitWorkAckMode,
operation_id: Option<meerkat_core::ops::OperationId>,
}
fn submit_work_turn_metadata(
render_metadata: Option<meerkat_core::types::RenderMetadata>,
interaction_id: Option<meerkat_core::interaction::InteractionId>,
) -> Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata> {
if render_metadata.is_none() && interaction_id.is_none() {
return None;
}
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
render_metadata,
transcript_identity: meerkat_core::types::TranscriptMessageIdentity {
interaction_id,
run_id: None,
},
..Default::default()
},
)
}
#[derive(Debug, Clone)]
enum SubmitWorkIngressAuthority {
Runtime {
agent_runtime_id: mob_dsl::AgentRuntimeId,
fence_token: mob_dsl::FenceToken,
generation: Option<mob_dsl::Generation>,
session_id: mob_dsl::SessionId,
work_id: mob_dsl::WorkId,
origin: mob_dsl::WorkOrigin,
},
PeerRuntime {
agent_runtime_id: mob_dsl::AgentRuntimeId,
fence_token: mob_dsl::FenceToken,
generation: Option<mob_dsl::Generation>,
work_id: mob_dsl::WorkId,
origin: mob_dsl::WorkOrigin,
},
}
impl SubmitWorkIngressAuthority {
fn from_transition(
transition: &mob_dsl::MobMachineTransition,
expected_runtime_id: &mob_dsl::AgentRuntimeId,
expected_fence_token: mob_dsl::FenceToken,
expected_generation: mob_dsl::Generation,
expected_work_id: &mob_dsl::WorkId,
expected_origin: mob_dsl::WorkOrigin,
) -> Result<Self, MobError> {
let mut resolved = None;
for effect in transition.effects() {
let authority = match effect {
mob_dsl::MobMachineEffect::RequestRuntimeIngress {
agent_runtime_id,
fence_token,
generation,
session_id,
work_id,
origin,
} => Self::Runtime {
agent_runtime_id: agent_runtime_id.clone(),
fence_token: *fence_token,
generation: *generation,
session_id: session_id.clone(),
work_id: work_id.clone(),
origin: *origin,
},
mob_dsl::MobMachineEffect::RequestPeerRuntimeIngress {
agent_runtime_id,
fence_token,
generation,
work_id,
origin,
} => Self::PeerRuntime {
agent_runtime_id: agent_runtime_id.clone(),
fence_token: *fence_token,
generation: *generation,
work_id: work_id.clone(),
origin: *origin,
},
_ => continue,
};
authority.verify_payload(
expected_runtime_id,
expected_fence_token,
expected_generation,
expected_work_id,
expected_origin,
)?;
if resolved.replace(authority).is_some() {
return Err(MobError::Internal(
"MobMachine SubmitWork emitted multiple generated runtime ingress authorities"
.to_string(),
));
}
}
resolved.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted SubmitWork but emitted no generated runtime ingress authority"
.to_string(),
)
})
}
fn variant(&self) -> &'static str {
match self {
Self::Runtime { .. } => "RequestRuntimeIngress",
Self::PeerRuntime { .. } => "RequestPeerRuntimeIngress",
}
}
fn agent_runtime_id(&self) -> &mob_dsl::AgentRuntimeId {
match self {
Self::Runtime {
agent_runtime_id, ..
}
| Self::PeerRuntime {
agent_runtime_id, ..
} => agent_runtime_id,
}
}
fn fence_token(&self) -> mob_dsl::FenceToken {
match self {
Self::Runtime { fence_token, .. } | Self::PeerRuntime { fence_token, .. } => {
*fence_token
}
}
}
fn generation(&self) -> Option<mob_dsl::Generation> {
match self {
Self::Runtime { generation, .. } | Self::PeerRuntime { generation, .. } => *generation,
}
}
fn work_id(&self) -> &mob_dsl::WorkId {
match self {
Self::Runtime { work_id, .. } | Self::PeerRuntime { work_id, .. } => work_id,
}
}
fn origin(&self) -> mob_dsl::WorkOrigin {
match self {
Self::Runtime { origin, .. } | Self::PeerRuntime { origin, .. } => *origin,
}
}
fn verify_payload(
&self,
expected_runtime_id: &mob_dsl::AgentRuntimeId,
expected_fence_token: mob_dsl::FenceToken,
expected_generation: mob_dsl::Generation,
expected_work_id: &mob_dsl::WorkId,
expected_origin: mob_dsl::WorkOrigin,
) -> Result<(), MobError> {
if self.agent_runtime_id() != expected_runtime_id
|| self.fence_token() != expected_fence_token
|| self.generation() != Some(expected_generation)
|| self.work_id() != expected_work_id
|| self.origin() != expected_origin
{
return Err(MobError::Internal(format!(
"generated {} authority did not match admitted SubmitWork payload",
self.variant()
)));
}
Ok(())
}
fn verify_member_ref(&self, member_ref: &MemberRef, context: &str) -> Result<(), MobError> {
match self {
Self::Runtime { session_id, .. } => {
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Err(MobError::Internal(format!(
"{context} requires a session-bound member for generated {} authority",
self.variant()
)));
};
let projected_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
if &projected_session_id != session_id {
return Err(MobError::Internal(format!(
"{context} generated {} authority session does not match MobMachine session binding",
self.variant()
)));
}
Ok(())
}
Self::PeerRuntime { .. } => match member_ref {
MemberRef::BackendPeer {
session_id: None, ..
} => Ok(()),
_ => Err(MobError::Internal(format!(
"{context} requires a peer-only member for generated {} authority",
self.variant()
))),
},
}
}
fn is_peer_runtime(&self) -> bool {
matches!(self, Self::PeerRuntime { .. })
}
}
#[cfg(not(target_arch = "wasm32"))]
const MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS: usize = 64;
const MAX_LIFECYCLE_NOTIFICATION_TASKS: usize = 16;
const MAX_PARALLEL_PEER_RETIRE_NOTIFICATIONS: usize = 64;
#[cfg(not(test))]
pub(super) const MAX_PENDING_PEER_DELIVERIES: usize = 1024;
#[cfg(test)]
pub(super) const MAX_PENDING_PEER_DELIVERIES: usize = 4;
#[cfg(test)]
pub(super) static SPAWN_PROVISIONED_COMMAND_DELAY_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
static FAIL_SESSION_INGRESS_DETACH_FOR_SESSION: std::sync::Mutex<Option<SessionId>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
static FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY: std::sync::Mutex<Option<AgentIdentity>> =
std::sync::Mutex::new(None);
#[cfg(test)]
pub(super) fn fail_session_ingress_detach_for_test(session_id: &SessionId) {
if let Ok(mut target) = FAIL_SESSION_INGRESS_DETACH_FOR_SESSION.lock() {
*target = Some(session_id.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_destroy_member_archive_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_destroy_remote_archive_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
#[cfg(test)]
pub(super) fn fail_after_remote_supervisor_revoked_for_test(agent_identity: &AgentIdentity) {
if let Ok(mut target) = FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY.lock() {
*target = Some(agent_identity.clone());
}
}
pub(super) fn render_member_comms_name(
mob_id: &str,
role: &str,
member: &str,
) -> Result<String, MobError> {
Ok(meerkat_core::MemberCommsName::new(mob_id, role, member)?.to_string())
}
fn observed_runtime_id(signal: &mob_dsl::MobMachineSignal) -> Option<&mob_dsl::AgentRuntimeId> {
match signal {
mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id, ..
}
| mob_dsl::MobMachineSignal::ObserveRuntimeRetired {
agent_runtime_id, ..
}
| mob_dsl::MobMachineSignal::ObserveRuntimeDestroyed {
agent_runtime_id, ..
} => Some(agent_runtime_id),
_ => None,
}
}
fn foreign_runtime_observation<'a>(
state: &mob_dsl::MobMachineState,
signal: &'a mob_dsl::MobMachineSignal,
) -> Option<&'a mob_dsl::AgentRuntimeId> {
let agent_runtime_id = observed_runtime_id(signal)?;
(!state.live_runtime_ids.contains(agent_runtime_id)).then_some(agent_runtime_id)
}
#[derive(Clone)]
enum WiringEndpoint {
Local {
entry: Box<RosterEntry>,
comms: Arc<dyn CoreCommsRuntime>,
spec: TrustedPeerDescriptor,
comms_name: String,
},
PeerOnly {
spec: TrustedPeerDescriptor,
binding: crate::RuntimeBinding,
},
}
#[derive(Clone)]
struct LocalBatchWiringEndpoint {
comms: Arc<dyn CoreCommsRuntime>,
spec: TrustedPeerDescriptor,
removal_key: String,
}
struct PeerMessageDeliveryPlan {
from: AgentIdentity,
to: AgentIdentity,
sender_comms: Arc<dyn CoreCommsRuntime>,
command: CommsCommand,
}
type PeerDeliveryId = u64;
pub(super) struct PeerDeliveryCompletion {
id: PeerDeliveryId,
}
pub(super) struct PeerDeliveryInflight {
from: AgentIdentity,
to: AgentIdentity,
cancel_token: tokio_util::sync::CancellationToken,
}
struct SupervisorPrivateTrustInstall {
peer_id: String,
epoch: u64,
removal_key: String,
}
#[derive(Debug)]
struct SupervisorPrivateTrustInstallError {
error: MobError,
new_trust_cleanup_failed: bool,
}
impl SupervisorPrivateTrustInstallError {
fn without_cleanup_failure(error: MobError) -> Self {
Self {
error,
new_trust_cleanup_failed: false,
}
}
#[cfg(feature = "runtime-adapter")]
fn with_failed_new_trust_cleanup(error: MobError) -> Self {
Self {
error,
new_trust_cleanup_failed: true,
}
}
}
impl std::fmt::Display for SupervisorPrivateTrustInstallError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.error, f)
}
}
impl From<MobError> for SupervisorPrivateTrustInstallError {
fn from(error: MobError) -> Self {
Self::without_cleanup_failure(error)
}
}
impl From<SupervisorPrivateTrustInstallError> for MobError {
fn from(value: SupervisorPrivateTrustInstallError) -> Self {
value.error
}
}
struct SupervisorAuthorityActivationError {
error: MobError,
rollback_succeeded: bool,
pending_authority_recorded: bool,
rollback_error: Option<String>,
}
struct SupervisorAuthorityLoad {
durable: crate::store::SupervisorAuthorityRecord,
}
struct SupervisorPendingRotationPersistence {
pending_authority_recorded: bool,
persisted_record: Option<crate::store::SupervisorAuthorityRecord>,
}
struct PendingSupervisorRotationPersistenceRequest<'a> {
current: &'a crate::store::SupervisorAuthorityRecord,
expected_durable: &'a crate::store::SupervisorAuthorityRecord,
pending: &'a crate::store::SupervisorAuthorityRecord,
operation_id: super::bridge_protocol::SupervisorRotationOperationId,
accepted_peer_ids: &'a BTreeSet<String>,
active_peer_ids: &'a BTreeSet<String>,
member_targets: &'a BTreeMap<String, super::bridge_protocol::BridgePeerSpec>,
}
#[cfg(feature = "runtime-adapter")]
struct SupervisorPrivateTrustRevokeRequest<'a> {
adapter: &'a meerkat_runtime::MeerkatMachine,
session_id: &'a SessionId,
comms: &'a dyn CoreCommsRuntime,
peer_id: String,
epoch: u64,
removal_key: String,
allow_absent_pending: bool,
}
struct PreparedSupervisorAuthorityPersistence {
transition: PreparedDslTransition,
authority: crate::store::SupervisorAuthorityPersistenceAuthority,
}
struct PreparedSupervisorAuthorityDeletion {
transition: PreparedDslTransition,
authority: crate::store::SupervisorAuthorityDeletionAuthority,
}
struct PreparedDslInput {
authority: mob_dsl::MobMachinePreparedAuthority,
effects: Vec<mob_dsl::MobMachineEffect>,
phase_changed: bool,
}
struct PreparedDslTransition {
authority: mob_dsl::MobMachinePreparedAuthority,
transition: mob_dsl::MobMachineTransition,
}
struct BatchWireTrustApplication {
edge: mob_dsl::WiringEdge,
identity: AgentIdentity,
peer_id: String,
comms: Arc<dyn CoreCommsRuntime>,
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
}
struct BatchWireTrustRollback {
edge: mob_dsl::WiringEdge,
identity: AgentIdentity,
peer_id: String,
comms: Arc<dyn CoreCommsRuntime>,
}
struct RetireTrustCleanupPlan {
retiring_comms: Option<Arc<dyn CoreCommsRuntime>>,
retiring_spec: Option<TrustedPeerDescriptor>,
machine_wired_peer_identities: BTreeSet<AgentIdentity>,
trust_unwire_authority_by_peer: BTreeMap<AgentIdentity, CommsTrustMutationAuthority>,
}
impl RetireTrustCleanupPlan {
fn empty() -> Self {
Self {
retiring_comms: None,
retiring_spec: None,
machine_wired_peer_identities: BTreeSet::new(),
trust_unwire_authority_by_peer: BTreeMap::new(),
}
}
fn has_peers(&self) -> bool {
!self.machine_wired_peer_identities.is_empty()
}
}
#[derive(Debug, Clone)]
enum WireTrustAuthority {
GraphAdded(MemberTrustHandoff),
RepairRequested(MemberTrustHandoff),
ExternalGraphAdded(CommsTrustMutationAuthority),
ExternalRepairRequested(CommsTrustMutationAuthority),
}
impl WireTrustAuthority {
fn dsl_added(&self) -> bool {
matches!(self, Self::GraphAdded(_) | Self::ExternalGraphAdded(_))
}
fn is_repair(&self) -> bool {
matches!(
self,
Self::RepairRequested(_) | Self::ExternalRepairRequested(_)
)
}
fn member_handoff(&self) -> Result<&MemberTrustHandoff, MobError> {
match self {
Self::GraphAdded(handoff) | Self::RepairRequested(handoff) => Ok(handoff),
Self::ExternalGraphAdded(_) | Self::ExternalRepairRequested(_) => {
Err(MobError::WiringError(
"external peer authority does not carry member peer handoff".to_string(),
))
}
}
}
fn external_authority(&self) -> Result<&CommsTrustMutationAuthority, MobError> {
match self {
Self::ExternalGraphAdded(authority) | Self::ExternalRepairRequested(authority) => {
Ok(authority)
}
Self::GraphAdded(_) | Self::RepairRequested(_) => Err(MobError::WiringError(
"member trust authority does not carry external peer authority".to_string(),
)),
}
}
}
#[derive(Debug, Clone)]
struct MemberTrustHandoff {
edge: mob_dsl::WiringEdge,
authority: MemberTrustAuthority,
operation: MemberTrustOperation,
}
#[derive(Debug, Clone)]
enum MemberTrustAuthority {
Wiring(crate::generated::protocol_mob_member_trust_wiring::MobMemberTrustWiringObligation),
Unwiring(
crate::generated::protocol_mob_member_trust_unwiring::MobMemberTrustUnwiringObligation,
),
Repair(crate::generated::protocol_mob_member_trust_wiring::MobMemberTrustWiringObligation),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MemberTrustOperation {
Wiring,
Unwiring,
Repair,
}
impl MemberTrustHandoff {
fn peer_id_for(&self, identity: &AgentIdentity) -> Result<&str, MobError> {
let identity = mob_dsl::AgentIdentity::from_domain(identity);
let actual = match &self.authority {
MemberTrustAuthority::Wiring(obligation) | MemberTrustAuthority::Repair(obligation) => {
if obligation.edge().a == identity {
Some(obligation.a_peer_id().0.as_str())
} else if obligation.edge().b == identity {
Some(obligation.b_peer_id().0.as_str())
} else {
None
}
}
MemberTrustAuthority::Unwiring(obligation) => {
if obligation.edge().a == identity {
Some(obligation.a_peer_id().0.as_str())
} else if obligation.edge().b == identity {
Some(obligation.b_peer_id().0.as_str())
} else {
None
}
}
};
actual.ok_or_else(|| {
MobError::WiringError(format!(
"generated member trust obligation does not cover '{identity:?}'"
))
})
}
fn require_peer_id_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
) -> Result<(), MobError> {
let actual = self.peer_id_for(identity)?;
if actual == expected_peer_id {
Ok(())
} else {
Err(MobError::WiringError(format!(
"generated member trust handoff peer id '{actual}' does not match resolved peer id '{expected_peer_id}' for '{identity}'"
)))
}
}
fn wiring_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Wiring(obligation) => {
crate::generated::protocol_mob_member_trust_wiring::wiring_authority_for_identity_with_live_authority(
obligation,
identity.as_str(),
expected_peer_id,
live_authority,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Repair(_) | MemberTrustAuthority::Unwiring(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot wire trust".to_string(),
))
}
}
}
fn unwiring_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Unwiring(obligation) => {
crate::generated::protocol_mob_member_trust_unwiring::unwiring_authority_for_identity(
obligation,
identity.as_str(),
expected_peer_id,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Repair(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot unwire trust".to_string(),
))
}
}
}
fn repair_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match &self.authority {
MemberTrustAuthority::Repair(obligation) => {
crate::generated::protocol_mob_member_trust_wiring::repair_authority_for_identity_with_live_authority(
obligation,
identity.as_str(),
expected_peer_id,
live_authority,
)
.map_err(MobError::WiringError)
}
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Unwiring(_) => {
Err(MobError::WiringError(
"generated member trust obligation cannot repair trust".to_string(),
))
}
}
}
fn add_authority_for(
&self,
identity: &AgentIdentity,
expected_peer_id: &str,
live_authority: &mob_dsl::MobMachineAuthority,
) -> Result<CommsTrustMutationAuthority, MobError> {
match self.operation {
MemberTrustOperation::Wiring => {
self.wiring_authority_for(identity, expected_peer_id, live_authority)
}
MemberTrustOperation::Repair => {
self.repair_authority_for(identity, expected_peer_id, live_authority)
}
MemberTrustOperation::Unwiring => Err(MobError::WiringError(
"generated member unwiring handoff cannot add trust".to_string(),
)),
}
}
}
fn resolve_binding(
binding: Option<crate::RuntimeBinding>,
backend: Option<crate::MobBackendKind>,
profile_backend: Option<crate::MobBackendKind>,
definition_default: crate::MobBackendKind,
agent_identity: &AgentIdentity,
) -> Result<crate::RuntimeBinding, MobError> {
if let Some(b) = binding {
return Ok(b);
}
let kind = backend.or(profile_backend).unwrap_or(definition_default);
match kind {
crate::MobBackendKind::Session => Ok(crate::RuntimeBinding::Session),
crate::MobBackendKind::External => Err(MobError::WiringError(format!(
"external backend requires explicit RuntimeBinding for '{agent_identity}'"
))),
}
}
fn normalize_runtime_mode_for_binding(
runtime_mode: crate::MobRuntimeMode,
binding: &crate::RuntimeBinding,
) -> crate::MobRuntimeMode {
match binding {
crate::RuntimeBinding::External { .. } => crate::MobRuntimeMode::TurnDriven,
crate::RuntimeBinding::Session => runtime_mode,
}
}
fn with_spawn_budget_limits(
mut req: meerkat_core::service::CreateSessionRequest,
limits: Option<meerkat_core::BudgetLimits>,
) -> meerkat_core::service::CreateSessionRequest {
if let Some(limits) = limits {
req.build
.get_or_insert_with(meerkat_core::service::SessionBuildOptions::default)
.budget_limits = Some(limits);
}
req
}
pub(super) fn admit_bridge_session_for_spawn(
req: &mut meerkat_core::service::CreateSessionRequest,
) -> SessionId {
if req.build.is_none() {
req.build = Some(meerkat_core::service::SessionBuildOptions::default());
}
let build = req
.build
.as_mut()
.expect("build options were initialized above");
if let Some(session) = build.resume_session.as_ref() {
return session.id().clone();
}
let session_id = SessionId::new();
build.resume_session = Some(meerkat_core::session::Session::with_id(session_id.clone()));
session_id
}
fn project_dsl_phase(phase: mob_dsl::MobPhase) -> MobState {
match phase {
mob_dsl::MobPhase::Running => MobState::Running,
mob_dsl::MobPhase::Stopped => MobState::Stopped,
mob_dsl::MobPhase::Completed => MobState::Completed,
mob_dsl::MobPhase::Destroyed => MobState::Destroyed,
}
}
pub(super) fn external_member_rebind_capability_from_member_ref(
member_ref: &MemberRef,
) -> mob_dsl::ExternalMemberRebindCapability {
match member_ref {
MemberRef::BackendPeer {
bootstrap_token, ..
} if bootstrap_token
.as_ref()
.is_some_and(|token| !token.is_empty()) =>
{
mob_dsl::ExternalMemberRebindCapability::Available
}
_ => mob_dsl::ExternalMemberRebindCapability::Unavailable,
}
}
fn remote_member_runtime_observed_state(
state: super::bridge_protocol::BridgeMemberRuntimeState,
) -> Result<mob_dsl::MobRemoteMemberRuntimeObservedState, MobError> {
use super::bridge_protocol::BridgeMemberRuntimeState as Wire;
use mob_dsl::MobRemoteMemberRuntimeObservedState as Observed;
Ok(match state {
Wire::Initializing => Observed::Initializing,
Wire::Idle => Observed::Idle,
Wire::Attached => Observed::Attached,
Wire::Running => Observed::Running,
Wire::Retired => Observed::Retired,
Wire::Stopped => Observed::Stopped,
Wire::Destroyed => Observed::Destroyed,
other => {
return Err(MobError::Internal(format!(
"unrecognized remote-member runtime wire state `{other}` cannot be classified for terminality"
)));
}
})
}
fn mob_bridge_rejection_cause(
cause: super::bridge_protocol::BridgeRejectionCause,
) -> mob_dsl::MobBridgeRejectionCause {
use super::bridge_protocol::BridgeRejectionCause as Wire;
use mob_dsl::MobBridgeRejectionCause as Mob;
match cause {
Wire::NotBound => Mob::NotBound,
Wire::StaleSupervisor => Mob::StaleSupervisor,
Wire::SenderMismatch => Mob::SenderMismatch,
Wire::AlreadyBound => Mob::AlreadyBound,
Wire::InvalidBootstrapToken => Mob::InvalidBootstrapToken,
Wire::UnsupportedProtocolVersion => Mob::UnsupportedProtocolVersion,
Wire::InvalidSupervisorSpec => Mob::InvalidSupervisorSpec,
Wire::InvalidPeerSpec => Mob::InvalidPeerSpec,
Wire::AddressMismatch => Mob::AddressMismatch,
Wire::Unsupported => Mob::Unsupported,
Wire::Internal => Mob::Internal,
_ => Mob::Internal,
}
}
fn render_fork_context(
source_member_id: &AgentIdentity,
messages: &[meerkat_core::types::Message],
) -> String {
use meerkat_core::types::Message;
let mut lines = Vec::new();
lines.push(format!(
"[Forked conversation context from member '{source_member_id}']"
));
for msg in messages {
match msg {
Message::System(s) => {
lines.push(format!("[system]: {}", s.content));
}
Message::SystemNotice(notice) => {
lines.push(format!(
"[system_notice]: {}",
notice.model_projection_text()
));
}
Message::User(u) => {
lines.push(format!("[user]: {}", u.text_content()));
}
Message::BlockAssistant(ba) => {
let text: String = ba
.blocks
.iter()
.filter_map(|b| match b {
meerkat_core::types::AssistantBlock::Text { text, .. }
| meerkat_core::types::AssistantBlock::Transcript { text, .. } => {
Some(text.as_str())
}
_ => None,
})
.collect::<Vec<_>>()
.join("");
if !text.is_empty() {
lines.push(format!("[assistant]: {text}"));
}
}
Message::ToolResults { results, .. } => {
for tr in results {
let text: String = tr
.content
.iter()
.filter_map(|b| match b {
meerkat_core::types::ContentBlock::Text { text, .. } => {
Some(text.as_str())
}
_ => None,
})
.collect::<Vec<_>>()
.join("");
if !text.is_empty() {
let preview = if text.len() > 200 {
let end = text
.char_indices()
.map(|(i, _)| i)
.take_while(|&i| i <= 200)
.last()
.unwrap_or(0);
format!("{}...", &text[..end])
} else {
text
};
lines.push(format!("[tool_result({})]: {preview}", tr.tool_use_id));
}
}
}
}
}
lines.push("[End of forked context]".to_string());
lines.join("\n")
}
pub(super) struct PendingSpawn {
pub(super) profile_name: ProfileName,
pub(super) agent_identity: AgentIdentity,
pub(super) admitted_bridge_session_id: SessionId,
pub(super) prompt: ContentInput,
pub(super) initial_turn_prompt: Option<ContentInput>,
pub(super) runtime_mode: crate::MobRuntimeMode,
pub(super) labels: std::collections::BTreeMap<String, String>,
pub(super) owner_bridge_session_id: Option<SessionId>,
pub(super) auto_wire_parent: bool,
pub(super) restore_wiring: Option<RestoreWiringPlan>,
pub(super) effective_profile_override: Option<crate::profile::Profile>,
pub(super) per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
pub(super) authorized_profile_material: AuthorizedSpawnProfileMaterial,
pub(super) continuity_intent: super::handle::SpawnContinuityIntent,
pub(super) progress: Arc<std::sync::Mutex<PendingSpawnProgress>>,
pub(super) pending_recipient_trust_peer_id: Option<String>,
pub(super) reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
}
#[derive(Debug, Default)]
pub(super) struct PendingSpawnProgress {
pub(super) bridge_session_id: Option<meerkat_core::types::SessionId>,
pub(super) operation_id: Option<meerkat_core::ops::OperationId>,
}
#[must_use = "mob spawn start authority must be consumed by pending spawn insertion"]
struct AuthorizedMobSpawnStart {
generated_can_start: generated_mob_command_capabilities::CommandPlanKind,
generated_owner: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
session_id: SessionId,
}
#[must_use = "started mob spawn authority must be consumed by PendingSpawnLineage insertion"]
struct AuthorizedMobSpawnStarted {
generated_owner: generated_mob_command_capabilities::CommandPlanKind,
generated_started: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
session_id: SessionId,
}
#[must_use = "mob spawn completion authority must be observed by pending spawn completion"]
struct AuthorizedMobSpawnCompleted {
generated_plan: generated_mob_command_capabilities::CommandPlanKind,
generated_effect: generated_mob_command_capabilities::CommandPlanKind,
agent_identity: AgentIdentity,
}
impl AuthorizedMobSpawnStart {
fn owner_session_id(&self) -> &SessionId {
debug_assert_eq!(
self.generated_can_start,
generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn
);
debug_assert_eq!(
self.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
&self.session_id
}
fn start(self, pending: &PendingSpawn) -> Result<AuthorizedMobSpawnStarted, MobError> {
debug_assert_eq!(
self.generated_can_start,
generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn
);
debug_assert_eq!(
self.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
if pending.agent_identity != self.agent_identity
|| pending.admitted_bridge_session_id != self.session_id
{
return Err(MobError::Internal(format!(
"MobMachine StageSpawn authority for '{}' / '{}' did not match pending spawn '{}' / '{}'",
self.agent_identity,
self.session_id,
pending.agent_identity,
pending.admitted_bridge_session_id
)));
}
Ok(AuthorizedMobSpawnStarted {
generated_owner: self.generated_owner,
generated_started: generated_mob_command_capabilities::CommandPlanKind::SpawnStarted,
agent_identity: self.agent_identity,
session_id: self.session_id,
})
}
}
#[derive(Clone, Debug)]
pub(super) struct PendingSpawnCleanupAnchor {
spawn_ticket: u64,
agent_identity: AgentIdentity,
session_id: meerkat_core::types::SessionId,
operation_id: meerkat_core::ops::OperationId,
reason: String,
}
#[derive(Clone, Debug, Default)]
pub(super) struct RestoreWiringPlan {
local_peers: Vec<AgentIdentity>,
external_peers: Vec<TrustedPeerDescriptor>,
}
struct RespawnSnapshot {
profile_name: ProfileName,
runtime_mode: crate::MobRuntimeMode,
labels: std::collections::BTreeMap<String, String>,
old_runtime_id: crate::ids::AgentRuntimeId,
old_fence_token: crate::ids::FenceToken,
restore_wiring: RestoreWiringPlan,
binding: crate::RuntimeBinding,
effective_profile_override: Option<crate::profile::Profile>,
cleanup_retry: bool,
}
struct FinalizeSpawnOutcome {
receipt: super::handle::MemberSpawnReceipt,
failed_restore_peer_ids: Vec<RespawnTopologyPeerId>,
}
struct SpawnFinalizeCtx {
profile_name: ProfileName,
agent_identity: AgentIdentity,
generation: crate::ids::Generation,
fence_token: crate::ids::FenceToken,
runtime_mode: crate::MobRuntimeMode,
prompt: ContentInput,
initial_turn_prompt: Option<ContentInput>,
labels: std::collections::BTreeMap<String, String>,
operation_id: meerkat_core::ops::OperationId,
owner_bridge_session_id: Option<SessionId>,
auto_wire_parent: bool,
restore_wiring: Option<RestoreWiringPlan>,
effective_profile_override: Option<crate::profile::Profile>,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
authorized_profile_material: AuthorizedSpawnProfileMaterial,
continuity_intent: super::handle::SpawnContinuityIntent,
}
struct SpawnAdmitted {
member_ref: MemberRef,
agent_runtime_id: crate::ids::AgentRuntimeId,
is_replacing: bool,
}
struct RespawnTopologyRestoreResolution {
result: mob_dsl::RespawnTopologyRestoreResultKind,
failed_peer_ids: Vec<RespawnTopologyPeerId>,
}
#[cfg(not(target_arch = "wasm32"))]
struct RemoteDestroyOutcome {
identity: AgentIdentity,
force_destroyed: bool,
orphaned: bool,
errors: Vec<String>,
}
struct RuntimeMetadataSnapshot {
supervisor: Option<crate::store::SupervisorAuthorityRecord>,
external_binding_overlays: Vec<crate::store::ExternalBindingOverlayRecord>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ExpectedRevokeCleanupFailure {
BridgeRejected {
cause: super::bridge_protocol::BridgeRejectionCause,
},
}
pub(super) struct MobActor {
pub(super) definition: Arc<MobDefinition>,
pub(super) roster: Arc<RwLock<RosterAuthority>>,
pub(super) events: Arc<dyn MobEventStore>,
pub(super) run_store: Arc<dyn MobRunStore>,
pub(super) provisioner: Arc<dyn MobProvisioner>,
pub(super) flow_engine: FlowEngine,
pub(super) has_orchestrator: bool,
pub(super) run_tasks: BTreeMap<RunId, tokio::task::JoinHandle<()>>,
pub(super) run_cancel_tokens: BTreeMap<RunId, (tokio_util::sync::CancellationToken, FlowId)>,
pub(super) flow_streams:
Arc<tokio::sync::Mutex<BTreeMap<RunId, mpsc::Sender<meerkat_core::ScopedAgentEvent>>>>,
pub(super) command_tx: mpsc::Sender<MobCommand>,
pub(super) tool_bundles: BTreeMap<String, Arc<dyn AgentToolDispatcher>>,
pub(super) default_llm_client: Option<Arc<dyn LlmClient>>,
pub(super) retired_event_index: Arc<RwLock<HashSet<String>>>,
pub(super) autonomous_initial_turns:
Arc<tokio::sync::Mutex<BTreeMap<AgentIdentity, InitialTurnHandle>>>,
pub(super) next_spawn_ticket: u64,
pub(super) next_fence_token: std::sync::atomic::AtomicU64,
pub(super) pending_spawns: PendingSpawnLineage,
pub(super) pending_spawn_cleanup_anchors: BTreeMap<u64, PendingSpawnCleanupAnchor>,
pub(super) edge_locks: Arc<super::edge_locks::EdgeLockRegistry>,
pub(super) lifecycle_tasks: tokio::task::JoinSet<Result<(), MobError>>,
pub(super) next_peer_delivery_ticket: PeerDeliveryId,
pub(super) peer_delivery_tasks: tokio::task::JoinSet<PeerDeliveryCompletion>,
pub(super) peer_delivery_inflight: BTreeMap<PeerDeliveryId, PeerDeliveryInflight>,
pub(super) peer_delivery_permits: Arc<tokio::sync::Semaphore>,
pub(super) session_service: Arc<dyn MobSessionService>,
#[cfg(feature = "runtime-adapter")]
pub(super) runtime_adapter: Option<Arc<meerkat_runtime::MeerkatMachine>>,
pub(super) restore_diagnostics:
Arc<RwLock<HashMap<AgentIdentity, super::handle::RestoreFailureDiagnostic>>>,
pub(super) member_revival_locks:
Arc<tokio::sync::Mutex<HashMap<SessionId, Arc<tokio::sync::Mutex<()>>>>>,
pub(super) runtime_metadata: Arc<dyn crate::store::MobRuntimeMetadataStore>,
pub(super) supervisor_bridge: Arc<super::MobSupervisorBridge>,
pub(super) spawn_policy: Arc<super::spawn_policy::SpawnPolicyService>,
pub(super) dsl_authority: mob_dsl::MobMachineAuthority,
pub(super) dsl_topology_epoch: Arc<std::sync::atomic::AtomicU64>,
pub(super) dsl_authority_owner_token: Arc<dyn std::any::Any + Send + Sync>,
pub(super) machine_state_watch_tx: tokio::sync::watch::Sender<mob_dsl::MobMachineState>,
pub(super) phase_watch_tx: tokio::sync::watch::Sender<MobState>,
pub(super) default_external_tools_provider: Option<crate::ExternalToolsProvider>,
pub(super) per_spawn_external_tools:
tokio::sync::RwLock<BTreeMap<AgentIdentity, Arc<dyn AgentToolDispatcher>>>,
pub(super) spawn_member_customizer: Option<Arc<dyn super::SpawnMemberCustomizer>>,
pub(super) realm_profile_store: Option<Arc<dyn crate::store::RealmProfileStore>>,
pub(super) composition_binding: super::composition::MobCompositionBinding,
pub(super) pending_routed_effects: Vec<super::composition::MobSeamEffect>,
pub(super) destroy_cleanup_active: bool,
}
struct AuthorizedPeerOnlyBind {
peer: TrustedPeerDescriptor,
response: super::bridge_protocol::BridgeBindResponse,
}
enum ActorBoundaryFlushDisposition {
Drained,
RetryAfterMachineClosure(MobError),
Fatal(MobError),
}
impl MobActor {
fn peer_only_member_control_error(
runtime_mode: crate::MobRuntimeMode,
action: &str,
) -> MobError {
MobError::UnsupportedForMode {
mode: runtime_mode,
reason: format!("{action} is not supported for peer-only members in phase 1"),
}
}
fn peer_only_spec_from_parts(
peer_id: &str,
address: &str,
context: &'static str,
pubkey: [u8; 32],
) -> Result<TrustedPeerDescriptor, MobError> {
let peer_name = address
.strip_prefix("inproc://")
.map(|value| value.split('?').next().unwrap_or(value).to_string())
.unwrap_or_else(|| format!("mob_member/backend_peer/{peer_id}"));
let result = TrustedPeerDescriptor::unsigned_with_pubkey(
peer_name,
peer_id.to_string(),
pubkey,
address.to_string(),
);
result.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid peer-only runtime spec: {error}"
))
})
}
fn peer_only_spec_for_binding(
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
match binding {
crate::RuntimeBinding::External {
peer_id,
address,
pubkey,
..
} => Self::peer_only_spec_from_parts(peer_id, address, context, *pubkey),
crate::RuntimeBinding::Session => Err(MobError::Internal(format!(
"{context}: peer-only runtime spec requested for session binding"
))),
}
}
fn peer_only_spec_from_member_endpoint(
endpoint: &mob_dsl::MemberPeerEndpoint,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
TrustedPeerDescriptor::unsigned_with_pubkey(
endpoint.name.0.clone(),
endpoint.peer_id.0.clone(),
endpoint.signing_key.0,
endpoint.address.0.clone(),
)
.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid MobMachine member peer endpoint authority: {error}"
))
})
}
fn member_peer_rebind_endpoint_from_transition(
transition: &mob_dsl::MobMachineTransition,
agent_identity: &mob_dsl::AgentIdentity,
context: &'static str,
) -> Result<mob_dsl::MemberPeerEndpoint, MobError> {
let mut authorized = None;
for effect in transition.effects() {
if let mob_dsl::MobMachineEffect::MemberPeerRebindAuthorized {
agent_identity: effect_identity,
peer_endpoint,
..
} = effect
{
if effect_identity != agent_identity {
continue;
}
if authorized.replace(peer_endpoint.clone()).is_some() {
return Err(MobError::WiringError(format!(
"{context}: duplicate generated member peer rebind authority for '{}'",
agent_identity.0
)));
}
}
}
authorized.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: missing generated member peer rebind authority for '{}'",
agent_identity.0
))
})
}
fn trusted_peer_removal_key(peer: &TrustedPeerDescriptor) -> String {
peer.peer_id.to_string()
}
fn supervisor_publish_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_publish::publish_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn supervisor_publish_cleanup_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_publish::cleanup_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn supervisor_revoke_authority(
obligation: &meerkat_runtime::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation,
) -> Result<CommsTrustMutationAuthority, String> {
meerkat_runtime::protocol_supervisor_trust_revoke::revoke_authority_for_peer(
obligation,
obligation.peer_id(),
)
}
fn unexpected_trust_mutation_result(
operation: &'static str,
result: CommsTrustMutationResult,
) -> SendError {
SendError::Internal(format!(
"{operation} returned unexpected trust mutation result: {result:?}"
))
}
async fn bind_generated_mob_trust_owner_for_authority(
&self,
comms: &(dyn CoreCommsRuntime + '_),
authority: &CommsTrustMutationAuthority,
) -> Result<(), SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn bind_generated_mob_trust_owner_for_authority_with_token(
comms: &(dyn CoreCommsRuntime + '_),
authority: &CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
if authority.is_mob_machine_source() {
comms
.install_generated_mob_trust_owner(Arc::clone(owner_token))
.await?;
}
Ok(())
}
async fn apply_trusted_peer_add(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<(), SendError> {
self.apply_trusted_peer_add_report(comms, peer, authority)
.await?;
Ok(())
}
async fn apply_trusted_peer_add_report(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
Self::apply_trusted_peer_add_with_owner_token_report(
comms,
peer,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn apply_trusted_peer_add_with_owner_token(
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
Self::apply_trusted_peer_add_with_owner_token_report(comms, peer, authority, owner_token)
.await?;
Ok(())
}
async fn apply_trusted_peer_add_with_owner_token_report(
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<bool, SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
&authority,
owner_token,
)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::AddTrustedPeer { peer, authority })
.await?
{
CommsTrustMutationResult::Added { created } => Ok(created),
result => Err(Self::unexpected_trust_mutation_result(
"add trusted peer",
result,
)),
}
}
async fn apply_trusted_peer_remove(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
Self::apply_trusted_peer_remove_with_owner_token(
comms,
peer_id,
authority,
&self.dsl_authority.generated_authority_owner_token(),
)
.await
}
async fn apply_trusted_peer_remove_with_owner_token(
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<bool, SendError> {
Self::bind_generated_mob_trust_owner_for_authority_with_token(
comms,
&authority,
owner_token,
)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::RemoveTrustedPeer { peer_id, authority })
.await?
{
CommsTrustMutationResult::Removed { removed } => Ok(removed),
result => Err(Self::unexpected_trust_mutation_result(
"remove trusted peer",
result,
)),
}
}
async fn apply_private_trusted_peer_add(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer: TrustedPeerDescriptor,
authority: CommsTrustMutationAuthority,
) -> Result<(), SendError> {
self.bind_generated_mob_trust_owner_for_authority(comms, &authority)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::AddPrivateTrustedPeer { peer, authority })
.await?
{
CommsTrustMutationResult::Added { .. } => Ok(()),
result => Err(Self::unexpected_trust_mutation_result(
"add private trusted peer",
result,
)),
}
}
async fn apply_private_trusted_peer_remove(
&self,
comms: &(dyn CoreCommsRuntime + '_),
peer_id: String,
authority: CommsTrustMutationAuthority,
) -> Result<bool, SendError> {
self.bind_generated_mob_trust_owner_for_authority(comms, &authority)
.await?;
match comms
.apply_trust_mutation(CommsTrustMutation::RemovePrivateTrustedPeer {
peer_id,
authority,
})
.await?
{
CommsTrustMutationResult::Removed { removed } => Ok(removed),
result => Err(Self::unexpected_trust_mutation_result(
"remove private trusted peer",
result,
)),
}
}
fn supervisor_spec_for_authority(
mob_id: &crate::MobId,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<TrustedPeerDescriptor, MobError> {
let participant_name = format!("{mob_id}/__mob_supervisor__");
let public_key = authority.keypair().public_key();
TrustedPeerDescriptor::unsigned_with_pubkey(
participant_name.clone(),
authority.public_peer_id.clone(),
*public_key.as_bytes(),
format!("inproc://{participant_name}"),
)
.map_err(|error| MobError::WiringError(format!("invalid supervisor spec: {error}")))
}
async fn install_supervisor_private_trust_for_session(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
previous_private_trust_removal_key: Option<&str>,
) -> Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError> {
let authority = self.supervisor_bridge.authority().await;
let spec = Self::supervisor_spec_for_authority(&self.definition.id, &authority)?;
Box::pin(self.install_supervisor_private_trust_for_session_authority(
session_id,
comms,
&authority,
spec,
None,
previous_private_trust_removal_key,
))
.await
}
#[cfg(feature = "runtime-adapter")]
async fn realize_supervisor_private_trust_revoke(
&self,
request: SupervisorPrivateTrustRevokeRequest<'_>,
) -> Result<bool, MobError> {
let SupervisorPrivateTrustRevokeRequest {
adapter,
session_id,
comms,
peer_id,
epoch,
removal_key,
allow_absent_pending,
} = request;
let revoke_transition = match adapter
.stage_supervisor_revoke(session_id, peer_id.clone(), epoch)
.await
{
Ok(transition) => transition,
Err(_) if allow_absent_pending => return Ok(false),
Err(error) => {
return Err(MobError::WiringError(format!(
"previous supervisor private trust revoke rejected for session '{session_id}': {error}"
)));
}
};
let revoke_freshness = adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
.map_err(|error| {
MobError::WiringError(format!(
"previous supervisor private trust revoke freshness unavailable for session '{session_id}': {error}"
))
})?;
let revoke_obligation =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
&revoke_transition,
revoke_freshness,
)
.into_iter()
.find(|obligation| obligation.peer_id() == &peer_id && obligation.epoch() == epoch)
.ok_or_else(|| {
MobError::WiringError(format!(
"previous supervisor private trust revoke for session '{session_id}' produced no generated revoke obligation"
))
})?;
if let Err(error) = self
.apply_private_trusted_peer_remove(
comms,
removal_key,
Self::supervisor_revoke_authority(&revoke_obligation)
.map_err(MobError::WiringError)?,
)
.await
{
let feedback = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
revoke_obligation.peer_id().clone(),
revoke_obligation.epoch(),
error.to_string(),
)
.await;
let mut reason = format!(
"previous supervisor private trust removal failed for session '{session_id}': {error}"
);
if let Err(feedback_error) = feedback {
reason.push_str(&format!("; revoke feedback failed: {feedback_error}"));
}
return Err(MobError::WiringError(reason));
}
adapter
.stage_supervisor_trust_revoked(
session_id,
revoke_obligation.peer_id().clone(),
revoke_obligation.epoch(),
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"previous supervisor private trust revoke feedback rejected for session '{session_id}': {error}"
))
})?;
Ok(true)
}
async fn install_supervisor_private_trust_for_session_authority(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
authority: &crate::store::SupervisorAuthorityRecord,
spec: TrustedPeerDescriptor,
previous_authority: Option<&crate::store::SupervisorAuthorityRecord>,
previous_private_trust_removal_key: Option<&str>,
) -> Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError> {
#[cfg(feature = "runtime-adapter")]
let Some(adapter) = self.runtime_adapter.as_ref() else {
return Err(MobError::Internal(format!(
"cannot publish supervisor private trust for session '{session_id}': runtime adapter unavailable"
))
.into());
};
#[cfg(not(feature = "runtime-adapter"))]
let _ = session_id;
#[cfg(not(feature = "runtime-adapter"))]
{
return Err(MobError::Internal(
"cannot publish supervisor private trust without runtime adapter".to_string(),
)
.into());
}
#[cfg(feature = "runtime-adapter")]
{
use meerkat_runtime::protocol_supervisor_trust_publish;
adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust local endpoint rejected for session '{session_id}': {error}"
))
})?;
let next_name = spec.name.as_str().to_owned();
let next_peer_id = spec.peer_id.as_str().to_owned();
let next_address = spec.address.to_string();
let next_signing_public_key =
meerkat_runtime::comms_drain::encode_supervisor_signing_public_key(spec.pubkey);
let next_epoch = authority.epoch;
let previous = adapter.supervisor_binding(session_id).await;
let already_bound = matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
name,
peer_id,
address,
signing_public_key,
epoch,
} if name == &next_name
&& peer_id == &next_peer_id
&& address == &next_address
&& signing_public_key == &next_signing_public_key
&& *epoch == next_epoch
);
let previous_peer_is_different = matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { peer_id, .. }
if peer_id != &next_peer_id
);
if matches!(
&previous,
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound
) && let Some(previous_authority) = previous_authority
{
let _ = self
.realize_supervisor_private_trust_revoke(SupervisorPrivateTrustRevokeRequest {
adapter: adapter.as_ref(),
session_id,
comms: comms.as_ref(),
peer_id: previous_authority.public_peer_id.clone(),
epoch: previous_authority.epoch,
removal_key: previous_private_trust_removal_key
.map(str::to_string)
.unwrap_or_else(|| previous_authority.public_peer_id.clone()),
allow_absent_pending: true,
})
.await?;
}
if previous_peer_is_different {
let (previous_peer_id, previous_epoch) = match &previous {
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
peer_id,
epoch,
..
} => (peer_id.clone(), *epoch),
_ => unreachable!("previous_peer_is_different only matches Bound"),
};
let previous_removal_key = previous_private_trust_removal_key
.map(str::to_string)
.unwrap_or_else(|| previous_peer_id.clone());
self.realize_supervisor_private_trust_revoke(SupervisorPrivateTrustRevokeRequest {
adapter: adapter.as_ref(),
session_id,
comms: comms.as_ref(),
peer_id: previous_peer_id,
epoch: previous_epoch,
removal_key: previous_removal_key,
allow_absent_pending: false,
})
.await?;
}
let stage_transition = if already_bound {
adapter
.stage_supervisor_trust_publish_request(
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publish request rejected for session '{session_id}': {error}"
))
})?
} else if previous_peer_is_different {
Self::stage_supervisor_bind_for_private_trust(
adapter,
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust bind rejected for session '{session_id}': {error}"
))
})?
} else {
match &previous {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => {
Self::stage_supervisor_bind_for_private_trust(
adapter,
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust bind rejected for session '{session_id}': {error}"
))
})?
}
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { .. } => {
adapter
.stage_supervisor_authorize(
session_id,
next_name.clone(),
next_peer_id.clone(),
next_address.clone(),
next_signing_public_key.clone(),
next_epoch,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust rotation rejected for session '{session_id}': {error}"
))
})?
}
_ => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' saw an unknown supervisor binding variant"
))
.into());
}
}
};
let publish_freshness = adapter
.supervisor_trust_publish_freshness_authority(session_id)
.await
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publish freshness unavailable for session '{session_id}': {error}"
))
})?;
let obligations = protocol_supervisor_trust_publish::extract_obligations_with_freshness(
&stage_transition,
publish_freshness,
);
let publish_obligation = match obligations.as_slice() {
[obligation] => obligation.clone(),
[] => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' produced no generated publish obligation"
))
.into());
}
_ => {
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' produced multiple generated publish obligations"
))
.into());
}
};
if publish_obligation.name() != &next_name
|| publish_obligation.peer_id() != &next_peer_id
|| publish_obligation.address() != &next_address
|| publish_obligation.signing_public_key().as_deref()
!= Some(next_signing_public_key.as_str())
|| publish_obligation.epoch() != next_epoch
{
return Err(MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' generated obligation did not match the staged supervisor binding"
))
.into());
}
let publish_spec =
meerkat_runtime::comms_drain::trusted_peer_descriptor_from_supervisor_publish_obligation(
&publish_obligation,
)
.map_err(|error| {
MobError::WiringError(format!(
"supervisor private trust publication for session '{session_id}' generated invalid trust descriptor: {error}"
))
})?;
let publish_peer_id = publish_obligation.peer_id().clone();
let publish_epoch = publish_obligation.epoch();
let publish_removal_key = Self::trusted_peer_removal_key(&publish_spec);
let publish_cleanup_authority =
Self::supervisor_publish_cleanup_authority(&publish_obligation)
.map_err(MobError::WiringError)?;
let rollback_binding = previous.clone();
if let Err(error) = self
.apply_private_trusted_peer_add(
comms.as_ref(),
publish_spec.clone(),
Self::supervisor_publish_authority(&publish_obligation)
.map_err(MobError::WiringError)?,
)
.await
{
let _ = adapter
.stage_supervisor_trust_publish_failed(
session_id,
publish_peer_id.clone(),
publish_epoch,
error.to_string(),
)
.await;
let new_trust_cleanup_failed = if !already_bound {
self.cleanup_supervisor_private_trust_publish_attempt(
session_id,
comms,
publish_cleanup_authority.clone(),
publish_removal_key.clone(),
"failed to clean up supervisor private trust after publish add failure",
)
.await
.is_err()
} else {
false
};
let rollback = if already_bound {
Ok(())
} else {
self.rollback_supervisor_private_trust_binding(
adapter,
session_id,
comms,
&rollback_binding,
&publish_peer_id,
publish_epoch,
)
.await
};
let mut reason = format!(
"supervisor private trust publication failed for session '{session_id}': {error}"
);
if let Err(rollback_error) = rollback {
reason.push_str(&format!("; rollback failed: {rollback_error}"));
}
let error = MobError::WiringError(reason);
return Err(if new_trust_cleanup_failed {
SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)
} else {
SupervisorPrivateTrustInstallError::without_cleanup_failure(error)
});
}
if let Err(error) = Self::stage_supervisor_trust_published_for_private_trust(
adapter,
session_id,
publish_peer_id.clone(),
publish_epoch,
)
.await
{
let new_trust_cleanup_failed = if !already_bound {
self.cleanup_supervisor_private_trust_publish_attempt(
session_id,
comms,
publish_cleanup_authority,
publish_removal_key.clone(),
"failed to clean up supervisor private trust after rejected publish ack",
)
.await
.is_err()
} else {
false
};
let rollback = if already_bound {
Ok(())
} else {
self.rollback_supervisor_private_trust_binding(
adapter,
session_id,
comms,
&rollback_binding,
&publish_peer_id,
publish_epoch,
)
.await
};
let mut reason = format!(
"supervisor private trust publication ack rejected for session '{session_id}': {error}"
);
if let Err(rollback_error) = rollback {
reason.push_str(&format!("; rollback failed: {rollback_error}"));
}
let error = MobError::WiringError(reason);
return Err(if new_trust_cleanup_failed {
SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)
} else {
SupervisorPrivateTrustInstallError::without_cleanup_failure(error)
});
}
Ok(SupervisorPrivateTrustInstall {
peer_id: next_peer_id,
epoch: next_epoch,
removal_key: publish_removal_key,
})
}
}
#[cfg(feature = "runtime-adapter")]
async fn stage_supervisor_trust_published_for_private_trust(
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
peer_id: String,
epoch: u64,
) -> Result<(), meerkat_runtime::meerkat_machine::SupervisorBindingStageError> {
#[cfg(target_arch = "wasm32")]
{
let adapter = Arc::clone(adapter);
let session_id = session_id.clone();
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let result = adapter
.stage_supervisor_trust_published(&session_id, peer_id, epoch)
.await;
let _ = reply_tx.send(result);
});
reply_rx.await.map_err(|_| {
meerkat_runtime::meerkat_machine::SupervisorBindingStageError::SessionRegistryBusy
})?
}
#[cfg(not(target_arch = "wasm32"))]
{
adapter
.stage_supervisor_trust_published(session_id, peer_id, epoch)
.await
}
}
#[cfg(feature = "runtime-adapter")]
async fn stage_supervisor_bind_for_private_trust(
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
name: String,
peer_id: String,
address: String,
signing_public_key: String,
epoch: u64,
) -> Result<
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineTransition,
meerkat_runtime::meerkat_machine::SupervisorBindingStageError,
> {
#[cfg(target_arch = "wasm32")]
{
let adapter = Arc::clone(adapter);
let session_id = session_id.clone();
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let result = adapter
.stage_supervisor_bind(
&session_id,
name,
peer_id,
address,
signing_public_key,
epoch,
)
.await;
let _ = reply_tx.send(result);
});
reply_rx.await.map_err(|_| {
meerkat_runtime::meerkat_machine::SupervisorBindingStageError::SessionRegistryBusy
})?
}
#[cfg(not(target_arch = "wasm32"))]
{
adapter
.stage_supervisor_bind(
session_id,
name,
peer_id,
address,
signing_public_key,
epoch,
)
.await
}
}
async fn cleanup_supervisor_private_trust_publish_attempt(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
authority: CommsTrustMutationAuthority,
removal_key: String,
context: &'static str,
) -> Result<(), MobError> {
if let Err(error) = self
.apply_private_trusted_peer_remove(comms.as_ref(), removal_key, authority)
.await
{
tracing::warn!(
%session_id,
%error,
context,
"failed to clean up supervisor private trust publish attempt"
);
return Err(MobError::from(error));
}
Ok(())
}
async fn cleanup_supervisor_private_trust_for_session(
&self,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
install: &SupervisorPrivateTrustInstall,
) {
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = self.runtime_adapter.as_ref() {
if let Err(error) = adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
{
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to stage local endpoint for supervisor private trust cleanup"
);
return;
}
let transition = match adapter
.stage_supervisor_revoke(session_id, install.peer_id.clone(), install.epoch)
.await
{
Ok(transition) => transition,
Err(error) => {
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to stage supervisor private trust cleanup"
);
return;
}
};
let revoke_freshness = match adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
{
Ok(authority) => authority,
Err(error) => {
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to build generated supervisor private trust cleanup freshness"
);
return;
}
};
let obligations =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(&transition, revoke_freshness);
let Some(obligation) = obligations.into_iter().find(|obligation| {
obligation.peer_id() == &install.peer_id && obligation.epoch() == install.epoch
}) else {
let reason =
"generated supervisor private trust cleanup effect was absent".to_string();
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
install.peer_id.clone(),
install.epoch,
reason.clone(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
reason,
"failed to stage supervisor private trust cleanup"
);
return;
};
if let Err(error) = self.apply_private_trusted_peer_remove(
comms.as_ref(),
install.removal_key.clone(),
match Self::supervisor_revoke_authority(&obligation) {
Ok(authority) => authority,
Err(error) => {
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
error.clone(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to build generated supervisor private trust cleanup authority"
);
return;
}
},
)
.await
{
let _ = adapter
.stage_supervisor_trust_revoke_failed(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
error.to_string(),
)
.await;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to clean up supervisor private trust"
);
return;
}
if let Err(error) = adapter
.stage_supervisor_trust_revoked(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
{
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
%error,
"failed to acknowledge supervisor private trust cleanup"
);
}
return;
}
let _ = comms;
tracing::warn!(
%session_id,
peer_id = %install.peer_id,
epoch = install.epoch,
"skipping supervisor private trust cleanup because generated runtime adapter authority is unavailable"
);
}
#[cfg(feature = "runtime-adapter")]
async fn rollback_supervisor_private_trust_binding(
&self,
adapter: &Arc<meerkat_runtime::MeerkatMachine>,
session_id: &SessionId,
comms: &Arc<dyn CoreCommsRuntime>,
previous: &meerkat_runtime::meerkat_machine::SupervisorBinding,
current_peer_id: &str,
current_epoch: u64,
) -> Result<(), MobError> {
adapter
.stage_local_endpoint_for_comms_runtime(session_id, comms.as_ref())
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
match previous {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => {
let transition = adapter
.stage_supervisor_revoke(session_id, current_peer_id.to_string(), current_epoch)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
let revoke_freshness = adapter
.supervisor_trust_revoke_freshness_authority(session_id)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
if let Some(obligation) =
meerkat_runtime::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
&transition,
revoke_freshness,
)
.into_iter()
.find(|obligation| {
obligation.peer_id().as_str() == current_peer_id
&& obligation.epoch() == current_epoch
})
{
adapter
.stage_supervisor_trust_revoked(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
}
Ok(())
}
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
name,
peer_id,
address,
signing_public_key,
epoch,
} => {
let current = adapter.supervisor_binding(session_id).await;
let transition = match current {
meerkat_runtime::meerkat_machine::SupervisorBinding::Unbound => adapter
.stage_supervisor_bind(
session_id,
name.clone(),
peer_id.clone(),
address.clone(),
signing_public_key.clone(),
*epoch,
)
.await,
meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { .. } => adapter
.stage_supervisor_authorize(
session_id,
name.clone(),
peer_id.clone(),
address.clone(),
signing_public_key.clone(),
*epoch,
)
.await,
other => {
return Err(MobError::WiringError(format!(
"supervisor private trust rollback for session '{session_id}' saw unsupported current binding {other:?}"
)));
}
}
.map_err(|error| MobError::WiringError(error.to_string()))?;
let publish_freshness = adapter
.supervisor_trust_publish_freshness_authority(session_id)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
let obligation =
meerkat_runtime::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
&transition,
publish_freshness,
)
.into_iter()
.find(|obligation| {
obligation.peer_id() == peer_id
&& obligation.epoch() == *epoch
&& obligation.signing_public_key().as_deref()
== Some(signing_public_key.as_str())
})
.ok_or_else(|| {
MobError::WiringError(format!(
"supervisor private trust rollback for session '{session_id}' produced no generated publish obligation"
))
})?;
let trusted_peer =
meerkat_runtime::comms_drain::trusted_peer_descriptor_from_supervisor_publish_obligation(
&obligation,
)
.map_err(MobError::WiringError)?;
self.apply_private_trusted_peer_add(
comms.as_ref(),
trusted_peer,
Self::supervisor_publish_authority(&obligation)
.map_err(MobError::WiringError)?,
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
adapter
.stage_supervisor_trust_published(
session_id,
obligation.peer_id().clone(),
obligation.epoch(),
)
.await
.map_err(|error| MobError::WiringError(error.to_string()))?;
Ok(())
}
_ => Err(MobError::WiringError(
"unknown supervisor binding variant during rollback".to_string(),
)),
}
}
async fn bridge_supervisor_payload(
&self,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let authority = self.supervisor_bridge.authority().await;
self.supervisor_payload_for_authority(&authority)
}
async fn bridge_supervisor_payload_for_recipient(
&self,
recipient: &TrustedPeerDescriptor,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let authority = self.supervisor_bridge.authority().await;
self.bridge_supervisor_payload_for_authority_and_recipient(&authority, recipient)
.await
}
async fn bridge_supervisor_payload_for_authority_and_recipient(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
recipient: &TrustedPeerDescriptor,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let spec = self
.supervisor_bridge
.supervisor_spec_for_authority_and_recipient(authority, recipient)
.await?;
Ok(super::bridge_protocol::BridgeSupervisorPayload {
supervisor: spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
})
}
fn supervisor_payload_for_authority(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<super::bridge_protocol::BridgeSupervisorPayload, MobError> {
let spec = Self::supervisor_spec_for_authority(&self.definition.id, authority)?;
Ok(super::bridge_protocol::BridgeSupervisorPayload {
supervisor: spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
})
}
fn bridge_bootstrap_token_from_binding(
binding: &crate::RuntimeBinding,
) -> Result<super::bridge_protocol::BridgeBootstrapToken, MobError> {
match binding {
crate::RuntimeBinding::External {
address,
bootstrap_token,
..
} => bootstrap_token
.as_ref()
.filter(|token| !token.is_empty())
.cloned()
.ok_or_else(|| {
MobError::WiringError(format!(
"external runtime binding for '{address}' is missing typed bootstrap_token field"
))
}),
crate::RuntimeBinding::Session => Err(MobError::Internal(
"bridge bootstrap token requested for session binding".to_string(),
)),
}
}
fn authorize_existing_peer_binding_for_rebind(
&mut self,
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<TrustedPeerDescriptor, MobError> {
let crate::RuntimeBinding::External {
peer_id,
address,
pubkey,
..
} = binding
else {
return Err(MobError::Internal(format!(
"{context}: peer-only rebind authority requested for session binding"
)));
};
let canonical_address = super::bridge_protocol::canonicalize_bridge_address(address);
let observed_peer =
Self::peer_only_spec_from_parts(peer_id, &canonical_address, context, *pubkey)?;
let affected_identities: BTreeSet<_> = self
.dsl_authority
.state()
.member_peer_ids
.iter()
.filter(|(_, existing_peer_id)| existing_peer_id.0 == *peer_id)
.map(|(identity, _)| identity.clone())
.collect();
if affected_identities.is_empty() {
return Err(MobError::WiringError(format!(
"{context}: peer-only rebind for '{peer_id}' requires MobMachine member peer authority"
)));
}
let mut authorized_peer: Option<TrustedPeerDescriptor> = None;
for identity in &affected_identities {
let expected_peer_endpoint = self
.dsl_authority
.state()
.member_peer_endpoints
.get(identity)
.cloned()
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: peer-only rebind for '{}' lacks MobMachine endpoint authority",
identity.0
))
})?;
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerRebind {
agent_identity: identity.clone(),
expected_peer_endpoint,
},
context,
)?;
let endpoint =
Self::member_peer_rebind_endpoint_from_transition(&transition, identity, context)?;
let peer = Self::peer_only_spec_from_member_endpoint(&endpoint, context)?;
if peer.name != observed_peer.name
|| peer.peer_id != observed_peer.peer_id
|| peer.address != observed_peer.address
|| peer.pubkey != observed_peer.pubkey
{
return Err(MobError::WiringError(format!(
"{context}: observed peer-only rebind endpoint for '{peer_id}' is outside generated MobMachine authority"
)));
}
if let Some(existing) = &authorized_peer
&& (existing.name != peer.name
|| existing.peer_id != peer.peer_id
|| existing.address != peer.address
|| existing.pubkey != peer.pubkey)
{
return Err(MobError::WiringError(format!(
"{context}: generated MobMachine peer rebind authority disagrees across identities for '{peer_id}'"
)));
}
authorized_peer = Some(peer);
}
authorized_peer.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: peer-only rebind for '{peer_id}' produced no generated authority"
))
})
}
async fn bind_peer_only_member_for_binding(
&mut self,
peer: &TrustedPeerDescriptor,
binding: &crate::RuntimeBinding,
) -> Result<AuthorizedPeerOnlyBind, MobError> {
let payload = self.bridge_supervisor_payload_for_recipient(peer).await?;
self.bind_peer_only_member_for_binding_with_payload(peer, binding, &payload)
.await
}
async fn bind_peer_only_member_for_binding_with_payload(
&mut self,
peer: &TrustedPeerDescriptor,
binding: &crate::RuntimeBinding,
payload: &super::bridge_protocol::BridgeSupervisorPayload,
) -> Result<AuthorizedPeerOnlyBind, MobError> {
let crate::RuntimeBinding::External {
peer_id,
address: _,
bootstrap_token: _,
pubkey: _,
} = binding
else {
return Err(MobError::Internal(
"bind requested for non-external runtime binding".to_string(),
));
};
let authorized_peer =
self.authorize_existing_peer_binding_for_rebind(binding, "bind_peer_only_member")?;
if authorized_peer.name != peer.name
|| authorized_peer.peer_id != peer.peer_id
|| authorized_peer.address != peer.address
|| authorized_peer.pubkey != peer.pubkey
{
return Err(MobError::WiringError(format!(
"bind requested for peer '{peer_id}' without matching MobMachine member peer authority"
)));
}
let bootstrap_token = Self::bridge_bootstrap_token_from_binding(binding)?;
let command = super::bridge_protocol::BridgeCommand::BindMember(
super::bridge_protocol::BridgeBindPayload {
supervisor: payload.supervisor.clone(),
epoch: payload.epoch,
protocol_version: payload.protocol_version,
expected_peer_id: authorized_peer.peer_id.to_string(),
expected_address: authorized_peer.address.to_string(),
bootstrap_token,
},
);
let bind: super::bridge_protocol::BridgeBindResponse = self
.send_bridge_command_typed(
&authorized_peer,
&command,
std::time::Duration::from_secs(60),
)
.await?;
let returned_address = super::bridge_protocol::canonicalize_bridge_address(&bind.address);
let authorized_peer_id = authorized_peer.peer_id.to_string();
let authorized_address = authorized_peer.address.to_string();
let expected_address =
super::bridge_protocol::canonicalize_bridge_address(&authorized_address);
if bind.peer_id != authorized_peer_id || returned_address != expected_address {
return Err(MobError::WiringError(format!(
"bind response changed authorized endpoint for peer '{peer_id}'"
)));
}
Ok(AuthorizedPeerOnlyBind {
peer: authorized_peer,
response: bind,
})
}
fn bridge_rejection_reply(
protocol_version: super::bridge_protocol::BridgeProtocolVersion,
value: &serde_json::Value,
) -> Option<super::bridge_protocol::BridgeRejectionReply> {
super::bridge_protocol::decode_bridge_rejection_reply(protocol_version, value)
}
fn bridge_rejection_error(rejection: super::bridge_protocol::BridgeRejectionReply) -> MobError {
MobError::from(rejection)
}
fn bridge_rejection_error_with_reason(
rejection: &super::bridge_protocol::BridgeRejectionReply,
reason: String,
) -> MobError {
match rejection.typed_cause() {
Some(cause) => MobError::BridgeCommandRejected { cause, reason },
None => MobError::WiringError(reason),
}
}
async fn persist_rebound_binding(
&mut self,
prior_binding: &crate::RuntimeBinding,
authorized_peer: &TrustedPeerDescriptor,
bind_response: &super::bridge_protocol::BridgeBindResponse,
) -> Result<(), MobError> {
let crate::RuntimeBinding::External {
peer_id: prior_peer_id,
address: prior_address,
pubkey,
..
} = prior_binding
else {
return Ok(());
};
let bootstrap_token = Some(Self::bridge_bootstrap_token_from_binding(prior_binding)?);
let authorized_peer_id = authorized_peer.peer_id.to_string();
let authorized_address = authorized_peer.address.to_string();
let canonical_authorized_address =
super::bridge_protocol::canonicalize_bridge_address(&authorized_address);
let expected_address = super::bridge_protocol::canonicalize_bridge_address(prior_address);
if authorized_peer_id != *prior_peer_id
|| canonical_authorized_address != expected_address
|| authorized_peer.pubkey != *pubkey
{
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' lacks matching generated MobMachine endpoint authority"
)));
}
let returned_address =
super::bridge_protocol::canonicalize_bridge_address(&bind_response.address);
if bind_response.peer_id != authorized_peer_id
|| returned_address != canonical_authorized_address
{
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' attempted to change MobMachine-authorized endpoint"
)));
}
let affected_identities: BTreeSet<_> = self
.dsl_authority
.state()
.member_peer_ids
.iter()
.filter(|(_, peer_id)| peer_id.0 == authorized_peer_id)
.map(|(identity, _)| identity.clone())
.collect();
if affected_identities.is_empty() {
return Err(MobError::WiringError(format!(
"rebound peer binding for '{prior_peer_id}' requires MobMachine member peer authority"
)));
}
let affected_domain_identities: BTreeSet<_> = affected_identities
.iter()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect();
let updated_entries = self
.roster
.write()
.await
.replace_backend_peer_binding_for_identities(
&affected_domain_identities,
&authorized_peer_id,
&canonical_authorized_address,
bootstrap_token.clone(),
);
for (identity, generation, pubkey) in updated_entries {
self.runtime_metadata
.upsert_external_binding_overlay(
&self.definition.id,
&crate::store::ExternalBindingOverlayRecord {
agent_identity: identity,
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: authorized_peer_id.clone(),
address: canonical_authorized_address.clone(),
pubkey,
bootstrap_token: None,
session_id: None,
}),
bootstrap_token: bootstrap_token.clone(),
status: crate::store::ExternalBindingOverlayStatus::Normalized,
updated_at: chrono::Utc::now(),
},
)
.await?;
}
Ok(())
}
async fn authorize_peer_only_member_ref_for_behavior(
&mut self,
member_ref: &MemberRef,
context: &'static str,
) -> Result<MemberRef, MobError> {
let Some(binding) = Self::runtime_binding_for_member_ref(member_ref) else {
return Ok(member_ref.clone());
};
let peer = Self::peer_only_spec_for_binding(&binding, context)?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(&binding))
.await?;
let bootstrap_token = Some(Self::bridge_bootstrap_token_from_binding(&binding)?);
Ok(MemberRef::BackendPeer {
peer_id: peer.peer_id.to_string(),
address: peer.address.to_string(),
pubkey: peer.pubkey,
bootstrap_token,
session_id: None,
})
}
async fn ensure_supervisor_authorized(
&mut self,
peer: &TrustedPeerDescriptor,
binding: Option<&crate::RuntimeBinding>,
) -> Result<TrustedPeerDescriptor, MobError> {
if let Some(current) = self.load_supervisor_authority().await?
&& let Some(pending) = current.pending_rotation.as_ref()
&& pending
.accepted_peer_ids
.iter()
.any(|peer_id| peer_id == &peer.peer_id.to_string())
{
return Err(Self::pending_supervisor_rotation_blocks_rebind(
¤t, pending, peer,
));
}
let payload = self.bridge_supervisor_payload_for_recipient(peer).await?;
let protocol_version = payload.protocol_version;
let command = super::bridge_protocol::BridgeCommand::AuthorizeSupervisor(payload);
self.record_pending_recipient_trust_obligation(peer, "ensure_supervisor_authorized")?;
let install = match self.supervisor_bridge.trust_recipient(peer).await {
Ok(install) => install,
Err(trust_error) => {
self.rollback_pending_recipient_trust_obligation(
peer,
"ensure_supervisor_authorized trust install failed",
)?;
return Err(trust_error);
}
};
let value = match self
.supervisor_bridge
.send_bridge_command(peer, &command, std::time::Duration::from_secs(60))
.await
{
Ok(value) => value,
Err(send_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, send_error)
.await);
}
};
if let Some(rejection) = Self::bridge_rejection_reply(protocol_version, &value) {
let should_rebind = match rejection.typed_cause() {
Some(cause) => match self.classify_bridge_rejection_recovery(cause) {
Ok(should_rebind) => should_rebind,
Err(classify_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, classify_error)
.await);
}
},
None => false,
};
if should_rebind && let Some(binding) = binding {
if let Some(current) = self.load_supervisor_authority().await?
&& let Some(pending) = current.pending_rotation.as_ref()
{
let pending_error =
Self::pending_supervisor_rotation_blocks_rebind(¤t, pending, peer);
return Err(self
.rollback_supervisor_recipient_trust(peer, install, pending_error)
.await);
}
let authorized_bind =
match self.bind_peer_only_member_for_binding(peer, binding).await {
Ok(authorized_bind) => authorized_bind,
Err(bind_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, bind_error)
.await);
}
};
self.resolve_pending_recipient_trust_obligation(
peer,
"ensure_supervisor_authorized rebind confirmed",
)?;
self.persist_rebound_binding(
binding,
&authorized_bind.peer,
&authorized_bind.response,
)
.await?;
return Ok(authorized_bind.peer);
}
return Err(self
.rollback_supervisor_recipient_trust(
peer,
install,
Self::bridge_rejection_error(rejection),
)
.await);
}
let _ack = match super::bridge_protocol::decode_bridge_ack(
&command,
value,
"authorize supervisor response",
) {
Ok(ack) => ack,
Err(decode_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, decode_error)
.await);
}
};
self.resolve_pending_recipient_trust_obligation(
peer,
"ensure_supervisor_authorized confirmed",
)?;
Ok(peer.clone())
}
fn record_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.record_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn record_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RecordPendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
fn resolve_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.resolve_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn resolve_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::ResolvePendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
fn rollback_pending_recipient_trust_obligation(
&mut self,
peer: &TrustedPeerDescriptor,
context: &str,
) -> Result<(), MobError> {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
&peer.peer_id.to_string(),
context,
)
}
fn rollback_pending_recipient_trust_obligation_for_peer_id(
&mut self,
peer_id: &str,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RollbackPendingRecipientTrust {
peer_id: mob_dsl::PeerId::from(peer_id.to_string()),
},
context,
)
}
async fn rollback_supervisor_recipient_trust(
&mut self,
peer: &TrustedPeerDescriptor,
install: super::supervisor_bridge::RecipientTrustInstall,
original_error: MobError,
) -> MobError {
if install == super::supervisor_bridge::RecipientTrustInstall::NewlyInstalled
&& let Err(untrust_error) = self.supervisor_bridge.untrust_recipient(peer).await
{
return MobError::WiringError(format!(
"supervisor authorization failed ({original_error}); additionally failed to roll back installed recipient trust: {untrust_error}"
));
}
if let Err(obligation_error) = self.rollback_pending_recipient_trust_obligation(
peer,
"rollback_supervisor_recipient_trust",
) {
return MobError::WiringError(format!(
"supervisor authorization failed ({original_error}); additionally failed to roll back the pending recipient-trust obligation: {obligation_error}"
));
}
original_error
}
async fn load_supervisor_authority_snapshot(
&self,
) -> Result<Option<SupervisorAuthorityLoad>, MobError> {
let Some(durable) = self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?
else {
return Ok(None);
};
Ok(Some(SupervisorAuthorityLoad { durable }))
}
async fn load_supervisor_authority(
&self,
) -> Result<Option<crate::store::SupervisorAuthorityRecord>, MobError> {
Ok(self
.load_supervisor_authority_snapshot()
.await?
.map(|loaded| loaded.durable))
}
fn pending_supervisor_rotation_blocks_rebind(
current: &crate::store::SupervisorAuthorityRecord,
pending: &crate::store::SupervisorPendingRotationRecord,
peer: &TrustedPeerDescriptor,
) -> MobError {
let operation = pending.operation_id.map_or_else(
|| "legacy operation awaiting migration".to_string(),
|operation_id| format!("operation {operation_id}"),
);
MobError::SupervisorRotationIncomplete {
previous_epoch: current.epoch,
attempted_epoch: pending.epoch,
attempted_public_peer_id: pending.public_peer_id.clone(),
rotated_peer_count: pending.accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation {operation} remains pending; refusing to reauthorize old authority for peer '{}'",
peer.peer_id
),
}
}
async fn send_bridge_command_typed<R: super::bridge_protocol::FromBridgeReply>(
&mut self,
peer: &TrustedPeerDescriptor,
command: &super::bridge_protocol::BridgeCommand,
timeout: std::time::Duration,
) -> Result<R, MobError> {
self.record_pending_recipient_trust_obligation(peer, "send_bridge_command_typed")?;
let install = match self.supervisor_bridge.trust_recipient(peer).await {
Ok(install) => install,
Err(trust_error) => {
self.rollback_pending_recipient_trust_obligation(
peer,
"send_bridge_command_typed trust install failed",
)?;
return Err(trust_error);
}
};
let value = match self
.supervisor_bridge
.send_bridge_command(peer, command, timeout)
.await
{
Ok(value) => value,
Err(send_error) => {
return Err(self
.rollback_supervisor_recipient_trust(peer, install, send_error)
.await);
}
};
if let Some(rejection) = Self::bridge_rejection_reply(command.protocol_version(), &value) {
return Err(self
.rollback_supervisor_recipient_trust(
peer,
install,
Self::bridge_rejection_error(rejection),
)
.await);
}
match super::bridge_protocol::decode_bridge_payload(command, value, "command") {
Ok(payload) => {
self.resolve_pending_recipient_trust_obligation(
peer,
"send_bridge_command_typed confirmed",
)?;
Ok(payload)
}
Err(decode_error) => Err(self
.rollback_supervisor_recipient_trust(peer, install, decode_error)
.await),
}
}
async fn observe_peer_only_binding(
&mut self,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<super::bridge_protocol::BridgeObservationResponse, MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "observe_peer_only_binding")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(binding))
.await?;
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::ObserveMember(payload);
self.send_bridge_command_typed(&peer, &command, timeout)
.await
}
async fn destroy_peer_only_binding(
&mut self,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<super::bridge_protocol::BridgeDestroyResponse, MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "destroy_peer_only_binding")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(binding))
.await?;
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::DestroyMember(payload);
self.send_bridge_command_typed(&peer, &command, timeout)
.await
}
async fn revoke_supervisor_for_retiring_entry(
&mut self,
entry: &RosterEntry,
binding: &crate::RuntimeBinding,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let peer = Self::peer_only_spec_for_binding(binding, "revoke_supervisor_for_binding")?;
if !self.remote_supervisor_revoked_for_entry(entry) {
let payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let command = super::bridge_protocol::BridgeCommand::RevokeSupervisor(payload);
match self
.send_bridge_command_typed::<super::bridge_protocol::BridgeAck>(
&peer, &command, timeout,
)
.await
{
Ok(_) => {}
Err(error) if Self::expected_revoke_cleanup_failure(&error).is_some() => {
tracing::debug!(
error = %error,
"supervisor revoke observed typed already-revoked terminality"
);
}
Err(error) => return Err(error),
}
self.record_remote_member_supervisor_revoked(entry).await?;
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_REMOTE_SUPERVISOR_REVOKED_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&entry.agent_identity)
{
target.take();
return Err(MobError::Internal(format!(
"fault-injected cancellation after remote supervisor revoke checkpoint for '{}'",
entry.agent_identity
)));
}
}
self.supervisor_bridge.untrust_recipient(&peer).await?;
Ok(())
}
async fn wire_peer_only_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
recipient_binding: Option<&crate::RuntimeBinding>,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let recipient = self
.ensure_supervisor_authorized(recipient, recipient_binding)
.await?;
let mob_peer_overlay =
self.mob_peer_overlay_for_recipient(&recipient, "wire_peer_only_recipient")?;
if !mob_peer_overlay
.peers()
.iter()
.any(|peer| peer.peer_id == peer_spec.peer_id)
{
return Err(MobError::WiringError(format!(
"wire_peer_only_recipient: peer '{}' is absent from MobMachine overlay for recipient '{}'",
peer_spec.peer_id, recipient.peer_id
)));
}
let authority = self.supervisor_bridge.authority().await;
let sup_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&recipient)
.await?;
let command = super::bridge_protocol::BridgeCommand::WireMember(
super::bridge_protocol::BridgePeerWiringPayload {
supervisor: sup_spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
peer_spec: peer_spec.clone().into(),
mob_peer_overlay: Some(mob_peer_overlay.bridge_handoff()),
},
);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(&recipient, &command, timeout)
.await?;
Ok(())
}
async fn unwire_peer_only_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
recipient_binding: Option<&crate::RuntimeBinding>,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let recipient = self
.ensure_supervisor_authorized(recipient, recipient_binding)
.await?;
let mob_peer_overlay =
self.mob_peer_overlay_for_recipient(&recipient, "unwire_peer_only_recipient")?;
if mob_peer_overlay
.peers()
.iter()
.any(|peer| peer.peer_id == peer_spec.peer_id)
{
return Err(MobError::WiringError(format!(
"unwire_peer_only_recipient: peer '{}' is still present in MobMachine overlay for recipient '{}'",
peer_spec.peer_id, recipient.peer_id
)));
}
let authority = self.supervisor_bridge.authority().await;
let sup_spec = self
.supervisor_bridge
.supervisor_spec_for_recipient(&recipient)
.await?;
let command = super::bridge_protocol::BridgeCommand::UnwireMember(
super::bridge_protocol::BridgePeerWiringPayload {
supervisor: sup_spec.into(),
epoch: authority.epoch,
protocol_version: authority.protocol_version,
peer_spec: peer_spec.clone().into(),
mob_peer_overlay: Some(mob_peer_overlay.bridge_handoff()),
},
);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(&recipient, &command, timeout)
.await?;
Ok(())
}
async fn unwire_retiring_peer_only_side(
&mut self,
retiring_entry: &RosterEntry,
retiring_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
timeout: std::time::Duration,
) -> Result<(), MobError> {
let Some(retiring_binding) = Self::runtime_binding_for_entry(retiring_entry) else {
return Ok(());
};
self.unwire_peer_only_recipient(retiring_spec, Some(&retiring_binding), peer_spec, timeout)
.await
}
fn mob_peer_overlay_for_recipient(
&mut self,
recipient: &TrustedPeerDescriptor,
context: &'static str,
) -> Result<super::provisioner::PeerOnlyTrustOverlay, MobError> {
let state = self.dsl_authority.state();
let expected_endpoint = mob_dsl::MemberPeerEndpoint::from(recipient);
let mut recipient_identity = None;
for (identity, endpoint) in &state.member_peer_endpoints {
if *endpoint == expected_endpoint {
if recipient_identity.replace(identity.clone()).is_some() {
return Err(MobError::WiringError(format!(
"{context}: recipient peer '{}' matches multiple MobMachine member endpoints",
recipient.peer_id
)));
}
}
}
let recipient_identity = recipient_identity.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: recipient peer '{}' is outside MobMachine member endpoint authority",
recipient.peer_id
))
})?;
let recipient_peer_id = recipient.peer_id.to_string();
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerOverlay {
agent_identity: recipient_identity.clone(),
expected_peer_endpoint: expected_endpoint,
},
context,
)?;
let obligation =
crate::generated::protocol_mob_member_peer_overlay::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_member_peer_overlay::MobTopologyFreshnessAuthority::from_live_topology_epoch(
self.dsl_topology_epoch.clone(),
Arc::clone(&self.dsl_authority_owner_token),
),
)
.into_iter()
.find(|obligation| {
obligation.agent_identity() == &recipient_identity
&& obligation.peer_id().0.as_str() == recipient_peer_id.as_str()
})
.ok_or_else(|| {
MobError::WiringError(format!(
"{context}: generated MobMachine peer overlay handoff missing for recipient '{}'",
recipient.peer_id
))
})?;
super::provisioner::PeerOnlyTrustOverlay::from_generated_mob_member_peer_overlay(
&obligation,
)
}
fn observation_is_terminal(
&mut self,
observation: &super::bridge_protocol::BridgeObservationResponse,
) -> Result<bool, MobError> {
let observed_state = remote_member_runtime_observed_state(observation.state)?;
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::ClassifyRemoteMemberRuntimeObservation { observed_state },
"classify_remote_member_runtime_observation",
)?;
let (effect_observed_state, terminality) = effects
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RemoteMemberRuntimeTerminalityClassified {
observed_state,
terminality,
} => Some((observed_state, terminality)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted remote-member runtime observation but emitted no terminality verdict"
.into(),
)
})?;
if effect_observed_state != observed_state {
return Err(MobError::Internal(format!(
"MobMachine remote-member terminality drift: input={observed_state:?}, effect={effect_observed_state:?}"
)));
}
Ok(matches!(
terminality,
mob_dsl::MobRemoteMemberRuntimeTerminality::Terminal
))
}
fn classify_bridge_rejection_recovery(
&mut self,
cause: super::bridge_protocol::BridgeRejectionCause,
) -> Result<bool, MobError> {
let rejection_cause = mob_bridge_rejection_cause(cause);
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::ClassifyBridgeRejectionRecovery { rejection_cause },
"classify_bridge_rejection_recovery",
)?;
let (effect_cause, recovery) = effects
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::BridgeRejectionRecoveryClassified {
rejection_cause,
recovery,
} => Some((rejection_cause, recovery)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted bridge rejection cause but emitted no recovery verdict"
.into(),
)
})?;
if effect_cause != rejection_cause {
return Err(MobError::Internal(format!(
"MobMachine bridge-rejection recovery drift: input={rejection_cause:?}, effect={effect_cause:?}"
)));
}
Ok(matches!(
recovery,
mob_dsl::MobBridgeRejectionRecovery::RebindRecover
))
}
fn issue_fence_token(&self) -> crate::ids::FenceToken {
let val = self
.next_fence_token
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
crate::ids::FenceToken::new(val)
}
fn next_fence_token_preview(&self) -> crate::ids::FenceToken {
crate::ids::FenceToken::new(
self.next_fence_token
.load(std::sync::atomic::Ordering::Relaxed),
)
}
fn invalid_transition_to(&self, target: MobState) -> MobError {
MobError::InvalidTransition {
from: self.state(),
to: target,
}
}
async fn restore_failure_for(
&self,
agent_identity: &AgentIdentity,
) -> Option<super::handle::RestoreFailureDiagnostic> {
self.restore_diagnostics
.read()
.await
.get(agent_identity)
.cloned()
}
async fn ensure_member_not_broken(
&self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&crate::ids::AgentIdentity::from(
agent_identity.as_str(),
));
let lifecycle = self
.dsl_authority
.state()
.member_lifecycle_for_identity(&dsl_identity);
if lifecycle.status == mob_dsl::MobMemberLifecycleStatus::Broken {
let diag = self.restore_failure_for(agent_identity).await.unwrap_or(
super::handle::RestoreFailureDiagnostic {
bridge_session_id: None,
reason: lifecycle
.error
.unwrap_or_else(|| "member restore failed".to_string()),
},
);
return Err(MobError::MemberRestoreFailed {
member_id: agent_identity.clone(),
session_id: diag.bridge_session_id,
reason: diag.reason,
});
}
Ok(())
}
fn dsl_state(&self) -> MobState {
project_dsl_phase(self.dsl_authority.state().lifecycle_phase)
}
fn destroy_admitted(&self) -> bool {
self.dsl_authority.state().destroy_admitted
}
fn state(&self) -> MobState {
if self.destroy_admitted() {
MobState::Destroyed
} else {
self.dsl_state()
}
}
fn publish_machine_state_projection(&self) {
self.dsl_topology_epoch.store(
self.dsl_authority.state().topology_epoch,
std::sync::atomic::Ordering::Release,
);
let _ = self
.machine_state_watch_tx
.send(self.dsl_authority.state().clone());
}
fn apply_dsl_input(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_input_collect_effects(input, context)
.map(|_| ())
}
fn apply_dsl_input_collect_effects(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<Vec<mob_dsl::MobMachineEffect>, MobError> {
Ok(self
.apply_dsl_input_collect_transition(input, context)?
.into_effects())
}
fn apply_dsl_input_collect_transition(
&mut self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<mob_dsl::MobMachineTransition, MobError> {
let input_debug = format!("{input:?}");
let transition = mob_dsl::MobMachineMutator::apply(&mut self.dsl_authority, input)
.map_err(|e| {
MobError::Internal(format!(
"DSL authority ({context}) rejected {input_debug}: {e}"
))
})?;
self.queue_routed_effects_from(transition.effects());
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(transition)
}
fn prepare_dsl_input(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<PreparedDslInput, MobError> {
self.prepare_dsl_inputs(std::slice::from_ref(&input), context)
}
fn prepare_dsl_input_transition(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<PreparedDslTransition, MobError> {
let input_debug = format!("{input:?}");
let mut authority = self.dsl_authority.prepare_authority();
let transition = mob_dsl::MobMachineMutator::apply(&mut authority, input).map_err(|e| {
MobError::Internal(format!(
"DSL authority prepare ({context}) rejected {input_debug}: {e}"
))
})?;
Ok(PreparedDslTransition {
authority,
transition,
})
}
fn prepare_dsl_signal_transition(
&self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<PreparedDslTransition, MobError> {
let signal_debug = format!("{signal:?}");
let mut authority = self.dsl_authority.prepare_authority();
let transition = authority.apply_signal(signal).map_err(|e| {
MobError::Internal(format!(
"DSL authority prepare ({context}): {e}; signal={signal_debug}; live_runtime_ids={:?}; runtime_fence_tokens={:?}",
self.dsl_authority.state().live_runtime_ids,
self.dsl_authority.state().runtime_fence_tokens,
))
})?;
Ok(PreparedDslTransition {
authority,
transition,
})
}
fn prepare_dsl_inputs(
&self,
inputs: &[mob_dsl::MobMachineInput],
context: &str,
) -> Result<PreparedDslInput, MobError> {
let mut authority = self.dsl_authority.prepare_authority();
let mut effects = Vec::new();
let mut phase_changed = false;
for input in inputs {
let transition = mob_dsl::MobMachineMutator::apply(&mut authority, input.clone())
.map_err(|e| {
let input_debug = format!("{input:?}");
MobError::Internal(format!(
"DSL authority prepare ({context}) rejected {input_debug}: {e}"
))
})?;
if transition.from_phase != transition.to_phase {
phase_changed = true;
}
effects.extend(transition.into_effects());
}
Ok(PreparedDslInput {
authority,
effects,
phase_changed,
})
}
fn commit_prepared_dsl_input(&mut self, prepared: PreparedDslInput) -> Result<(), MobError> {
let effects = prepared.effects;
let phase_changed = prepared.phase_changed;
self.dsl_authority
.commit_prepared_authority(prepared.authority)
.map_err(|error| {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(())
}
async fn commit_prepared_dsl_input_after<T, F, Fut>(
&mut self,
prepared: PreparedDslInput,
effect: F,
) -> Result<T, MobError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, MobError>>,
{
let effects = prepared.effects;
let phase_changed = prepared.phase_changed;
let result = self
.dsl_authority
.commit_prepared_authority_after(prepared.authority, effect)
.await
.map_err(|error| match error {
mob_dsl::MobMachinePreparedCommitEffectError::Commit(error) => {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
}
mob_dsl::MobMachinePreparedCommitEffectError::Effect(error) => error,
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(result)
}
fn commit_prepared_dsl_transition(
&mut self,
prepared: PreparedDslTransition,
) -> Result<(), MobError> {
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
self.dsl_authority
.commit_prepared_authority(prepared.authority)
.map_err(|error| {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(())
}
async fn commit_prepared_dsl_transition_after<T, F, Fut>(
&mut self,
prepared: PreparedDslTransition,
effect: F,
) -> Result<T, MobError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, MobError>>,
{
let phase_changed = prepared.transition.from_phase != prepared.transition.to_phase;
let effects = prepared.transition.effects().to_vec();
let result = self
.dsl_authority
.commit_prepared_authority_after(prepared.authority, effect)
.await
.map_err(|error| match error {
mob_dsl::MobMachinePreparedCommitEffectError::Commit(error) => {
MobError::Internal(format!("DSL authority prepared commit rejected: {error}"))
}
mob_dsl::MobMachinePreparedCommitEffectError::Effect(error) => error,
})?;
self.queue_routed_effects_from(&effects);
if phase_changed {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(result)
}
fn prepare_supervisor_authority_persistence(
&self,
input: mob_dsl::MobMachineInput,
record: &crate::store::SupervisorAuthorityRecord,
context: &str,
) -> Result<PreparedSupervisorAuthorityPersistence, MobError> {
let prepared = self.prepare_dsl_input_transition(input, context)?;
let authority = crate::store::SupervisorAuthorityPersistenceAuthority::from_transition(
record,
&prepared.transition,
)?;
Ok(PreparedSupervisorAuthorityPersistence {
transition: prepared,
authority,
})
}
fn prepare_supervisor_authority_deletion(
&self,
record: &crate::store::SupervisorAuthorityRecord,
context: &str,
) -> Result<PreparedSupervisorAuthorityDeletion, MobError> {
let prepared = self.prepare_dsl_input_transition(
record.dsl_clear_authority_for_destroy_input(),
context,
)?;
let authority = crate::store::SupervisorAuthorityDeletionAuthority::from_transition(
record,
&prepared.transition,
)?;
Ok(PreparedSupervisorAuthorityDeletion {
transition: prepared,
authority,
})
}
fn supervisor_authority_record_is_machine_authorized(
&self,
record: &crate::store::SupervisorAuthorityRecord,
) -> bool {
let state = self.dsl_authority.state();
let peer_id = record.dsl_peer_id();
let signing_key = record.dsl_signing_key();
let protocol_version = record.dsl_protocol_version();
let current_matches = state.supervisor_authority_peer_id.as_ref() == Some(&peer_id)
&& state.supervisor_authority_signing_key == Some(signing_key)
&& state.supervisor_authority_epoch == Some(record.epoch)
&& state.supervisor_authority_protocol_version == Some(protocol_version.clone());
let pending_matches = state.supervisor_pending_authority_peer_id.as_ref() == Some(&peer_id)
&& state.supervisor_pending_authority_signing_key == Some(signing_key)
&& state.supervisor_pending_authority_epoch == Some(record.epoch)
&& state.supervisor_pending_authority_protocol_version == Some(protocol_version);
current_matches || pending_matches
}
fn supervisor_bridge_authority_for_record(
&self,
record: &crate::store::SupervisorAuthorityRecord,
) -> Result<crate::store::SupervisorAuthorityBridgeAuthority, MobError> {
Ok(
crate::store::SupervisorAuthorityBridgeAuthority::from_machine_state(
record,
self.dsl_authority.state(),
)?,
)
}
fn require_lifecycle_journal_effect(
transition: &mob_dsl::MobMachineTransition,
kind: mob_dsl::MobLifecycleJournalKind,
context: &str,
) -> Result<(), MobError> {
if transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::AppendLifecycleJournal {
kind: effect_kind,
agent_identity: None,
agent_runtime_id: None,
fence_token: None,
generation: None,
session_id: None,
}
if *effect_kind == kind
)
}) {
Ok(())
} else {
Err(MobError::Internal(format!(
"MobMachine {context} produced no generated {kind:?} lifecycle journal authority"
)))
}
}
#[allow(clippy::too_many_arguments)]
fn require_member_lifecycle_journal_effect(
transition: &mob_dsl::MobMachineTransition,
kind: mob_dsl::MobLifecycleJournalKind,
agent_identity: &AgentIdentity,
agent_runtime_id: &crate::ids::AgentRuntimeId,
fence_token: Option<crate::ids::FenceToken>,
generation: crate::ids::Generation,
session_id: Option<mob_dsl::SessionId>,
context: &str,
) -> Result<(), MobError> {
let expected_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let expected_runtime_id = mob_dsl::AgentRuntimeId::from_domain(agent_runtime_id);
let expected_fence = fence_token.map(mob_dsl::FenceToken::from_domain);
let expected_generation = mob_dsl::Generation::from_domain(generation);
let expected_session = session_id;
if transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::AppendLifecycleJournal {
kind: effect_kind,
agent_identity: Some(effect_identity),
agent_runtime_id: Some(effect_runtime_id),
fence_token: effect_fence,
generation: Some(effect_generation),
session_id: effect_session,
} if *effect_kind == kind
&& effect_identity == &expected_identity
&& effect_runtime_id == &expected_runtime_id
&& effect_fence == &expected_fence
&& effect_generation == &expected_generation
&& effect_session == &expected_session
)
}) {
Ok(())
} else {
Err(MobError::Internal(format!(
"MobMachine {context} produced no generated {kind:?} lifecycle journal authority for member '{agent_identity}'"
)))
}
}
fn operator_action_recorded_event_from_generated_effect(
transition: &mob_dsl::MobMachineTransition,
context: &str,
) -> Result<MobEventKind, MobError> {
let mut authorized = transition
.effects()
.iter()
.filter_map(|effect| match effect {
mob_dsl::MobMachineEffect::AppendOperatorActionProvenance {
tool_name,
principal_token,
caller_provenance,
audit_invocation_id,
} => Some(MobEventKind::OperatorActionRecorded {
tool_name: tool_name.clone(),
principal_token: principal_token.clone(),
caller_provenance: caller_provenance.clone(),
audit_invocation_id: audit_invocation_id.clone(),
}),
_ => None,
});
let Some(event) = authorized.next() else {
return Err(MobError::Internal(format!(
"MobMachine {context} produced no generated operator provenance journal authority"
)));
};
if authorized.next().is_some() {
return Err(MobError::Internal(format!(
"MobMachine {context} produced multiple generated operator provenance journal authorities"
)));
}
Ok(event)
}
fn prepare_command_admission(
&self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<PreparedDslInput, MobError> {
self.prepare_dsl_input(input, context).map_err(|error| {
tracing::debug!(
context,
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(target)
})
}
fn probe_command_admission(
&self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<(), MobError> {
self.prepare_command_admission(input, target, context)
.map(|_| ())
}
fn apply_command_admission(
&mut self,
input: mob_dsl::MobMachineInput,
target: MobState,
context: &str,
) -> Result<Vec<mob_dsl::MobMachineEffect>, MobError> {
self.apply_dsl_input_collect_effects(input, context)
.map_err(|error| {
tracing::debug!(
context,
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(target)
})
}
fn machine_member_peer_spec_for(
&self,
identity: &AgentIdentity,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.member_peer_endpoints
.get(&dsl_identity)
.map(|endpoint| Self::peer_only_spec_from_member_endpoint(endpoint, context))
.transpose()
}
fn roster_member_peer_spec_for(
&self,
entry: &RosterEntry,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let Some(peer_id) = entry.peer_id else {
return Ok(None);
};
let Some(public_key) = entry.transport_public_key.as_deref() else {
return Ok(None);
};
let pubkey = meerkat_comms::PubKey::from_pubkey_string(public_key).map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid retained peer key for '{}': {error}",
entry.agent_identity
))
})?;
let comms_name = self.comms_name_for(entry)?;
TrustedPeerDescriptor::unsigned_with_pubkey(
comms_name.clone(),
peer_id.to_string(),
*pubkey.as_bytes(),
format!("inproc://{comms_name}"),
)
.map(Some)
.map_err(|error| {
MobError::WiringError(format!(
"{context}: invalid retained peer descriptor for '{}': {error}",
entry.agent_identity
))
})
}
async fn retained_member_peer_spec_from_wired_peer_trust(
&self,
entry: &RosterEntry,
peer_identities: &BTreeSet<AgentIdentity>,
context: &'static str,
) -> Result<Option<TrustedPeerDescriptor>, MobError> {
let expected_name = self.comms_name_for(entry)?;
let mut retained = None;
for peer_identity in peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let Some(comms) = self.provisioner_comms(&peer_entry.member_ref).await else {
continue;
};
let trusted_peers = comms
.trusted_peer_projection_snapshot_for_source(
meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind::MobMachineMemberTrustWiring,
)
.await
.map_err(|error| {
MobError::WiringError(format!(
"{context}: failed to read retained trusted peers for '{}': {error}",
peer_entry.agent_identity
))
})?;
for peer in trusted_peers {
if peer.name.as_str() != expected_name {
continue;
}
if let Some(previous) = retained.replace(peer.clone())
&& previous.peer_id != peer.peer_id
{
return Err(MobError::WiringError(format!(
"{context}: retained peer descriptor for '{}' disagrees across wired peers",
entry.agent_identity
)));
}
}
}
Ok(retained)
}
fn effects_include_wiring_graph_change(effects: &[mob_dsl::MobMachineEffect]) -> bool {
effects
.iter()
.any(|effect| matches!(effect, mob_dsl::MobMachineEffect::WiringGraphChanged { .. }))
}
fn wire_members_disposition_from_effects(
effects: &[mob_dsl::MobMachineEffect],
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<bool, MobError> {
let graph_changed = Self::effects_include_wiring_graph_change(effects);
let repair_requested = effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::WiringTrustRepairRequested {
edge: effect_edge
} if effect_edge == edge
)
});
match (graph_changed, repair_requested) {
(true, false) => Ok(true),
(false, true) => Ok(false),
(false, false) => Err(MobError::WiringError(format!(
"{context} produced no generated wiring graph or repair authority for edge {edge:?}"
))),
(true, true) => Err(MobError::WiringError(format!(
"{context} produced conflicting generated wiring graph and repair authority for edge {edge:?}"
))),
}
}
fn member_trust_wiring_handoff_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
self.member_trust_wiring_handoff_from_transition_with_freshness(
transition,
edge,
crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority::from_live_member_trust_authority(
&self.dsl_authority,
),
context,
operation,
)
}
fn member_trust_wiring_handoff_from_prepared_batch_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
freshness_authority: crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
self.member_trust_wiring_handoff_from_transition_with_freshness(
transition,
edge,
freshness_authority,
context,
operation,
)
}
fn member_trust_wiring_handoff_from_transition_with_freshness(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
mob_topology_freshness_authority: crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
let obligation =
crate::generated::protocol_mob_member_trust_wiring::extract_obligations_with_freshness(
transition,
mob_topology_freshness_authority,
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced no generated member wiring trust obligation"
))
})?;
let authority = match operation {
MemberTrustOperation::Wiring => MemberTrustAuthority::Wiring(obligation),
MemberTrustOperation::Repair => MemberTrustAuthority::Repair(obligation),
MemberTrustOperation::Unwiring => {
return Err(MobError::WiringError(
"member unwiring cannot use a wiring trust obligation".to_string(),
));
}
};
Ok(MemberTrustHandoff {
edge: edge.clone(),
authority,
operation,
})
}
fn wire_external_authority_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::ExternalPeerEdge,
context: &str,
) -> Result<WireTrustAuthority, MobError> {
let effects = transition.effects();
let graph_changed = Self::effects_include_wiring_graph_change(effects);
let wiring_obligation =
crate::generated::protocol_mob_external_peer_trust_wiring::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_external_peer_trust_wiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge);
let repair_obligation =
crate::generated::protocol_mob_external_peer_trust_repair::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_external_peer_trust_repair::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge);
let repair_requested = repair_obligation.is_some();
match (graph_changed, repair_requested) {
(true, false) => {
let obligation = wiring_obligation.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced external graph change without generated wiring trust obligation"
))
})?;
let expected_peer_id = edge.endpoint.peer_id.0.as_str();
Ok(WireTrustAuthority::ExternalGraphAdded(
crate::generated::protocol_mob_external_peer_trust_wiring::wiring_authority_for_peer(
&obligation,
expected_peer_id,
)
.map_err(MobError::WiringError)?,
))
}
(false, true) => {
let obligation = repair_obligation.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced external repair marker without generated repair trust obligation"
))
})?;
let expected_peer_id = edge.endpoint.peer_id.0.as_str();
Ok(WireTrustAuthority::ExternalRepairRequested(
crate::generated::protocol_mob_external_peer_trust_repair::repair_authority_for_peer(
&obligation,
expected_peer_id,
)
.map_err(MobError::WiringError)?,
))
}
(false, false) => Err(MobError::WiringError(format!(
"{context} produced no generated external-peer trust authority"
))),
(true, true) => Err(MobError::WiringError(format!(
"{context} produced conflicting generated external-peer trust authority"
))),
}
}
fn unwire_members_authority_from_transition(
&self,
transition: &mob_dsl::MobMachineTransition,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let obligation = crate::generated::protocol_mob_member_trust_unwiring::extract_obligations_with_freshness(
transition,
crate::generated::protocol_mob_member_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(format!(
"{context} produced no generated member unwiring trust obligation"
))
})?;
Ok(MemberTrustHandoff {
edge: edge.clone(),
authority: MemberTrustAuthority::Unwiring(obligation),
operation: MemberTrustOperation::Unwiring,
})
}
fn authorize_member_trust_wiring(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
operation: MemberTrustOperation,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustWiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.member_trust_wiring_handoff_from_transition(&transition, edge, context, operation)
}
fn authorize_member_trust_unwiring(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustUnwiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn authorize_member_trust_cleanup(
&mut self,
edge: &mob_dsl::WiringEdge,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustCleanup {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn authorize_member_trust_cleanup_observed(
&mut self,
edge: &mob_dsl::WiringEdge,
a_identity: &AgentIdentity,
a_peer_id: &str,
b_identity: &AgentIdentity,
b_peer_id: &str,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberTrustCleanupObserved {
edge: edge.clone(),
a_identity: mob_dsl::AgentIdentity::from_domain(a_identity),
a_peer_id: mob_dsl::PeerId(a_peer_id.to_string()),
b_identity: mob_dsl::AgentIdentity::from_domain(b_identity),
b_peer_id: mob_dsl::PeerId(b_peer_id.to_string()),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
#[allow(clippy::too_many_arguments)]
fn authorize_retiring_member_trust_cleanup_observed(
&mut self,
edge: &mob_dsl::WiringEdge,
retiring_entry: &RosterEntry,
a_identity: &AgentIdentity,
a_peer_id: &str,
b_identity: &AgentIdentity,
b_peer_id: &str,
context: &str,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeRetiringMemberTrustCleanupObserved {
edge: edge.clone(),
a_identity: mob_dsl::AgentIdentity::from_domain(a_identity),
a_peer_id: mob_dsl::PeerId(a_peer_id.to_string()),
b_identity: mob_dsl::AgentIdentity::from_domain(b_identity),
b_peer_id: mob_dsl::PeerId(b_peer_id.to_string()),
agent_identity: mob_dsl::AgentIdentity::from_domain(&retiring_entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&retiring_entry.agent_runtime_id,
),
fence_token: mob_dsl::FenceToken::from_domain(retiring_entry.fence_token),
generation: mob_dsl::Generation::from_domain(retiring_entry.generation),
},
context,
)?;
self.unwire_members_authority_from_transition(&transition, edge, context)
}
fn preview_dsl_input(
&self,
input: mob_dsl::MobMachineInput,
context: &str,
) -> Result<mob_dsl::MobMachineState, MobError> {
let input_debug = format!("{input:?}");
let mut authority =
mob_dsl::MobMachineAuthority::recover_from_state(self.dsl_authority.state().clone())
.map_err(|error| {
MobError::Internal(format!(
"DSL authority preview ({context}) could not recover state: {error}"
))
})?;
let transition = mob_dsl::MobMachineMutator::apply(&mut authority, input).map_err(|e| {
MobError::Internal(format!(
"DSL authority preview ({context}) rejected {input_debug}: {e}"
))
})?;
let _ = transition;
Ok(authority.state().clone())
}
fn apply_dsl_signal(
&mut self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<(), MobError> {
self.apply_dsl_signal_collect_transition(signal, context)
.map(|_| ())
}
fn apply_dsl_signal_collect_transition(
&mut self,
signal: mob_dsl::MobMachineSignal,
context: &str,
) -> Result<mob_dsl::MobMachineTransition, MobError> {
let signal_debug = format!("{signal:?}");
let transition = self
.dsl_authority
.apply_signal(signal)
.map_err(|e| {
MobError::Internal(format!(
"DSL authority ({context}): {e}; signal={signal_debug}; live_runtime_ids={:?}; runtime_fence_tokens={:?}",
self.dsl_authority.state().live_runtime_ids,
self.dsl_authority.state().runtime_fence_tokens,
))
})?;
self.queue_routed_effects_from(transition.effects());
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
Ok(transition)
}
fn queue_routed_effects_from(&mut self, effects: &[mob_dsl::MobMachineEffect]) {
for effect in effects {
if let Some(seam_effect) = super::composition::lift_routed_effect(effect) {
self.pending_routed_effects.push(seam_effect);
}
}
}
fn runtime_retire_blocked_on_pending_detach(
state: &mob_dsl::MobMachineState,
effect: &super::composition::MobSeamEffect,
) -> bool {
matches!(
effect.body(),
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_runtime_id,
..
} if state
.pending_session_ingress_detach_runtime_ids
.contains(agent_runtime_id)
)
}
fn has_exact_queued_runtime_retire(
&self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
) -> Result<bool, MobError> {
let mut exact_count = 0usize;
for effect in &self.pending_routed_effects {
let mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: queued_identity,
agent_runtime_id: queued_runtime_id,
session_id: queued_session_id,
} = effect.body()
else {
continue;
};
if queued_runtime_id != agent_runtime_id {
continue;
}
if queued_identity != agent_identity || queued_session_id != session_id {
return Err(MobError::Internal(format!(
"queued runtime-retire correlation for '{}' does not match machine retry authority",
agent_runtime_id.0
)));
}
exact_count += 1;
}
if exact_count > 1 {
return Err(MobError::Internal(format!(
"multiple queued runtime-retire effects exist for '{}'",
agent_runtime_id.0
)));
}
Ok(exact_count == 1)
}
fn discard_queued_runtime_retire_for(&mut self, agent_runtime_id: &mob_dsl::AgentRuntimeId) {
self.pending_routed_effects.retain(|effect| {
!matches!(
effect.body(),
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_runtime_id: queued_runtime_id,
..
} if queued_runtime_id == agent_runtime_id
)
});
}
fn ensure_runtime_retire_route_after_detach(
&mut self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
context: &str,
) -> Result<(), MobError> {
if self
.dsl_authority
.state()
.live_runtime_ids
.contains(agent_runtime_id)
{
if !self.has_exact_queued_runtime_retire(
agent_identity,
agent_runtime_id,
session_id,
)? {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RetryRuntimeRetire {
agent_identity: agent_identity.clone(),
agent_runtime_id: agent_runtime_id.clone(),
},
context,
)?;
}
} else {
self.discard_queued_runtime_retire_for(agent_runtime_id);
}
Ok(())
}
async fn ensure_destroy_runtime_retire_route_after_detach(
&mut self,
agent_identity: &mob_dsl::AgentIdentity,
agent_runtime_id: &mob_dsl::AgentRuntimeId,
session_id: &mob_dsl::SessionId,
domain_session_id: &SessionId,
context: &str,
) -> Result<(), MobError> {
if self
.retirement_archive_already_complete(domain_session_id)
.await?
{
self.discard_queued_runtime_retire_for(agent_runtime_id);
return Ok(());
}
self.ensure_runtime_retire_route_after_detach(
agent_identity,
agent_runtime_id,
session_id,
context,
)
}
pub(super) async fn flush_routed_effects(&mut self) -> Result<(), MobError> {
use super::composition::{
dispatch_refusal_to_mob_error, dispatch_routed_effect, refusal_feedback_input,
};
use meerkat_runtime::composition::DispatchRefusal;
while let Some(effect) = self.pending_routed_effects.first().cloned() {
if Self::runtime_retire_blocked_on_pending_detach(self.dsl_authority.state(), &effect) {
tracing::debug!(
mob_id = %self.definition.id,
effect = ?effect.body(),
"deferring runtime retirement until session-ingress detach acknowledgement"
);
return Ok(());
}
match dispatch_routed_effect(&self.composition_binding, effect.clone()).await {
Ok(_outcome) => {
self.pending_routed_effects.remove(0);
}
Err(DispatchRefusal::ConsumerRefused { error, .. })
if !self.destroy_cleanup_active =>
{
let feedback = refusal_feedback_input(&effect, &error)?;
let transition = self.apply_dsl_input_collect_transition(
feedback,
"close_routed_effect_consumer_refusal",
)?;
let closed = closed_runtime_effect_refusal_from_transition(&transition)?;
self.pending_routed_effects.remove(0);
if let Some(agent_identity) = closed.broken_member.as_ref() {
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(closed.session_id.clone()),
reason: format!("{} [{}]", closed.reason, closed.refusal_code),
},
);
}
tracing::warn!(
mob_id = %self.definition.id,
kind = %closed.kind,
bridge_session_id = %closed.session_id,
refusal_code = %closed.refusal_code,
reason = %closed.reason,
queued_remaining = self.pending_routed_effects.len(),
"runtime consumer refusal closed through generated MobMachine feedback"
);
return Err(closed.into_mob_error());
}
Err(refusal) => {
return Err(dispatch_refusal_to_mob_error(refusal));
}
}
}
Ok(())
}
fn classify_actor_boundary_flush(
result: Result<(), MobError>,
) -> ActorBoundaryFlushDisposition {
match result {
Ok(()) => ActorBoundaryFlushDisposition::Drained,
Err(error) if error.is_closed_runtime_effect_refusal() => {
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(error)
}
Err(error) => ActorBoundaryFlushDisposition::Fatal(error),
}
}
fn discard_pending_routed_effects_for_session(&mut self, session_id: &SessionId) {
let dsl_session_id = mob_dsl::SessionId::from_domain(session_id);
self.pending_routed_effects.retain(|effect| {
routed_effect_session_scope_dsl(effect.body()) != Some(&dsl_session_id)
});
}
fn retiring_runtime_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
self.dsl_authority
.state()
.member_state_markers
.iter()
.filter_map(|(runtime_id, member_state)| match member_state {
mob_dsl::MobMemberState::Retiring => Some(runtime_id.0.clone()),
mob_dsl::MobMemberState::Active => None,
})
.collect()
}
fn pending_kickoff_member_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
self.dsl_authority
.state()
.member_kickoff_pending
.iter()
.chain(self.dsl_authority.state().member_kickoff_starting.iter())
.chain(
self.dsl_authority
.state()
.member_kickoff_callback_pending
.iter(),
)
.map(|id| id.0.clone())
.collect()
}
fn ready_runtime_ids_from_dsl(&self) -> std::collections::BTreeSet<String> {
self.dsl_authority
.state()
.member_startup_runtime_ready
.iter()
.chain(self.dsl_authority.state().member_startup_ready.iter())
.map(|runtime_id| runtime_id.0.clone())
.collect()
}
fn machine_projection_for_identity(
&self,
agent_identity: &crate::ids::AgentIdentity,
) -> super::state::MobMemberMachineProjection {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl = self.dsl_authority.state();
let runtime_id = dsl.identity_to_runtime.get(&dsl_identity).cloned();
let state_marker = runtime_id
.as_ref()
.and_then(|runtime_id| dsl.member_state_markers.get(runtime_id).copied());
let live_runtime = runtime_id
.as_ref()
.is_some_and(|runtime_id| dsl.live_runtime_ids.contains(runtime_id));
let bound_session_id = dsl.member_session_bindings.get(&dsl_identity).cloned();
super::state::MobMemberMachineProjection {
runtime_id,
state_marker,
live_runtime,
bound_session_id,
}
}
fn machine_bridge_session_id_for_identity(
&self,
agent_identity: &crate::ids::AgentIdentity,
) -> Option<SessionId> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
self.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.and_then(|session_id| SessionId::parse(&session_id.0).ok())
}
fn project_member_ref_session_binding(
member_ref: &MemberRef,
current_bridge_session_id: Option<SessionId>,
) -> Option<MemberRef> {
match member_ref {
MemberRef::Session { .. } => {
current_bridge_session_id.map(MemberRef::from_bridge_session_id)
}
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
..
} => Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: address.clone(),
pubkey: *pubkey,
bootstrap_token: bootstrap_token.clone(),
session_id: current_bridge_session_id,
}),
}
}
fn machine_member_ref_for_behavior(
&self,
entry: &RosterEntry,
context: &str,
) -> Result<MemberRef, MobError> {
let bridge_session_id = self.machine_bridge_session_id_for_identity(&entry.agent_identity);
Self::project_member_ref_session_binding(&entry.member_ref, bridge_session_id).ok_or_else(
|| {
MobError::Internal(format!(
"{context} requires MobMachine session binding for '{}'",
entry.agent_identity
))
},
)
}
async fn machine_member_material(
&mut self,
agent_identity: &AgentIdentity,
include_session_details: bool,
) -> Result<CanonicalMemberSnapshotMaterial, MobError> {
let roster_entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let domain_identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let current_bridge_session_id =
self.machine_bridge_session_id_for_identity(&domain_identity);
let machine_runtime = self
.dsl_authority
.state()
.member_runtime_material_for_identity(&dsl_identity)
.map(|material| material.to_domain_for_identity(&domain_identity));
let member_present = roster_entry.is_some();
let (output_preview, tokens_used) = match current_bridge_session_id.as_ref() {
None => (None, 0),
Some(bridge_session_id) if include_session_details => {
match self.session_service.read(bridge_session_id).await {
Ok(view) => (
view.state.last_assistant_text.clone(),
view.billing.total_tokens,
),
Err(meerkat_core::service::SessionError::NotFound { .. }) => {
let _ = self
.record_missing_member_bridge_session(
agent_identity,
bridge_session_id,
"member_status",
)
.await;
(None, 0)
}
Err(_) => (None, 0),
}
}
Some(_) => (None, 0),
};
let machine_lifecycle = self
.dsl_authority
.state()
.member_lifecycle_for_identity(&dsl_identity);
let kickoff = kickoff_snapshot_from_machine_state(
agent_identity.as_str(),
self.dsl_authority.state(),
roster_entry
.as_ref()
.and_then(|entry| entry.kickoff.as_ref()),
);
Ok(MobMemberLifecycleProjection::materialize(
MobMemberLifecycleInput {
member_present,
machine_lifecycle,
output_preview,
tokens_used,
agent_identity: domain_identity,
agent_runtime_id: machine_runtime
.as_ref()
.map(|(agent_runtime_id, _)| agent_runtime_id.clone()),
fence_token: machine_runtime.map(|(_, fence_token)| fence_token),
current_bridge_session_id,
peer_connectivity: None,
kickoff,
},
))
}
async fn record_missing_member_bridge_session(
&mut self,
agent_identity: &AgentIdentity,
bridge_session_id: &SessionId,
context: &'static str,
) -> Option<String> {
let domain_identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let current_binding_matches = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.is_some_and(|session_id| session_id.0 == bridge_session_id.to_string());
if !current_binding_matches {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
"observed missing bridge session for a stale/non-current member binding"
);
return None;
}
if let Some(reason) = self
.dsl_authority
.state()
.member_restore_failures
.get(&dsl_identity)
.cloned()
{
return Some(reason);
}
let reason = format!("missing bridge session snapshot for '{bridge_session_id}'");
if let Err(error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::RecoverMemberRestoreFailure {
agent_identity: dsl_identity,
reason: reason.clone(),
},
"record_missing_member_bridge_session",
) {
let fallback = format!("{reason}; failed to record restore failure: {error}");
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
error = %error,
"MobMachine rejected member restore-failure observation"
);
return Some(fallback);
}
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
context,
reason = %reason,
"member bridge session is missing; marked member broken"
);
Some(reason)
}
async fn revive_member_live_materialization(
&mut self,
entry: &RosterEntry,
member_ref: &MemberRef,
bridge_session_id: &SessionId,
) -> Result<(), MobError> {
let revival_lock = self.member_revival_lock_for(bridge_session_id).await;
let _revival_guard = revival_lock.lock().await;
match self
.session_service
.has_live_session(bridge_session_id)
.await
{
Ok(true) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %entry.agent_identity,
bridge_session_id = %bridge_session_id,
"member live materialization already present after acquiring revival lock"
);
return Ok(());
}
Ok(false) | Err(meerkat_core::service::SessionError::NotFound { .. }) => {}
Err(error) => return Err(MobError::SessionError(error)),
}
let agent_identity = entry.agent_identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let stored_session_present = if self.session_service.supports_persistent_sessions() {
self.session_service
.load_persisted_session_metadata(bridge_session_id)
.await
.map_err(MobError::SessionError)?
.is_some()
} else {
false
};
let (observation, reason) = if stored_session_present {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotPresent,
format!("live session materialization missing for '{bridge_session_id}'"),
)
} else {
(
mob_dsl::MemberLiveMaterializationObservationKind::DurableSnapshotMissing,
format!("missing bridge session snapshot for '{bridge_session_id}'"),
)
};
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::ClassifyMemberLiveMaterialization {
agent_identity: dsl_identity.clone(),
observation,
reason: reason.clone(),
},
"classify_member_live_materialization",
)?;
let (effect_observation, verdict) = transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::MemberLiveMaterializationClassified {
agent_identity: effect_identity,
observation,
verdict,
..
} if effect_identity == &dsl_identity => Some((*observation, *verdict)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted member live-materialization observation but emitted no revival verdict"
.into(),
)
})?;
if effect_observation != observation {
return Err(MobError::Internal(format!(
"MobMachine member-revival drift: input={observation:?}, effect={effect_observation:?}"
)));
}
match verdict {
mob_dsl::MemberRevivalVerdictKind::BrokenRecorded => {
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %reason,
"member live materialization is unrecoverable; MobMachine recorded terminal restore failure"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity,
session_id: Some(bridge_session_id.clone()),
reason,
})
}
mob_dsl::MemberRevivalVerdictKind::ReviveAuthorized => {
let materialization = match self
.session_service
.load_persisted_session(bridge_session_id)
.await
{
Ok(Some(stored_session)) => {
self.materialize_revived_member_session(
entry,
member_ref,
bridge_session_id,
stored_session,
)
.await
}
Ok(None) => Err(MobError::Internal(format!(
"durable snapshot for bridge session '{bridge_session_id}' vanished \
between the metadata presence probe and revival materialization"
))),
Err(error) => Err(MobError::SessionError(error)),
};
match materialization {
Ok(()) => {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalSucceeded {
agent_identity: dsl_identity.clone(),
},
"resolve_member_revival_succeeded",
)?;
self.restore_diagnostics
.write()
.await
.remove(&agent_identity);
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
"machine-authorized revival rebuilt the member's live session"
);
Ok(())
}
Err(error) => {
if revival_error_means_session_already_live(&error, bridge_session_id) {
match self
.session_service
.has_live_session(bridge_session_id)
.await
{
Ok(true) => {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalSucceeded {
agent_identity: dsl_identity.clone(),
},
"resolve_member_revival_already_live",
)?;
self.restore_diagnostics
.write()
.await
.remove(&agent_identity);
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
"machine-authorized revival found an already-live session; treating materialization as idempotent success"
);
return Ok(());
}
Ok(false)
| Err(meerkat_core::service::SessionError::NotFound { .. }) => {}
Err(error) => return Err(MobError::SessionError(error)),
}
}
let failure_reason = format!(
"machine-authorized revival of bridge session '{bridge_session_id}' failed: {error}"
);
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResolveMemberRevivalFailed {
agent_identity: dsl_identity,
reason: failure_reason.clone(),
},
"resolve_member_revival_failed",
)?;
self.restore_diagnostics.write().await.insert(
agent_identity.clone(),
super::handle::RestoreFailureDiagnostic {
bridge_session_id: Some(bridge_session_id.clone()),
reason: failure_reason.clone(),
},
);
tracing::error!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %bridge_session_id,
reason = %failure_reason,
"machine-authorized revival failed terminally; member is broken"
);
Err(MobError::MemberRestoreFailed {
member_id: agent_identity,
session_id: Some(bridge_session_id.clone()),
reason: failure_reason,
})
}
}
}
}
}
async fn member_revival_lock_for(
&self,
bridge_session_id: &SessionId,
) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = self.member_revival_locks.lock().await;
locks
.entry(bridge_session_id.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
async fn materialize_revived_member_session(
&mut self,
entry: &RosterEntry,
member_ref: &MemberRef,
bridge_session_id: &SessionId,
stored_session: meerkat_core::session::Session,
) -> Result<(), MobError> {
let agent_identity = entry.agent_identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let profile = if let Some(p) = entry.effective_profile_override.clone() {
p
} else {
self.definition
.resolve_profile(&entry.role, self.realm_profile_store.as_ref())
.await?
};
self.authorize_spawn_profile_material(
&agent_identity,
&entry.role,
&profile,
"revive_member_profile_authority",
)?;
let per_spawn_overlay = self
.per_spawn_external_tools
.read()
.await
.get(&agent_identity)
.cloned();
let external_tools = self.external_tools_for_profile(&profile, per_spawn_overlay)?;
let mut config = build::build_resumed_agent_config(build::BuildResumedAgentConfigParams {
base: build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &entry.role,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context: None,
labels: Some(entry.labels.clone()),
additional_instructions: None,
shell_env: None,
mob_tool_authority_context: None,
inherited_tool_filter: None,
tool_access_policy: None,
system_prompt_override: None,
},
expected_session_id: bridge_session_id,
resumed_session: stored_session,
})
.await?;
config.keep_alive = entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
let prompt = ContentInput::from(self.fallback_spawn_prompt(&entry.role, &agent_identity));
let req = build::to_create_session_request(&config, prompt);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
agent_identity.as_str(),
)?;
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let dsl_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
let owner_transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::RecoverMemberSessionBinding {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
bridge_session_id: dsl_session_id.clone(),
replacing,
},
"revive_member_session_provision_owner",
)?;
let owner_authorized = owner_transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::SessionProvisionOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id,
} if effect_identity == &dsl_identity && session_id == &dsl_session_id
)
});
if !owner_authorized {
return Err(MobError::Internal(format!(
"MobMachine produced no session provision operation owner for revived member '{agent_identity}'"
)));
}
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
match adapter
.update_peer_ingress_context(bridge_session_id, false, None)
.await
{
Ok(_) => {}
Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(error) => {
return Err(MobError::Internal(format!(
"failed to detach stale peer ingress for revived session '{bridge_session_id}': {error}"
)));
}
}
}
let receipt = self
.provisioner
.provision_member(ProvisionMemberRequest {
create_session: req,
binding: crate::RuntimeBinding::Session,
peer_name,
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(bridge_session_id.clone()),
})
.await?;
let revived_session_id =
receipt
.member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"revival provisioned a non-session member for '{agent_identity}'"
))
})?;
if &revived_session_id != bridge_session_id {
return Err(MobError::Internal(format!(
"revival provisioned bridge session '{revived_session_id}' for machine binding '{bridge_session_id}'"
)));
}
self.ensure_mob_comms_drain(&agent_identity, member_ref)
.await?;
let plan = self.machine_restore_wiring_plan(&agent_identity)?;
let mut failed_restore_peer_ids: Vec<RespawnTopologyPeerId> = Vec::new();
for peer_identity in plan.local_peers {
if peer_identity == agent_identity {
continue;
}
let peer_agent_identity = crate::ids::AgentIdentity::from(peer_identity.as_str());
if let Err(error) = self
.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::Local(peer_agent_identity),
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_identity,
%error,
"revival: failed to restore machine-owned local peer edge"
);
failed_restore_peer_ids.push(RespawnTopologyPeerId::from(peer_identity.as_str()));
}
}
for peer_spec in plan.external_peers {
let peer_id = RespawnTopologyPeerId::from(peer_spec.peer_id.as_str());
if let Err(error) = self
.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::External(peer_spec.clone()),
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_spec.name,
%error,
"revival: failed to restore machine-owned external peer edge"
);
failed_restore_peer_ids.push(peer_id);
}
}
let resolution =
self.resolve_respawn_topology_restore_result(&agent_identity, failed_restore_peer_ids)?;
if resolution.result == mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed {
tracing::warn!(
agent_identity = %agent_identity,
failed_peer_ids = ?resolution.failed_peer_ids,
"revival completed with degraded machine-owned topology edges"
);
}
Ok(())
}
fn active_machine_member_ids_for_profile(
&self,
profile_name: &ProfileName,
excluded_identity: &AgentIdentity,
) -> Vec<AgentIdentity> {
let dsl = self.dsl_authority.state();
dsl.member_profile_names
.iter()
.filter_map(|(identity, machine_profile_name)| {
if machine_profile_name.as_str() != profile_name.as_str()
|| identity.0.as_str() == excluded_identity.as_str()
|| !MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
{
return None;
}
Some(AgentIdentity::from(identity.0.as_str()))
})
.collect()
}
async fn project_member_list_from_machine(
&mut self,
include_retiring: bool,
) -> Vec<MobMemberListEntry> {
let entries_by_identity: BTreeMap<_, _> = {
let roster = self.roster.read().await;
roster
.list_all()
.cloned()
.map(|entry| (entry.agent_identity.clone(), entry))
.collect()
};
let machine_state = self.dsl_authority.state().clone();
let mut projected = Vec::with_capacity(machine_state.identity_to_runtime.len());
for identity in machine_state.identity_to_runtime.keys() {
let Some(entry) =
super::handle::MobHandle::project_member_list_entry_from_machine_identity(
identity,
entries_by_identity.get(&AgentIdentity::from(identity.0.as_str())),
&machine_state,
)
else {
continue;
};
if !include_retiring && entry.status == super::handle::MobMemberStatus::Retiring {
continue;
}
projected.push(entry);
}
projected
}
fn mob_handle_for_tools(&self) -> MobHandle {
MobHandle {
command_tx: self.command_tx.clone(),
roster: self.roster.clone(),
definition: self.definition.clone(),
events: self.events.clone(),
run_store: self.run_store.clone(),
flow_streams: self.flow_streams.clone(),
session_service: self.session_service.clone(),
#[cfg(feature = "runtime-adapter")]
runtime_adapter: self.runtime_adapter.clone(),
restore_diagnostics: self.restore_diagnostics.clone(),
supervisor_bridge: self.supervisor_bridge.clone(),
machine_state_watch_rx: self.machine_state_watch_tx.subscribe(),
phase_watch_rx: self.phase_watch_tx.subscribe(),
realtime_session_factory: None,
}
}
async fn persist_kickoff_state(
&self,
agent_identity: &AgentIdentity,
phase: crate::roster::MobMemberKickoffPhase,
error: Option<String>,
) -> Result<(), MobError> {
let kickoff = crate::roster::MobMemberKickoffSnapshot {
phase,
error,
updated_at: SystemTime::now(),
};
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberKickoffUpdated {
member: AgentIdentity::from(agent_identity.as_str()),
kickoff: kickoff.clone(),
},
})
.await
.map_err(MobError::from)?;
self.roster
.write()
.await
.set_kickoff(agent_identity, Some(kickoff));
Ok(())
}
fn kickoff_phase_from_dsl(
phase: mob_dsl::KickoffPhase,
) -> crate::roster::MobMemberKickoffPhase {
match phase {
mob_dsl::KickoffPhase::Pending => crate::roster::MobMemberKickoffPhase::Pending,
mob_dsl::KickoffPhase::Starting => crate::roster::MobMemberKickoffPhase::Starting,
mob_dsl::KickoffPhase::Started => crate::roster::MobMemberKickoffPhase::Started,
mob_dsl::KickoffPhase::CallbackPending => {
crate::roster::MobMemberKickoffPhase::CallbackPending
}
mob_dsl::KickoffPhase::Failed => crate::roster::MobMemberKickoffPhase::Failed,
mob_dsl::KickoffPhase::Cancelled => crate::roster::MobMemberKickoffPhase::Cancelled,
}
}
pub(super) fn kickoff_notice_intent(
intent: crate::machines::mob_machine::KickoffIntent,
) -> &'static str {
use crate::machines::mob_machine::KickoffIntent;
match intent {
KickoffIntent::Pending => "mob.kickoff_pending",
KickoffIntent::Starting => "mob.kickoff_starting",
KickoffIntent::Started => "mob.kickoff_started",
KickoffIntent::CallbackPending => "mob.kickoff_callback_pending",
KickoffIntent::Failed => "mob.kickoff_failed",
KickoffIntent::Cancelled => "mob.kickoff_cancelled",
}
}
async fn clear_kickoff_state(&mut self, agent_identity: &AgentIdentity) {
match self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffClear {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"clear_kickoff_state",
)
.await
{
Ok(_) => {
self.roster.write().await.set_kickoff(agent_identity, None);
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
%error,
"kickoff clear failed"
);
}
}
}
async fn fail_startup_to_stopped(&mut self, failure_label: &'static str) {
if let Err(stop_error) = self.stop_all_autonomous_members().await {
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
"failed cleaning up autonomous host loops after startup error"
);
}
if let Err(error) =
self.apply_dsl_input(mob_dsl::MobMachineInput::Stop, "stop_after_startup_failure")
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
failure_label,
"authority rejected Stop after startup failure"
);
}
}
async fn apply_kickoff_input(
&mut self,
agent_identity: &AgentIdentity,
input: mob_dsl::MobMachineInput,
context: &'static str,
) -> Result<bool, MobError> {
let transition = match mob_dsl::MobMachineMutator::apply(&mut self.dsl_authority, input) {
Ok(transition) => transition,
Err(error) => return Err(Self::kickoff_rejection_error(context, error)),
};
self.queue_routed_effects_from(transition.effects());
self.publish_machine_state_projection();
for effect in transition.into_effects() {
match effect {
mob_dsl::MobMachineEffect::PersistKickoffUpdate {
member_id: _,
phase,
} => {
let phase = Self::kickoff_phase_from_dsl(phase);
self.persist_kickoff_state(agent_identity, phase, None)
.await?;
}
mob_dsl::MobMachineEffect::PersistKickoffFailureUpdate {
member_id: _,
phase,
error,
} => {
let phase = Self::kickoff_phase_from_dsl(phase);
self.persist_kickoff_state(agent_identity, phase, Some(error))
.await?;
}
mob_dsl::MobMachineEffect::EmitKickoffLifecycleNotice {
member_id: _,
intent,
} => {
let notice_intent = Self::kickoff_notice_intent(intent);
if let Err(error) = self
.notify_kickoff_event(agent_identity, notice_intent)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
intent = %intent,
"failed to emit kickoff lifecycle notice"
);
}
}
mob_dsl::MobMachineEffect::RequestRuntimeBinding { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeIngress { .. }
| mob_dsl::MobMachineEffect::RequestPeerRuntimeIngress { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeRetire { .. }
| mob_dsl::MobMachineEffect::RequestRuntimeDestroy { .. } => {}
_ => {}
}
}
Ok(true)
}
fn kickoff_rejection_error(context: &'static str, error: impl ToString) -> MobError {
MobError::MobMachineRejected {
context,
reason: error.to_string(),
}
}
#[cfg(test)]
fn machine_active_run_count(&self) -> u32 {
self.run_cancel_tokens.len() as u32
}
#[cfg(test)]
fn machine_orchestrator_snapshot(&self, phase: MobState) -> Option<MobOrchestratorSnapshot> {
if !self.has_orchestrator {
return None;
}
let coordinator_bound = self.dsl_authority.state().coordinator_bound;
Some(MobOrchestratorSnapshot {
phase,
coordinator_bound,
pending_spawn_count: self.dsl_authority.state().pending_spawn_count as u32,
active_flow_count: self.machine_active_run_count(),
topology_revision: 0,
supervisor_active: coordinator_bound,
})
}
fn require_member_operation_eligible(&self) -> Result<(), MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ClassifyMemberOperationEligibility {},
"member_operation_eligibility",
)?;
let mut admission = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::MemberOperationEligibilityResolved {
admission: kind,
} = effect
{
if admission.replace(*kind).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple member-operation eligibility verdicts".into(),
));
}
}
}
match admission {
Some(mob_dsl::MobMemberOperationEligibilityKind::Admitted) => Ok(()),
Some(mob_dsl::MobMemberOperationEligibilityKind::DeniedNotRunning) => {
Err(self.invalid_transition_to(MobState::Running))
}
None => Err(MobError::Internal(
"MobMachine emitted no member-operation eligibility verdict".into(),
)),
}
}
fn drain_completed_peer_delivery_tasks(&mut self) {
while let Some(result) = self.peer_delivery_tasks.try_join_next() {
match result {
Ok(completion) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Err(error) => {
tracing::debug!(error = %error, "peer delivery task failed");
if self.peer_delivery_tasks.is_empty() {
self.peer_delivery_inflight.clear();
}
}
}
}
}
fn spawn_peer_message_delivery(
&mut self,
plan: PeerMessageDeliveryPlan,
reply_tx: tokio::sync::oneshot::Sender<Result<meerkat_core::comms::SendReceipt, MobError>>,
) {
let permit = match self.peer_delivery_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
let _ = reply_tx.send(Err(MobError::Internal(format!(
"peer message delivery lane saturated (limit={MAX_PENDING_PEER_DELIVERIES})"
))));
return;
}
};
let id = self.next_peer_delivery_ticket;
self.next_peer_delivery_ticket = self.next_peer_delivery_ticket.wrapping_add(1);
let cancel_token = tokio_util::sync::CancellationToken::new();
self.peer_delivery_inflight.insert(
id,
PeerDeliveryInflight {
from: plan.from.clone(),
to: plan.to.clone(),
cancel_token: cancel_token.clone(),
},
);
self.peer_delivery_tasks.spawn(async move {
let _permit = permit;
let result = tokio::select! {
result = plan.sender_comms.send(plan.command) => result.map_err(MobError::from),
() = cancel_token.cancelled() => Err(MobError::Internal(
"peer message delivery canceled because mob topology or lifecycle changed".to_string(),
)),
};
let _ = reply_tx.send(result);
PeerDeliveryCompletion { id }
});
}
async fn cancel_pending_peer_deliveries(&mut self, reason: &'static str) {
self.cancel_peer_deliveries_matching(reason, |_| true).await;
}
async fn cancel_peer_deliveries_for_member(
&mut self,
member: &AgentIdentity,
reason: &'static str,
) {
self.cancel_peer_deliveries_matching(reason, |delivery| {
&delivery.from == member || &delivery.to == member
})
.await;
}
async fn cancel_peer_deliveries_for_edge(
&mut self,
a: &AgentIdentity,
b: &AgentIdentity,
reason: &'static str,
) {
self.cancel_peer_deliveries_matching(reason, |delivery| {
(&delivery.from == a && &delivery.to == b) || (&delivery.from == b && &delivery.to == a)
})
.await;
}
async fn cancel_peer_deliveries_matching(
&mut self,
reason: &'static str,
mut predicate: impl FnMut(&PeerDeliveryInflight) -> bool,
) {
let ids = self
.peer_delivery_inflight
.iter()
.filter_map(|(id, delivery)| predicate(delivery).then_some(*id))
.collect::<BTreeSet<_>>();
if ids.is_empty() {
return;
}
tracing::debug!(
mob_id = %self.definition.id,
pending = ids.len(),
reason,
"canceling pending peer delivery tasks"
);
for id in &ids {
if let Some(delivery) = self.peer_delivery_inflight.get(id) {
delivery.cancel_token.cancel();
}
}
while ids
.iter()
.any(|id| self.peer_delivery_inflight.contains_key(id))
{
match self.peer_delivery_tasks.join_next().await {
Some(Ok(completion)) => {
self.peer_delivery_inflight.remove(&completion.id);
}
Some(Err(error)) => {
tracing::debug!(error = %error, "peer delivery task failed during cancellation");
}
None => break,
}
}
for id in ids {
self.peer_delivery_inflight.remove(&id);
}
}
async fn notify_orchestrator_lifecycle(&mut self, message: String) -> Result<(), MobError> {
self.drain_completed_lifecycle_tasks()?;
let Some(orchestrator) = &self.definition.orchestrator else {
return Ok(());
};
let Some(orchestrator_entry) = self
.roster
.read()
.await
.by_profile(&orchestrator.profile)
.next()
.cloned()
else {
return Ok(());
};
while self.lifecycle_tasks.len() >= MAX_LIFECYCLE_NOTIFICATION_TASKS {
match self.lifecycle_tasks.join_next().await {
Some(join_result) => Self::surface_lifecycle_task_outcome(join_result)?,
None => break,
}
}
let provisioner = self.provisioner.clone();
let runtime_mode = orchestrator_entry.runtime_mode;
let agent_identity = orchestrator_entry.agent_identity;
let bridge_session_id = self.machine_bridge_session_id_for_identity(&agent_identity);
let Some(member_ref) = Self::project_member_ref_session_binding(
&orchestrator_entry.member_ref,
bridge_session_id,
) else {
return Err(MobError::WiringError(format!(
"orchestrator lifecycle notification for '{agent_identity}' could not project an addressable member binding"
)));
};
match runtime_mode {
crate::MobRuntimeMode::AutonomousHost => {
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Err(MobError::WiringError(format!(
"orchestrator lifecycle notification for '{agent_identity}' has no bridge session for autonomous-host injection"
)));
};
let Some(injector) = provisioner
.interaction_event_injector(bridge_session_id)
.await
else {
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "orchestrator lifecycle notification",
});
};
let task_identity = agent_identity.clone();
self.lifecycle_tasks.spawn(async move {
injector
.inject(
message.into(),
meerkat_core::PlainEventSource::Rpc,
meerkat_core::types::HandlingMode::Queue,
None,
)
.map_err(|error| {
MobError::Internal(format!(
"orchestrator lifecycle inject failed for '{task_identity}': {error}"
))
})
});
}
crate::MobRuntimeMode::TurnDriven => {
let task_member_ref = member_ref.clone();
self.lifecycle_tasks.spawn(async move {
provisioner
.start_turn(
&task_member_ref,
meerkat_core::service::StartTurnRequest {
injected_context: Vec::new(),
prompt: message.into(),
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::default(
),
},
)
.await
});
}
}
Ok(())
}
fn drain_completed_lifecycle_tasks(&mut self) -> Result<(), MobError> {
let mut surfaced: Option<MobError> = None;
while let Some(join_result) = self.lifecycle_tasks.try_join_next() {
if let Err(error) = Self::surface_lifecycle_task_outcome(join_result) {
if surfaced.is_none() {
surfaced = Some(error);
}
}
}
match surfaced {
Some(error) => Err(error),
None => Ok(()),
}
}
fn surface_lifecycle_task_outcome(
join_result: Result<Result<(), MobError>, tokio::task::JoinError>,
) -> Result<(), MobError> {
match join_result {
Ok(Ok(())) => Ok(()),
Ok(Err(delivery_error)) => {
tracing::warn!(
error = %delivery_error,
"orchestrator lifecycle delivery failed"
);
Err(delivery_error)
}
Err(join_error) if join_error.is_cancelled() => Ok(()),
Err(join_error) => Err(MobError::Internal(format!(
"orchestrator lifecycle delivery task did not complete: {join_error}"
))),
}
}
fn retire_event_key(
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> String {
crate::ids::AgentRuntimeId::new(agent_identity.clone(), generation).to_string()
}
fn pending_spawn_maps_aligned(&self) -> bool {
self.pending_spawn_alignment_violation().is_none()
}
fn pending_spawn_alignment_violation(&self) -> Option<String> {
let expected = if self.has_orchestrator {
Some(self.dsl_authority.state().pending_spawn_count as usize)
} else {
None
};
if let Some(message) = self.pending_spawns.alignment_violation(expected) {
return Some(message);
}
if self.has_orchestrator {
let dsl_pending = self
.dsl_authority
.state()
.pending_spawn_sessions
.iter()
.map(|(identity, session_id)| (identity.0.clone(), session_id.0.clone()))
.collect::<BTreeMap<_, _>>();
let local_pending = self.pending_spawns.member_session_pairs();
if dsl_pending != local_pending {
return Some(format!(
"pending admission mismatch: dsl={dsl_pending:?}, local={local_pending:?}"
));
}
}
None
}
fn ensure_pending_spawn_alignment(&self, context: &str) -> Result<(), MobError> {
if let Some(message) = self.pending_spawn_alignment_violation() {
return Err(MobError::Internal(format!(
"{context}: pending spawn alignment violation: {message}"
)));
}
Ok(())
}
fn debug_assert_pending_spawn_alignment(&self) {
debug_assert!(
self.pending_spawn_maps_aligned(),
"pending spawn alignment must hold across pending maps and orchestrator count"
);
}
fn insert_pending_spawn(
&mut self,
spawn_ticket: u64,
pending: PendingSpawn,
task: tokio::task::JoinHandle<()>,
started: AuthorizedMobSpawnStarted,
) {
debug_assert_eq!(pending.agent_identity, started.agent_identity);
debug_assert_eq!(pending.admitted_bridge_session_id, started.session_id);
debug_assert_eq!(
started.generated_owner,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
debug_assert_eq!(
started.generated_started,
generated_mob_command_capabilities::CommandPlanKind::SpawnStarted
);
let impact = self.pending_spawns.insert(spawn_ticket, pending, task);
if let PendingSpawnInsertImpact::Collided { replaced_identity } = impact {
if let Some(replaced_identity) = replaced_identity.as_ref() {
if let Ok(completed) = self.complete_orchestrator_spawn(
Some(spawn_ticket),
replaced_identity,
"pending spawn slot collision replaced existing entry",
) {
debug_assert_eq!(completed.agent_identity, *replaced_identity);
}
}
tracing::warn!(
spawn_ticket,
"pending spawn slot collision replaced existing entry"
);
}
self.debug_assert_pending_spawn_alignment();
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
spawn_ticket,
message = %message,
"pending spawn alignment violated after insert"
);
}
}
fn take_pending_spawn_slot(
&mut self,
spawn_ticket: u64,
) -> (Option<PendingSpawn>, Option<tokio::task::JoinHandle<()>>) {
self.pending_spawns
.take_slot(spawn_ticket)
.map_or((None, None), |slot| (Some(slot.spawn), slot.task))
}
fn complete_pending_spawn_slot(
&mut self,
spawn_ticket: u64,
context: &'static str,
) -> (Option<PendingSpawn>, Option<tokio::task::JoinHandle<()>>) {
let (pending, task) = self.take_pending_spawn_slot(spawn_ticket);
if pending.is_some() || task.is_some() {
if let Some(pending) = pending.as_ref() {
if let Ok(completed) = self.complete_orchestrator_spawn(
Some(spawn_ticket),
&pending.agent_identity,
context,
) {
debug_assert_eq!(completed.agent_identity, pending.agent_identity);
}
}
}
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
spawn_ticket,
context,
message = %message,
"pending spawn alignment violated after completion"
);
}
(pending, task)
}
fn stage_orchestrator_spawn(
&mut self,
agent_identity: &AgentIdentity,
session_id: &SessionId,
) -> Result<AuthorizedMobSpawnStart, MobError> {
let dsl_agent_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let dsl_session_id = mob_dsl::SessionId::from_domain(session_id);
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::StageSpawn {
agent_identity: dsl_agent_identity.clone(),
session_id: dsl_session_id.clone(),
},
"stage_spawn",
)?;
let authorized = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::PendingSpawnOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id: effect_session_id,
} if effect_identity == &dsl_agent_identity && effect_session_id == &dsl_session_id
)
});
if !authorized {
return Err(MobError::Internal(format!(
"MobMachine StageSpawn did not authorize pending operation owner for '{agent_identity}'"
)));
}
Ok(AuthorizedMobSpawnStart {
generated_can_start: generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn,
generated_owner:
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart,
agent_identity: agent_identity.clone(),
session_id: session_id.clone(),
})
}
fn apply_generated_self_owned_operation_owner(
provision_request: &mut ProvisionMemberRequest,
generated_owner: &AuthorizedMobSpawnStart,
) -> Result<(), MobError> {
if !matches!(provision_request.binding, crate::RuntimeBinding::Session)
|| provision_request.owner_bridge_session_id.is_some()
|| provision_request.ops_registry.is_some()
{
return Ok(());
}
provision_request.generated_self_owned_operation_owner =
Some(generated_owner.owner_session_id().clone());
Ok(())
}
fn preview_spawn_admission(
&self,
agent_identity: &AgentIdentity,
authorized_profile_material: &AuthorizedSpawnProfileMaterial,
bridge_session_id: Option<&SessionId>,
) -> Result<(), MobError> {
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
self.preview_dsl_input(
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&crate::ids::AgentRuntimeId::initial(domain_identity),
),
fence_token: mob_dsl::FenceToken::from_domain(self.next_fence_token_preview()),
generation: mob_dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: bridge_session_id.map(mob_dsl::SessionId::from_domain),
replacing,
},
"spawn_command_admission",
)
.map(|_| ())
.map_err(|_| self.invalid_transition_to(MobState::Running))
}
fn preview_spawn_command_admission(
&self,
agent_identity: &AgentIdentity,
) -> Result<(), MobError> {
self.require_member_operation_eligible()?;
self.probe_member_admission(agent_identity)
}
fn probe_member_admission(&self, agent_identity: &AgentIdentity) -> Result<(), MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ProbeMemberAdmission {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"probe_member_admission",
)?;
let mut verdict = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::MemberAdmissionProbed { verdict: kind, .. } = effect {
if verdict.replace(*kind).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple member-admission verdicts".into(),
));
}
}
}
match verdict {
Some(mob_dsl::MemberAdmissionVerdictKind::Admitted) => Ok(()),
Some(mob_dsl::MemberAdmissionVerdictKind::DuplicateRejected) => {
Err(MobError::MemberAlreadyExists(agent_identity.clone()))
}
None => Err(MobError::Internal(
"MobMachine emitted no member-admission verdict".into(),
)),
}
}
fn compute_respawn_generation(
&self,
agent_identity: &AgentIdentity,
) -> Result<crate::ids::Generation, MobError> {
let prepared = self.prepare_dsl_input(
mob_dsl::MobMachineInput::ComputeRespawnGeneration {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"compute_respawn_generation",
)?;
let mut next_generation = None;
for effect in &prepared.effects {
if let mob_dsl::MobMachineEffect::RespawnGenerationComputed {
next_generation: next,
..
} = effect
{
if next_generation.replace(next.0).is_some() {
return Err(MobError::Internal(
"MobMachine emitted multiple respawn generations".into(),
));
}
}
}
match next_generation {
Some(next) => Ok(crate::ids::Generation::new(next)),
None => Err(MobError::Internal(
"MobMachine emitted no respawn generation".into(),
)),
}
}
fn authorize_spawn_profile_material(
&mut self,
agent_identity: &AgentIdentity,
profile_name: &ProfileName,
profile: &crate::profile::Profile,
context: &str,
) -> Result<AuthorizedSpawnProfileMaterial, MobError> {
let (input, expected) =
authorize_spawn_profile_input(agent_identity, profile_name, profile)?;
let transition = self.apply_dsl_input_collect_transition(input, context)?;
require_authorized_effect(&transition, &expected, context)?;
Ok(expected)
}
fn preview_run_flow_command_admission(&self, run_id: &RunId) -> Result<(), MobError> {
self.prepare_command_admission(
mob_dsl::MobMachineInput::RunFlow {
run_id: mob_dsl::RunId::from(run_id.to_string()),
step_ids: Default::default(),
ordered_steps: Vec::new(),
step_status: Default::default(),
output_recorded: Default::default(),
step_condition_results: Default::default(),
step_has_conditions: Default::default(),
step_dependencies: Default::default(),
step_dependency_modes: Default::default(),
step_branches: Default::default(),
step_collection_policies: Default::default(),
step_quorum_thresholds: Default::default(),
step_target_counts: Default::default(),
step_target_success_counts: Default::default(),
step_target_terminal_failure_counts: Default::default(),
escalation_threshold: 0,
max_step_retries: 0,
max_active_nodes: 0,
max_active_frames: 0,
max_frame_depth: 0,
},
MobState::Running,
"run_flow_command_admission",
)
.map(|_| ())
}
fn resolve_respawn_topology_restore_result(
&mut self,
agent_identity: &AgentIdentity,
failed_restore_peer_ids: Vec<RespawnTopologyPeerId>,
) -> Result<RespawnTopologyRestoreResolution, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from(agent_identity.as_str());
let dsl_failed_peer_ids = failed_restore_peer_ids
.iter()
.map(|peer_id| mob_dsl::RespawnTopologyPeerId::from(peer_id.as_str()))
.collect::<Vec<_>>();
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::ResolveRespawnTopologyRestore {
agent_identity: dsl_identity.clone(),
failed_peer_ids: dsl_failed_peer_ids.clone(),
},
"respawn_topology_restore_result",
)?;
let resolution = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::RespawnTopologyRestoreResolved {
agent_identity: effect_identity,
result,
failed_peer_ids,
} if effect_identity == dsl_identity => Some((result, failed_peer_ids)),
_ => None,
})
.ok_or_else(|| {
MobError::Internal(
"MobMachine accepted respawn topology feedback but emitted no typed result"
.into(),
)
})?;
let (result, effect_failed_peer_ids) = resolution;
if effect_failed_peer_ids != dsl_failed_peer_ids {
return Err(MobError::Internal(
"MobMachine respawn topology feedback echoed inconsistent failed peer ids".into(),
));
}
let failed_peer_ids = effect_failed_peer_ids
.into_iter()
.map(|peer_id| RespawnTopologyPeerId::from(peer_id.0.as_str()))
.collect::<Vec<_>>();
match result {
mob_dsl::RespawnTopologyRestoreResultKind::Completed if failed_peer_ids.is_empty() => {
Ok(RespawnTopologyRestoreResolution {
result,
failed_peer_ids,
})
}
mob_dsl::RespawnTopologyRestoreResultKind::Completed => Err(MobError::Internal(
"MobMachine classified respawn topology restore as completed with failed peers"
.into(),
)),
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed
if !failed_peer_ids.is_empty() =>
{
Ok(RespawnTopologyRestoreResolution {
result,
failed_peer_ids,
})
}
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed => {
Err(MobError::Internal(
"MobMachine classified respawn topology restore as failed without failed peers"
.into(),
))
}
}
}
#[allow(clippy::too_many_arguments)]
fn resolve_submit_work_rejection_in_authority(
authority: &mut mob_dsl::MobMachineAuthority,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
origin: WorkOrigin,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let transition = match mob_dsl::MobMachineMutator::apply(
authority,
mob_dsl::MobMachineInput::ResolveSubmitWorkRejection {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
origin: dsl_origin,
},
) {
Ok(transition) => transition,
Err(err) => {
return MobError::Internal(format!(
"MobMachine rejected SubmitWork and failed to resolve typed rejection: {err}"
));
}
};
let reason = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::SubmitWorkRejected {
agent_runtime_id,
reason,
expected_fence_token,
actual_fence_token,
..
} if agent_runtime_id == *dsl_runtime_id => {
Some((reason, expected_fence_token, actual_fence_token))
}
_ => None,
});
match reason {
Some((mob_dsl::SubmitWorkRejectReasonKind::MobNotRunning, _, _)) => {
MobError::InvalidTransition {
from: current_state,
to: MobState::Running,
}
}
Some((mob_dsl::SubmitWorkRejectReasonKind::MemberNotFound, _, _)) => {
MobError::MemberNotFound(agent_identity.clone())
}
Some((
mob_dsl::SubmitWorkRejectReasonKind::StaleFenceToken,
Some(expected),
Some(actual),
)) => MobError::StaleFenceToken {
runtime_id: runtime_id.clone(),
expected: FenceToken::new(expected.0),
actual: FenceToken::new(actual.0),
},
Some((mob_dsl::SubmitWorkRejectReasonKind::StaleFenceToken, _, _)) => {
MobError::Internal(
"MobMachine rejected SubmitWork as stale without fence-token feedback".into(),
)
}
Some((mob_dsl::SubmitWorkRejectReasonKind::NotExternallyAddressable, _, _)) => {
MobError::NotExternallyAddressable(agent_identity.clone())
}
None => MobError::Internal(
"MobMachine rejected SubmitWork without typed rejection feedback".into(),
),
}
}
#[allow(clippy::too_many_arguments)]
fn resolve_submit_work_projection_missing_or_rejection(
authority: &mut mob_dsl::MobMachineAuthority,
declared_submit_work_admitted: bool,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
origin: WorkOrigin,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
if declared_submit_work_admitted {
return MobError::Internal(format!(
"MobMachine admitted SubmitWork for '{agent_identity}' but the roster projection has no member entry"
));
}
Self::resolve_submit_work_rejection_in_authority(
authority,
dsl_identity,
dsl_runtime_id,
dsl_fence_token,
runtime_id,
origin,
agent_identity,
current_state,
)
}
async fn resolve_spawn_policy_via_machine(
&mut self,
identity: &AgentIdentity,
) -> Result<Option<super::spawn_policy::SpawnSpec>, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let (policy_enabled, revision) = {
let state = self.dsl_authority.state();
(state.spawn_policy_enabled, state.spawn_policy_revision)
};
if !policy_enabled {
return Ok(None);
}
let observed = self.spawn_policy.observe_resolution(identity).await;
let profile_name = observed
.as_ref()
.map(|spec| spec.profile.as_str().to_owned());
let runtime_mode = observed
.as_ref()
.and_then(|spec| spec.runtime_mode.map(mob_dsl::SpawnPolicyRuntimeMode::from));
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::ResolveSpawnPolicy {
agent_identity: dsl_identity.clone(),
revision,
profile_name,
runtime_mode,
},
"resolve_spawn_policy",
)?;
let recorded = transition.effects().iter().find_map(|effect| match effect {
mob_dsl::MobMachineEffect::SpawnPolicyResolutionRecorded {
agent_identity,
revision: effect_revision,
profile_name,
runtime_mode,
} if *agent_identity == dsl_identity && *effect_revision == revision => {
Some((profile_name.clone(), *runtime_mode))
}
_ => None,
});
match recorded {
Some((Some(profile), runtime_mode)) => Ok(Some(super::spawn_policy::SpawnSpec {
profile: crate::ids::ProfileName::from(profile),
runtime_mode: runtime_mode.map(crate::MobRuntimeMode::from),
})),
Some((None, None)) => Ok(None),
Some((None, Some(_))) => Err(MobError::Internal(
"MobMachine recorded spawn-policy runtime mode without profile".into(),
)),
None => Err(MobError::Internal(
"MobMachine accepted spawn-policy resolution but emitted no typed feedback".into(),
)),
}
}
fn resolve_cancel_all_work_rejection_in_authority(
authority: &mut mob_dsl::MobMachineAuthority,
dsl_identity: &mob_dsl::AgentIdentity,
dsl_runtime_id: &mob_dsl::AgentRuntimeId,
dsl_fence_token: mob_dsl::FenceToken,
runtime_id: &AgentRuntimeId,
agent_identity: &AgentIdentity,
current_state: MobState,
) -> MobError {
let transition = match mob_dsl::MobMachineMutator::apply(
authority,
mob_dsl::MobMachineInput::ResolveCancelAllWorkRejection {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
},
) {
Ok(transition) => transition,
Err(err) => {
return MobError::Internal(format!(
"MobMachine rejected CancelAllWork and failed to resolve typed rejection: {err}"
));
}
};
let reason = transition
.into_effects()
.into_iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::CancelAllWorkRejected {
agent_runtime_id,
reason,
expected_fence_token,
actual_fence_token,
} if agent_runtime_id == *dsl_runtime_id => {
Some((reason, expected_fence_token, actual_fence_token))
}
_ => None,
});
match reason {
Some((mob_dsl::CancelAllWorkRejectReasonKind::MobNotRunning, _, _)) => {
MobError::InvalidTransition {
from: current_state,
to: MobState::Running,
}
}
Some((mob_dsl::CancelAllWorkRejectReasonKind::MemberNotFound, _, _)) => {
MobError::MemberNotFound(agent_identity.clone())
}
Some((
mob_dsl::CancelAllWorkRejectReasonKind::StaleFenceToken,
Some(expected),
Some(actual),
)) => MobError::StaleFenceToken {
runtime_id: runtime_id.clone(),
expected: FenceToken::new(expected.0),
actual: FenceToken::new(actual.0),
},
Some((mob_dsl::CancelAllWorkRejectReasonKind::StaleFenceToken, _, _)) => {
MobError::Internal(
"MobMachine rejected CancelAllWork as stale without fence-token feedback"
.into(),
)
}
None => MobError::Internal(
"MobMachine rejected CancelAllWork without typed rejection feedback".into(),
),
}
}
fn preview_policy_spawn_submit_work_admission(
&self,
agent_identity: &AgentIdentity,
authorized_profile_material: &AuthorizedSpawnProfileMaterial,
work_ref: &WorkRef,
origin: WorkOrigin,
) -> Result<(), MobError> {
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let domain_runtime_id =
crate::ids::AgentRuntimeId::new(domain_identity, crate::ids::Generation::INITIAL);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&domain_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(self.next_fence_token_preview());
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let mut authority =
mob_dsl::MobMachineAuthority::recover_from_state(self.dsl_authority.state().clone())
.map_err(|error| {
MobError::Internal(format!(
"spawn preview could not recover DSL authority state: {error}"
))
})?;
let preview_bridge_session_id = mob_dsl::SessionId::from_domain(&SessionId::new());
let begin_spawn = mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
generation: mob_dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
profile_material_digest: authorized_profile_material.profile_material_digest.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(preview_bridge_session_id.clone()),
replacing: replacing.clone(),
};
mob_dsl::MobMachineMutator::apply(&mut authority, begin_spawn)
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
let commit_membership = mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
generation: mob_dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
profile_material_digest: authorized_profile_material.profile_material_digest.clone(),
external_addressable: authorized_profile_material.external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(preview_bridge_session_id),
replacing,
};
mob_dsl::MobMachineMutator::apply(&mut authority, commit_membership)
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: mob_dsl::WorkId::from_work_ref(work_ref),
origin: mob_dsl::WorkOrigin::from(origin),
},
)
.map(|_| ())
.map_err(|_| {
Self::resolve_submit_work_rejection_in_authority(
&mut authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&domain_runtime_id,
origin,
agent_identity,
self.state(),
)
})
}
fn complete_orchestrator_spawn(
&mut self,
spawn_ticket: Option<u64>,
agent_identity: &AgentIdentity,
context: &'static str,
) -> Result<AuthorizedMobSpawnCompleted, MobError> {
let dsl_agent_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let transition = match self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::CompleteSpawn {
agent_identity: dsl_agent_identity,
},
"complete_spawn",
) {
Ok(transition) => transition,
Err(error) => {
if let Some(spawn_ticket) = spawn_ticket {
tracing::warn!(
spawn_ticket,
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
} else {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
}
return Err(error);
}
};
let completed = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::EmitMemberLifecycleNotice {
kind: mob_dsl::MemberLifecycleKind::Spawned
}
)
});
if !completed {
let error = MobError::Internal(format!(
"MobMachine CompleteSpawn did not authorize spawn completion notice for '{agent_identity}'"
));
if let Some(spawn_ticket) = spawn_ticket {
tracing::warn!(
spawn_ticket,
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
} else {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
context,
"failed to reconcile generated pending-spawn snapshot"
);
}
return Err(error);
}
let completed = AuthorizedMobSpawnCompleted {
generated_plan:
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart,
generated_effect: generated_mob_command_capabilities::CommandPlanKind::SpawnEffect,
agent_identity: agent_identity.clone(),
};
debug_assert_eq!(
completed.generated_effect,
generated_mob_command_capabilities::CommandPlanKind::SpawnEffect
);
debug_assert_eq!(
completed.generated_plan,
generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart
);
Ok(completed)
}
async fn flow_tracker_alignment_violation(&self) -> Option<String> {
let snapshot = self.flow_tracker_snapshot().await;
let run_task_ids = snapshot.run_task_ids;
let run_token_ids = snapshot.cancel_token_ids;
if run_task_ids != run_token_ids {
return Some(format!(
"run task/token tracker mismatch: tasks={run_task_ids:?}, tokens={run_token_ids:?}"
));
}
let stream_ids = snapshot.stream_ids;
let unknown_streams = stream_ids
.iter()
.filter(|run_id| !run_task_ids.contains(*run_id))
.cloned()
.collect::<Vec<_>>();
if !unknown_streams.is_empty() {
return Some(format!(
"flow stream tracker contains unknown runs: {unknown_streams:?}"
));
}
None
}
async fn flow_tracker_snapshot(&self) -> super::MobFlowTrackerSnapshot {
super::MobFlowTrackerSnapshot {
run_task_ids: self
.run_tasks
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
cancel_token_ids: self
.run_cancel_tokens
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
stream_ids: self
.flow_streams
.lock()
.await
.keys()
.cloned()
.collect::<std::collections::BTreeSet<_>>(),
tracked_flows: self
.run_cancel_tokens
.iter()
.map(|(run_id, (_, flow_id))| (run_id.clone(), flow_id.clone()))
.collect(),
}
}
async fn ensure_flow_tracker_alignment(&self, context: &str) -> Result<(), MobError> {
if let Some(message) = self.flow_tracker_alignment_violation().await {
return Err(MobError::Internal(format!(
"{context}: flow tracker alignment violation: {message}"
)));
}
Ok(())
}
async fn cleanup_namespace(&self) -> Result<(), MobError> {
Ok(())
}
fn fallback_spawn_prompt(
&self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
) -> String {
format!(
"You have been spawned as '{}' (role: {}) in mob '{}'.",
agent_identity, profile_name, self.definition.id
)
}
#[cfg(feature = "runtime-adapter")]
async fn start_autonomous_member(
&mut self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
prompt: meerkat_core::types::ContentInput,
) -> Result<(), MobError> {
self.ensure_autonomous_runtime_ready(agent_identity, member_ref)
.await?;
let startup_marker = {
let roster = self.roster.read().await;
roster
.get_by_identity(&AgentIdentity::from(agent_identity.as_str()))
.map(|entry| {
(
mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
)
})
}
.ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' missing roster entry for startup readiness"
))
})?;
if !self
.dsl_authority
.state()
.member_startup_ready
.contains(&startup_marker.0)
{
self.apply_dsl_input(
mob_dsl::MobMachineInput::StartupMarkReady {
agent_runtime_id: startup_marker.0,
fence_token: startup_marker.1,
},
"start_autonomous_member/startup_mark_ready",
)?;
}
let bridge_session_id = member_ref.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' must be session-backed"
))
})?;
let adapter = self.runtime_adapter.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' requires admission-capable substrate (runtime adapter)"
))
})?;
{
use meerkat_runtime::{Input, InputHeader, PromptInput};
let input = Input::Prompt(PromptInput {
injected_context: Vec::new(),
header: InputHeader {
id: meerkat_core::lifecycle::InputId::new(),
timestamp: chrono::Utc::now(),
source: meerkat_runtime::InputOrigin::Operator,
durability: meerkat_runtime::InputDurability::Durable,
visibility: meerkat_runtime::InputVisibility::default(),
idempotency_key: None,
supersession_key: None,
correlation_id: None,
},
content: prompt,
typed_turn_appends: Vec::new(),
turn_metadata: None,
});
let (_outcome, completion_handle) = adapter
.accept_input_with_completion(bridge_session_id, input)
.await
.map_err(|e| {
MobError::Internal(format!(
"autonomous prompt admission failed for '{agent_identity}': {e}"
))
})?;
let log_id = agent_identity.clone();
let completion_command_tx = self.command_tx.clone();
let handle = tokio::spawn(async move {
if let Some(h) = completion_handle {
let outcome = h.wait().await;
let (ack_tx, ack_rx) = oneshot::channel();
if completion_command_tx
.send(MobCommand::KickoffOutcomeResolved {
agent_identity: log_id.clone(),
outcome,
ack_tx,
})
.await
.is_err()
{
tracing::warn!(
agent_identity = %log_id,
"mob actor dropped before kickoff outcome could be recorded"
);
} else {
let _ = ack_rx.await;
}
}
});
self.autonomous_initial_turns
.lock()
.await
.insert(agent_identity.clone(), InitialTurnHandle { handle });
}
tracing::debug!(agent_identity = %agent_identity, "autonomous member started");
Ok(())
}
async fn ensure_autonomous_runtime_ready(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
self.ensure_mob_comms_drain(agent_identity, member_ref)
.await?;
self.ensure_autonomous_dispatch_capability(agent_identity, member_ref)
.await
}
async fn ensure_mob_comms_drain(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
#[cfg(all(not(target_arch = "wasm32"), feature = "runtime-adapter"))]
{
let Some(bridge_session_id) = member_ref.bridge_session_id() else {
return Ok(());
};
if let Some(adapter) = self.runtime_adapter.clone()
&& let Some(comms_runtime) = self.provisioner.comms_runtime(member_ref).await
{
let mob_id =
meerkat_runtime::meerkat_machine::dsl::MobId::from(self.definition.id.as_ref());
let spawned = adapter
.maybe_spawn_mob_comms_drain(bridge_session_id, comms_runtime, mob_id)
.await
.map_err(|err| {
MobError::Internal(format!(
"mob comms drain spawn failed for session {bridge_session_id}: {err}"
))
})?;
if spawned {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %bridge_session_id,
"updated peer ingress for mob member"
);
}
}
}
#[cfg(any(target_arch = "wasm32", not(feature = "runtime-adapter")))]
{
let _ = (agent_identity, member_ref);
}
Ok(())
}
async fn teardown_session_runtime_bindings_from_machine(&self) {
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
let session_ids = self
.dsl_authority
.state()
.member_session_bindings
.values()
.filter_map(|session_id| SessionId::parse(&session_id.0).ok())
.collect::<Vec<_>>();
for session_id in session_ids {
adapter.unregister_session(&session_id).await;
}
}
}
async fn ensure_autonomous_dispatch_capability_for_provisioner(
provisioner: &Arc<dyn MobProvisioner>,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
let bridge_session_id = member_ref.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"autonomous member '{agent_identity}' must be session-backed for injector dispatch"
))
})?;
if provisioner
.interaction_event_injector(bridge_session_id)
.await
.is_none()
{
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous member dispatch",
});
}
Ok(())
}
#[cfg(feature = "runtime-adapter")]
async fn resolve_kickoff_outcome(
&mut self,
agent_identity: &AgentIdentity,
outcome: Result<
meerkat_runtime::completion::CompletionOutcome,
meerkat_runtime::completion::CompletionWaitError,
>,
) -> Result<(), MobError> {
let outcome = match outcome {
Ok(outcome) => outcome,
Err(error) => {
return self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffResolveFailed {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
error: format!("runtime completion waiter failed: {error}"),
},
"resolve_kickoff_outcome_wait_error",
)
.await
.map(|_| ());
}
};
if let meerkat_runtime::completion::CompletionOutcome::CallbackPending { tool_name, args } =
&outcome
{
tracing::debug!(
agent_identity = %agent_identity,
tool_name = %tool_name,
args = ?args,
"autonomous kickoff reached callback-pending boundary"
);
}
let _ = self
.apply_kickoff_input(
agent_identity,
match outcome {
meerkat_runtime::completion::CompletionOutcome::Completed(_)
| meerkat_runtime::completion::CompletionOutcome::CompletedWithFinalizationFailure {
..
}
| meerkat_runtime::completion::CompletionOutcome::CompletedWithoutResult => {
mob_dsl::MobMachineInput::KickoffResolveStarted {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
}
}
meerkat_runtime::completion::CompletionOutcome::CallbackPending { .. } => {
mob_dsl::MobMachineInput::KickoffResolveCallbackPending {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
}
}
meerkat_runtime::completion::CompletionOutcome::Cancelled => {
mob_dsl::MobMachineInput::KickoffCancelRequested {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
}
}
meerkat_runtime::completion::CompletionOutcome::Abandoned {
reason: error,
..
}
| meerkat_runtime::completion::CompletionOutcome::AbandonedWithError {
reason: error,
..
}
| meerkat_runtime::completion::CompletionOutcome::RuntimeTerminated {
reason: error,
..
} => {
mob_dsl::MobMachineInput::KickoffResolveFailed {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
error,
}
}
},
"resolve_kickoff_outcome",
)
.await?;
Ok(())
}
async fn maybe_mark_kickoff_cancelled(&mut self, agent_identity: &AgentIdentity) {
if let Err(error) = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffCancelRequested {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"maybe_mark_kickoff_cancelled",
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
"failed to apply kickoff cancellation transition"
);
}
}
async fn ensure_autonomous_dispatch_capability(
&self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
agent_identity,
member_ref,
)
.await
}
async fn stop_autonomous_member(
&mut self,
agent_identity: &AgentIdentity,
member_ref: &MemberRef,
) -> Result<(), MobError> {
let kickoff_handle = self
.autonomous_initial_turns
.lock()
.await
.remove(agent_identity);
if let Some(handle) = kickoff_handle {
self.maybe_mark_kickoff_cancelled(agent_identity).await;
handle.abort_and_join().await;
}
if let Err(error) = self.provisioner.interrupt_member(member_ref).await
&& !matches!(
error,
MobError::SessionError(meerkat_core::service::SessionError::NotFound { .. })
)
{
return Err(error);
}
#[cfg(feature = "runtime-adapter")]
if let (Some(adapter), Some(session_id)) =
(&self.runtime_adapter, member_ref.bridge_session_id())
{
adapter.abort_comms_drain(session_id).await.map_err(|err| {
MobError::Internal(format!(
"failed to abort comms drain for stopped member session {session_id}: {err}"
))
})?;
}
let mut still_active = false;
for _ in 0..40 {
match self.provisioner.is_member_active(member_ref).await? {
Some(true) => tokio::time::sleep(std::time::Duration::from_millis(25)).await,
_ => {
still_active = false;
break;
}
}
still_active = true;
}
if still_active {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"autonomous member stop polling exhausted before member became idle"
);
}
Ok(())
}
async fn stop_all_autonomous_members(&mut self) -> Result<(), MobError> {
let entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost)
.cloned()
.collect::<Vec<_>>()
};
if entries.is_empty() {
return Ok(());
}
let mut first_error: Option<MobError> = None;
for entry in entries {
let result = self.stop_autonomous_member_entry(entry).await;
if let Err((agent_identity, error)) = result {
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
"failed stopping autonomous member"
);
if first_error.is_none() {
first_error = Some(error);
}
}
}
if let Some(error) = first_error {
return Err(error);
}
Ok(())
}
async fn stop_autonomous_member_entry(
&mut self,
entry: RosterEntry,
) -> Result<(), (AgentIdentity, MobError)> {
self.stop_autonomous_member(&entry.agent_identity, &entry.member_ref)
.await
.map_err(|error| (entry.agent_identity, error))
}
async fn ensure_autonomous_runtimes_from_roster(&self) -> Result<(), MobError> {
let broken_members = self
.dsl_authority
.state()
.member_restore_failures
.keys()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<HashSet<_>>();
let entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| {
entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& !broken_members.contains(&entry.agent_identity)
})
.cloned()
.collect::<Vec<_>>()
};
if entries.is_empty() {
}
let mut first_error: Option<MobError> = None;
let all_entries = {
let roster = self.roster.read().await;
roster
.list()
.filter(|entry| {
entry.runtime_mode != crate::MobRuntimeMode::AutonomousHost
&& !broken_members.contains(&entry.agent_identity)
})
.cloned()
.collect::<Vec<_>>()
};
for entry in &all_entries {
let ensure_result = tokio::time::timeout(std::time::Duration::from_secs(5), async {
self.provisioner
.ensure_runtime_session_state(&entry.member_ref)
.await?;
self.ensure_mob_comms_drain(&entry.agent_identity, &entry.member_ref)
.await
})
.await;
let result = match ensure_result {
Ok(result) => result,
Err(_elapsed) => {
tracing::warn!(
agent_identity = %entry.agent_identity,
"timed out ensuring mob comms drain ready"
);
if first_error.is_none() {
first_error = Some(MobError::ReadyWaitTimedOut {
pending_member_ids: vec![AgentIdentity::from(
entry.agent_identity.as_str(),
)],
});
}
continue;
}
};
if let Err(error) = result {
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed ensuring mob comms drain ready"
);
if first_error.is_none() {
first_error = Some(error);
}
}
}
for entry in &entries {
let ensure_result = tokio::time::timeout(
std::time::Duration::from_secs(5),
self.ensure_autonomous_runtime_ready(&entry.agent_identity, &entry.member_ref),
)
.await;
let result = match ensure_result {
Ok(result) => result,
Err(_elapsed) => {
tracing::warn!(
agent_identity = %entry.agent_identity,
"timed out ensuring autonomous runtime ready"
);
if first_error.is_none() {
first_error = Some(MobError::ReadyWaitTimedOut {
pending_member_ids: vec![AgentIdentity::from(
entry.agent_identity.as_str(),
)],
});
}
continue;
}
};
if let Err(error) = result {
tracing::warn!(
agent_identity = %entry.agent_identity,
error = %error,
"failed ensuring autonomous runtime ready"
);
if first_error.is_none() {
first_error = Some(error);
}
}
}
if let Some(error) = first_error {
return Err(error);
}
Ok(())
}
pub(super) async fn run(mut self, mut command_rx: mpsc::Receiver<MobCommand>) {
if matches!(self.dsl_state(), MobState::Running) {
if let Err(error) = self.restore_generated_member_operation_bindings().await {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"failed to restore generated mob member operation bindings during actor startup; entering Stopped"
);
self.fail_startup_to_stopped("mob member operation binding restore failure")
.await;
}
}
if matches!(self.state(), MobState::Running) {
if let Err(error) = self.ensure_autonomous_runtimes_from_roster().await {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
"failed to start autonomous host loops during actor startup; entering Stopped"
);
self.fail_startup_to_stopped("autonomous runtime startup failure")
.await;
}
}
let mut deferred_commands = VecDeque::new();
loop {
self.drain_completed_peer_delivery_tasks();
let cmd = if let Some(cmd) = deferred_commands.pop_front() {
cmd
} else if let Some(cmd) = command_rx.recv().await {
cmd
} else {
break;
};
tracing::debug!(command_kind = cmd.kind(), "MobActor received command");
match cmd {
MobCommand::Spawn {
spec,
spawn_source,
owner_bridge_session_id,
ops_registry,
reply_tx,
} => {
tracing::debug!(
member_id = %spec.identity,
profile = %spec.role_name,
owner_bound = owner_bridge_session_id.is_some(),
"MobActor received Spawn command"
);
Box::pin(self.enqueue_spawn(
*spec,
spawn_source,
owner_bridge_session_id,
ops_registry,
reply_tx,
))
.await;
}
MobCommand::SpawnProvisioned {
spawn_ticket,
result,
} => {
let mut completions = vec![(spawn_ticket, result)];
loop {
match command_rx.try_recv() {
Ok(MobCommand::SpawnProvisioned {
spawn_ticket,
result,
}) => completions.push((spawn_ticket, result)),
Ok(other) => {
deferred_commands.push_back(other);
break;
}
Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break,
Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => break,
}
}
Box::pin(self.handle_spawn_provisioned_batch(completions)).await;
}
MobCommand::Retire {
agent_identity,
reply_tx,
} => {
let result = match self
.cancel_pending_spawns_for_member(
&agent_identity,
"retire command received",
)
.await
{
Ok(canceled) => {
if canceled > 0 {
tracing::info!(
agent_identity = %agent_identity,
canceled,
"retire canceled pending spawn lineage before roster retirement"
);
}
self.handle_retire(agent_identity).await
}
Err(error) => Err(error),
};
let _ = reply_tx.send(result);
}
MobCommand::Respawn {
agent_identity,
initial_message,
reply_tx,
} => {
let result =
Box::pin(self.handle_respawn(agent_identity, initial_message)).await;
let _ = reply_tx.send(result);
}
MobCommand::RetireAll { reply_tx } => {
let result = self.retire_all_members("retire_all").await;
let _ = reply_tx.send(result);
}
MobCommand::SubmitWork { payload, reply_tx } => {
tracing::debug!(
agent_identity = %payload.runtime_id.identity,
runtime_id = %payload.runtime_id,
work_ref = %payload.work_ref,
origin = ?payload.origin,
handling_mode = ?payload.handling_mode,
ack_mode = ?payload.ack_mode,
"MobActor handling SubmitWork command"
);
match Box::pin(self.handle_submit_work(payload)).await {
Ok(SubmitWorkDispatchCompletion::Completed) => {
let _ = reply_tx.send(Ok(()));
}
Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id: _,
member_ref,
req,
}) => {
Self::spawn_turn_admission_reply(
self.provisioner.clone(),
member_ref,
req,
reply_tx,
);
}
Ok(SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref,
req,
}) => {
Self::spawn_turn_completed_reply(
self.provisioner.clone(),
member_ref,
req,
reply_tx,
);
}
Err(error) => {
let _ = reply_tx.send(Err(error));
}
}
}
MobCommand::SendPeerMessage {
from,
to,
content,
handling_mode,
reply_tx,
} => {
match Box::pin(self.prepare_send_peer_message(from, to, content, handling_mode))
.await
{
Ok(plan) => {
self.spawn_peer_message_delivery(plan, reply_tx);
}
Err(error) => {
let _ = reply_tx.send(Err(error));
}
}
}
MobCommand::DeclareMemberOutboundTaint {
identity,
taint,
reply_tx,
} => {
let result =
Box::pin(self.declare_member_outbound_taint(identity, taint)).await;
let _ = reply_tx.send(result);
}
#[cfg(feature = "runtime-adapter")]
MobCommand::KickoffOutcomeResolved {
agent_identity,
outcome,
ack_tx,
} => {
if let Err(error) = self.resolve_kickoff_outcome(&agent_identity, outcome).await
{
tracing::warn!(
agent_identity = %agent_identity,
error = %error,
"failed to persist kickoff outcome"
);
}
let _ = ack_tx.send(());
}
MobCommand::RunFlow {
flow_id,
activation_params,
scoped_event_tx,
reply_tx,
} => {
let result = self
.handle_run_flow(flow_id, activation_params, scoped_event_tx)
.await;
let _ = reply_tx.send(result);
}
MobCommand::PreviewRunFlowAdmission { reply_tx } => {
let result = self.preview_run_flow_command_admission(&RunId::new());
let _ = reply_tx.send(result);
}
MobCommand::CancelFlow { run_id, reply_tx } => {
let result = self.handle_cancel_flow(run_id).await;
let _ = reply_tx.send(result);
}
MobCommand::FlowStatus { run_id, reply_tx } => {
let result = self
.run_store
.get_run(&run_id)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowRunCommand {
run_id,
command,
context,
reply_tx,
} => {
let result = self
.commit_flow_run_command_in_actor(&run_id, *command, context)
.await;
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowTerminalization {
run_id,
flow_id,
target,
command,
context,
reply_tx,
} => {
let result = self
.commit_flow_terminalization_in_actor(
run_id, flow_id, target, *command, context,
)
.await;
let _ = reply_tx.send(result);
}
MobCommand::CommitFlowFrameStorePlan {
run_id,
plan,
reply_tx,
} => {
let result = self
.commit_flow_frame_store_plan_in_actor(&run_id, *plan)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ProjectMachineInput { input, reply_tx } => {
let result = self
.apply_dsl_input(*input, "project_machine_input")
.map(|()| self.dsl_authority.state().clone());
let _ = reply_tx.send(result);
}
MobCommand::ApplyMachineInputEffects { input, reply_tx } => {
let result =
self.apply_dsl_input_collect_effects(*input, "apply_machine_input_effects");
let _ = reply_tx.send(result);
}
MobCommand::PreviewMachineInput { input, reply_tx } => {
let result = self.preview_dsl_input(*input, "preview_machine_input");
let _ = reply_tx.send(result);
}
MobCommand::QueryMachineState { reply_tx } => {
let _ = reply_tx.send(self.dsl_authority.state().clone());
}
#[cfg(test)]
MobCommand::AuthorizeMemberTrustCleanupForTest { edge, reply_tx } => {
let result = self
.authorize_member_trust_cleanup(
&edge,
"authorize_member_trust_cleanup_for_test",
)
.and_then(|handoff| match handoff.authority {
MemberTrustAuthority::Unwiring(obligation) => Ok(obligation),
MemberTrustAuthority::Wiring(_) | MemberTrustAuthority::Repair(_) => {
Err(MobError::WiringError(
"member trust cleanup returned non-unwiring authority"
.to_string(),
))
}
});
let _ = reply_tx.send(result);
}
MobCommand::ApplyExternalPeerReciprocalTrust {
key,
target_comms,
peer,
reply_tx,
} => {
let result = self
.apply_external_peer_reciprocal_trust(key, target_comms, peer)
.await;
let _ = reply_tx.send(result);
}
MobCommand::ProjectMachineSignal { signal } => {
let foreign_runtime_id =
foreign_runtime_observation(self.dsl_authority.state(), &signal).cloned();
if let Err(error) = self.apply_dsl_signal(signal, "project_machine_signal") {
if let Some(agent_runtime_id) = foreign_runtime_id {
tracing::debug!(
?agent_runtime_id,
error = %error,
"ignored foreign runtime lifecycle observation"
);
} else {
tracing::error!(
error = %error,
"typed composition signal projection failed"
);
}
}
}
MobCommand::RecordMissingMemberBridgeSession {
agent_identity,
bridge_session_id,
} => {
let _ = self
.record_missing_member_bridge_session(
&agent_identity,
&bridge_session_id,
"ready_wait",
)
.await;
}
MobCommand::FlowFinished { run_id } => {
if let Err(error) = self
.handle_flow_cleanup(run_id, "flow finished cleanup")
.await
{
tracing::error!(error = %error, "flow finished cleanup failed");
}
}
MobCommand::FlowCanceledCleanup {
run_id,
terminalized,
} => {
if !terminalized {
tracing::error!(
%run_id,
"flow canceled cleanup refused: spawned cancellation path did not prove persisted terminalization"
);
continue;
}
if let Err(error) = self
.handle_flow_cleanup(run_id, "flow canceled cleanup")
.await
{
tracing::error!(error = %error, "flow canceled cleanup failed");
}
}
#[cfg(test)]
MobCommand::FlowTrackerCounts { reply_tx } => {
let snapshot = self.flow_tracker_snapshot().await;
let tasks = snapshot.run_task_ids.len();
let tokens = snapshot.cancel_token_ids.len();
let _ = reply_tx.send((tasks, tokens));
}
#[cfg(test)]
MobCommand::OrchestratorSnapshot { reply_tx } => {
let phase = self.state();
let _ = reply_tx.send(
self.machine_orchestrator_snapshot(phase)
.unwrap_or_default(),
);
}
#[cfg(test)]
MobCommand::LifecycleSnapshot { reply_tx } => {
let _ = reply_tx.send(super::state::MobLifecycleSnapshot {
phase: self.state(),
active_run_count: self.machine_active_run_count(),
cleanup_pending: false,
});
}
#[cfg(test)]
MobCommand::LifecycleNotificationBurst {
count,
message,
reply_tx,
} => {
let mut burst_result = Ok(());
for index in 0..count {
if let Err(error) = self
.notify_orchestrator_lifecycle(format!("{message} #{index}"))
.await
{
burst_result = Err(error);
break;
}
}
let _ = reply_tx.send(burst_result);
}
#[cfg(test)]
MobCommand::DslT2Snapshot { reply_tx } => {
let dsl = self.dsl_authority.state();
let _ = reply_tx.send(super::state::MobDslT2Snapshot {
destroy_admitted: dsl.destroy_admitted,
flow_authority_schema_version: dsl.flow_authority_schema_version,
owner_bridge_session_id: dsl.owner_bridge_session_id.clone(),
owner_bridge_destroy_on_archive: dsl.owner_bridge_destroy_on_archive,
implicit_delegation_mob: dsl.implicit_delegation_mob,
supervisor_authority_peer_id: dsl.supervisor_authority_peer_id.clone(),
supervisor_authority_signing_key: dsl.supervisor_authority_signing_key,
supervisor_authority_epoch: dsl.supervisor_authority_epoch,
supervisor_authority_protocol_version: dsl
.supervisor_authority_protocol_version
.clone(),
supervisor_pending_authority_peer_id: dsl
.supervisor_pending_authority_peer_id
.clone(),
supervisor_pending_authority_signing_key: dsl
.supervisor_pending_authority_signing_key,
supervisor_pending_authority_epoch: dsl.supervisor_pending_authority_epoch,
supervisor_pending_authority_protocol_version: dsl
.supervisor_pending_authority_protocol_version
.clone(),
supervisor_pending_authority_operation_id: dsl
.supervisor_pending_authority_operation_id
.clone(),
supervisor_pending_authority_member_target_names: dsl
.supervisor_pending_authority_member_target_names
.clone(),
supervisor_pending_authority_member_target_addresses: dsl
.supervisor_pending_authority_member_target_addresses
.clone(),
supervisor_pending_authority_accepted_peer_ids: dsl
.supervisor_pending_authority_accepted_peer_ids
.clone(),
pending_recipient_trust: dsl.pending_recipient_trust.clone(),
member_state_markers: dsl.member_state_markers.clone(),
wiring_edges: dsl.wiring_edges.clone(),
external_peer_edges: dsl.external_peer_edges.clone(),
external_peer_edges_by_key: dsl.external_peer_edges_by_key.clone(),
identity_to_runtime: dsl.identity_to_runtime.clone(),
identity_runtime_generations: dsl.identity_runtime_generations.clone(),
identity_runtime_fence_tokens: dsl.identity_runtime_fence_tokens.clone(),
member_profile_names: dsl.member_profile_names.clone(),
member_runtime_modes: dsl.member_runtime_modes.clone(),
member_peer_ids: dsl.member_peer_ids.clone(),
member_peer_endpoints: dsl.member_peer_endpoints.clone(),
member_restore_failures: dsl.member_restore_failures.clone(),
member_restore_failure_codes: dsl.member_restore_failure_codes.clone(),
runtime_retire_refusal_codes: dsl.runtime_retire_refusal_codes.clone(),
runtime_retire_refusal_reasons: dsl.runtime_retire_refusal_reasons.clone(),
runtime_retire_pending_sessions: dsl
.runtime_retire_pending_sessions
.clone(),
remote_runtime_retired_ids: dsl.remote_runtime_retired_ids.clone(),
remote_supervisor_revoked_ids: dsl.remote_supervisor_revoked_ids.clone(),
member_revival_pending: dsl.member_revival_pending.clone(),
member_session_bindings: dsl.member_session_bindings.clone(),
spawn_exec_phase: dsl.spawn_exec_phase.clone(),
pending_spawn_sessions: dsl.pending_spawn_sessions.clone(),
pending_session_ingress_detach_runtime_ids: dsl
.pending_session_ingress_detach_runtime_ids
.clone(),
topology_epoch: dsl.topology_epoch,
spawn_policy_enabled: dsl.spawn_policy_enabled,
spawn_policy_revision: dsl.spawn_policy_revision,
spawn_policy_resolution_revision: dsl
.spawn_policy_resolution_revision
.clone(),
spawn_policy_resolution_profiles: dsl
.spawn_policy_resolution_profiles
.clone(),
spawn_policy_resolution_runtime_modes: dsl
.spawn_policy_resolution_runtime_modes
.clone(),
spawn_policy_resolution_absent: dsl.spawn_policy_resolution_absent.clone(),
spawn_profile_authority_profile_names: dsl
.spawn_profile_authority_profile_names
.clone(),
spawn_profile_authority_models: dsl.spawn_profile_authority_models.clone(),
spawn_profile_authority_material_digests: dsl
.spawn_profile_authority_material_digests
.clone(),
spawn_profile_authority_tool_config_digests: dsl
.spawn_profile_authority_tool_config_digests
.clone(),
spawn_profile_authority_skills_digests: dsl
.spawn_profile_authority_skills_digests
.clone(),
spawn_profile_authority_provider_params_digests: dsl
.spawn_profile_authority_provider_params_digests
.clone(),
spawn_profile_authority_output_schema_digests: dsl
.spawn_profile_authority_output_schema_digests
.clone(),
spawn_profile_authority_external_addressable: dsl
.spawn_profile_authority_external_addressable
.clone(),
orphan_budget: dsl.orphan_budget,
topology_default_policy: dsl.topology_default_policy,
external_member_rebind_capability: dsl
.external_member_rebind_capability
.clone(),
desired_members: dsl.desired_members.clone(),
members_to_spawn: dsl.members_to_spawn.clone(),
members_to_retire: dsl.members_to_retire.clone(),
});
}
MobCommand::StartupKickoffSnapshot { reply_tx } => {
let _ = reply_tx.send(super::state::MobStartupKickoffSnapshot {
pending_kickoff_member_ids: self.pending_kickoff_member_ids_from_dsl(),
ready_runtime_ids: self.ready_runtime_ids_from_dsl(),
});
}
MobCommand::ProjectMemberList {
include_retiring,
reply_tx,
} => {
let _ = reply_tx.send(
self.project_member_list_from_machine(include_retiring)
.await,
);
}
MobCommand::ProjectMemberStatus {
agent_identity,
reply_tx,
} => {
let _ = reply_tx.send(
self.machine_member_material(&agent_identity, true)
.await
.map(|material| material.to_snapshot()),
);
}
MobCommand::MemberMachineProjection {
agent_identity,
reply_tx,
} => {
let _ = reply_tx.send(self.machine_projection_for_identity(&agent_identity));
}
MobCommand::Stop { reply_tx } => {
let result = if self.state() == MobState::Destroyed {
Err(self.invalid_transition_to(MobState::Stopped))
} else {
match self.probe_command_admission(
mob_dsl::MobMachineInput::Stop,
MobState::Stopped,
"stop_command_admission",
) {
Ok(()) => {
let mut stop_result =
self.fail_all_pending_spawns("mob is stopping").await;
if stop_result.is_ok() {
self.cancel_pending_peer_deliveries("mob is stopping").await;
if let Err(error) = self
.notify_orchestrator_lifecycle(format!(
"Mob '{}' is stopping.",
self.definition.id
))
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"stop encountered orchestrator lifecycle delivery error"
);
stop_result = Err(error);
}
self.provisioner.cancel_all_checkpointers().await;
}
if stop_result.is_ok() {
let loop_result = self.stop_all_autonomous_members().await;
if let Err(error) = loop_result {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"stop encountered autonomous loop cleanup error"
);
if stop_result.is_ok() {
stop_result = Err(error);
}
}
}
if stop_result.is_ok() {
if self.has_orchestrator
&& let Err(error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::StopOrchestrator,
"stop_orchestrator",
)
{
stop_result = Err(MobError::Internal(format!(
"orchestrator StopOrchestrator transition failed during stop: {error}"
)));
}
if stop_result.is_ok()
&& let Err(error) = self.apply_command_admission(
mob_dsl::MobMachineInput::Stop,
MobState::Stopped,
"stop_input",
)
{
stop_result = Err(error);
}
}
if stop_result.is_err() {
self.provisioner.rearm_all_checkpointers().await;
}
stop_result
}
Err(error) => Err(error),
}
};
let _ = reply_tx.send(result);
}
MobCommand::ResumeLifecycle { reply_tx } => {
let result = match self.probe_command_admission(
mob_dsl::MobMachineInput::Resume,
MobState::Running,
"resume_command_admission",
) {
Ok(()) => {
self.provisioner.rearm_all_checkpointers().await;
if let Err(error) = self.ensure_autonomous_runtimes_from_roster().await
{
if let Err(stop_error) = self.stop_all_autonomous_members().await {
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
"resume cleanup failed while stopping autonomous loops"
);
}
self.provisioner.cancel_all_checkpointers().await;
Err(error)
} else {
let mut resume_result: Result<(), MobError> = Ok(());
if self.has_orchestrator
&& let Err(error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::ResumeOrchestrator,
"resume_orchestrator",
)
{
resume_result = Err(MobError::Internal(format!(
"orchestrator ResumeOrchestrator transition failed during resume: {error}"
)));
}
if resume_result.is_ok()
&& let Err(error) = self.apply_command_admission(
mob_dsl::MobMachineInput::Resume,
MobState::Running,
"resume_input",
)
{
resume_result = Err(error);
}
if let Err(error) = resume_result {
if let Err(stop_error) =
self.stop_all_autonomous_members().await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %stop_error,
"resume transition rollback failed while stopping autonomous loops"
);
}
self.provisioner.cancel_all_checkpointers().await;
Err(error)
} else {
Ok(())
}
}
}
Err(error) => Err(error),
};
let _ = reply_tx.send(result);
}
MobCommand::Complete { reply_tx } => {
let result = match self.probe_command_admission(
mob_dsl::MobMachineInput::Complete,
MobState::Completed,
"complete_command_admission",
) {
Ok(()) => match self.fail_all_pending_spawns("mob is completing").await {
Ok(()) => {
self.cancel_pending_peer_deliveries("mob is completing")
.await;
self.handle_complete().await
}
Err(error) => Err(error),
},
Err(error) => Err(error),
};
let _ = reply_tx.send(result);
}
MobCommand::Destroy { reply_tx } => {
let result = self.handle_destroy().await;
let destroy_succeeded = result.is_ok();
let _ = reply_tx.send(result);
if destroy_succeeded {
self.lifecycle_tasks.abort_all();
while self.lifecycle_tasks.join_next().await.is_some() {}
break;
}
}
MobCommand::Reset { reply_tx } => {
let prior_state = self.state();
if let Err(error) = self.probe_command_admission(
mob_dsl::MobMachineInput::Reset,
MobState::Running,
"reset_command_admission",
) {
let _ = reply_tx.send(Err(error));
continue;
}
if let Err(error) = self.fail_all_pending_spawns("mob is resetting").await {
let _ = reply_tx.send(Err(error));
continue;
}
self.cancel_pending_peer_deliveries("mob is resetting")
.await;
let result = self.handle_reset(prior_state).await;
let _ = reply_tx.send(result);
}
MobCommand::RotateSupervisor { reply_tx } => {
let result = self.handle_rotate_supervisor().await;
let _ = reply_tx.send(result);
}
MobCommand::PollEvents {
after_cursor,
limit,
reply_tx,
} => {
let result = self
.events
.poll(after_cursor, limit)
.await
.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::ReplayAllEvents { reply_tx } => {
let result = self.events.replay_all().await.map_err(MobError::from);
let _ = reply_tx.send(result);
}
MobCommand::RecordOperatorActionProvenance {
tool_name,
authority_context,
reply_tx,
} => {
let result = async {
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordOperatorActionProvenance {
tool_name,
principal_token: authority_context.principal_token().clone(),
caller_provenance: authority_context.caller_provenance().cloned(),
audit_invocation_id: authority_context
.audit_invocation_id()
.map(ToOwned::to_owned),
},
"record_operator_action_provenance",
)?;
let kind = Self::operator_action_recorded_event_from_generated_effect(
&prepared.transition,
"record_operator_action_provenance",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
Ok(())
}
.await;
let _ = reply_tx.send(result);
}
MobCommand::ForceCancel {
agent_identity,
reply_tx,
} => {
let result = self.handle_force_cancel(agent_identity).await;
let _ = reply_tx.send(result);
}
MobCommand::Wire {
local,
target,
reply_tx,
} => {
let result = self.handle_wire(local, target).await;
let _ = reply_tx.send(result);
}
MobCommand::WireMembersBatch { edges, reply_tx } => {
let result = self.handle_wire_members_batch(edges).await;
let _ = reply_tx.send(result);
}
MobCommand::Unwire {
local,
target,
reply_tx,
} => {
let result = self.handle_unwire(local, target).await;
let _ = reply_tx.send(result);
}
MobCommand::CancelAllWork {
runtime_id,
fence_token,
reply_tx,
} => {
let result = self.handle_cancel_all_work(runtime_id, fence_token).await;
let _ = reply_tx.send(result);
}
MobCommand::SetSpawnPolicy { policy, reply_tx } => {
let enabled = policy.is_some();
let result = self.apply_dsl_input(
mob_dsl::MobMachineInput::SetSpawnPolicy { enabled },
"set_spawn_policy",
)
.map_err(|error| {
MobError::Internal(format!(
"SetSpawnPolicy rejected by MobMachine guards before shell policy write: {error}"
))
});
if result.is_ok() {
self.spawn_policy.set(policy).await;
}
let _ = reply_tx.send(result);
}
MobCommand::QueryPhase { reply_tx } => {
let _ = reply_tx.send(self.state());
}
MobCommand::Shutdown { reply_tx } => {
if let Err(error) = self.probe_command_admission(
mob_dsl::MobMachineInput::Shutdown,
MobState::Stopped,
"shutdown_command_admission",
) {
let _ = reply_tx.send(Err(error));
continue;
}
let mut result = self
.fail_all_pending_spawns("mob runtime is shutting down")
.await;
if result.is_err() {
let _ = reply_tx.send(result);
continue;
}
if result.is_ok() {
self.cancel_pending_peer_deliveries("mob runtime is shutting down")
.await;
}
if let Err(error) = self.cancel_all_flow_tasks().await {
tracing::warn!(error = %error, "shutdown flow cancellation encountered errors");
if result.is_ok() {
result = Err(error);
}
}
if let Err(error) = self.stop_all_autonomous_members().await {
tracing::warn!(error = %error, "shutdown loop stop encountered errors");
if result.is_ok() {
result = Err(error);
}
}
self.teardown_session_runtime_bindings_from_machine().await;
self.supervisor_bridge.shutdown().await;
self.lifecycle_tasks.abort_all();
while self.lifecycle_tasks.join_next().await.is_some() {}
if let Err(error) = self.apply_command_admission(
mob_dsl::MobMachineInput::Shutdown,
MobState::Stopped,
"shutdown_input",
) {
tracing::warn!(error = %error, "shutdown admission apply failed");
if result.is_ok() {
result = Err(error);
}
}
let _ = reply_tx.send(result);
break;
}
}
if self.destroy_admitted()
&& !self.destroy_cleanup_active
&& !self.pending_routed_effects.is_empty()
{
tracing::debug!(
mob_id = %self.definition.id,
queued_remaining = self.pending_routed_effects.len(),
"retaining destroy cleanup routed effects for explicit destroy retry"
);
continue;
}
loop {
let result = self.flush_routed_effects().await;
match Self::classify_actor_boundary_flush(result) {
ActorBoundaryFlushDisposition::Drained => break,
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
queued_remaining = self.pending_routed_effects.len(),
"composition consumer refusal was machine-closed; draining independent routed-effect tails"
);
}
ActorBoundaryFlushDisposition::Fatal(error) => {
tracing::error!(
mob_id = %self.definition.id,
error = %error,
queued_remaining = self.pending_routed_effects.len(),
"composition dispatch failed; terminating mob actor task"
);
return;
}
}
}
}
}
fn pending_spawn_cleanup_anchor_for_slot(
slot: &super::pending_spawn_lineage::PendingSpawnSlot,
reason: &str,
) -> Option<PendingSpawnCleanupAnchor> {
let snapshot = {
let progress = slot
.spawn
.progress
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
progress
.bridge_session_id
.clone()
.zip(progress.operation_id.clone())
};
snapshot.map(|(session_id, operation_id)| PendingSpawnCleanupAnchor {
spawn_ticket: slot.ticket,
agent_identity: slot.spawn.agent_identity.clone(),
session_id,
operation_id,
reason: reason.to_string(),
})
}
async fn cleanup_pending_spawn_anchor(
&self,
anchor: &PendingSpawnCleanupAnchor,
) -> Result<(), MobError> {
self.provisioner
.abort_member_provision(
&MemberRef::from_bridge_session_id(anchor.session_id.clone()),
&anchor.operation_id,
&anchor.reason,
)
.await
}
fn pending_spawn_cleanup_error(context: &str, errors: Vec<String>) -> MobError {
MobError::Internal(format!(
"{context}: pending spawn cleanup incomplete: {}",
errors.join("; ")
))
}
async fn drain_pending_spawn_cleanup_anchors(&mut self, context: &str) -> Result<(), MobError> {
if self.pending_spawn_cleanup_anchors.is_empty() {
return Ok(());
}
let anchors = self
.pending_spawn_cleanup_anchors
.values()
.cloned()
.collect::<Vec<_>>();
let mut errors = Vec::new();
for anchor in anchors {
match self.cleanup_pending_spawn_anchor(&anchor).await {
Ok(()) => {
self.pending_spawn_cleanup_anchors
.remove(&anchor.spawn_ticket);
tracing::info!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
operation_id = %anchor.operation_id,
"retried pending spawn cleanup anchor successfully"
);
}
Err(error) => {
errors.push(format!(
"{} ticket {} operation {}: {error}",
anchor.agent_identity, anchor.spawn_ticket, anchor.operation_id
));
tracing::warn!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
operation_id = %anchor.operation_id,
error = %error,
"pending spawn cleanup anchor still failed"
);
}
}
}
if errors.is_empty() {
Ok(())
} else {
Err(Self::pending_spawn_cleanup_error(context, errors))
}
}
#[cfg(feature = "runtime-adapter")]
fn authorize_restored_member_operation_owner(
&mut self,
entry: &RosterEntry,
bridge_session_id: &SessionId,
) -> Result<SessionId, MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let dsl_session_id = mob_dsl::SessionId::from_domain(bridge_session_id);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let transition = self.apply_dsl_signal_collect_transition(
mob_dsl::MobMachineSignal::RecoverMemberSessionBinding {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
bridge_session_id: dsl_session_id.clone(),
replacing,
},
"restore_member_operation_owner_binding",
)?;
let authorized = transition.effects().iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::SessionProvisionOperationOwnerAuthorized {
agent_identity: effect_identity,
session_id,
} if effect_identity == &dsl_identity && session_id == &dsl_session_id
)
});
if !authorized {
return Err(MobError::Internal(format!(
"MobMachine restore did not authorize operation owner for '{}'",
entry.agent_identity
)));
}
Ok(bridge_session_id.clone())
}
#[cfg(feature = "runtime-adapter")]
fn authorize_peer_only_operation_owner_from_machine(
&mut self,
agent_identity: &AgentIdentity,
peer_id: &str,
context: &'static str,
) -> Result<(SessionId, mob_dsl::MemberPeerEndpoint), MobError> {
let owner_bridge_session_id = self
.dsl_authority
.state()
.owner_bridge_session_id
.clone()
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MobMachine owner bridge authority"
))
})?;
let owner_bridge_session_id =
SessionId::parse(&owner_bridge_session_id.0).map_err(|error| {
MobError::Internal(format!(
"{context}: MobMachine has invalid owner bridge session for peer-only member '{agent_identity}': {error}"
))
})?;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let expected_peer_endpoint = self
.dsl_authority
.state()
.member_peer_endpoints
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MobMachine member peer endpoint authority"
))
})?;
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeMemberPeerRebind {
agent_identity: dsl_identity.clone(),
expected_peer_endpoint: expected_peer_endpoint.clone(),
},
context,
)?;
let authorized_peer_endpoint = transition
.effects()
.iter()
.find_map(|effect| match effect {
mob_dsl::MobMachineEffect::MemberPeerRebindAuthorized {
agent_identity: effect_identity,
peer_id: effect_peer_id,
peer_endpoint,
} if effect_identity == &dsl_identity
&& effect_peer_id.0 == peer_id
&& peer_endpoint == &expected_peer_endpoint =>
{
Some(peer_endpoint.clone())
}
_ => None,
})
.ok_or_else(|| {
MobError::Internal(format!(
"{context}: MobMachine did not authorize peer-only operation owner for '{agent_identity}'"
))
})?;
Ok((owner_bridge_session_id, authorized_peer_endpoint))
}
#[cfg(feature = "runtime-adapter")]
async fn generated_peer_only_operation_owner_context(
&mut self,
agent_identity: &AgentIdentity,
binding: &crate::RuntimeBinding,
context: &'static str,
) -> Result<
(
SessionId,
Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
),
MobError,
> {
let crate::RuntimeBinding::External { peer_id, .. } = binding else {
return Err(MobError::Internal(format!(
"{context}: peer-only operation owner requested for non-external binding"
)));
};
let observed_peer = Self::peer_only_spec_for_binding(binding, context)?;
let (owner_bridge_session_id, authorized_endpoint) = self
.authorize_peer_only_operation_owner_from_machine(agent_identity, peer_id, context)?;
let authorized_peer =
Self::peer_only_spec_from_member_endpoint(&authorized_endpoint, context)?;
if authorized_peer.name != observed_peer.name
|| authorized_peer.peer_id != observed_peer.peer_id
|| authorized_peer.address != observed_peer.address
|| authorized_peer.pubkey != observed_peer.pubkey
{
return Err(MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' does not match MobMachine peer endpoint authority"
)));
}
let adapter = self.runtime_adapter.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"{context}: peer-only operation owner for '{agent_identity}' requires MeerkatMachine runtime authority"
))
})?;
let bindings = adapter
.prepare_local_session_bindings(owner_bridge_session_id.clone())
.await
.map_err(|error| {
MobError::Internal(format!(
"{context}: failed to prepare MeerkatMachine operation bindings for peer-only member '{agent_identity}': {error}"
))
})?;
if bindings.session_id() != &owner_bridge_session_id {
return Err(MobError::Internal(format!(
"{context}: MeerkatMachine operation bindings for peer-only member '{agent_identity}' returned session '{}' but expected '{owner_bridge_session_id}'",
bindings.session_id()
)));
}
if !meerkat_runtime::session_runtime_bindings_have_machine_authority(&bindings) {
return Err(MobError::Internal(format!(
"{context}: MeerkatMachine operation bindings for peer-only member '{agent_identity}' lacked machine authority"
)));
}
Ok((
owner_bridge_session_id,
Arc::clone(bindings.ops_lifecycle()),
))
}
#[cfg(feature = "runtime-adapter")]
fn authorize_restored_peer_only_operation_owner(
&mut self,
entry: &RosterEntry,
) -> Result<Option<SessionId>, MobError> {
let MemberRef::BackendPeer {
peer_id,
session_id: None,
..
} = &entry.member_ref
else {
return Ok(None);
};
if self.dsl_authority.state().owner_bridge_session_id.is_none() {
return Ok(None);
}
let (owner_bridge_session_id, _) = self.authorize_peer_only_operation_owner_from_machine(
&entry.agent_identity,
peer_id,
"restore_peer_only_operation_owner_binding",
)?;
Ok(Some(owner_bridge_session_id))
}
#[cfg(feature = "runtime-adapter")]
async fn restore_generated_member_operation_bindings(&mut self) -> Result<(), MobError> {
let Some(adapter) = self.runtime_adapter.clone() else {
return Ok(());
};
let entries = self.roster.read().await.list().cloned().collect::<Vec<_>>();
for entry in entries {
let generated_owner_session_id =
if let Some(bridge_session_id) = entry.member_ref.bridge_session_id().cloned() {
self.authorize_restored_member_operation_owner(&entry, &bridge_session_id)?
} else {
let Some(owner_bridge_session_id) =
self.authorize_restored_peer_only_operation_owner(&entry)?
else {
continue;
};
owner_bridge_session_id
};
if matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
) && !self
.dsl_authority
.state()
.owner_bridge_session_id
.as_ref()
.and_then(|session_id| SessionId::parse(&session_id.0).ok())
.is_some_and(|owner_session_id| owner_session_id == generated_owner_session_id)
{
return Err(MobError::Internal(format!(
"peer-only operation owner restore for '{}' lost MobMachine owner bridge authority",
entry.agent_identity
)));
}
let bindings = adapter
.prepare_local_session_bindings(generated_owner_session_id.clone())
.await
.map_err(|error| {
MobError::Internal(format!(
"restore operation owner binding failed for member '{}': {error}",
entry.agent_identity
))
})?;
if bindings.session_id() != &generated_owner_session_id {
return Err(MobError::Internal(format!(
"restore operation owner binding returned session '{}' for member '{}' generated owner '{}'",
bindings.session_id(),
entry.agent_identity,
generated_owner_session_id
)));
}
if !meerkat_runtime::session_runtime_bindings_have_machine_authority(&bindings) {
return Err(MobError::Internal(format!(
"restore operation owner binding lacked MeerkatMachine authority for member '{}'",
entry.agent_identity
)));
}
self.provisioner
.bind_member_owner_context(
&entry.member_ref,
generated_owner_session_id,
Arc::clone(bindings.ops_lifecycle()),
)
.await?;
}
Ok(())
}
#[cfg(not(feature = "runtime-adapter"))]
async fn restore_generated_member_operation_bindings(&self) -> Result<(), MobError> {
Ok(())
}
async fn fail_all_pending_spawns(&mut self, reason: &str) -> Result<(), MobError> {
self.drain_pending_spawn_cleanup_anchors(reason).await?;
if self.pending_spawns.is_empty() {
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
reason,
message = %message,
"pending spawn alignment violated with no local pending slots to drain"
);
}
return Ok(());
}
let mut cleanup_errors = Vec::new();
for mut slot in self.pending_spawns.drain_all() {
let spawn_ticket = slot.ticket;
let agent_identity = slot.spawn.agent_identity.clone();
let dsl_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let task_handle = slot.task.take();
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::CancelPendingSpawn {
agent_identity: dsl_identity,
},
"fail_all_pending_spawns",
) {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
} else {
let generated_failed =
generated_mob_command_capabilities::CommandPlanKind::FailSpawn;
debug_assert_eq!(
generated_failed,
generated_mob_command_capabilities::CommandPlanKind::FailSpawn
);
}
if let Err(error) = self.abort_pending_spawn_slot(&slot, reason).await {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
}
slot.fail(&format!("spawn canceled for '{agent_identity}': {reason}"));
if let Some(handle) = task_handle {
handle.abort();
let _ = handle.await;
}
tracing::debug!(
spawn_ticket,
agent_identity = %agent_identity,
"failed pending spawn due to lifecycle transition"
);
}
debug_assert!(
self.pending_spawns.is_empty(),
"all pending spawn slots should be drained during lifecycle transition"
);
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
message = %message,
"pending spawn alignment still violated after lifecycle drain"
);
}
if !cleanup_errors.is_empty() {
return Err(Self::pending_spawn_cleanup_error(reason, cleanup_errors));
}
self.drain_pending_spawn_cleanup_anchors(reason).await?;
Ok(())
}
async fn abort_pending_spawn_slot(
&mut self,
slot: &super::pending_spawn_lineage::PendingSpawnSlot,
reason: &str,
) -> Result<(), MobError> {
let Some(anchor) = Self::pending_spawn_cleanup_anchor_for_slot(slot, reason) else {
return Ok(());
};
if let Err(error) = self.cleanup_pending_spawn_anchor(&anchor).await {
self.pending_spawn_cleanup_anchors
.insert(anchor.spawn_ticket, anchor.clone());
tracing::warn!(
spawn_ticket = anchor.spawn_ticket,
agent_identity = %anchor.agent_identity,
operation_id = %anchor.operation_id,
error = %error,
"failed to abort pending member provision during lifecycle drain; retained cleanup anchor"
);
return Err(MobError::Internal(format!(
"pending spawn cleanup failed for '{}': {error}",
anchor.agent_identity
)));
}
self.pending_spawn_cleanup_anchors
.remove(&anchor.spawn_ticket);
Ok(())
}
async fn cancel_pending_spawns_for_member(
&mut self,
agent_identity: &AgentIdentity,
reason: &str,
) -> Result<usize, MobError> {
self.drain_pending_spawn_cleanup_anchors(reason).await?;
let slots = self.pending_spawns.take_for_member(agent_identity);
if slots.is_empty() {
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
agent_identity = %agent_identity,
reason,
message = %message,
"pending spawn alignment violated while canceling member-specific pending spawns"
);
}
return Ok(0);
}
let canceled = slots.len();
let mut cleanup_errors = Vec::new();
for slot in slots {
let spawn_ticket = slot.ticket;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
slot.spawn.agent_identity.as_str(),
));
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::CancelPendingSpawn {
agent_identity: dsl_identity,
},
"cancel_pending_spawns_for_member",
) {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
} else {
let generated_failed =
generated_mob_command_capabilities::CommandPlanKind::FailSpawn;
debug_assert_eq!(
generated_failed,
generated_mob_command_capabilities::CommandPlanKind::FailSpawn
);
}
if let Err(error) = self.abort_pending_spawn_slot(&slot, reason).await {
cleanup_errors.push(format!("{agent_identity} ticket {spawn_ticket}: {error}"));
}
slot.fail(&format!("spawn canceled for '{agent_identity}': {reason}"));
tracing::debug!(
spawn_ticket,
agent_identity = %agent_identity,
"canceled pending spawn for member lifecycle command"
);
}
self.debug_assert_pending_spawn_alignment();
if let Some(message) = self.pending_spawn_alignment_violation() {
tracing::error!(
agent_identity = %agent_identity,
message = %message,
"pending spawn alignment violated after member-specific cancellation"
);
}
if cleanup_errors.is_empty() {
Ok(canceled)
} else {
Err(Self::pending_spawn_cleanup_error(reason, cleanup_errors))
}
}
fn customize_spawn_spec(
&self,
spawn_source: super::handle::SpawnSource,
spawner: Option<&(AgentIdentity, AgentRuntimeId)>,
spec: &mut super::handle::SpawnMemberSpec,
) -> Result<(), MobError> {
if let Some(customizer) = self.spawn_member_customizer.as_ref() {
let ctx = super::handle::SpawnCustomizationContext {
mob_id: self.definition.id.clone(),
spawn_source,
spawner_identity: spawner.map(|(identity, _)| identity.clone()),
spawner_runtime_id: spawner.map(|(_, runtime_id)| runtime_id.clone()),
requested_profile: spec.role_name.clone(),
};
customizer.customize_spawn(&ctx, spec)?;
}
Ok(())
}
async fn enqueue_spawn(
&mut self,
mut spec: super::handle::SpawnMemberSpec,
spawn_source: super::handle::SpawnSource,
owner_bridge_session_id: Option<SessionId>,
ops_registry: Option<Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>>,
reply_tx: oneshot::Sender<Result<super::handle::MemberSpawnReceipt, MobError>>,
) {
let spawner = match owner_bridge_session_id.as_ref() {
Some(owner_session_id) => self.spawner_for_bridge_session(owner_session_id).await,
None => None,
};
if let Err(error) = self.customize_spawn_spec(spawn_source, spawner.as_ref(), &mut spec) {
let _ = reply_tx.send(Err(error));
return;
}
let allow_reserved_flow_identity = spawn_source.allows_reserved_flow_identity();
let super::handle::SpawnMemberSpec {
role_name: profile_name,
identity,
initial_message,
runtime_mode,
backend,
binding,
context,
labels,
launch_mode,
tool_access_policy,
budget_split_policy: _budget_split_policy,
budget_limits,
auto_wire_parent,
additional_instructions,
shell_env,
inherited_tool_filter,
override_profile,
auth_binding,
external_tools: per_spawn_external_tools,
system_prompt_override,
continuity_intent,
} = spec;
let agent_identity = AgentIdentity::from(identity.as_str());
if let Err(error) = self.preview_spawn_command_admission(&agent_identity) {
let _ = reply_tx.send(Err(error));
return;
}
let resume_bridge_session_id = launch_mode.resume_bridge_session_id().cloned();
let fork_spec = match launch_mode {
crate::launch::MemberLaunchMode::Fork {
source_member_id,
fork_context,
} => Some((source_member_id, fork_context)),
_ => None,
};
let prepare_result = async {
if agent_identity.is_system_reserved() && !allow_reserved_flow_identity {
return Err(MobError::WiringError(format!(
"meerkat id '{agent_identity}' uses reserved system identifier namespace"
)));
}
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn start"
);
{
let roster = self.roster.read().await;
if roster.get(&agent_identity).is_some() {
return Err(MobError::MemberAlreadyExists(agent_identity.clone()));
}
}
if !self.definition.profiles.contains_key(&profile_name) {
return Err(MobError::ProfileNotFound(profile_name.clone()));
}
let effective_profile_override = override_profile.clone();
let mut profile = if let Some(p) = override_profile {
p
} else {
self.definition
.resolve_profile(&profile_name, self.realm_profile_store.as_ref())
.await?
};
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn profile resolved"
);
if inherited_tool_filter.is_some() && effective_profile_override.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn authorizing profile material"
);
let authorized_profile_material = self.authorize_spawn_profile_material(
&agent_identity,
&profile_name,
&profile,
"enqueue_spawn_profile_authority",
)?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn profile material authorized"
);
let selected_runtime_mode = runtime_mode.unwrap_or(profile.runtime_mode);
let profile_external_addressable = authorized_profile_material.external_addressable;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn resolving external tools"
);
if let Some(resume_id) = resume_bridge_session_id {
let member_ref = MemberRef::from_bridge_session_id(resume_id.clone());
let is_active = self
.provisioner
.is_member_active(&member_ref)
.await
.map_err(|e| {
MobError::Internal(format!(
"resume bridge session check failed for '{agent_identity}': {e}"
))
})?;
if is_active.unwrap_or(false) {
if selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& self.provisioner.interaction_event_injector(&resume_id).await.is_none()
{
return Err(MobError::MissingMemberCapability {
member_id: agent_identity.clone(),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous member resume",
});
}
let has_wiring = self.definition.wiring.auto_wire_orchestrator
|| !self.definition.wiring.role_wiring.is_empty();
if has_wiring
&& self
.provisioner
.comms_runtime(&member_ref)
.await
.is_none()
{
return Err(MobError::Internal(format!(
"resumed session '{resume_id}' has no comms runtime for '{agent_identity}'"
)));
}
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let resolved_labels = labels.unwrap_or_default();
return Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
Some(member_ref),
None,
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override.clone(),
per_spawn_external_tools.clone(),
authorized_profile_material.clone(),
continuity_intent.clone(),
));
}
if self.session_service.supports_persistent_sessions() {
let stored_session = self
.session_service
.load_persisted_session(&resume_id)
.await
.map_err(MobError::from)?
.ok_or_else(|| {
MobError::Internal(format!(
"missing durable session snapshot for '{resume_id}'"
))
})?;
let external_tools =
self.external_tools_for_profile(&profile, per_spawn_external_tools.clone())?;
let mut config = build::build_resumed_agent_config(
build::BuildResumedAgentConfigParams {
base: build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: labels.clone(),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter: inherited_tool_filter.clone(),
tool_access_policy: tool_access_policy.clone(),
system_prompt_override: system_prompt_override.clone(),
},
expected_session_id: &resume_id,
resumed_session: stored_session,
},
)
.await?;
config.keep_alive =
selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits.clone());
let selected_binding = resolve_binding(
binding.clone(),
backend,
profile.backend,
self.definition.backend.default,
&agent_identity,
)?;
let selected_runtime_mode =
normalize_runtime_mode_for_binding(selected_runtime_mode, &selected_binding);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
let provision_request = ProvisionMemberRequest {
create_session: req,
binding: selected_binding,
peer_name,
owner_bridge_session_id: owner_bridge_session_id.clone(),
ops_registry: ops_registry.clone(),
generated_self_owned_operation_owner: None,
};
let resolved_labels = labels.unwrap_or_default();
return Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
None::<MemberRef>,
Some(provision_request),
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override.clone(),
per_spawn_external_tools.clone(),
authorized_profile_material.clone(),
continuity_intent.clone(),
));
}
return Err(MobError::Internal(format!(
"resumed session '{resume_id}' not found or inactive for '{agent_identity}'"
)));
}
let fork_context_text = if let Some((source_member_id, fork_context)) = fork_spec {
let source_session_id = {
let roster = self.roster.read().await;
let source_entry = roster.get(&source_member_id).ok_or_else(|| {
MobError::MemberNotFound(source_member_id.clone())
})?;
source_entry
.member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"fork source '{source_member_id}' has no session"
))
})?
};
let query = match fork_context {
crate::launch::ForkContext::FullHistory => {
meerkat_core::service::SessionHistoryQuery::default()
}
crate::launch::ForkContext::LastMessages { count } => {
let view = self
.session_service
.read(&source_session_id)
.await
.map_err(|e| {
MobError::Internal(format!(
"failed to read source session metadata for fork from '{source_member_id}': {e}"
))
})?;
let total = view.state.message_count;
let offset = total.saturating_sub(count as usize);
meerkat_core::service::SessionHistoryQuery {
offset,
limit: Some(count as usize),
}
}
};
let history = meerkat_core::service::SessionServiceHistoryExt::read_history(
self.session_service.as_ref(),
&source_session_id,
query,
)
.await
.map_err(|e| {
MobError::Internal(format!(
"failed to read source session history for fork from '{source_member_id}': {e}"
))
})?;
Some(render_fork_context(&source_member_id, &history.messages))
} else {
None
};
let external_tools =
self.external_tools_for_profile(&profile, per_spawn_external_tools.clone())?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn external tools resolved"
);
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: labels.clone(),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
})
.await?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
profile = %profile_name,
"MobActor::enqueue_spawn agent config built"
);
config.keep_alive =
selected_runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
if let Some(ref cref) = auth_binding {
config.auth_binding = Some(cref.clone());
}
let base_prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, &agent_identity))
});
let prompt = if let Some(fork_text) = fork_context_text {
let mut blocks = vec![meerkat_core::types::ContentBlock::Text {
text: format!("{fork_text}\n\n"),
}];
blocks.extend(base_prompt.into_blocks());
ContentInput::Blocks(blocks)
} else {
base_prompt
};
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits);
let selected_binding = resolve_binding(
binding,
backend,
profile.backend,
self.definition.backend.default,
&agent_identity,
)?;
let selected_runtime_mode =
normalize_runtime_mode_for_binding(selected_runtime_mode, &selected_binding);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
let provision_request = ProvisionMemberRequest {
create_session: req,
binding: selected_binding,
peer_name,
owner_bridge_session_id: owner_bridge_session_id.clone(),
ops_registry: ops_registry.clone(),
generated_self_owned_operation_owner: None,
};
let resolved_labels = labels.unwrap_or_default();
Ok((
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
profile_external_addressable,
resolved_labels,
None::<MemberRef>,
Some(provision_request),
owner_bridge_session_id.clone(),
auto_wire_parent,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
))
}
.await;
let (
profile_name,
agent_identity,
prompt,
initial_turn_prompt,
selected_runtime_mode,
_external_addressable,
resolved_labels,
resume_member_ref,
maybe_provision_request,
spawn_owner_bridge_session_id,
auto_wire_parent,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
) = match prepare_result {
Ok(prepared) => prepared,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
if let Some(member_ref) = resume_member_ref {
let Some(bridge_session_id) = member_ref.bridge_session_id().cloned() else {
let _ = reply_tx.send(Err(MobError::Internal(format!(
"resumed member '{agent_identity}' has no bridge session id"
))));
return;
};
if let Err(error) = self.preview_spawn_admission(
&agent_identity,
&authorized_profile_material,
Some(&bridge_session_id),
) {
let _ = reply_tx.send(Err(error));
return;
}
if let (Some(owner_bridge_session_id), Some(ops_registry)) =
(owner_bridge_session_id.clone(), ops_registry.clone())
&& let Err(error) = self
.provisioner
.bind_member_owner_context(&member_ref, owner_bridge_session_id, ops_registry)
.await
{
let _ = reply_tx.send(Err(error));
return;
}
let operation_id = self
.provisioner
.active_operation_id_for_member(&member_ref)
.await
.ok_or_else(|| {
MobError::Internal(format!(
"resumed member '{agent_identity}' has no tracked mob child operation"
))
});
let operation_id = match operation_id {
Ok(operation_id) => operation_id,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
let provision =
PendingProvision::new(member_ref, agent_identity.clone(), self.provisioner.clone());
let fence = self.issue_fence_token();
let result = Box::pin(self.finalize_spawn_from_pending(
&profile_name,
&agent_identity,
crate::ids::Generation::INITIAL,
fence,
selected_runtime_mode,
prompt,
initial_turn_prompt,
resolved_labels,
provision,
operation_id,
spawn_owner_bridge_session_id,
auto_wire_parent,
None,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
))
.await
.map(|outcome| outcome.receipt);
let _ = reply_tx.send(result);
return;
}
let Some(mut provision_request) = maybe_provision_request else {
let _ = reply_tx.send(Err(MobError::Internal(
"provision_request missing for normal spawn path".into(),
)));
return;
};
let admitted_bridge_session_id =
admit_bridge_session_for_spawn(&mut provision_request.create_session);
let spawn_bridge_session_id = match &provision_request.binding {
crate::RuntimeBinding::Session => Some(&admitted_bridge_session_id),
crate::RuntimeBinding::External { .. } => None,
};
if let Err(error) = self.preview_spawn_admission(
&agent_identity,
&authorized_profile_material,
spawn_bridge_session_id,
) {
let _ = reply_tx.send(Err(error));
return;
}
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let spawn_member_identity = agent_identity.clone();
let spawn_member_identity_for_log = spawn_member_identity.clone();
let spawn_runtime_mode = selected_runtime_mode;
let pending_progress = Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default()));
let generated_self_owned_operation_owner =
match self.stage_orchestrator_spawn(&agent_identity, &admitted_bridge_session_id) {
Ok(authority) => authority,
Err(error) => {
let _ = reply_tx.send(Err(error));
return;
}
};
if let Err(error) = Self::apply_generated_self_owned_operation_owner(
&mut provision_request,
&generated_self_owned_operation_owner,
) {
let _ = reply_tx.send(Err(error));
return;
}
let pending_recipient_trust_peer_id = match &provision_request.binding {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
crate::RuntimeBinding::Session => None,
};
if let Some(peer_id) = pending_recipient_trust_peer_id.as_deref()
&& let Err(error) = self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"enqueue_spawn external provision",
)
{
let _ = reply_tx.send(Err(error));
return;
}
let pending = PendingSpawn {
profile_name,
agent_identity,
admitted_bridge_session_id,
prompt,
initial_turn_prompt,
runtime_mode: selected_runtime_mode,
labels: resolved_labels,
owner_bridge_session_id: spawn_owner_bridge_session_id,
auto_wire_parent,
restore_wiring: None,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
progress: pending_progress.clone(),
pending_recipient_trust_peer_id,
reply_tx,
};
let provisioner = self.provisioner.clone();
let command_tx = self.command_tx.clone();
let task = tokio::spawn(async move {
let panic_member_identity = spawn_member_identity.clone();
let provision_result = std::panic::AssertUnwindSafe(async {
let spawn_receipt = provisioner.provision_member(provision_request).await?;
if let Some(bridge_session_id) =
spawn_receipt.member_ref.bridge_session_id().cloned()
{
let mut progress = pending_progress
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
progress.bridge_session_id = Some(bridge_session_id);
progress.operation_id = Some(spawn_receipt.operation_id.clone());
}
#[cfg(test)]
{
let delay_ms = SPAWN_PROVISIONED_COMMAND_DELAY_MS
.load(std::sync::atomic::Ordering::Relaxed);
if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
}
if spawn_runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&provisioner,
&spawn_member_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) =
provisioner.retire_member(&spawn_receipt.member_ref).await
{
return Err(MobError::Internal(format!(
"autonomous capability check failed for '{spawn_member_identity}': {capability_error}; cleanup retire failed for member '{:?}': {retire_error}",
spawn_receipt.member_ref
)));
}
return Err(capability_error);
}
Ok(spawn_receipt)
})
.catch_unwind()
.await;
let provision_result = match provision_result {
Ok(result) => result,
Err(_) => Err(MobError::Internal(format!(
"spawn provisioning task panicked for '{panic_member_identity}'"
))),
};
if let Err(send_error) = command_tx
.send(MobCommand::SpawnProvisioned {
spawn_ticket,
result: provision_result,
})
.await
&& let MobCommand::SpawnProvisioned {
result: Ok(spawn_receipt),
..
} = send_error.0
&& let Err(cleanup_error) =
provisioner.retire_member(&spawn_receipt.member_ref).await
{
tracing::warn!(
spawn_ticket,
member_ref = ?spawn_receipt.member_ref,
error = %cleanup_error,
"spawn completion dropped; failed cleanup retire for provisioned member"
);
}
});
let spawn_started = match generated_self_owned_operation_owner.start(&pending) {
Ok(started) => started,
Err(error) => {
let _ = pending.reply_tx.send(Err(error));
task.abort();
return;
}
};
self.insert_pending_spawn(spawn_ticket, pending, task, spawn_started);
if let Err(error) = self.ensure_pending_spawn_alignment("enqueue_spawn post-insert") {
tracing::error!(
spawn_ticket,
error = %error,
"pending spawn alignment check failed after enqueue; canceling all pending spawns"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated after enqueue")
.await
{
tracing::error!(
spawn_ticket,
error = %cleanup_error,
"pending spawn cleanup failed after enqueue alignment violation"
);
}
return;
}
tracing::debug!(
spawn_ticket,
agent_identity = %spawn_member_identity_for_log,
runtime_mode = ?spawn_runtime_mode,
"MobActor::enqueue_spawn queued provisioning task"
);
}
async fn handle_spawn_provisioned_batch(
&mut self,
completions: Vec<(u64, Result<super::handle::MemberSpawnReceipt, MobError>)>,
) {
tracing::debug!(
completion_count = completions.len(),
"MobActor::handle_spawn_provisioned_batch start"
);
if let Err(error) = self.ensure_pending_spawn_alignment("spawn batch preflight") {
tracing::error!(
error = %error,
"pending spawn alignment check failed before spawn completion batch"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated before spawn batch")
.await
{
tracing::error!(
error = %cleanup_error,
"pending spawn cleanup failed after pre-batch alignment violation"
);
}
return;
}
let mut pending_items = Vec::with_capacity(completions.len());
for (spawn_ticket, result) in completions {
tracing::debug!(
spawn_ticket,
"MobActor::handle_spawn_provisioned_batch completing pending slot"
);
let (pending, task_handle) =
self.complete_pending_spawn_slot(spawn_ticket, "spawn provisioned batch");
let Some(pending) = pending else {
tracing::warn!(spawn_ticket, "received spawn completion for unknown ticket");
if let Some(handle) = task_handle {
handle.abort();
tracing::warn!(
spawn_ticket,
"received spawn completion for unknown pending metadata but found task handle"
);
}
if let Ok(spawn_receipt) = result {
let orphan = PendingProvision::new(
spawn_receipt.member_ref,
AgentIdentity::from("__unknown_ticket__"),
self.provisioner.clone(),
);
if let Err(error) = orphan.rollback().await {
tracing::warn!(
spawn_ticket,
error = %error,
"unknown spawn completion cleanup failed"
);
}
}
continue;
};
pending_items.push((pending, result));
}
for (pending, result) in pending_items {
tracing::debug!("MobActor::handle_spawn_provisioned_batch finalizing pending spawn");
let PendingSpawn {
profile_name,
agent_identity,
admitted_bridge_session_id: _,
prompt,
initial_turn_prompt,
runtime_mode,
labels,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
progress: _,
pending_recipient_trust_peer_id,
reply_tx,
} = pending;
if let Some(peer_id) = pending_recipient_trust_peer_id.as_deref() {
let close_result = if result.is_ok() {
self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn provisioned batch confirmed",
)
} else {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn provisioned batch failed",
)
};
if let Err(error) = close_result {
tracing::error!(
peer_id,
error = %error,
"failed to close pending recipient-trust obligation after provision result"
);
}
}
let reply = match result {
Ok(spawn_receipt) => {
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Err(retire_error) = provision.rollback().await {
Err(MobError::Internal(format!(
"spawn completed while mob state changed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)))
} else {
Err(error)
}
} else {
let fence = self.issue_fence_token();
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_spawn_provisioned_batch calling finalize_spawn_from_pending"
);
Box::pin(self.finalize_spawn_from_pending(
&profile_name,
&agent_identity,
crate::ids::Generation::INITIAL,
fence,
runtime_mode,
prompt,
initial_turn_prompt,
labels,
provision,
spawn_receipt.operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
))
.await
.map(|outcome| outcome.receipt)
}
}
Err(error) => Err(error),
};
let _ = reply_tx.send(reply);
}
if let Err(error) = self.ensure_pending_spawn_alignment("spawn batch completion") {
tracing::error!(
error = %error,
"pending spawn alignment check failed after spawn completion batch"
);
if let Err(cleanup_error) = self
.fail_all_pending_spawns("pending spawn alignment violated after spawn batch")
.await
{
tracing::error!(
error = %cleanup_error,
"pending spawn cleanup failed after spawn batch alignment violation"
);
}
}
}
async fn spawn_from_policy_inline(
&mut self,
agent_identity: &AgentIdentity,
spawn_spec: super::spawn_policy::SpawnSpec,
work_ref: &WorkRef,
origin: WorkOrigin,
) -> Result<super::handle::MemberSpawnReceipt, MobError> {
self.ensure_pending_spawn_alignment("spawn_from_policy_inline preflight")?;
let requested_identity = AgentIdentity::from(agent_identity.as_str());
let mut member_spec =
super::handle::SpawnMemberSpec::new(spawn_spec.profile, requested_identity.clone());
member_spec.runtime_mode = spawn_spec.runtime_mode;
let super::handle::SpawnMemberSpec {
role_name: profile_name,
identity: _,
initial_message,
runtime_mode,
backend,
binding,
context,
labels,
launch_mode: _,
tool_access_policy,
budget_split_policy: _budget_split_policy,
budget_limits,
auto_wire_parent: _,
additional_instructions,
shell_env,
inherited_tool_filter,
override_profile,
auth_binding,
external_tools: per_spawn_external_tools,
system_prompt_override,
continuity_intent,
} = member_spec;
if agent_identity.is_system_reserved() {
return Err(MobError::WiringError(format!(
"meerkat id '{agent_identity}' uses reserved system identifier namespace"
)));
}
self.probe_member_admission(agent_identity)?;
{
let roster = self.roster.read().await;
if roster.get(agent_identity).is_some() {
return Err(MobError::MemberAlreadyExists(agent_identity.clone()));
}
}
let mut profile = if let Some(p) = override_profile.clone() {
p
} else {
self.definition
.resolve_profile(&profile_name, self.realm_profile_store.as_ref())
.await?
};
if inherited_tool_filter.is_some() && override_profile.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let authorized_profile_material = self.authorize_spawn_profile_material(
agent_identity,
&profile_name,
&profile,
"policy_spawn_profile_authority",
)?;
self.preview_policy_spawn_submit_work_admission(
&requested_identity,
&authorized_profile_material,
work_ref,
origin,
)?;
let runtime_mode = runtime_mode.unwrap_or(profile.runtime_mode);
let selected_binding = resolve_binding(
binding,
backend,
profile.backend,
self.definition.backend.default,
agent_identity,
)?;
let runtime_mode = normalize_runtime_mode_for_binding(runtime_mode, &selected_binding);
let external_tools =
self.external_tools_for_profile(&profile, per_spawn_external_tools.clone())?;
let labels = labels.unwrap_or_default();
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &profile_name,
agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context,
labels: Some(labels.clone()),
additional_instructions,
shell_env,
mob_tool_authority_context: None,
inherited_tool_filter,
tool_access_policy,
system_prompt_override,
})
.await?;
config.keep_alive = runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
if let Some(ref cref) = auth_binding {
config.auth_binding = Some(cref.clone());
}
let prompt = initial_message.clone().unwrap_or_else(|| {
ContentInput::from(self.fallback_spawn_prompt(&profile_name, agent_identity))
});
let initial_turn_prompt = initial_message.as_ref().map(|_| prompt.clone());
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, budget_limits);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
agent_identity.as_str(),
)?;
let mut provision_request = ProvisionMemberRequest {
create_session: req,
binding: selected_binding,
peer_name,
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: None,
};
let admitted_bridge_session_id =
admit_bridge_session_for_spawn(&mut provision_request.create_session);
let spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let generated_self_owned_operation_owner =
self.stage_orchestrator_spawn(agent_identity, &admitted_bridge_session_id)?;
Self::apply_generated_self_owned_operation_owner(
&mut provision_request,
&generated_self_owned_operation_owner,
)?;
let inline_pending_trust_peer_id = match &provision_request.binding {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
crate::RuntimeBinding::Session => None,
};
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline external provision",
)?;
}
let (pending_reply_tx, _pending_reply_rx) = oneshot::channel();
let pending = PendingSpawn {
profile_name: profile_name.clone(),
agent_identity: agent_identity.clone(),
admitted_bridge_session_id,
prompt: prompt.clone(),
initial_turn_prompt: initial_turn_prompt.clone(),
runtime_mode,
labels: labels.clone(),
owner_bridge_session_id: None,
auto_wire_parent: false,
restore_wiring: None,
effective_profile_override: override_profile.clone(),
per_spawn_external_tools: per_spawn_external_tools.clone(),
authorized_profile_material: authorized_profile_material.clone(),
continuity_intent: continuity_intent.clone(),
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default())),
pending_recipient_trust_peer_id: None,
reply_tx: pending_reply_tx,
};
let pending_task = tokio::spawn(async {
std::future::pending::<()>().await;
});
let spawn_started = generated_self_owned_operation_owner.start(&pending)?;
self.insert_pending_spawn(spawn_ticket, pending, pending_task, spawn_started);
if let Err(error) =
self.ensure_pending_spawn_alignment("spawn_from_policy_inline staged pending")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated while staging inline policy spawn"
);
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline staging failed",
)?;
}
self.fail_all_pending_spawns(
"pending spawn alignment violated while staging inline policy spawn",
)
.await?;
return Err(error);
}
let spawn_result = Box::pin(async {
let spawn_receipt = self.provisioner.provision_member(provision_request).await?;
if let Some(peer_id) = inline_pending_trust_peer_id.as_deref() {
self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline provision confirmed",
)?;
}
if runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
agent_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) = self
.provisioner
.retire_member(&spawn_receipt.member_ref)
.await
{
return Err(MobError::Internal(format!(
"autonomous capability check failed for '{agent_identity}': {capability_error}; cleanup retire failed: {retire_error}"
)));
}
return Err(capability_error);
}
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Err(retire_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"policy spawn completed while mob state changed for '{agent_identity}': {error}; cleanup retire failed: {retire_error}"
)));
}
return Err(error);
}
let fence = self.issue_fence_token();
Box::pin(self.finalize_spawn_from_pending(
&profile_name,
agent_identity,
crate::ids::Generation::INITIAL,
fence,
runtime_mode,
prompt,
initial_turn_prompt,
labels,
provision,
spawn_receipt.operation_id,
None,
false,
None,
override_profile, per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
))
.await
.map(|outcome| outcome.receipt)
})
.await;
let (_pending, task_handle) =
self.complete_pending_spawn_slot(spawn_ticket, "policy inline spawn completion");
if let Some(handle) = task_handle {
handle.abort();
}
if spawn_result.is_err()
&& let Some(peer_id) = inline_pending_trust_peer_id.as_deref()
{
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"spawn_from_policy_inline provision failed",
)?;
}
if let Err(error) =
self.ensure_pending_spawn_alignment("spawn_from_policy_inline completion")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated after inline policy spawn completion"
);
self.fail_all_pending_spawns(
"pending spawn alignment violated after inline policy spawn completion",
)
.await?;
return Err(error);
}
spawn_result
}
fn fold_spawn_exec_abort(
&mut self,
dsl_identity: &mob_dsl::AgentIdentity,
agent_identity: &AgentIdentity,
original: MobError,
context: &str,
) -> MobError {
match self.apply_dsl_input(
mob_dsl::MobMachineInput::AbortSpawnExec {
agent_identity: dsl_identity.clone(),
},
context,
) {
Ok(()) => original,
Err(abort_error) => MobError::Internal(format!(
"spawn pre-commit unwind for '{agent_identity}' ({context}): {original}; spawn-exec phase reset (AbortSpawnExec) failed: {abort_error}"
)),
}
}
#[allow(clippy::too_many_arguments)]
async fn finalize_spawn_from_pending(
&mut self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
fence_token: crate::ids::FenceToken,
runtime_mode: crate::MobRuntimeMode,
prompt: ContentInput,
initial_turn_prompt: Option<ContentInput>,
labels: std::collections::BTreeMap<String, String>,
provision: PendingProvision,
operation_id: meerkat_core::ops::OperationId,
owner_bridge_session_id: Option<SessionId>,
auto_wire_parent: bool,
restore_wiring: Option<RestoreWiringPlan>,
effective_profile_override: Option<crate::profile::Profile>,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
authorized_profile_material: AuthorizedSpawnProfileMaterial,
continuity_intent: super::handle::SpawnContinuityIntent,
) -> Result<FinalizeSpawnOutcome, MobError> {
tracing::debug!(
agent_identity = %agent_identity,
profile = %profile_name,
runtime_mode = ?runtime_mode,
"MobActor::finalize_spawn_from_pending start"
);
let ctx = Box::new(SpawnFinalizeCtx {
profile_name: profile_name.clone(),
agent_identity: agent_identity.clone(),
generation,
fence_token,
runtime_mode,
prompt,
initial_turn_prompt,
labels,
operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material,
continuity_intent,
});
let admitted = Box::pin(self.finalize_spawn_admit(&ctx, provision)).await?;
Box::pin(self.finalize_spawn_activate(ctx, admitted)).await
}
async fn finalize_spawn_admit(
&mut self,
ctx: &SpawnFinalizeCtx,
provision: PendingProvision,
) -> Result<SpawnAdmitted, MobError> {
let profile_name = &ctx.profile_name;
let agent_identity = &ctx.agent_identity;
let generation = ctx.generation;
let fence_token = ctx.fence_token;
let runtime_mode = ctx.runtime_mode;
let authorized_profile_material = &ctx.authorized_profile_material;
let labels = &ctx.labels;
let continuity_intent = &ctx.continuity_intent;
let identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let agent_runtime_id = crate::ids::AgentRuntimeId::new(identity.clone(), generation);
let overlay_record =
self.external_binding_overlay_record(&identity, generation, provision.member_ref());
let external_addressable = authorized_profile_material.external_addressable;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
let bridge_session_id = provision
.member_ref()
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain);
let replacing = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let is_replacing = replacing.is_some();
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending preparing DSL Spawn"
);
self.apply_dsl_input(
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(fence_token),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::from(runtime_mode),
bridge_session_id: bridge_session_id.clone(),
replacing: replacing.clone(),
},
"finalize_spawn_admit_begin_spawn_exec",
)?;
let prepared_spawn = match self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: dsl_identity.clone(),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(fence_token),
generation: mob_dsl::Generation::from_domain(generation),
profile_material_digest: authorized_profile_material
.profile_material_digest
.clone(),
external_addressable,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::from(runtime_mode),
bridge_session_id: bridge_session_id.clone(),
replacing,
},
"finalize_spawn_admit_commit_membership",
) {
Ok(prepared) => prepared,
Err(error) => {
return Err(self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error,
"finalize_spawn_admit_commit_membership",
));
}
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit prepared membership commit"
);
if let Err(error) = Self::require_member_lifecycle_journal_effect(
&prepared_spawn.transition,
mob_dsl::MobLifecycleJournalKind::MemberSpawned,
&identity,
&agent_runtime_id,
Some(fence_token),
generation,
bridge_session_id.clone(),
"finalize_spawn_admit_commit_membership",
) {
return Err(self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error,
"finalize_spawn_admit_commit_membership",
));
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit validated lifecycle journal"
);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending resolving supervisor comms"
);
let supervisor_private_trust_install = if agent_identity.is_flow_member_namespace() {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending skipped supervisor private trust for run-scoped flow member"
);
None
} else if let (Some(session_id), Some(comms)) = (
provision.member_ref().bridge_session_id().cloned(),
self.provisioner_comms(provision.member_ref()).await,
) {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %session_id,
"MobActor::finalize_spawn_from_pending installing supervisor private trust"
);
match Box::pin(async {
self.install_supervisor_private_trust_for_session(&session_id, &comms, None)
.await
})
.await
{
Ok(install) => {
tracing::debug!(
agent_identity = %agent_identity,
session_id = %session_id,
"MobActor::finalize_spawn_from_pending installed supervisor private trust"
);
Some((session_id, comms, install))
}
Err(error) => {
let error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error.into(),
"finalize_spawn_admit_trust",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn supervisor private trust failed for '{agent_identity}': {error}; archive compensation failed: {rollback_error}"
)));
}
return Err(error);
}
}
} else {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending skipped supervisor private trust"
);
None
};
if let Some(overlay_record) = overlay_record.as_ref() {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending upserting overlay"
);
if let Err(error) = self
.runtime_metadata
.upsert_external_binding_overlay(&self.definition.id, overlay_record)
.await
{
if let Some((session_id, comms, install)) =
supervisor_private_trust_install.as_ref()
{
Box::pin(
self.cleanup_supervisor_private_trust_for_session(
session_id, comms, install,
),
)
.await;
}
let error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
error.into(),
"finalize_spawn_admit_overlay",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn overlay upsert failed for '{agent_identity}': {error}; archive compensation failed: {rollback_error}"
)));
}
return Err(error);
}
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending appending spawn event"
);
if let Err(append_error) = self
.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberSpawned({
let mut event = crate::event::MemberSpawnedEvent::new(
identity.clone(),
generation,
fence_token,
agent_runtime_id.clone(),
profile_name.clone(),
)
.with_bridge_member_ref(Some(Self::sanitized_member_ref(
provision.member_ref(),
)));
event.runtime_mode = runtime_mode;
event.labels = labels.clone();
event.continuity_intent = continuity_intent.clone();
event.effective_profile_override = ctx.effective_profile_override.clone();
event
}),
})
.await
{
if overlay_record.is_some() {
let _ = self
.delete_external_binding_overlay_for_member(&identity, generation)
.await;
}
if let Some((session_id, comms, install)) = supervisor_private_trust_install.as_ref() {
Box::pin(
self.cleanup_supervisor_private_trust_for_session(session_id, comms, install),
)
.await;
}
let append_error = self.fold_spawn_exec_abort(
&dsl_identity,
agent_identity,
MobError::from(append_error),
"finalize_spawn_admit_append",
);
if let Err(rollback_error) = provision.rollback().await {
return Err(MobError::Internal(format!(
"spawn append failed for '{agent_identity}': {append_error}; archive compensation failed: {rollback_error}"
)));
}
return Err(append_error);
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committing membership"
);
self.commit_prepared_dsl_transition(prepared_spawn)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committed membership"
);
let member_ref = provision.commit()?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit committed provision"
);
self.restore_diagnostics
.write()
.await
.remove(agent_identity);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_admit cleared diagnostics"
);
Ok(SpawnAdmitted {
member_ref,
agent_runtime_id,
is_replacing,
})
}
async fn finalize_spawn_activate(
&mut self,
ctx: Box<SpawnFinalizeCtx>,
admitted: SpawnAdmitted,
) -> Result<FinalizeSpawnOutcome, MobError> {
let SpawnFinalizeCtx {
profile_name,
agent_identity,
generation,
fence_token,
runtime_mode,
prompt,
initial_turn_prompt,
labels,
operation_id,
owner_bridge_session_id,
auto_wire_parent,
restore_wiring,
effective_profile_override,
per_spawn_external_tools,
authorized_profile_material: _,
continuity_intent: _,
} = *ctx;
let SpawnAdmitted {
member_ref,
agent_runtime_id,
is_replacing,
} = admitted;
let profile_name = &profile_name;
let agent_identity = &agent_identity;
let identity = crate::ids::AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
let (peer_descriptor, transport_public_key) =
if let Some(session_id) = member_ref.bridge_session_id() {
match self.session_service.comms_runtime(session_id).await {
Some(runtime) => {
let public_key_bytes = runtime.public_key_bytes();
let descriptor = match public_key_bytes {
Some(_) => {
let comms_name = render_member_comms_name(
self.definition.id.as_str(),
profile_name.as_str(),
identity.as_str(),
)?;
Some(
self.provisioner
.trusted_peer_spec_for_operation(
&member_ref,
&operation_id,
&comms_name,
"",
)
.await?,
)
}
None => None,
};
let public_key = public_key_bytes
.map(|pubkey| meerkat_comms::PubKey::new(pubkey).to_pubkey_string());
(descriptor, public_key)
}
None => (None, None),
}
} else if let MemberRef::BackendPeer {
peer_id,
address,
pubkey,
..
} = &member_ref
{
let descriptor =
Self::peer_only_spec_from_parts(peer_id, address, "finalize_spawn", *pubkey)?;
(Some(descriptor), None)
} else {
(None, None)
};
let peer_id = peer_descriptor
.as_ref()
.map(|descriptor| descriptor.peer_id);
if let Some(descriptor) = peer_descriptor.as_ref() {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl_identity.clone(),
peer_endpoint: mob_dsl::MemberPeerEndpoint::from(descriptor),
},
"finalize_spawn_register_member_peer",
)?;
}
{
let mut roster = self.roster.write().await;
roster.add_member(crate::roster::RosterAddEntry {
agent_identity: identity.clone(),
generation,
fence_token,
agent_runtime_id: agent_runtime_id.clone(),
role: profile_name.clone(),
runtime_mode,
member_ref: Self::sanitized_member_ref(&member_ref),
peer_id,
transport_public_key,
labels: labels.clone(),
effective_profile_override: effective_profile_override.clone(),
});
}
{
let mut per_spawn = self.per_spawn_external_tools.write().await;
if let Some(dispatcher) = per_spawn_external_tools {
per_spawn.insert(identity.clone(), dispatcher);
} else {
per_spawn.remove(&identity);
}
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::SetExternalMemberRebindCapability {
agent_identity: dsl_identity.clone(),
capability: external_member_rebind_capability_from_member_ref(&member_ref),
},
"finalize_spawn_set_external_member_rebind_capability",
)?;
if runtime_mode == crate::MobRuntimeMode::AutonomousHost {
if is_replacing {
self.clear_kickoff_state(agent_identity).await;
}
let _ = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffMarkPending {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"finalize_spawn_kickoff_mark_pending",
)
.await?;
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_from_pending roster updated"
);
let mut planned_wiring_targets = if agent_identity.is_flow_member_namespace() {
Vec::new()
} else {
Box::pin(self.spawn_wiring_targets(profile_name, agent_identity)).await
};
if auto_wire_parent
&& let Some(parent_target) = self
.resolve_auto_wire_parent_target(owner_bridge_session_id.as_ref(), agent_identity)
.await
&& !planned_wiring_targets.contains(&parent_target)
{
planned_wiring_targets.push(parent_target);
}
if let Some(plan) = restore_wiring.as_ref() {
planned_wiring_targets.retain(|target| !plan.local_peers.contains(target));
}
let mut wired_spawn_targets: Vec<AgentIdentity> = Vec::new();
for target in &planned_wiring_targets {
let target_identity = crate::ids::AgentIdentity::from(target.as_str());
let local_meerkat = agent_identity.clone();
match Box::pin(self.handle_wire(
local_meerkat,
super::handle::PeerTarget::Local(target_identity),
))
.await
{
Ok(()) => wired_spawn_targets.push(target.clone()),
Err(wire_error) => {
let surfaced_wire_error = match wire_error {
MobError::WiringError(_) => wire_error,
other => MobError::WiringError(other.to_string()),
};
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
generation,
profile_name,
&member_ref,
&wired_spawn_targets,
&planned_wiring_targets,
))
.await
{
return Err(MobError::Internal(format!(
"spawn wire fan-out failed for '{agent_identity}': {surfaced_wire_error}; rollback failed: {rollback_error}"
)));
}
return Err(surfaced_wire_error);
}
}
}
#[cfg(feature = "runtime-adapter")]
if runtime_mode == crate::MobRuntimeMode::AutonomousHost {
let _ = self
.apply_kickoff_input(
agent_identity,
mob_dsl::MobMachineInput::KickoffMarkStarting {
member_id: mob_dsl::AgentIdentity::from_domain(agent_identity),
},
"finalize_spawn_kickoff_mark_starting",
)
.await?;
if let Err(binding_error) = Box::pin(self.flush_routed_effects()).await {
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
generation,
profile_name,
&member_ref,
&wired_spawn_targets,
&planned_wiring_targets,
))
.await
{
return Err(MobError::Internal(format!(
"spawn runtime binding failed for '{agent_identity}': {binding_error}; rollback failed: {rollback_error}"
)));
}
return Err(binding_error);
}
if let Err(start_error) =
Box::pin(self.start_autonomous_member(agent_identity, &member_ref, prompt)).await
{
self.clear_kickoff_state(agent_identity).await;
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
generation,
profile_name,
&member_ref,
&wired_spawn_targets,
&planned_wiring_targets,
))
.await
{
return Err(MobError::Internal(format!(
"spawn host-loop start failed for '{agent_identity}': {start_error}; rollback failed: {rollback_error}"
)));
}
return Err(start_error);
}
}
if runtime_mode == crate::MobRuntimeMode::TurnDriven
&& !agent_identity.is_flow_member_namespace()
{
#[cfg(all(not(target_arch = "wasm32"), feature = "runtime-adapter"))]
if let (Some(adapter), Some(bridge_session_id)) =
(self.runtime_adapter.clone(), member_ref.bridge_session_id())
{
let comms_runtime = self.provisioner.comms_runtime(&member_ref).await;
if std::env::var_os("RKAT_TRACE_COMMS_DRAIN_BIND").is_some()
&& let Some(runtime) = comms_runtime.as_ref()
{
tracing::info!(
agent_identity = %agent_identity,
session_id = %bridge_session_id,
comms_ptr = ?Arc::as_ptr(runtime),
"mob turn-driven spawn binding comms drain"
);
}
if let Some(comms_runtime) = comms_runtime {
let mob_id = meerkat_runtime::meerkat_machine::dsl::MobId::from(
self.definition.id.as_ref(),
);
Box::pin(adapter.maybe_spawn_mob_comms_drain(
bridge_session_id,
comms_runtime,
mob_id,
))
.await
.map_err(|err| {
MobError::Internal(format!(
"mob comms drain spawn failed for session {bridge_session_id}: {err}"
))
})?;
}
}
if let Some(initial_turn_prompt) = initial_turn_prompt {
if let Err(start_error) = Box::pin(self.dispatch_turn_driven_spawn_initial_turn(
agent_identity,
&agent_runtime_id,
fence_token,
&operation_id,
initial_turn_prompt,
))
.await
{
if let Err(rollback_error) = Box::pin(self.rollback_failed_spawn(
agent_identity,
generation,
profile_name,
&member_ref,
&wired_spawn_targets,
&planned_wiring_targets,
))
.await
{
return Err(MobError::Internal(format!(
"turn-driven spawn initial turn failed for '{agent_identity}': {start_error}; rollback failed: {rollback_error}"
)));
}
return Err(start_error);
}
}
}
let mut failed_restore_peer_ids: Vec<RespawnTopologyPeerId> = Vec::new();
if let Some(plan) = restore_wiring {
for peer_identity in plan.local_peers {
if peer_identity == *agent_identity {
continue;
}
let peer_agent_identity = crate::ids::AgentIdentity::from(peer_identity.as_str());
if let Err(error) = Box::pin(self.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::Local(peer_agent_identity),
))
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_identity,
%error,
"respawn: failed to restore machine-owned local peer edge"
);
failed_restore_peer_ids
.push(RespawnTopologyPeerId::from(peer_identity.as_str()));
}
}
for peer_spec in plan.external_peers {
let peer_id = RespawnTopologyPeerId::from(peer_spec.peer_id.as_str());
if let Err(error) = Box::pin(self.handle_wire(
agent_identity.clone(),
super::handle::PeerTarget::External(peer_spec.clone()),
))
.await
{
tracing::warn!(
agent_identity = %agent_identity,
peer = %peer_spec.name,
%error,
"respawn: failed to restore machine-owned external peer edge"
);
failed_restore_peer_ids.push(peer_id);
}
}
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::CommitSpawnActivation {
agent_identity: dsl_identity.clone(),
},
"finalize_spawn_activate_commit_activation",
)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::finalize_spawn_activate done"
);
Ok(FinalizeSpawnOutcome {
receipt: super::handle::MemberSpawnReceipt {
member_ref,
operation_id,
},
failed_restore_peer_ids,
})
}
async fn spawn_wiring_targets(
&self,
profile_name: &ProfileName,
agent_identity: &AgentIdentity,
) -> Vec<AgentIdentity> {
let mut targets = Vec::new();
if self.definition.wiring.auto_wire_orchestrator
&& let Some(orchestrator) = &self.definition.orchestrator
&& profile_name != &orchestrator.profile
{
let orchestrator_ids =
self.active_machine_member_ids_for_profile(&orchestrator.profile, agent_identity);
for orchestrator_id in orchestrator_ids {
if orchestrator_id != *agent_identity && !targets.contains(&orchestrator_id) {
targets.push(orchestrator_id);
}
}
}
for rule in &self.definition.wiring.role_wiring {
let target_profile = if &rule.a == profile_name {
Some(&rule.b)
} else if &rule.b == profile_name {
Some(&rule.a)
} else {
None
};
if let Some(target_profile) = target_profile {
let target_ids =
self.active_machine_member_ids_for_profile(target_profile, agent_identity);
for target_id in target_ids {
if !targets.contains(&target_id) {
targets.push(target_id);
}
}
}
}
targets
}
async fn resolve_auto_wire_parent_target(
&self,
owner_bridge_session_id: Option<&SessionId>,
spawned_member_identity: &AgentIdentity,
) -> Option<AgentIdentity> {
let owner_bridge_session_id = owner_bridge_session_id?;
let dsl_session_id = mob_dsl::SessionId::from_domain(owner_bridge_session_id);
let dsl = self.dsl_authority.state();
dsl.member_session_bindings
.iter()
.find(|(identity, bound_session_id)| {
**bound_session_id == dsl_session_id
&& identity.0.as_str() != spawned_member_identity.as_str()
&& !dsl.member_restore_failures.contains_key(*identity)
&& MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
})
.map(|(identity, _)| AgentIdentity::from(identity.0.as_str()))
}
async fn spawner_for_bridge_session(
&self,
owner_bridge_session_id: &SessionId,
) -> Option<(AgentIdentity, AgentRuntimeId)> {
let dsl_session_id = mob_dsl::SessionId::from_domain(owner_bridge_session_id);
let dsl = self.dsl_authority.state();
dsl.member_session_bindings
.iter()
.find_map(|(identity, bound_session_id)| {
if *bound_session_id != dsl_session_id
|| !MobMemberLifecycleProjection::is_active_machine_lifecycle(
&dsl.member_lifecycle_for_identity(identity),
)
{
return None;
}
let agent_identity = AgentIdentity::from(identity.0.as_str());
let (agent_runtime_id, _) = dsl
.member_runtime_material_for_identity(identity)?
.to_domain_for_identity(&agent_identity);
Some((agent_identity, agent_runtime_id))
})
}
async fn handle_force_cancel(&mut self, agent_identity: AgentIdentity) -> Result<(), MobError> {
let prepared = self.prepare_command_admission(
mob_dsl::MobMachineInput::ForceCancel {
agent_identity: mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
agent_identity.as_str(),
)),
},
MobState::Running,
"force_cancel",
)?;
let member_ref = {
let roster = self.roster.read().await;
roster
.get(&agent_identity)
.map(|entry| entry.member_ref.clone())
};
if let Some(member_ref) = member_ref {
self.provisioner.interrupt_member(&member_ref).await?;
} else {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"MobMachine admitted force-cancel without a roster projection; committing machine authority without mechanical interrupt"
);
}
self.commit_prepared_dsl_input(prepared)?;
Ok(())
}
async fn handle_wire(
&mut self,
local: AgentIdentity,
target: super::handle::PeerTarget,
) -> Result<(), MobError> {
let peer_identity = match target {
super::handle::PeerTarget::Local(id) => id,
super::handle::PeerTarget::ExternalName(name) => {
return Err(MobError::WiringError(format!(
"wire external peer '{name}' requires external_binding"
)));
}
super::handle::PeerTarget::ExternalBinding(binding) => {
let descriptor = Self::trusted_peer_descriptor_from_external_binding(binding)?;
return self.handle_wire_external(local, descriptor).await;
}
super::handle::PeerTarget::External(descriptor) => {
return self.handle_wire_external(local, descriptor).await;
}
};
let local_identity = AgentIdentity::from(local.as_str());
let peer_member_identity = AgentIdentity::from(peer_identity.as_str());
let dsl_a = mob_dsl::AgentIdentity::from_domain(&local_identity);
let dsl_b = mob_dsl::AgentIdentity::from_domain(&peer_identity);
let edge = mob_dsl::WiringEdge::new(dsl_a, dsl_b);
self.probe_command_admission(
mob_dsl::MobMachineInput::WireMembers { edge: edge.clone() },
MobState::Running,
"wire_members_command_admission",
)?;
if local_identity == peer_identity {
return Err(MobError::WiringError(format!(
"wire requires distinct members (got '{local}')"
)));
}
self.ensure_member_not_broken(&local).await?;
self.ensure_member_not_broken(&peer_member_identity).await?;
let (local_entry, peer_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&local)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?,
roster
.get(&peer_member_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(peer_member_identity.clone()))?,
)
};
let dsl_has_edge = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
let local_endpoint = self.resolve_wiring_endpoint(&local_entry, "wire").await?;
let peer_endpoint = self.resolve_wiring_endpoint(&peer_entry, "wire").await?;
if dsl_has_edge {
let authority = self.apply_wire_members_idempotent(&edge)?;
if !authority.is_repair() {
return Err(MobError::WiringError(
"idempotent wire repair did not produce generated repair authority".to_string(),
));
}
let handoff = authority.member_handoff()?;
match (&local_endpoint, &peer_endpoint) {
(
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) => {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.repair_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
handoff.require_peer_id_for(&local, &local_key)?;
if let Err(error) = self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.repair_authority_for(
&local,
&local_key,
&self.dsl_authority,
)?,
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_repair_rollback_trust_authority",
)
.await;
}
return Err(MobError::from(error));
}
}
(
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) => {
self.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
.map_err(|error| {
tracing::debug!(
mob_id = %self.definition.id,
%error,
"peer-only trust repair failed before reciprocal side"
);
error
})?;
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
false,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
}
(
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) => {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.repair_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_repair_rollback_trust_authority",
)
.await;
}
return Err(error);
}
}
(
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) => {
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&local, &local_key)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.repair_authority_for(
&local,
&local_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => return Err(MobError::from(error)),
};
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_repair_rollback_trust_authority",
)
.await;
}
return Err(error);
}
}
}
return Ok(());
}
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::both(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
if let (
WiringEndpoint::Local {
comms: local_comms,
spec: local_spec,
..
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let peer_key = Self::trusted_peer_removal_key(peer_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_key)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.wiring_authority_for(
&peer_member_identity,
&peer_key,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
if local_trust_created {
self.rollback_peer_only_trust(
&edge,
local_comms.as_ref(),
&peer_member_identity,
&peer_key,
"wire_members_peer_only_event_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::peer(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::Local {
comms: peer_comms,
spec: peer_spec,
..
},
) = (&local_endpoint, &peer_endpoint)
{
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let local_key = Self::trusted_peer_removal_key(local_spec);
handoff.require_peer_id_for(&local, &local_key)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.wiring_authority_for(&local, &local_key, &self.dsl_authority)?,
)
.await
{
Ok(created) => created,
Err(error) => {
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
if peer_trust_created {
self.rollback_peer_only_trust(
&edge,
peer_comms.as_ref(),
&local,
&local_key,
"wire_members_peer_only_event_rollback_trust_authority",
)
.await;
}
self.rollback_peer_only_wire(
&edge,
dsl_added,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
peer_spec,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
let (local_comms, local_spec) = match local_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire requires local session comms runtime for '{local}'"
)));
}
};
let (peer_comms, peer_spec) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire requires local session comms runtime for '{peer_identity}'"
)));
}
};
let authority = self.apply_wire_members_idempotent(&edge)?;
let dsl_added = authority.dsl_added();
let handoff = authority.member_handoff()?;
let local_peer_id = Self::trusted_peer_removal_key(&local_spec);
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
handoff.require_peer_id_for(&peer_member_identity, &peer_peer_id)?;
let local_trust_created = match self
.apply_trusted_peer_add_report(
local_comms.as_ref(),
peer_spec.clone(),
handoff.wiring_authority_for(
&peer_member_identity,
&peer_peer_id,
&self.dsl_authority,
)?,
)
.await
{
Ok(created) => created,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
false,
false,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
handoff.require_peer_id_for(&local, &local_peer_id)?;
let peer_trust_created = match self
.apply_trusted_peer_add_report(
peer_comms.as_ref(),
local_spec.clone(),
handoff.wiring_authority_for(&local, &local_peer_id, &self.dsl_authority)?,
)
.await
{
Ok(created) => created,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
false,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
if let Err(err) = self
.notify_peer_added(&peer_comms, &local_spec, &peer_member_identity, &peer_entry)
.await
{
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(err);
}
if let Err(err) = self
.notify_peer_added(&local_comms, &peer_spec, &local, &local_entry)
.await
{
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(err);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersWired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(err) => {
self.rollback_wire_side_effects(
&edge,
dsl_added,
local_trust_created,
peer_trust_created,
&local_comms,
&peer_comms,
&local_peer_id,
&peer_peer_id,
handoff,
)
.await;
return Err(MobError::from(err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
async fn handle_wire_members_batch(
&mut self,
requested_edges: Vec<(AgentIdentity, AgentIdentity)>,
) -> Result<super::handle::MobWireMembersBatchReport, MobError> {
let requested = requested_edges.len();
let mut normalized_edges = BTreeSet::new();
let mut endpoint_ids = BTreeSet::new();
for (left, right) in requested_edges {
if left == right {
return Err(MobError::WiringError(format!(
"wire_members_batch requires distinct members (got '{left}')"
)));
}
let (a, b) = if left <= right {
(left, right)
} else {
(right, left)
};
endpoint_ids.insert(a.clone());
endpoint_ids.insert(b.clone());
normalized_edges.insert((a, b));
}
if normalized_edges.is_empty() {
return Ok(super::handle::MobWireMembersBatchReport {
requested,
already_wired: Vec::new(),
wired: Vec::new(),
});
}
let broken_members = self
.dsl_authority
.state()
.member_restore_failures
.keys()
.map(|identity| AgentIdentity::from(identity.0.as_str()))
.collect::<BTreeSet<_>>();
for identity in &endpoint_ids {
if broken_members.contains(identity) {
return Err(MobError::WiringError(format!(
"wire_members_batch cannot wire broken member '{identity}'"
)));
}
}
let entries = {
let roster = self.roster.read().await;
let mut entries = BTreeMap::new();
for identity in &endpoint_ids {
let entry = roster
.get(identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(identity.clone()))?;
entries.insert(identity.clone(), entry);
}
entries
};
let mut endpoints = BTreeMap::new();
for (identity, entry) in entries {
match self
.resolve_wiring_endpoint(&entry, "wire_members_batch")
.await?
{
WiringEndpoint::Local { comms, spec, .. } => {
let removal_key = Self::trusted_peer_removal_key(&spec);
endpoints.insert(
identity,
LocalBatchWiringEndpoint {
comms,
spec,
removal_key,
},
);
}
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"wire_members_batch only supports local session-backed members (got '{identity}')"
)));
}
}
}
let mut already_wired = Vec::new();
let mut repair_edges = Vec::new();
let mut to_add = Vec::new();
for (a, b) in normalized_edges {
let event_edge = crate::event::MemberWireEdge {
a: a.clone(),
b: b.clone(),
};
let dsl_edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&a),
mob_dsl::AgentIdentity::from_domain(&b),
);
if self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &dsl_edge)
{
already_wired.push(event_edge.clone());
repair_edges.push((event_edge, dsl_edge));
} else {
to_add.push((event_edge, dsl_edge));
}
}
let mut wired = Vec::with_capacity(to_add.len());
let mut trust_applications = Vec::with_capacity((to_add.len() + repair_edges.len()) * 2);
if !to_add.is_empty() {
let prepared_batch_authority =
crate::generated::protocol_mob_member_trust_wiring::MobTopologyPreparedBatchAuthority::from_live_authority(
&self.dsl_authority,
);
let mut prepared_authority = self.dsl_authority.prepare_authority();
let mut batch_effects = Vec::new();
let mut phase_changed = false;
for (event_edge, dsl_edge) in &to_add {
let input = mob_dsl::MobMachineInput::WireMembersWithTrust {
edge: dsl_edge.clone(),
a_identity: dsl_edge.a.clone(),
b_identity: dsl_edge.b.clone(),
};
let transition = match mob_dsl::MobMachineMutator::apply(
&mut prepared_authority,
input,
) {
Ok(transition) => transition,
Err(error) => {
return Err(MobError::Internal(format!(
"DSL authority (wire_members_batch) rejected WireMembersWithTrust for edge {dsl_edge:?}: {error}"
)));
}
};
let freshness_authority = prepared_batch_authority
.freshness_for_prepared_transitions(&self.dsl_authority, [&transition])
.map_err(MobError::WiringError)?;
let handoff = match self.member_trust_wiring_handoff_from_prepared_batch_transition(
&transition,
dsl_edge,
freshness_authority,
"wire_members_batch",
MemberTrustOperation::Wiring,
) {
Ok(handoff) => handoff,
Err(error) => return Err(error),
};
trust_applications.extend(self.batch_wire_trust_applications_from_handoff(
&endpoints, event_edge, dsl_edge, &handoff,
)?);
if transition.from_phase != transition.to_phase {
phase_changed = true;
}
for effect in transition.effects() {
if let mob_dsl::MobMachineEffect::EmitWiringLifecycleNotice {
kind: mob_dsl::WiringLifecycleKind::Wired,
edge,
} = effect
{
wired.push(crate::event::MemberWireEdge {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
});
}
}
batch_effects.extend(transition.into_effects());
}
let prepared = PreparedDslInput {
authority: prepared_authority,
effects: batch_effects,
phase_changed,
};
let events = self.events.clone();
let mob_id = self.definition.id.clone();
let wired_for_event = wired.clone();
let stored = self
.commit_prepared_dsl_input_after(prepared, move || async move {
let event = NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MembersWiredBatch {
edges: wired_for_event,
},
};
events.append(event).await.map_err(MobError::from)
})
.await?;
self.roster.write().await.apply_event(&stored);
}
for (event_edge, dsl_edge) in &repair_edges {
let handoff = self.authorize_batch_member_trust_repair(dsl_edge)?;
trust_applications.extend(self.batch_wire_trust_applications_from_handoff(
&endpoints, event_edge, dsl_edge, &handoff,
)?);
}
let owner_token = self.dsl_authority.generated_authority_owner_token();
self.apply_batch_wire_trust_applications(trust_applications, &owner_token)
.await?;
tracing::info!(
mob_id = %self.definition.id,
requested,
wired = wired.len(),
already_wired = already_wired.len(),
participants = endpoints.len(),
"wire_members_batch materialized local topology"
);
Ok(super::handle::MobWireMembersBatchReport {
requested,
already_wired,
wired,
})
}
async fn declare_member_outbound_taint(
&mut self,
identity: AgentIdentity,
taint: Option<meerkat_core::comms::SenderContentTaint>,
) -> Result<(), MobError> {
self.require_member_operation_eligible()?;
self.ensure_member_not_broken(&identity).await?;
let entry = {
let roster = self.roster.read().await;
roster
.get(&identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(identity.clone()))?
};
if let Some(comms) = self.provisioner_comms(&entry.member_ref).await {
return comms.set_outbound_content_taint(taint).map_err(|error| {
match error {
meerkat_core::comms::SendError::Unsupported(_) => {
MobError::MissingMemberCapability {
member_id: identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "declare member outbound taint",
}
}
other => MobError::CommsError(other),
}
});
}
let Some(binding) = Self::runtime_binding_for_entry(&entry) else {
return Err(MobError::MissingMemberCapability {
member_id: identity.clone(),
capability: crate::error::MobMemberCapability::OutboundCommsRuntime,
context: "declare member outbound taint",
});
};
let peer = Self::peer_only_spec_for_binding(&binding, "declare_member_outbound_taint")?;
let peer = self
.ensure_supervisor_authorized(&peer, Some(&binding))
.await?;
let sup_payload = self.bridge_supervisor_payload_for_recipient(&peer).await?;
let payload = super::bridge_protocol::BridgeOutboundTaintPayload {
supervisor: sup_payload.supervisor,
epoch: sup_payload.epoch,
protocol_version: sup_payload.protocol_version,
taint,
};
let command = super::bridge_protocol::BridgeCommand::DeclareMemberOutboundTaint(payload);
let _ack: super::bridge_protocol::BridgeAck = self
.send_bridge_command_typed(&peer, &command, std::time::Duration::from_secs(10))
.await?;
Ok(())
}
async fn prepare_send_peer_message(
&mut self,
from: AgentIdentity,
to: AgentIdentity,
content: ContentInput,
handling_mode: meerkat_core::types::HandlingMode,
) -> Result<PeerMessageDeliveryPlan, MobError> {
self.require_member_operation_eligible()?;
if from == to {
return Err(MobError::WiringError(format!(
"peer message requires distinct members (got '{from}')"
)));
}
self.ensure_member_not_broken(&from).await?;
self.ensure_member_not_broken(&to).await?;
let (sender_entry, recipient_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&from)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(from.clone()))?,
roster
.get(&to)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(to.clone()))?,
)
};
let edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(from.as_str())),
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(to.as_str())),
);
let wired_by_machine = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
if !wired_by_machine {
return Err(MobError::WiringError(format!(
"peer message requires wired members '{from}' and '{to}'"
)));
}
let sender_comms = self
.provisioner_comms(&sender_entry.member_ref)
.await
.ok_or_else(|| {
MobError::WiringError(format!(
"peer message requires sender comms runtime for '{from}'"
))
})?;
let recipient_endpoint = self
.resolve_wiring_endpoint(&recipient_entry, "peer_message")
.await?;
let recipient_spec = match recipient_endpoint {
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
};
let route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
let (body, blocks) = match content {
ContentInput::Text(body) => (body, None),
ContentInput::Blocks(blocks) => {
(meerkat_core::types::text_content(&blocks), Some(blocks))
}
};
Ok(PeerMessageDeliveryPlan {
from,
to,
sender_comms,
command: CommsCommand::PeerMessage {
to: route,
body,
blocks,
content_taint: None,
handling_mode,
},
})
}
fn apply_wire_members_idempotent(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<WireTrustAuthority, MobError> {
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::WireMembers { edge: edge.clone() },
"wire_members",
)?;
let graph_added =
Self::wire_members_disposition_from_effects(&effects, edge, "wire_members")?;
let handoff = self.authorize_member_trust_wiring(
edge,
"wire_members_trust_authority",
if graph_added {
MemberTrustOperation::Wiring
} else {
MemberTrustOperation::Repair
},
)?;
if graph_added {
Ok(WireTrustAuthority::GraphAdded(handoff))
} else {
Ok(WireTrustAuthority::RepairRequested(handoff))
}
}
fn authorize_batch_member_trust_repair(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<MemberTrustHandoff, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::WireMembersWithTrust {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
"wire_members_batch_repair_request",
)?;
let graph_added = Self::wire_members_disposition_from_effects(
transition.effects(),
edge,
"wire_members_batch_repair_request",
)?;
if graph_added {
return Err(MobError::WiringError(
"wire_members_batch repair unexpectedly produced a generated graph change"
.to_string(),
));
}
self.authorize_member_trust_wiring(
edge,
"wire_members_batch_repair_trust_authority",
MemberTrustOperation::Repair,
)
}
fn batch_wire_trust_applications_from_handoff(
&self,
endpoints: &BTreeMap<AgentIdentity, LocalBatchWiringEndpoint>,
event_edge: &crate::event::MemberWireEdge,
dsl_edge: &mob_dsl::WiringEdge,
handoff: &MemberTrustHandoff,
) -> Result<Vec<BatchWireTrustApplication>, MobError> {
let left = endpoints.get(&event_edge.a).ok_or_else(|| {
MobError::WiringError(format!(
"wire_members_batch missing endpoint '{}'",
event_edge.a
))
})?;
let right = endpoints.get(&event_edge.b).ok_or_else(|| {
MobError::WiringError(format!(
"wire_members_batch missing endpoint '{}'",
event_edge.b
))
})?;
let right_peer_id = right.removal_key.clone();
handoff.require_peer_id_for(&event_edge.b, &right_peer_id)?;
let left_trust_authority =
handoff.add_authority_for(&event_edge.b, &right_peer_id, &self.dsl_authority)?;
let left_peer_id = left.removal_key.clone();
handoff.require_peer_id_for(&event_edge.a, &left_peer_id)?;
let right_trust_authority =
handoff.add_authority_for(&event_edge.a, &left_peer_id, &self.dsl_authority)?;
Ok(vec![
BatchWireTrustApplication {
edge: dsl_edge.clone(),
identity: event_edge.b.clone(),
peer_id: right_peer_id,
comms: left.comms.clone(),
peer: right.spec.clone(),
authority: left_trust_authority,
},
BatchWireTrustApplication {
edge: dsl_edge.clone(),
identity: event_edge.a.clone(),
peer_id: left_peer_id,
comms: right.comms.clone(),
peer: left.spec.clone(),
authority: right_trust_authority,
},
])
}
async fn apply_batch_wire_trust_applications(
&mut self,
applications: Vec<BatchWireTrustApplication>,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), MobError> {
let mut installed = Vec::new();
for application in applications {
let rollback = BatchWireTrustRollback {
edge: application.edge.clone(),
identity: application.identity.clone(),
peer_id: application.peer_id.clone(),
comms: application.comms.clone(),
};
match Self::apply_trusted_peer_add_with_owner_token_report(
application.comms.as_ref(),
application.peer,
application.authority,
owner_token,
)
.await
{
Ok(created) => {
if created {
installed.push(rollback);
}
}
Err(error) => {
self.rollback_batch_wire_trust_applications(installed, owner_token)
.await;
return Err(MobError::from(error));
}
}
}
Ok(())
}
async fn rollback_batch_wire_trust_applications(
&mut self,
installed: Vec<BatchWireTrustRollback>,
owner_token: &Arc<dyn std::any::Any + Send + Sync>,
) {
for rollback in installed.into_iter().rev() {
let handoff = match self.authorize_member_trust_unwiring(
&rollback.edge,
"wire_members_batch_trust_failure_rollback",
) {
Ok(handoff) => handoff,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch could not authorize trust rollback after trust failure"
);
continue;
}
};
let authority =
match handoff.unwiring_authority_for(&rollback.identity, &rollback.peer_id) {
Ok(authority) => authority,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch could not derive generated trust rollback authority"
);
continue;
}
};
if let Err(error) = Self::apply_trusted_peer_remove_with_owner_token(
rollback.comms.as_ref(),
rollback.peer_id,
authority,
owner_token,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"wire_members_batch failed to rollback previously installed trust after trust failure"
);
}
}
}
async fn rollback_peer_only_trust(
&mut self,
edge: &mob_dsl::WiringEdge,
comms: &(dyn CoreCommsRuntime + '_),
identity: &AgentIdentity,
removal_key: &str,
context: &str,
) {
let rollback_handoff = match self.authorize_member_trust_unwiring(edge, context) {
Ok(handoff) => handoff,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to obtain generated trust removal authority"
);
return;
}
};
let authority = match rollback_handoff.unwiring_authority_for(identity, removal_key) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to build generated trust removal authority"
);
return;
}
};
if let Err(err) = self
.apply_trusted_peer_remove(comms, removal_key.to_string(), authority)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"peer-only wire rollback: failed to remove trust"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn rollback_peer_only_wire(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_added: bool,
installed_sides: WiringSides,
_local_identity: &AgentIdentity,
_peer_identity: &AgentIdentity,
local_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
) {
if !dsl_added {
return;
}
if let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"peer_only_wire_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to revert DSL wire"
);
return;
}
if installed_sides.has_local()
&& let Err(error) = self
.unwire_peer_only_recipient(
local_spec,
None,
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to unwire local recipient"
);
}
if installed_sides.has_peer()
&& let Err(error) = self
.unwire_peer_only_recipient(
peer_spec,
None,
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only wire rollback: failed to unwire peer recipient"
);
}
}
#[allow(clippy::too_many_arguments)]
async fn rollback_peer_only_unwire(
&mut self,
edge: &mob_dsl::WiringEdge,
sides_to_rewire: WiringSides,
_local_identity: &AgentIdentity,
_peer_identity: &AgentIdentity,
local_spec: &TrustedPeerDescriptor,
local_binding: &crate::RuntimeBinding,
peer_spec: &TrustedPeerDescriptor,
peer_binding: &crate::RuntimeBinding,
) {
if let Err(error) = self.apply_wire_members_idempotent(edge) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to restore MobMachine wire"
);
return;
}
if sides_to_rewire.has_local() {
if let Err(error) = self
.wire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to rewire local recipient"
);
}
}
if sides_to_rewire.has_peer() {
if let Err(error) = self
.wire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"peer-only unwire rollback: failed to rewire peer recipient"
);
}
}
}
#[allow(clippy::too_many_arguments)]
async fn rollback_wire_side_effects(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_added: bool,
installed_local_trust: bool,
installed_peer_trust: bool,
local_comms: &Arc<dyn CoreCommsRuntime>,
peer_comms: &Arc<dyn CoreCommsRuntime>,
local_peer_id: &str,
peer_peer_id: &str,
_handoff: &MemberTrustHandoff,
) {
let local_identity = AgentIdentity::from(edge.a.0.as_str());
let peer_identity = AgentIdentity::from(edge.b.0.as_str());
let rollback_handoff = if installed_local_trust || installed_peer_trust {
match self
.authorize_member_trust_unwiring(edge, "wire_members_rollback_trust_authority")
{
Ok(handoff) => Some(handoff),
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to obtain generated trust removal authority"
);
None
}
}
} else {
None
};
if installed_local_trust {
if let Err(err) = self.apply_trusted_peer_remove(
local_comms.as_ref(),
peer_peer_id.to_string(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"wire rollback has no generated local trust removal authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.unwiring_authority_for(&peer_identity, peer_peer_id)
}) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to build generated local trust removal authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to remove local trust"
);
}
}
if installed_peer_trust {
if let Err(err) = self.apply_trusted_peer_remove(
peer_comms.as_ref(),
local_peer_id.to_string(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"wire rollback has no generated peer trust removal authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.unwiring_authority_for(&local_identity, local_peer_id)
}) {
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to build generated peer trust removal authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to remove peer trust"
);
}
}
if dsl_added {
if let Err(err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"wire_members_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"wire rollback: failed to revert DSL wire"
);
}
}
}
async fn handle_unwire(
&mut self,
local: AgentIdentity,
target: super::handle::PeerTarget,
) -> Result<(), MobError> {
let peer_identity = match target {
super::handle::PeerTarget::Local(id) => {
let target_identity =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(id.as_str()));
let target_is_member = self
.dsl_authority
.state()
.identity_to_runtime
.contains_key(&target_identity);
let external_peer_name =
meerkat_core::comms::PeerName::new(id.as_str().to_string())
.ok()
.filter(|peer_name| {
let local_identity = AgentIdentity::from(local.as_str());
self.external_peer_edge_for_name(&local_identity, peer_name)
.is_some()
});
if !target_is_member && let Some(peer_name) = external_peer_name {
return self.handle_unwire_external(local, peer_name, None).await;
}
id
}
super::handle::PeerTarget::ExternalName(peer_name) => {
return self.handle_unwire_external(local, peer_name, None).await;
}
super::handle::PeerTarget::ExternalBinding(binding) => {
let descriptor = Self::trusted_peer_descriptor_from_external_binding(binding)?;
let peer_name = descriptor.name.clone();
return self
.handle_unwire_external(local, peer_name, Some(descriptor))
.await;
}
super::handle::PeerTarget::External(descriptor) => {
let peer_name = descriptor.name.clone();
return self
.handle_unwire_external(local, peer_name, Some(descriptor))
.await;
}
};
let local_identity = AgentIdentity::from(local.as_str());
if local_identity == peer_identity {
return Err(MobError::WiringError(format!(
"unwire requires distinct peers (got '{local}')"
)));
}
let peer_member_identity = AgentIdentity::from(peer_identity.as_str());
let dsl_a = mob_dsl::AgentIdentity::from_domain(&local_identity);
let dsl_b = mob_dsl::AgentIdentity::from_domain(&peer_identity);
let edge = mob_dsl::WiringEdge::new(dsl_a, dsl_b);
let (local_entry, peer_entry) = {
let roster = self.roster.read().await;
(
roster
.get(&local)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?,
roster
.get(&peer_member_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(peer_member_identity.clone()))?,
)
};
let dsl_has_edge = self
.dsl_authority
.state()
.wiring_edges
.iter()
.any(|existing| existing == &edge);
let local_endpoint = self.resolve_wiring_endpoint(&local_entry, "unwire").await?;
let peer_endpoint = self.resolve_wiring_endpoint(&peer_entry, "unwire").await?;
if let (
WiringEndpoint::PeerOnly {
spec: local_spec,
binding: local_binding,
},
WiringEndpoint::PeerOnly {
spec: peer_spec,
binding: peer_binding,
},
) = (&local_endpoint, &peer_endpoint)
{
if !dsl_has_edge {
return Err(MobError::WiringError(format!(
"peer-only unwire for '{local}' <-> '{peer_identity}' requires MobMachine wiring authority"
)));
}
self.cancel_peer_deliveries_for_edge(
&local,
&peer_member_identity,
"members are unwiring",
)
.await;
let _unwire_handoff = self.apply_unwire_members_idempotent(&edge)?;
if let Err(error) = self
.unwire_peer_only_recipient(
local_spec,
Some(local_binding),
peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_unwire(
&edge,
WiringSides::empty(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(error);
}
if let Err(error) = self
.unwire_peer_only_recipient(
peer_spec,
Some(peer_binding),
local_spec,
std::time::Duration::from_secs(10),
)
.await
{
self.rollback_peer_only_unwire(
&edge,
WiringSides::local(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(error);
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersUnwired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(error) => {
self.rollback_peer_only_unwire(
&edge,
WiringSides::both(),
&local,
&peer_member_identity,
local_spec,
local_binding,
peer_spec,
peer_binding,
)
.await;
return Err(MobError::from(error));
}
};
self.roster.write().await.apply_event(&stored);
return Ok(());
}
let (local_comms, local_spec) = match local_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"unwire requires local session comms runtime for '{local}'"
)));
}
};
let (peer_comms, peer_spec) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (comms, spec),
WiringEndpoint::PeerOnly { .. } => {
return Err(MobError::WiringError(format!(
"unwire requires local session comms runtime for '{peer_identity}'"
)));
}
};
let local_peer_id = Self::trusted_peer_removal_key(&local_spec);
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
if !dsl_has_edge {
let local_has_stale_trust = local_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == peer_peer_id);
let peer_has_stale_trust = peer_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == local_peer_id);
if local_has_stale_trust || peer_has_stale_trust {
return Err(MobError::WiringError(format!(
"unwire for '{local}' <-> '{peer_identity}' requires MobMachine wiring authority"
)));
}
return Ok(());
}
self.cancel_peer_deliveries_for_edge(&local, &peer_member_identity, "members are unwiring")
.await;
let unwire_handoff = self.apply_unwire_members_idempotent(&edge)?;
let dsl_removed = true;
let mut removed_local_trust = false;
let mut removed_peer_trust = false;
let mut sent_unwired_from_local = false;
let mut sent_unwired_from_peer = false;
if let Err(err) = self
.notify_peer_event(
"mob.peer_unwired",
&peer_spec,
&peer_member_identity,
&peer_entry,
&local_comms,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(err);
}
sent_unwired_from_local = true;
if let Err(err) = self
.notify_peer_event(
"mob.peer_unwired",
&local_spec,
&local,
&local_entry,
&peer_comms,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(err);
}
sent_unwired_from_peer = true;
unwire_handoff.require_peer_id_for(&peer_member_identity, &peer_peer_id)?;
if let Err(err) = self
.apply_trusted_peer_remove(
local_comms.as_ref(),
peer_peer_id.clone(),
unwire_handoff.unwiring_authority_for(&peer_member_identity, &peer_peer_id)?,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
removed_local_trust = true;
unwire_handoff.require_peer_id_for(&local, &local_peer_id)?;
if let Err(err) = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
local_peer_id.clone(),
unwire_handoff.unwiring_authority_for(&local, &local_peer_id)?,
)
.await
{
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
removed_peer_trust = true;
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MembersUnwired {
a: AgentIdentity::from(edge.a.0.as_str()),
b: AgentIdentity::from(edge.b.0.as_str()),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(err) => {
self.rollback_unwire_side_effects(
&edge,
dsl_removed,
removed_local_trust,
removed_peer_trust,
sent_unwired_from_local,
sent_unwired_from_peer,
&local_comms,
&peer_comms,
&local_spec,
&peer_spec,
&local,
&peer_member_identity,
&local_entry,
&peer_entry,
&unwire_handoff,
)
.await;
return Err(MobError::from(err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
fn apply_unwire_members_idempotent(
&mut self,
edge: &mob_dsl::WiringEdge,
) -> Result<MemberTrustHandoff, MobError> {
let handoff =
self.authorize_member_trust_unwiring(edge, "unwire_members_trust_authority")?;
let effects = self.apply_dsl_input_collect_effects(
mob_dsl::MobMachineInput::UnwireMembers { edge: edge.clone() },
"unwire_members",
)?;
if Self::effects_include_wiring_graph_change(&effects) {
Ok(handoff)
} else {
Err(MobError::WiringError(format!(
"unwire_members produced no generated wiring graph authority for edge {edge:?}"
)))
}
}
fn external_peer_edge(
local_identity: &AgentIdentity,
spec: &TrustedPeerDescriptor,
) -> mob_dsl::ExternalPeerEdge {
mob_dsl::ExternalPeerEdge::new(
mob_dsl::AgentIdentity::from_domain(local_identity),
mob_dsl::ExternalPeerEndpoint::from(spec),
)
}
fn external_peer_key(
local_identity: &AgentIdentity,
peer_name: &meerkat_core::comms::PeerName,
) -> mob_dsl::ExternalPeerKey {
mob_dsl::ExternalPeerKey::new(
mob_dsl::AgentIdentity::from_domain(local_identity),
mob_dsl::PeerName::from(peer_name.as_str()),
)
}
fn external_peer_key_for_edge(edge: &mob_dsl::ExternalPeerEdge) -> mob_dsl::ExternalPeerKey {
mob_dsl::ExternalPeerKey::new(edge.local.clone(), edge.endpoint.name.clone())
}
fn external_peer_edge_for_name(
&self,
local_identity: &AgentIdentity,
peer_name: &meerkat_core::comms::PeerName,
) -> Option<mob_dsl::ExternalPeerEdge> {
let key = Self::external_peer_key(local_identity, peer_name);
self.dsl_authority
.state()
.external_peer_edges_by_key
.get(&key)
.cloned()
}
fn apply_wire_external_peer_idempotent(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<WireTrustAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"wire_external_peer",
)?;
self.wire_external_authority_from_transition(&transition, edge, "wire_external_peer")
}
async fn apply_external_peer_reciprocal_trust(
&mut self,
key: mob_dsl::ExternalPeerKey,
target_comms: Arc<dyn CoreCommsRuntime>,
peer: TrustedPeerDescriptor,
) -> Result<(), MobError> {
let local_identity = AgentIdentity::from(key.local.0.as_str());
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::AuthorizeExternalPeerReciprocalTrust {
key: key.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&local_identity),
},
"apply_external_peer_reciprocal_trust",
)?;
let Some(obligation) =
crate::generated::protocol_mob_external_peer_reciprocal_trust::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_reciprocal_trust::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.key() == &key)
else {
return Err(MobError::WiringError(
"MobMachine produced no external reciprocal trust obligation".to_string(),
));
};
let target_peer_id = target_comms.peer_id().ok_or_else(|| {
MobError::WiringError(
"external reciprocal trust target runtime did not expose peer_id".to_string(),
)
})?;
if target_peer_id.to_string() != obligation.edge().endpoint.peer_id.0 {
return Err(MobError::WiringError(format!(
"external reciprocal trust target peer_id {target_peer_id} does not match MobMachine external edge peer_id {}",
obligation.edge().endpoint.peer_id.0
)));
}
let peer_id = obligation.peer_id().0.clone();
let authority = crate::generated::protocol_mob_external_peer_reciprocal_trust::reciprocal_wiring_authority_for_peer(
&obligation,
&peer_id,
)
.map_err(MobError::WiringError)?;
self.apply_trusted_peer_add(target_comms.as_ref(), peer, authority)
.await
.map_err(MobError::CommsError)
}
fn apply_unwire_external_peer_idempotent(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<Option<CommsTrustMutationAuthority>, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::UnwireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"unwire_external_peer",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects) {
return Ok(None);
}
let obligation =
crate::generated::protocol_mob_external_peer_trust_unwiring::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(self.dsl_topology_epoch.clone(), Arc::clone(&self.dsl_authority_owner_token)),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(
"unwire_external_peer produced graph authority without generated unwiring trust obligation"
.to_string(),
)
})?;
Ok(Some(
crate::generated::protocol_mob_external_peer_trust_unwiring::unwiring_authority_for_peer(
&obligation,
edge.endpoint.peer_id.0.as_str(),
)
.map_err(MobError::WiringError)?,
))
}
fn apply_cleanup_retiring_external_peer(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<CommsTrustMutationAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::CleanupRetiringExternalPeer {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"cleanup_retiring_external_peer",
)?;
let obligation = crate::generated::protocol_mob_external_peer_trust_unwiring::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_external_peer_trust_unwiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(
self.dsl_topology_epoch.clone(),
Arc::clone(&self.dsl_authority_owner_token),
),
)
.into_iter()
.find(|obligation| obligation.edge() == edge)
.ok_or_else(|| {
MobError::WiringError(
"retiring external cleanup produced no generated unwiring trust obligation"
.to_string(),
)
})?;
crate::generated::protocol_mob_external_peer_trust_unwiring::unwiring_authority_for_peer(
&obligation,
edge.endpoint.peer_id.0.as_str(),
)
.map_err(MobError::WiringError)
}
fn apply_restore_retiring_external_peer(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<WireTrustAuthority, MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RestoreRetiringExternalPeer {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"restore_retiring_external_peer",
)?;
self.wire_external_authority_from_transition(
&transition,
edge,
"restore_retiring_external_peer",
)
}
fn apply_cleanup_retiring_external_peer_observed_absent(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<(), MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::CleanupRetiringExternalPeerObservedAbsent {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"cleanup_retiring_external_peer_observed_absent",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects)
|| effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::ExternalPeerTrustUnwiringRequested { .. }
)
})
{
return Err(MobError::WiringError(
"observed-absent external cleanup must change topology without publishing live trust work"
.to_string(),
));
}
Ok(())
}
fn apply_restore_retiring_external_peer_observed_absent(
&mut self,
entry: &RosterEntry,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
) -> Result<(), MobError> {
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RestoreRetiringExternalPeerObservedAbsent {
key: key.clone(),
edge: edge.clone(),
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"restore_retiring_external_peer_observed_absent",
)?;
let effects = transition.effects();
if !Self::effects_include_wiring_graph_change(effects)
|| effects.iter().any(|effect| {
matches!(
effect,
mob_dsl::MobMachineEffect::ExternalPeerTrustWiringRequested { .. }
)
})
{
return Err(MobError::WiringError(
"observed-absent external restore must change topology without publishing live trust work"
.to_string(),
));
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn rollback_unwire_side_effects(
&mut self,
edge: &mob_dsl::WiringEdge,
dsl_removed: bool,
removed_local_trust: bool,
removed_peer_trust: bool,
sent_unwired_from_local: bool,
sent_unwired_from_peer: bool,
local_comms: &Arc<dyn CoreCommsRuntime>,
peer_comms: &Arc<dyn CoreCommsRuntime>,
local_spec: &TrustedPeerDescriptor,
peer_spec: &TrustedPeerDescriptor,
local_member_identity: &AgentIdentity,
peer_member_identity: &AgentIdentity,
local_entry: &RosterEntry,
peer_entry: &RosterEntry,
_handoff: &MemberTrustHandoff,
) {
let rollback_handoff = if dsl_removed {
match self.apply_wire_members_idempotent(edge) {
Ok(authority) => match authority.member_handoff() {
Ok(handoff) => Some(handoff.clone()),
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: generated wire authority did not include member handoff"
);
None
}
},
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-submit DSL wire for generated trust authority"
);
None
}
}
} else {
None
};
if removed_local_trust {
let peer_key = Self::trusted_peer_removal_key(peer_spec);
if let Err(err) = self.apply_trusted_peer_add(
local_comms.as_ref(),
peer_spec.clone(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"unwire rollback has no generated local trust add authority"
.to_string(),
)
})
.and_then(|handoff| {
handoff.add_authority_for(
peer_member_identity,
&peer_key,
&self.dsl_authority,
)
})
{
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to build generated local trust add authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-install local trust"
);
}
}
if removed_peer_trust {
let local_key = Self::trusted_peer_removal_key(local_spec);
if let Err(err) = self.apply_trusted_peer_add(
peer_comms.as_ref(),
local_spec.clone(),
match rollback_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(
"unwire rollback has no generated peer trust add authority".to_string(),
)
})
.and_then(|handoff| {
handoff.add_authority_for(
local_member_identity,
&local_key,
&self.dsl_authority,
)
})
{
Ok(authority) => authority,
Err(err) => {
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to build generated peer trust add authority"
);
return;
}
},
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to re-install peer trust"
);
}
}
if sent_unwired_from_local {
if let Err(err) = self
.notify_peer_added(local_comms, peer_spec, peer_member_identity, peer_entry)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to send compensating peer_added from local side"
);
}
}
if sent_unwired_from_peer {
if let Err(err) = self
.notify_peer_added(peer_comms, local_spec, local_member_identity, local_entry)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
%err,
"unwire rollback: failed to send compensating peer_added from peer side"
);
}
}
}
fn trusted_peer_descriptor_from_external_binding(
binding: super::handle::ExternalPeerBindingSpec,
) -> Result<TrustedPeerDescriptor, MobError> {
let name = PeerName::new(binding.name.clone()).map_err(|error| {
MobError::WiringError(format!(
"external binding has invalid peer name '{}': {error}",
binding.name
))
})?;
let resolved = binding
.identity
.resolve()
.map_err(|error| MobError::WiringError(error.to_string()))?;
let _address = PeerAddress::parse(&binding.address).map_err(|error| {
MobError::WiringError(format!(
"external binding has invalid peer address '{}': {error}",
binding.address
))
})?;
TrustedPeerDescriptor::unsigned_with_pubkey(
name.as_str().to_string(),
resolved.peer_id.to_string(),
resolved.pubkey,
binding.address,
)
.map_err(|error| MobError::WiringError(format!("external binding is invalid: {error}")))
}
async fn handle_wire_external(
&mut self,
local: AgentIdentity,
spec: TrustedPeerDescriptor,
) -> Result<(), MobError> {
TrustedPeerDescriptor::validate_pubkey_for_peer_id(spec.peer_id, &spec.pubkey).map_err(
|error| MobError::WiringError(format!("external peer descriptor is invalid: {error}")),
)?;
let local_identity = AgentIdentity::from(local.as_str());
let external_identity = AgentIdentity::from(spec.name.as_str());
if local_identity == external_identity {
return Err(MobError::WiringError(format!(
"wire requires distinct members (got '{local}')"
)));
}
let edge = Self::external_peer_edge(&local_identity, &spec);
let key = Self::external_peer_key_for_edge(&edge);
self.probe_command_admission(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
MobState::Running,
"wire_external_peer_command_admission",
)?;
let member_ref = {
let roster = self.roster.read().await;
let entry = roster
.get(&local)
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?;
entry.member_ref.clone()
};
let comms = self.provisioner_comms(&member_ref).await.ok_or_else(|| {
MobError::WiringError(format!(
"wire requires comms runtime for '{local}' (external peer wire)"
))
})?;
let authority = self.apply_wire_external_peer_idempotent(&key, &edge)?;
let removal_key = Self::trusted_peer_removal_key(&spec);
if authority.is_repair() {
self.apply_trusted_peer_add(
comms.as_ref(),
spec.clone(),
authority.external_authority()?.clone(),
)
.await?;
return Ok(());
}
let dsl_added = authority.dsl_added();
if let Err(error) = self
.apply_trusted_peer_add(
comms.as_ref(),
spec.clone(),
authority.external_authority()?.clone(),
)
.await
{
self.rollback_external_wire_dsl(&key, &edge, dsl_added)
.await;
return Err(MobError::from(error));
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerWired {
local: local_identity,
spec: spec.clone(),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_err) => {
let rollback_handoff = if dsl_added {
match self.apply_unwire_external_peer_idempotent(&key, &edge) {
Ok(Some(handoff)) => Some(handoff),
Ok(None) => None,
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
%error,
"failed to obtain generated external unwiring authority after event append failure"
);
None
}
}
} else {
None
};
if let Some(rollback_handoff) = rollback_handoff {
if let Err(rollback_err) = self
.apply_trusted_peer_remove(
comms.as_ref(),
removal_key.clone(),
rollback_handoff,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external trust install after event append failure"
);
}
} else {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
"external trust install rollback skipped without generated unwiring authority"
);
}
return Err(MobError::from(append_err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
async fn rollback_external_wire_dsl(
&mut self,
key: &mob_dsl::ExternalPeerKey,
edge: &mob_dsl::ExternalPeerEdge,
dsl_added: bool,
) {
if dsl_added
&& let Err(error) = self.apply_dsl_input(
mob_dsl::MobMachineInput::UnwireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"external_wire_rollback",
)
{
tracing::warn!(
mob_id = %self.definition.id,
%error,
"external wire rollback: failed to revert DSL wire"
);
}
}
async fn handle_unwire_external(
&mut self,
local: AgentIdentity,
peer_name: meerkat_core::comms::PeerName,
stale_cleanup_spec: Option<TrustedPeerDescriptor>,
) -> Result<(), MobError> {
let local_identity = AgentIdentity::from(local.as_str());
let member_ref = {
let roster = self.roster.read().await;
let entry = roster
.get(&local)
.ok_or_else(|| MobError::MemberNotFound(local.clone()))?;
entry.member_ref.clone()
};
let authority_edge = self
.external_peer_edge_for_name(&local_identity, &peer_name)
.or_else(|| {
stale_cleanup_spec
.as_ref()
.map(|spec| Self::external_peer_edge(&local_identity, spec))
});
let prior_spec = authority_edge
.as_ref()
.and_then(|edge| {
self.external_peer_edge_for_name(&local_identity, &peer_name)
.filter(|machine_edge| machine_edge == edge)
})
.map(|edge| Self::trusted_peer_descriptor_from_machine_endpoint(&edge.endpoint))
.transpose()?;
let Some(prior_spec) = prior_spec else {
let unwire_handoff = match authority_edge.as_ref() {
Some(edge) => {
let key = Self::external_peer_key_for_edge(edge);
self.apply_unwire_external_peer_idempotent(&key, edge)?
}
None => None,
};
let dsl_removed = unwire_handoff.is_some();
if let Some(spec) = stale_cleanup_spec
&& let Some(comms) = self.provisioner_comms(&member_ref).await
{
let removal_key = Self::trusted_peer_removal_key(&spec);
let has_stale_trust = comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == removal_key);
if !dsl_removed {
if has_stale_trust {
return Err(MobError::WiringError(format!(
"external unwire for '{local}' -> '{peer_name}' requires MobMachine external peer authority"
)));
}
return Ok(());
}
if let Err(error) = self.apply_trusted_peer_remove(
comms.as_ref(),
removal_key.clone(),
unwire_handoff
.as_ref()
.ok_or_else(|| {
MobError::WiringError(format!(
"external stale cleanup for '{local}' -> '{peer_name}' has no generated unwiring handoff"
))
})?
.clone(),
)
.await
{
if dsl_removed && let Some(edge) = authority_edge.as_ref() {
let key = Self::external_peer_key_for_edge(edge);
if let Err(rollback_err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::WireExternalPeer {
key,
edge: edge.clone(),
},
"external_unwire_stale_cleanup_rollback",
) {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external DSL unwire after stale cleanup failure"
);
}
}
return Err(MobError::from(error));
}
}
return Ok(());
};
let edge = authority_edge
.unwrap_or_else(|| Self::external_peer_edge(&local_identity, &prior_spec));
let key = Self::external_peer_key_for_edge(&edge);
let comms = self.provisioner_comms(&member_ref).await.ok_or_else(|| {
MobError::WiringError(format!(
"unwire requires comms runtime for '{local}' (external peer unwire)"
))
})?;
let Some(unwire_handoff) = self.apply_unwire_external_peer_idempotent(&key, &edge)? else {
return Err(MobError::WiringError(format!(
"external unwire for '{local}' -> '{peer_name}' was not authorized by MobMachine"
)));
};
let dsl_removed = true;
let prior_removal_key = Self::trusted_peer_removal_key(&prior_spec);
if let Err(error) = self
.apply_trusted_peer_remove(comms.as_ref(), prior_removal_key.clone(), unwire_handoff)
.await
{
if dsl_removed
&& let Err(rollback_err) = self.apply_dsl_input(
mob_dsl::MobMachineInput::WireExternalPeer {
key: key.clone(),
edge: edge.clone(),
},
"external_unwire_trust_remove_rollback",
)
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external DSL unwire after trust removal failure"
);
}
return Err(MobError::from(error));
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerUnwired {
local: local_identity,
peer_name: peer_name.clone(),
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_err) => {
if dsl_removed {
match self.apply_wire_external_peer_idempotent(&key, &edge) {
Ok(rollback_authority) => {
let rollback_authority =
rollback_authority.external_authority()?.clone();
if let Err(rollback_err) = self
.apply_trusted_peer_add(
comms.as_ref(),
prior_spec.clone(),
rollback_authority,
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to rollback external trust removal after event append failure"
);
}
}
Err(rollback_err) => {
tracing::warn!(
mob_id = %self.definition.id,
local = %local,
error = %rollback_err,
"failed to obtain generated external wiring authority after event append failure"
);
}
}
}
return Err(MobError::from(append_err));
}
};
self.roster.write().await.apply_event(&stored);
Ok(())
}
async fn handle_retire(&mut self, agent_identity: AgentIdentity) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_retire preflight")?;
self.handle_retire_inner(&agent_identity, false, false)
.await?;
self.ensure_pending_spawn_alignment("handle_retire completion")
}
async fn retirement_archive_already_complete(
&self,
session_id: &SessionId,
) -> Result<bool, MobError> {
if self.session_service.has_live_session(session_id).await? {
return Ok(false);
}
if self
.session_service
.load_persisted_session(session_id)
.await?
.is_some()
{
return Ok(false);
}
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = self.runtime_adapter.as_ref()
&& adapter
.archive_runtime_residue_present(session_id)
.await
.map_err(|error| {
MobError::Internal(format!(
"pending retirement runtime-residue observation failed for '{session_id}': {error}"
))
})?
{
return Ok(false);
}
self.session_service
.session_known_to_archive_authority(session_id)
.await
.map_err(MobError::from)
}
async fn detach_session_ingress_for_mob_destroy(
&mut self,
session_id: &SessionId,
obligation: MobDestroyingSessionIngressObligation,
) -> Result<(), MobError> {
use crate::generated::protocol_mob_destroying_session_ingress::{
submit_session_ingress_detach_failed_for_mob_destroy,
submit_session_ingress_detached_for_mob_destroy,
};
let detach_result = self.detach_runtime_session_ingress(session_id).await;
match detach_result {
Ok(()) => {
submit_session_ingress_detached_for_mob_destroy(&mut self.dsl_authority, obligation)
.map_err(|error| {
MobError::Internal(format!(
"MobMachine SessionIngressDetachedForMobDestroy transition rejected: {error}"
))
})?;
self.publish_machine_state_projection();
Ok(())
}
Err(error) => {
let reason = error.to_string();
let _ = submit_session_ingress_detach_failed_for_mob_destroy(
&mut self.dsl_authority,
obligation,
reason.clone(),
);
self.publish_machine_state_projection();
Err(MobError::Internal(format!(
"mob_destroying_session_ingress detach failed for {session_id}: {reason}"
)))
}
}
}
fn destroy_ingress_detach_session_id(
entry: &RosterEntry,
releasing: &mob_dsl::SessionId,
) -> Result<SessionId, MobError> {
let session_id = SessionId::parse(&releasing.0).map_err(|_| {
MobError::Internal(format!(
"destroy retire for member '{}' has invalid bridge session binding '{}'",
entry.agent_identity, releasing.0
))
})?;
let roster_session_id = entry.bridge_session_id().ok_or_else(|| {
MobError::Internal(format!(
"destroy retire for member '{}' has a pending machine detach but no roster bridge session",
entry.agent_identity
))
})?;
if roster_session_id != &session_id {
return Err(MobError::Internal(format!(
"destroy retire for member '{}' has mismatched machine/roster bridge sessions ('{}' != '{}')",
entry.agent_identity, session_id, roster_session_id
)));
}
Ok(session_id)
}
async fn detach_runtime_session_ingress(&self, session_id: &SessionId) -> Result<(), MobError> {
#[cfg(test)]
if let Ok(mut target) = FAIL_SESSION_INGRESS_DETACH_FOR_SESSION.lock()
&& target.as_ref() == Some(session_id)
{
target.take();
return Err(MobError::Internal(format!(
"fault-injected session-ingress detach failure for {session_id}"
)));
}
#[cfg(feature = "runtime-adapter")]
if let Some(adapter) = &self.runtime_adapter {
match adapter
.update_peer_ingress_context(session_id, false, None)
.await
{
Ok(_) => {}
Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(err) => {
return Err(MobError::Internal(format!(
"failed to detach peer ingress for session {session_id}: {err}"
)));
}
}
let owner = adapter.peer_ingress_owner(session_id).await;
if !matches!(owner, meerkat_runtime::PeerIngressOwner::Unattached) {
return Err(MobError::Internal(format!(
"peer ingress owner remained attached after detach: {owner:?}"
)));
}
}
let _ = session_id;
Ok(())
}
async fn admit_member_retire_for_destroy(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await?
{
return Ok(());
}
let agent_identity = &entry.agent_identity;
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let bound_session_id = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let pending_retire_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned();
let session_id_for_route = bound_session_id.or_else(|| pending_retire_session.clone());
if self
.dsl_authority
.state()
.pending_session_ingress_detach_runtime_ids
.contains(&dsl_runtime_id)
{
let transition = self.apply_dsl_input_collect_transition(
mob_dsl::MobMachineInput::RequestPendingSessionIngressDetachForMobDestroy {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: dsl_runtime_id.clone(),
},
"destroy_request_pending_session_ingress_detach",
)?;
let obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&transition,
);
let obligation = match obligations.as_slice() {
[obligation] => obligation.clone(),
[] => {
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' produced no generated session ingress obligation"
)));
}
_ => {
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' produced multiple generated session ingress obligations"
)));
}
};
if obligation.mob_id() != &mob_dsl::MobId::from_domain(&self.definition.id)
|| obligation.agent_runtime_id() != &dsl_runtime_id
{
return Err(MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' generated an obligation for a different mob/runtime"
)));
}
let pending_retire_session = pending_retire_session.as_ref().ok_or_else(|| {
MobError::Internal(format!(
"MobMachine pending destroy detach for member '{agent_identity}' has no runtime-retirement session correlation"
))
})?;
let detach_session_id =
Self::destroy_ingress_detach_session_id(entry, pending_retire_session)?;
self.detach_session_ingress_for_mob_destroy(&detach_session_id, obligation)
.await?;
self.ensure_destroy_runtime_retire_route_after_detach(
&dsl_identity,
&dsl_runtime_id,
pending_retire_session,
&detach_session_id,
"destroy_retry_runtime_retire_after_detach_ack",
)
.await?;
return self.flush_routed_effects().await;
}
let already_retiring = matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyMemberRetire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
session_id: session_id_for_route.clone(),
},
"destroy_mark_member_retiring",
)?;
if !already_retiring {
let journal_kind = if session_id_for_route.is_some() {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding
} else {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
};
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"destroy_mark_member_retiring",
)?;
self.append_retirement_started_event_for_entry(
entry,
journal_kind,
session_id_for_route.clone(),
)
.await?;
}
let obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared.transition,
);
self.commit_prepared_dsl_transition(prepared)?;
match (session_id_for_route.as_ref(), obligations.as_slice()) {
(Some(session_id), [obligation]) => {
if obligation.mob_id() != &mob_dsl::MobId::from_domain(&self.definition.id)
|| obligation.agent_runtime_id() != &dsl_runtime_id
{
return Err(MobError::Internal(format!(
"MobMachine fresh destroy detach for member '{agent_identity}' generated an obligation for a different mob/runtime"
)));
}
let detach_session_id = Self::destroy_ingress_detach_session_id(entry, session_id)?;
self.detach_session_ingress_for_mob_destroy(&detach_session_id, obligation.clone())
.await?;
self.ensure_destroy_runtime_retire_route_after_detach(
&dsl_identity,
&dsl_runtime_id,
session_id,
&detach_session_id,
"destroy_runtime_retire_after_fresh_detach_ack",
)
.await?;
}
(None, []) => {}
(Some(_), []) => {
return Err(MobError::Internal(format!(
"MobMachine session-bound destroy retire for member '{agent_identity}' produced no generated session ingress obligation"
)));
}
(None, _) => {
return Err(MobError::Internal(format!(
"MobMachine peer-only destroy retire for member '{agent_identity}' unexpectedly produced a session ingress obligation"
)));
}
(Some(_), _) => {
return Err(MobError::Internal(format!(
"MobMachine session-bound destroy retire for member '{agent_identity}' produced multiple session ingress obligations"
)));
}
}
self.flush_routed_effects().await
}
async fn handle_retire_inner(
&mut self,
agent_identity: &AgentIdentity,
bulk: bool,
preserve_realtime_binding: bool,
) -> Result<(), MobError> {
tracing::debug!(
agent_identity = %agent_identity,
bulk,
preserve_realtime_binding,
"MobActor::handle_retire_inner start"
);
let entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let Some(entry) = entry else {
self.apply_command_admission(
mob_dsl::MobMachineInput::RetireAbsent {
agent_identity: mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(
agent_identity.as_str(),
)),
},
MobState::Running,
"handle_retire_inner_absent",
)?;
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"retire requested for unknown meerkat id; MobMachine accepted RetireAbsent"
);
return Ok(());
};
tracing::debug!(
agent_identity = %agent_identity,
member_ref = ?entry.member_ref,
runtime_id = %entry.agent_runtime_id,
"MobActor::handle_retire_inner loaded roster entry"
);
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let detach_session_id = if preserve_realtime_binding {
None
} else {
self.machine_bridge_session_id_for_identity(&domain_identity)
};
let releasing = if preserve_realtime_binding {
None
} else {
self.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned()
};
let session_id_for_route = releasing.clone().or_else(|| {
self.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned()
});
let retirement_started_releasing = releasing.clone();
let retirement_started_journal_kind = match (
retirement_started_releasing.is_some(),
session_id_for_route.is_some(),
) {
(true, true) => mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing,
(false, true) => {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding
}
(false, false) => mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly,
(true, false) => {
return Err(MobError::Internal(format!(
"retirement for '{agent_identity}' releases a session without a routed session target"
)));
}
};
let retire_input = mob_dsl::MobMachineInput::Retire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
agent_identity: dsl_identity,
generation: mob_dsl::Generation::from_domain(entry.generation),
releasing,
session_id: session_id_for_route.clone(),
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner checking prior retire event"
);
let retire_event_already_present = self
.retire_event_exists(agent_identity, entry.generation)
.await?;
tracing::debug!(
agent_identity = %agent_identity,
retire_event_already_present,
"MobActor::handle_retire_inner checked prior retire event"
);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner planning trust cleanup"
);
let trust_cleanup_plan = self
.member_retire_trust_cleanup_plan(agent_identity, &entry)
.await?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner planned trust cleanup"
);
let preserve_topology_for_respawn = preserve_realtime_binding;
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let cleanup_retry = retire_event_already_present
|| matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
if cleanup_retry {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
"retrying member retire cleanup from retained roster anchor"
);
let mut retire_route_retried_after_detach = false;
if self
.dsl_authority
.state()
.pending_session_ingress_detach_runtime_ids
.contains(&dsl_runtime_id)
{
self.admit_member_retire_for_destroy(&entry).await?;
retire_route_retried_after_detach = true;
}
if !retire_route_retried_after_detach
&& self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.contains_key(&dsl_runtime_id)
{
let pending_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.ok_or_else(|| {
MobError::Internal(format!(
"pending runtime retirement for '{agent_identity}' disappeared during retry observation"
))
})?;
let pending_session = SessionId::parse(&pending_session.0).map_err(|error| {
MobError::Internal(format!(
"pending runtime retirement for '{agent_identity}' has invalid session id '{}': {error}",
pending_session.0
))
})?;
if self
.retirement_archive_already_complete(&pending_session)
.await?
{
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
bridge_session_id = %pending_session,
"pending retirement route already discharged by archived session authority"
);
} else {
self.apply_dsl_input(
mob_dsl::MobMachineInput::RetryRuntimeRetire {
agent_identity: mob_dsl::AgentIdentity::from_domain(agent_identity),
agent_runtime_id: dsl_runtime_id.clone(),
},
"retry_runtime_retire_after_consumer_refusal",
)?;
self.flush_routed_effects().await?;
}
}
} else {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner preparing Retire transition"
);
let prepared_retire = self
.prepare_dsl_input_transition(retire_input, "handle_retire_inner_mark_retiring")?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner prepared Retire transition"
);
Self::require_member_lifecycle_journal_effect(
&prepared_retire.transition,
retirement_started_journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"handle_retire_inner_mark_retiring",
)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner validated Retire transition journal"
);
if !retire_event_already_present
&& let Err(error) = self
.append_retirement_started_event_for_entry(
&entry,
retirement_started_journal_kind,
session_id_for_route.clone(),
)
.await
{
return Err(error);
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner appended retirement-start event"
);
let mut detach_obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared_retire.transition,
);
self.commit_prepared_dsl_transition(prepared_retire)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner committed Retire transition"
);
if let (Some(obligation), Some(session_id)) =
(detach_obligations.pop(), detach_session_id)
{
self.detach_session_ingress_for_mob_destroy(&session_id, obligation)
.await?;
}
if let Err(error) = self.flush_routed_effects().await {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
%error,
"pre-disposal routed-effect flush failed; retaining member for retry"
);
return Err(error);
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner flushed routed effects"
);
}
self.cancel_peer_deliveries_for_member(agent_identity, "member retirement admitted")
.await;
let canceled = self
.cancel_pending_spawns_for_member(
agent_identity,
"member retirement superseded pending spawn",
)
.await?;
if canceled > 0 {
tracing::info!(
agent_identity = %agent_identity,
canceled,
"retirement canceled pending spawn lineage after durable admission"
);
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner building disposal context"
);
let ctx = self
.disposal_context_from_entry(
agent_identity,
&entry,
trust_cleanup_plan,
preserve_topology_for_respawn,
)
.await;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner built disposal context"
);
let mut policy: Box<dyn ErrorPolicy> = if bulk {
Box::new(BulkBestEffort)
} else {
Box::new(WarnAndContinue)
};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner disposing member"
);
let mut report = self.dispose_member(&ctx, policy.as_mut()).await;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_retire_inner disposed member"
);
if let Some((step, error)) = report.aborted_at.take() {
if step == DisposalStep::CleanupMachineTopology {
return Err(error);
}
report.aborted_at = Some((step, error));
}
if let Some((step, MobError::RetirementTopologyIncomplete(reason))) = &report.aborted_at
&& *step == DisposalStep::NotifyPeers
{
return Err(MobError::RetirementTopologyIncomplete(reason.clone()));
}
if let Some((_, error)) = report
.skipped
.iter()
.find(|(step, _)| *step == DisposalStep::ArchiveSession)
{
return Err(MobError::Internal(format!(
"disposal completed but ArchiveSession failed: {error}"
)));
}
if let Some((step, error)) = &report.aborted_at
&& *step == DisposalStep::ArchiveSession
{
return Err(MobError::Internal(format!(
"disposal aborted at ArchiveSession: {error}"
)));
}
if let Some(binding) = Self::runtime_binding_for_entry(&entry) {
if !self.remote_runtime_retired_for_entry(&entry) {
return Err(MobError::Internal(format!(
"peer-only retirement for '{}' reached supervisor revoke without a durable remote-runtime checkpoint",
entry.agent_identity
)));
}
self.revoke_supervisor_for_retiring_entry(
&entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to revoke supervisor authority from retiring peer-only member '{}': {error}",
entry.agent_identity
))
})?;
}
self.delete_external_binding_overlay_for_member(&entry.agent_identity, entry.generation)
.await?;
self.observe_member_retirement_archived(&ctx).await?;
self.dispose_remove_from_roster(&ctx).await;
Ok(())
}
async fn handle_respawn(
&mut self,
agent_identity: AgentIdentity,
initial_message: Option<ContentInput>,
) -> Result<super::handle::MemberRespawnReceipt, super::handle::MobRespawnError> {
use super::handle::{MemberRespawnReceipt, MobRespawnError};
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn start"
);
self.ensure_pending_spawn_alignment("handle_respawn preflight")
.map_err(MobRespawnError::from)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn preflight aligned"
);
let snapshot = {
let roster = self.roster.read().await;
let entry = roster
.get(&agent_identity)
.cloned()
.ok_or_else(|| MobError::MemberNotFound(agent_identity.clone()))?;
let restore_wiring = self
.machine_restore_wiring_plan(&agent_identity)
.map_err(MobRespawnError::from)?;
let binding = match &entry.member_ref {
crate::event::MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
..
} => crate::RuntimeBinding::External {
peer_id: peer_id.clone(),
address: address.clone(),
bootstrap_token: bootstrap_token.clone(),
pubkey: *pubkey,
},
crate::event::MemberRef::Session { .. } => crate::RuntimeBinding::Session,
};
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let cleanup_retry = matches!(
self.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id),
Some(mob_dsl::MobMemberState::Retiring)
);
RespawnSnapshot {
profile_name: entry.role.clone(),
runtime_mode: entry.runtime_mode,
labels: entry.labels.clone(),
old_runtime_id: entry.agent_runtime_id.clone(),
old_fence_token: entry.fence_token,
restore_wiring,
binding,
effective_profile_override: entry.effective_profile_override,
cleanup_retry,
}
};
tracing::debug!(
agent_identity = %agent_identity,
old_runtime_id = %snapshot.old_runtime_id,
runtime_mode = ?snapshot.runtime_mode,
cleanup_retry = snapshot.cleanup_retry,
"MobActor::handle_respawn captured snapshot"
);
let original_identity = AgentIdentity::from(agent_identity.as_str());
let mut replacement_spec = super::handle::SpawnMemberSpec::new(
snapshot.profile_name.clone(),
original_identity.clone(),
);
replacement_spec.initial_message = initial_message;
replacement_spec.runtime_mode = Some(snapshot.runtime_mode);
replacement_spec.binding = Some(snapshot.binding.clone());
replacement_spec.labels = Some(snapshot.labels.clone());
replacement_spec.override_profile = snapshot.effective_profile_override.clone();
self.customize_spawn_spec(
super::handle::SpawnSource::Respawn,
None,
&mut replacement_spec,
)
.map_err(MobRespawnError::from)?;
if replacement_spec.identity != original_identity {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn identity from '{original_identity}' to '{}'",
replacement_spec.identity
))));
}
if replacement_spec.role_name != snapshot.profile_name {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn profile for '{original_identity}' from '{}' to '{}'",
snapshot.profile_name, replacement_spec.role_name
))));
}
if replacement_spec.binding.as_ref() != Some(&snapshot.binding) {
return Err(MobRespawnError::from(MobError::Internal(format!(
"spawn customizer cannot change respawn runtime binding for '{original_identity}'"
))));
}
let super::handle::SpawnMemberSpec {
role_name: _,
identity: _,
initial_message: replacement_initial_message,
runtime_mode: _,
backend: _,
binding: _,
context: replacement_context,
labels: replacement_labels,
launch_mode: _,
tool_access_policy: replacement_tool_access_policy,
budget_split_policy: _budget_split_policy,
budget_limits: replacement_budget_limits,
auto_wire_parent: _,
additional_instructions: replacement_additional_instructions,
shell_env: replacement_shell_env,
inherited_tool_filter: replacement_inherited_tool_filter,
override_profile: replacement_profile_override,
auth_binding: replacement_auth_binding,
external_tools: replacement_external_tools,
system_prompt_override: replacement_system_prompt_override,
continuity_intent: replacement_continuity_intent,
} = replacement_spec;
let replacement_labels = replacement_labels.unwrap_or_default();
if !snapshot.cleanup_retry {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn previewing Respawn admission"
);
self.preview_dsl_input(
mob_dsl::MobMachineInput::Respawn {
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(
&snapshot.old_runtime_id,
),
},
"handle_respawn_admission",
)
.map_err(|_| MobRespawnError::from(self.invalid_transition_to(MobState::Running)))?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn previewed Respawn admission"
);
}
#[cfg(feature = "runtime-adapter")]
let respawn_peer_only_owner_context =
if matches!(snapshot.binding, crate::RuntimeBinding::External { .. }) {
Some(
self.generated_peer_only_operation_owner_context(
&original_identity,
&snapshot.binding,
"respawn_peer_only_operation_owner",
)
.await
.map_err(MobRespawnError::from)?,
)
} else {
None
};
#[cfg(not(feature = "runtime-adapter"))]
let respawn_peer_only_owner_context: Option<(
SessionId,
Arc<dyn meerkat_core::ops_lifecycle::OpsLifecycleRegistry>,
)> = None;
let replacement_generation = self
.compute_respawn_generation(&agent_identity)
.map_err(MobRespawnError::from)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn retiring previous member"
);
if let Err(error) = self.handle_retire_inner(&agent_identity, false, true).await {
let roster_still_contains_member = {
let roster = self.roster.read().await;
roster.get(&agent_identity).is_some()
};
if roster_still_contains_member {
return Err(MobRespawnError::from(error));
}
let mut cleanup_report = super::handle::PreviousMemberCleanupReport {
identity: AgentIdentity::from(agent_identity.as_str()),
agent_runtime_id: snapshot.old_runtime_id.clone(),
fence_token: snapshot.old_fence_token,
retire_attempted: true,
retire_error: Some(error.to_string()),
confirmatory_observation_attempted: false,
confirmatory_observation: None,
destroy_attempted: false,
destroy_error: None,
};
match &snapshot.binding {
crate::RuntimeBinding::External { .. } => {
cleanup_report.confirmatory_observation_attempted = true;
match self
.observe_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_millis(750),
)
.await
{
Ok(observation) => {
cleanup_report.confirmatory_observation =
Some(format!("state={}", observation.state));
let observation_is_terminal = self
.observation_is_terminal(&observation)
.map_err(MobRespawnError::from)?;
if !observation_is_terminal {
cleanup_report.destroy_attempted = true;
if let Err(destroy_error) = self
.destroy_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_secs(5),
)
.await
{
cleanup_report.destroy_error = Some(destroy_error.to_string());
return Err(MobRespawnError::PreviousMemberCleanupAmbiguous {
report: cleanup_report,
});
}
}
}
Err(observe_error) => {
cleanup_report.confirmatory_observation =
Some(observe_error.to_string());
cleanup_report.destroy_attempted = true;
if let Err(destroy_error) = self
.destroy_peer_only_binding(
&snapshot.binding,
std::time::Duration::from_secs(5),
)
.await
{
cleanup_report.destroy_error = Some(destroy_error.to_string());
return Err(MobRespawnError::PreviousMemberCleanupAmbiguous {
report: cleanup_report,
});
}
}
}
}
crate::RuntimeBinding::Session => {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %agent_identity,
retire_error = %error,
"respawn proceeding after session-bound retire removed the stale roster anchor"
);
}
}
}
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn retired previous member"
);
self.clear_kickoff_state(&agent_identity).await;
let (prompt, initial_turn_prompt) = match replacement_initial_message {
Some(message) => {
let prompt = message;
(prompt.clone(), Some(prompt))
}
None => (
ContentInput::from(
self.fallback_spawn_prompt(&snapshot.profile_name, &agent_identity),
),
None,
),
};
let mut profile = if let Some(p) = replacement_profile_override.clone() {
p
} else {
self.definition
.resolve_profile(&snapshot.profile_name, self.realm_profile_store.as_ref())
.await?
};
if replacement_inherited_tool_filter.is_some() && replacement_profile_override.is_none() {
build::open_profile_tool_categories_for_inherited_filter(&mut profile);
}
let replacement_authorized_profile_material = self
.authorize_spawn_profile_material(
&agent_identity,
&snapshot.profile_name,
&profile,
"respawn_profile_authority",
)
.map_err(MobRespawnError::from)?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn authorized replacement profile material"
);
let external_tools =
self.external_tools_for_profile(&profile, replacement_external_tools.clone())?;
let mut config = build::build_agent_config(build::BuildAgentConfigParams {
mob_id: &self.definition.id,
profile_name: &snapshot.profile_name,
agent_identity: &agent_identity,
profile: &profile,
definition: &self.definition,
external_tools,
context: replacement_context,
labels: Some(replacement_labels.clone()),
additional_instructions: replacement_additional_instructions,
shell_env: replacement_shell_env,
mob_tool_authority_context: None,
inherited_tool_filter: replacement_inherited_tool_filter,
tool_access_policy: replacement_tool_access_policy,
system_prompt_override: replacement_system_prompt_override,
})
.await?;
config.keep_alive = snapshot.runtime_mode == crate::MobRuntimeMode::AutonomousHost;
if let Some(ref client) = self.default_llm_client {
config.llm_client_override = Some(client.clone());
}
if let Some(ref cref) = replacement_auth_binding {
config.auth_binding = Some(cref.clone());
}
let req = build::to_create_session_request(&config, prompt.clone());
let req = with_spawn_budget_limits(req, replacement_budget_limits);
let peer_name = render_member_comms_name(
self.definition.id.as_str(),
snapshot.profile_name.as_str(),
agent_identity.as_str(),
)?;
let mut provision_request = ProvisionMemberRequest {
create_session: req,
binding: snapshot.binding.clone(),
peer_name,
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: None,
};
if let Some((owner_bridge_session_id, ops_registry)) = respawn_peer_only_owner_context {
provision_request.owner_bridge_session_id = Some(owner_bridge_session_id);
provision_request.ops_registry = Some(ops_registry);
}
let admitted_bridge_session_id =
admit_bridge_session_for_spawn(&mut provision_request.create_session);
tracing::debug!(
agent_identity = %agent_identity,
bridge_session_id = %admitted_bridge_session_id,
"MobActor::handle_respawn admitted replacement bridge session"
);
let respawn_spawn_ticket = self.next_spawn_ticket;
self.next_spawn_ticket = self.next_spawn_ticket.wrapping_add(1);
let generated_self_owned_operation_owner = self
.stage_orchestrator_spawn(&agent_identity, &admitted_bridge_session_id)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to stage respawn replacement spawn: {error}"),
})?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn staged replacement spawn"
);
Self::apply_generated_self_owned_operation_owner(
&mut provision_request,
&generated_self_owned_operation_owner,
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to authorize respawn replacement operation owner: {error}"),
})?;
let respawn_pending_trust_peer_id = match &snapshot.binding {
crate::RuntimeBinding::External { peer_id, .. } => Some(peer_id.clone()),
crate::RuntimeBinding::Session => None,
};
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
self.record_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn external provision",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
let (respawn_inline_reply_tx, _respawn_inline_reply_rx) = oneshot::channel();
let respawn_pending = PendingSpawn {
profile_name: snapshot.profile_name.clone(),
agent_identity: agent_identity.clone(),
admitted_bridge_session_id,
prompt: prompt.clone(),
initial_turn_prompt: initial_turn_prompt.clone(),
runtime_mode: snapshot.runtime_mode,
labels: replacement_labels.clone(),
owner_bridge_session_id: None,
auto_wire_parent: false,
restore_wiring: (!snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty())
.then_some(snapshot.restore_wiring.clone()),
effective_profile_override: replacement_profile_override.clone(),
per_spawn_external_tools: replacement_external_tools.clone(),
authorized_profile_material: replacement_authorized_profile_material.clone(),
continuity_intent: replacement_continuity_intent.clone(),
progress: Arc::new(std::sync::Mutex::new(PendingSpawnProgress::default())),
pending_recipient_trust_peer_id: None,
reply_tx: respawn_inline_reply_tx,
};
let respawn_inline_task = tokio::spawn(async {
std::future::pending::<()>().await;
});
let spawn_started = generated_self_owned_operation_owner
.start(&respawn_pending)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!("failed to start respawn replacement spawn: {error}"),
})?;
self.insert_pending_spawn(
respawn_spawn_ticket,
respawn_pending,
respawn_inline_task,
spawn_started,
);
if let Err(error) = self.ensure_pending_spawn_alignment("handle_respawn staged replacement")
{
tracing::error!(
agent_identity = %agent_identity,
error = %error,
"pending spawn alignment violated while staging respawn replacement"
);
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn staging failed",
)
.map_err(|rollback_error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: rollback_error.to_string(),
})?;
}
if let Err(cleanup_error) = self
.fail_all_pending_spawns(
"pending spawn alignment violated while staging respawn replacement",
)
.await
{
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: cleanup_error.to_string(),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
});
}
let replacement_result: Result<super::handle::MemberSpawnReceipt, MobRespawnError> = Box::pin(async {
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn provisioning replacement member"
);
let spawn_receipt = self
.provisioner
.provision_member(provision_request)
.await
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
if let Some(peer_id) = respawn_pending_trust_peer_id.as_deref() {
self.resolve_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn provision confirmed",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
tracing::debug!(
agent_identity = %agent_identity,
member_ref = ?spawn_receipt.member_ref,
"MobActor::handle_respawn provisioned replacement member"
);
if snapshot.runtime_mode == crate::MobRuntimeMode::AutonomousHost
&& let Err(capability_error) =
Self::ensure_autonomous_dispatch_capability_for_provisioner(
&self.provisioner,
&agent_identity,
&spawn_receipt.member_ref,
)
.await
{
if let Err(retire_error) = self
.provisioner
.retire_member(&spawn_receipt.member_ref)
.await
{
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!(
"autonomous capability check failed: {capability_error}; cleanup retire failed: {retire_error}"
),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: capability_error.to_string(),
});
}
let provision = PendingProvision::new(
spawn_receipt.member_ref.clone(),
agent_identity.clone(),
self.provisioner.clone(),
);
if let Err(error) = self.require_member_operation_eligible() {
if let Err(retire_error) = provision.rollback().await {
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: format!(
"mob state changed before respawn finalization: {error}; cleanup retire failed: {retire_error}"
),
});
}
return Err(MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
});
}
if !snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty()
{
tracing::info!(
agent_identity = %agent_identity,
local_peers = ?snapshot.restore_wiring.local_peers,
external_peers = ?snapshot.restore_wiring.external_peers,
"respawn: restoring peer wiring during replacement finalization"
);
}
let respawn_fence = self.issue_fence_token();
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn finalizing replacement spawn"
);
let finalized = Box::pin(self.finalize_spawn_from_pending(
&snapshot.profile_name,
&agent_identity,
replacement_generation,
respawn_fence,
snapshot.runtime_mode,
prompt,
initial_turn_prompt,
replacement_labels,
provision,
spawn_receipt.operation_id,
None,
false,
(!snapshot.restore_wiring.local_peers.is_empty()
|| !snapshot.restore_wiring.external_peers.is_empty())
.then_some(snapshot.restore_wiring.clone()),
replacement_profile_override,
replacement_external_tools,
replacement_authorized_profile_material,
replacement_continuity_intent,
))
.await
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn finalized replacement spawn"
);
tracing::debug!(
agent_identity = %agent_identity,
"MobActor::handle_respawn resolving topology restore"
);
let topology_restore = self
.resolve_respawn_topology_restore_result(
&agent_identity,
finalized.failed_restore_peer_ids,
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
tracing::debug!(
agent_identity = %agent_identity,
result = ?topology_restore.result,
"MobActor::handle_respawn resolved topology restore"
);
match topology_restore.result {
mob_dsl::RespawnTopologyRestoreResultKind::Completed => Ok(finalized.receipt),
mob_dsl::RespawnTopologyRestoreResultKind::TopologyRestoreFailed => {
Err(MobRespawnError::TopologyRestoreFailed {
receipt: super::handle::MemberRespawnReceipt::new(
AgentIdentity::from(agent_identity.as_str()),
crate::ids::AgentRuntimeId::new(
AgentIdentity::from(agent_identity.as_str()),
replacement_generation,
),
snapshot.old_fence_token,
respawn_fence,
),
failed_peer_ids: topology_restore.failed_peer_ids,
})
}
}
})
.await;
let (_respawn_pending, respawn_task) =
self.complete_pending_spawn_slot(respawn_spawn_ticket, "respawn replacement spawn");
if let Some(handle) = respawn_task {
handle.abort();
}
if replacement_result.is_err()
&& let Some(peer_id) = respawn_pending_trust_peer_id.as_deref()
{
self.rollback_pending_recipient_trust_obligation_for_peer_id(
peer_id,
"handle_respawn provision failed",
)
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
}
self.ensure_pending_spawn_alignment("handle_respawn completion")
.map_err(|error| MobRespawnError::SpawnAfterRetire {
identity: AgentIdentity::from(agent_identity.as_str()),
reason: error.to_string(),
})?;
let _replacement = replacement_result?;
Ok(MemberRespawnReceipt::new(
AgentIdentity::from(agent_identity.as_str()),
crate::ids::AgentRuntimeId::new(
AgentIdentity::from(agent_identity.as_str()),
replacement_generation,
),
snapshot.old_fence_token,
self.roster
.read()
.await
.get(&agent_identity)
.map(|entry| entry.fence_token)
.unwrap_or(snapshot.old_fence_token),
))
}
fn machine_wired_peer_identities_for(
&self,
identity: &AgentIdentity,
) -> BTreeSet<AgentIdentity> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
let state = self.dsl_authority.state();
let mut wired_to = self.machine_member_wired_peer_identities_for(identity);
wired_to.extend(
state
.external_peer_edges
.iter()
.filter(|edge| edge.local == dsl_identity)
.map(|edge| AgentIdentity::from(edge.endpoint.name.0.as_str())),
);
wired_to
}
fn machine_member_wired_peer_identities_for(
&self,
identity: &AgentIdentity,
) -> BTreeSet<AgentIdentity> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.wiring_edges
.iter()
.filter_map(|edge| {
if edge.a == dsl_identity {
Some(AgentIdentity::from(edge.b.0.as_str()))
} else if edge.b == dsl_identity {
Some(AgentIdentity::from(edge.a.0.as_str()))
} else {
None
}
})
.collect()
}
fn machine_external_peer_edges_for(
&self,
identity: &AgentIdentity,
) -> Vec<mob_dsl::ExternalPeerEdge> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(identity);
self.dsl_authority
.state()
.external_peer_edges
.iter()
.filter(|edge| edge.local == dsl_identity)
.cloned()
.collect()
}
async fn cleanup_retiring_external_peer_edges(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
let local = AgentIdentity::from(entry.agent_identity.as_str());
let external_edges = self.machine_external_peer_edges_for(&entry.agent_identity);
if external_edges.is_empty() {
return Ok(());
}
let comms = self.provisioner_comms(&entry.member_ref).await;
for edge in external_edges {
let peer_name = meerkat_core::comms::PeerName::new(edge.endpoint.name.0.clone())
.map_err(|error| {
MobError::WiringError(format!(
"retire external cleanup has invalid peer name '{}': {error}",
edge.endpoint.name.0
))
})?;
let prior_spec = Self::trusted_peer_descriptor_from_machine_endpoint(&edge.endpoint)?;
let key = Self::external_peer_key_for_edge(&edge);
match comms.as_ref() {
Some(comms) => {
let unwiring_authority =
self.apply_cleanup_retiring_external_peer(entry, &key, &edge)?;
let prior_removal_key = Self::trusted_peer_removal_key(&prior_spec);
if let Err(error) = self
.apply_trusted_peer_remove(
comms.as_ref(),
prior_removal_key,
unwiring_authority,
)
.await
{
self.apply_restore_retiring_external_peer(entry, &key, &edge)
.map_err(|rollback_error| {
MobError::WiringError(format!(
"retiring external trust removal failed: {error}; machine rollback failed: {rollback_error}"
))
})?;
return Err(MobError::from(error));
}
}
None => {
self.apply_cleanup_retiring_external_peer_observed_absent(entry, &key, &edge)?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %entry.agent_identity,
peer_id = %edge.endpoint.peer_id.0,
"retiring external cleanup observed no surviving local comms runtime; committed no-effect machine topology cleanup"
);
}
}
let event = NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::ExternalPeerUnwired {
local: local.clone(),
peer_name,
},
};
let stored = match self.events.append(event).await {
Ok(stored) => stored,
Err(append_error) => {
match comms.as_ref() {
Some(comms) => {
let rollback =
self.apply_restore_retiring_external_peer(entry, &key, &edge)?;
let rollback_authority = rollback.external_authority()?.clone();
self.apply_trusted_peer_add(
comms.as_ref(),
prior_spec,
rollback_authority,
)
.await
.map_err(|rollback_error| {
MobError::WiringError(format!(
"retiring external event append failed: {append_error}; trust rollback failed: {rollback_error}"
))
})?;
}
None => self.apply_restore_retiring_external_peer_observed_absent(
entry, &key, &edge,
)?,
}
return Err(MobError::from(append_error));
}
};
self.roster.write().await.apply_event(&stored);
}
Ok(())
}
async fn member_retire_trust_cleanup_plan(
&mut self,
agent_identity: &AgentIdentity,
entry: &RosterEntry,
) -> Result<RetireTrustCleanupPlan, MobError> {
let retiring_identity = AgentIdentity::from(agent_identity.as_str());
let machine_wired_peer_identities =
self.machine_member_wired_peer_identities_for(&retiring_identity);
if machine_wired_peer_identities.is_empty() {
return Ok(RetireTrustCleanupPlan::empty());
}
let dsl_identity = mob_dsl::AgentIdentity::from_domain(agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id);
let exact_runtime_is_retiring = self
.dsl_authority
.state()
.identity_to_runtime
.get(&dsl_identity)
== Some(&dsl_runtime_id)
&& self
.dsl_authority
.state()
.member_state_markers
.get(&dsl_runtime_id)
== Some(&mob_dsl::MobMemberState::Retiring);
let durable_retiring_spec = if exact_runtime_is_retiring {
self.machine_member_peer_spec_for(
&retiring_identity,
"retire trust authority durable retiring endpoint",
)?
} else {
None
};
let (retiring_spec, retiring_comms) = if let Some(durable_retiring_spec) =
durable_retiring_spec
{
let retiring_comms = match self
.resolve_wiring_endpoint(entry, "retire trust authority retiring member")
.await
{
Ok(WiringEndpoint::Local { comms, .. }) => Some(comms),
Ok(WiringEndpoint::PeerOnly { .. }) => {
Some(self.supervisor_bridge.runtime_core().await)
}
Err(error) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
error = %error,
"retire trust authority using durable endpoint without a live retiring comms runtime"
);
None
}
};
(durable_retiring_spec, retiring_comms)
} else {
match self
.resolve_wiring_endpoint(entry, "retire trust authority retiring member")
.await
{
Ok(WiringEndpoint::Local { spec, comms, .. }) => (spec, Some(comms)),
Ok(WiringEndpoint::PeerOnly { spec, .. }) => {
(spec, Some(self.supervisor_bridge.runtime_core().await))
}
Err(error) => {
let retained_spec = match self.machine_member_peer_spec_for(
&retiring_identity,
"retire trust authority retiring member",
)? {
Some(spec) => Some(spec),
None => match self.roster_member_peer_spec_for(
entry,
"retire trust authority retiring member",
)? {
Some(spec) => Some(spec),
None => {
self.retained_member_peer_spec_from_wired_peer_trust(
entry,
&machine_wired_peer_identities,
"retire trust authority retiring member",
)
.await?
}
},
};
match retained_spec {
Some(spec) => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
"retire trust authority using retained generated peer endpoint for member without live comms runtime"
);
(spec, None)
}
None => {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %retiring_identity,
error = %error,
"retire trust authority found no retained peer trust to clean up for member without live comms runtime"
);
return Ok(RetireTrustCleanupPlan {
retiring_comms: None,
retiring_spec: None,
machine_wired_peer_identities,
trust_unwire_authority_by_peer: BTreeMap::new(),
});
}
}
}
}
};
let retiring_peer_id = Self::trusted_peer_removal_key(&retiring_spec);
let mut authorities = BTreeMap::new();
for peer_identity in &machine_wired_peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let peer_spec = match self
.resolve_wiring_endpoint(&peer_entry, "member_retire_trust_authority peer")
.await
{
Ok(WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. }) => {
spec
}
Err(error) => {
let retained_spec = match self.machine_member_peer_spec_for(
peer_identity,
"member_retire_trust_authority peer",
)? {
Some(spec) => Some(spec),
None => self.roster_member_peer_spec_for(
&peer_entry,
"member_retire_trust_authority peer",
)?,
};
retained_spec.ok_or(error)?
}
};
let peer_peer_id = Self::trusted_peer_removal_key(&peer_spec);
let peer_identity_for_cleanup = peer_identity.clone();
let edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(&retiring_identity),
mob_dsl::AgentIdentity::from_domain(peer_identity),
);
let handoff = match self.authorize_member_trust_unwiring(
&edge,
"member_retire_trust_authority",
) {
Ok(handoff) => handoff,
Err(unwiring_error) => match self.authorize_member_trust_cleanup_observed(
&edge,
&retiring_identity,
&retiring_peer_id,
&peer_identity_for_cleanup,
&peer_peer_id,
"member_retire_trust_authority_observed",
) {
Ok(handoff) => handoff,
Err(observed_error) => self
.authorize_retiring_member_trust_cleanup_observed(
&edge,
entry,
&retiring_identity,
&retiring_peer_id,
&peer_identity_for_cleanup,
&peer_peer_id,
"member_retire_trust_authority_retiring_observed",
)
.map_err(|retiring_error| {
MobError::WiringError(format!(
"member retire trust cleanup failed: {unwiring_error}; observed restore-failure cleanup failed: {observed_error}; observed retiring-runtime cleanup failed: {retiring_error}"
))
})?,
},
};
let authority =
handoff.unwiring_authority_for(&retiring_identity, &retiring_peer_id)?;
authorities.insert(peer_identity.clone(), authority);
}
Ok(RetireTrustCleanupPlan {
retiring_comms,
retiring_spec: Some(retiring_spec),
machine_wired_peer_identities,
trust_unwire_authority_by_peer: authorities,
})
}
async fn disposal_context_from_entry(
&self,
agent_identity: &AgentIdentity,
entry: &RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
preserve_machine_topology: bool,
) -> DisposalContext {
let retiring_key = self
.provisioner_comms(&entry.member_ref)
.await
.and_then(|comms| comms.public_key());
DisposalContext {
agent_identity: agent_identity.clone(),
entry: entry.clone(),
retiring_key,
retiring_comms: trust_cleanup_plan.retiring_comms,
retiring_spec: trust_cleanup_plan.retiring_spec,
preserve_machine_topology,
machine_wired_peer_identities: trust_cleanup_plan.machine_wired_peer_identities,
trust_unwire_authority_by_peer: trust_cleanup_plan.trust_unwire_authority_by_peer,
}
}
async fn dispose_member(
&mut self,
ctx: &DisposalContext,
policy: &mut dyn ErrorPolicy,
) -> DisposalReport {
let mut report = DisposalReport::new();
for &step in &DisposalStep::ORDERED {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
"MobActor::dispose_member executing step"
);
match self.execute_step(step, ctx).await {
Ok(()) => {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
"MobActor::dispose_member completed step"
);
report.completed.push(step);
}
Err(error) => {
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
step = %step,
error = %error,
"MobActor::dispose_member step failed"
);
if policy.on_step_error(step, &error, ctx) {
report.skipped.push((step, error));
} else {
report.aborted_at = Some((step, error));
break;
}
}
}
}
let archive_failed = report
.skipped
.iter()
.any(|(step, _)| *step == DisposalStep::ArchiveSession)
|| matches!(
report.aborted_at.as_ref(),
Some((DisposalStep::ArchiveSession, _))
);
self.dispose_prune_edge_locks(ctx).await;
if archive_failed {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
"retaining durable retiring roster entry after ArchiveSession failure"
);
return report;
}
report
}
async fn dispose_member_for_destroy_steps(&mut self, ctx: &DisposalContext) -> DisposalReport {
let mut report = DisposalReport::new();
let mut policy = AbortOnError;
for &step in &DisposalStep::ORDERED {
match self.execute_destroy_step(step, ctx).await {
Ok(()) => report.completed.push(step),
Err(error) => {
if policy.on_step_error(step, &error, ctx) {
report.skipped.push((step, error));
} else {
report.aborted_at = Some((step, error));
break;
}
}
}
}
report
}
async fn dispose_member_for_destroy(&mut self, ctx: &DisposalContext) -> DisposalReport {
let mut report = self.dispose_member_for_destroy_steps(ctx).await;
if report.completed.contains(&DisposalStep::ArchiveSession)
&& let Err(error) = self.record_destroy_member_retirement_archived(ctx).await
{
report.aborted_at = Some((DisposalStep::ArchiveSession, error));
}
report
}
async fn execute_destroy_step(
&mut self,
step: DisposalStep,
ctx: &DisposalContext,
) -> Result<(), MobError> {
match step {
DisposalStep::StopHostLoop => self.dispose_stop_host_loop_for_destroy(ctx).await,
_ => self.execute_step(step, ctx).await,
}
}
fn destroy_disposal_failure(report: &DisposalReport) -> Option<String> {
if let Some((step, error)) = &report.aborted_at {
return Some(format!("disposal aborted at {step}: {error}"));
}
report
.skipped
.first()
.map(|(step, error)| format!("disposal completed but {step} failed: {error}"))
}
fn cleanup_retired_member_machine_wiring(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
if ctx.machine_wired_peer_identities.is_empty() {
return Ok(());
}
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let retiring_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id);
let cleanup_inputs = ctx
.machine_wired_peer_identities
.iter()
.map(|peer_identity| {
let peer_identity = mob_dsl::AgentIdentity::from_domain(peer_identity);
let edge =
mob_dsl::WiringEdge::new(retiring_identity.clone(), peer_identity.clone());
mob_dsl::MobMachineInput::CleanupRetiringMemberWiring {
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
edge,
agent_identity: retiring_identity.clone(),
agent_runtime_id: retiring_runtime_id.clone(),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
}
})
.collect::<Vec<_>>();
let prepared =
self.prepare_dsl_inputs(&cleanup_inputs, "member_retire_machine_wiring_cleanup")?;
self.commit_prepared_dsl_input(prepared)?;
Ok(())
}
fn cleanup_member_machine_wiring_edge(
&mut self,
a: &AgentIdentity,
b: &AgentIdentity,
context: &'static str,
) -> Result<(), MobError> {
let input = mob_dsl::MobMachineInput::UnwireMembers {
edge: mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(a),
mob_dsl::AgentIdentity::from_domain(b),
),
};
let prepared = self.prepare_dsl_input_transition(input, context)?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
fn cleanup_retiring_member_machine_wiring_edge(
&mut self,
ctx: &DisposalContext,
peer_identity: &AgentIdentity,
context: &'static str,
) -> Result<mob_dsl::WiringEdge, MobError> {
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let peer_identity = mob_dsl::AgentIdentity::from_domain(peer_identity);
let edge = mob_dsl::WiringEdge::new(retiring_identity.clone(), peer_identity.clone());
self.apply_dsl_input(
mob_dsl::MobMachineInput::CleanupRetiringMemberWiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
agent_identity: retiring_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
},
context,
)?;
Ok(edge)
}
fn restore_retiring_member_machine_wiring_edge(
&mut self,
ctx: &DisposalContext,
edge: mob_dsl::WiringEdge,
context: &'static str,
) -> Result<(), MobError> {
let retiring_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
self.apply_dsl_input(
mob_dsl::MobMachineInput::RestoreRetiringMemberWiring {
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
edge,
agent_identity: retiring_identity,
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
},
context,
)?;
Ok(())
}
async fn execute_step(
&mut self,
step: DisposalStep,
ctx: &DisposalContext,
) -> Result<(), MobError> {
match step {
DisposalStep::StopHostLoop => self.dispose_stop_host_loop(ctx).await,
DisposalStep::NotifyPeers => self.dispose_notify_peers(ctx).await,
DisposalStep::CleanupMachineTopology => {
if ctx.preserve_machine_topology {
Ok(())
} else {
self.cleanup_retiring_external_peer_edges(&ctx.entry)
.await?;
self.cleanup_retired_member_machine_wiring(ctx)
}
}
DisposalStep::ArchiveSession => {
if Self::runtime_binding_for_entry(&ctx.entry).is_some() {
if self.remote_runtime_retired_for_entry(&ctx.entry) {
return Ok(());
}
self.dispose_archive_session(ctx).await?;
self.record_remote_member_runtime_retired(&ctx.entry).await
} else {
self.dispose_archive_session(ctx).await
}
}
}
}
async fn dispose_stop_host_loop(&mut self, ctx: &DisposalContext) -> Result<(), MobError> {
if ctx.entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
self.stop_autonomous_member(&ctx.agent_identity, &ctx.entry.member_ref)
.await?;
}
self.apply_kickoff_input(
&ctx.agent_identity,
mob_dsl::MobMachineInput::KickoffQuiesced {
member_id: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
},
"dispose_stop_host_loop",
)
.await?;
Ok(())
}
async fn dispose_stop_host_loop_for_destroy(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
if ctx.entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
let session_already_gone = {
#[cfg(feature = "runtime-adapter")]
{
match (
&self.runtime_adapter,
ctx.entry.member_ref.bridge_session_id(),
) {
(Some(adapter), Some(session_id)) => {
!adapter.contains_session(session_id).await
}
_ => false,
}
}
#[cfg(not(feature = "runtime-adapter"))]
{
false
}
};
if !session_already_gone {
self.stop_autonomous_member(&ctx.agent_identity, &ctx.entry.member_ref)
.await?;
}
}
self.apply_kickoff_input(
&ctx.agent_identity,
mob_dsl::MobMachineInput::KickoffQuiesced {
member_id: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
},
"dispose_stop_host_loop_for_destroy",
)
.await?;
Ok(())
}
async fn dispose_notify_peers(&mut self, ctx: &DisposalContext) -> Result<(), MobError> {
if ctx.machine_wired_peer_identities.is_empty() {
return Ok(());
}
let Some(retiring_spec) = ctx.retiring_spec.as_ref() else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
"dispose_notify_peers: skipping lifecycle notice and trust cleanup because retiring endpoint is absent"
);
return Ok(());
};
let peer_identities = ctx
.machine_wired_peer_identities
.iter()
.cloned()
.collect::<Vec<_>>();
let mut local_jobs = Vec::new();
let mut local_peer_specs = Vec::new();
let mut peer_only_jobs = Vec::new();
for peer_identity in peer_identities {
let peer_entry = {
let roster = self.roster.read().await;
roster.get_by_identity(&peer_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: skipping absent peer"
);
continue;
};
let endpoint = match self
.resolve_wiring_endpoint(&peer_entry, "dispose_notify_peers")
.await
{
Ok(endpoint) => endpoint,
Err(error) if Self::runtime_binding_for_entry(&peer_entry).is_none() => {
let retained_spec = self.machine_member_peer_spec_for(
&peer_identity,
"dispose_notify_peers retained local endpoint",
)?;
let retained_spec = match retained_spec {
Some(spec) => Some(spec),
None => self.roster_member_peer_spec_for(
&peer_entry,
"dispose_notify_peers retained local endpoint",
)?,
}
.ok_or_else(|| {
MobError::RetirementTopologyIncomplete(format!(
"failed to resolve machine-wired peer '{peer_identity}' for trust cleanup and no retained endpoint exists: {error}"
))
})?;
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
%error,
"dispose_notify_peers: retaining machine endpoint for peer without a live local comms runtime"
);
if ctx.preserve_machine_topology {
continue;
}
local_peer_specs.push((peer_identity, retained_spec));
continue;
}
Err(error) => {
return Err(MobError::RetirementTopologyIncomplete(format!(
"failed to resolve machine-wired peer '{peer_identity}' for trust cleanup: {error}"
)));
}
};
match endpoint {
WiringEndpoint::Local { comms, spec, .. } => {
let Some(authority) = ctx
.trust_unwire_authority_by_peer
.get(&peer_identity)
.cloned()
else {
return Err(MobError::RetirementTopologyIncomplete(format!(
"dispose_notify_peers missing generated retire trust handoff for '{peer_identity}'"
)));
};
local_peer_specs.push((peer_identity.clone(), spec.clone()));
local_jobs.push((peer_identity, spec, comms, authority));
}
WiringEndpoint::PeerOnly { spec, binding } => {
peer_only_jobs.push((peer_identity, spec, binding));
}
}
}
let actor = &*self;
let retiring_key = Self::trusted_peer_removal_key(retiring_spec);
let retiring_comms = ctx.retiring_comms.clone();
let mut local_tasks = FuturesUnordered::new();
for (peer_identity, recipient_spec, recipient_comms, authority) in local_jobs {
let retiring_key = retiring_key.clone();
let retiring_comms = retiring_comms.clone();
let retiring_spec = retiring_spec.clone();
let retired_id = ctx.agent_identity.clone();
let retired_entry = ctx.entry.clone();
local_tasks.push(async move {
if let Some(retiring_comms) = retiring_comms.as_ref() {
if let Err(error) = actor
.notify_peer_retired(
&recipient_spec,
&retired_id,
&retired_entry,
&retiring_spec,
retiring_comms,
)
.await
{
if Self::is_peer_destroying_admission_rejection(&error) {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: peer rejected lifecycle notice (already retiring)"
);
} else {
tracing::warn!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
error = %error,
"dispose_notify_peers: lifecycle notice failed; continuing with critical trust cleanup"
);
}
} else {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: lifecycle notice sent"
);
}
} else {
tracing::debug!(
mob_id = %actor.definition.id,
agent_identity = %retired_id,
peer_id = %peer_identity,
"dispose_notify_peers: skipping lifecycle notice because retiring member has no live comms runtime"
);
}
actor
.apply_trusted_peer_remove(recipient_comms.as_ref(), retiring_key, authority)
.await
.map_err(|error| {
MobError::RetirementTopologyIncomplete(format!(
"failed to remove retiring member trust from '{peer_identity}': {error}"
))
})?;
Ok(())
});
}
let mut topology_error = None;
while let Some(result) = local_tasks.next().await {
if let Err(error) = result
&& topology_error.is_none()
{
topology_error = Some(error);
}
}
drop(local_tasks);
if topology_error.is_none() {
for (peer_identity, peer_spec) in local_peer_specs {
let cleanup_edge = match self.cleanup_retiring_member_machine_wiring_edge(
ctx,
&peer_identity,
"dispose_notify_peers_local_machine_wiring_cleanup",
) {
Ok(edge) => edge,
Err(error) => {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"failed to reserve machine-wired edge for retiring peer-only cleanup of '{peer_identity}': {error}"
)));
continue;
}
};
if let Err(error) = self
.unwire_retiring_peer_only_side(
&ctx.entry,
retiring_spec,
&peer_spec,
std::time::Duration::from_secs(10),
)
.await
{
if let Err(rollback_error) = self.restore_retiring_member_machine_wiring_edge(
ctx,
cleanup_edge,
"dispose_notify_peers_local_machine_wiring_rollback",
) {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"retiring peer-only unwire failed: {error}; machine wiring rollback failed: {rollback_error}"
)));
} else {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"retiring peer-only member failed to unwire local peer '{peer_identity}': {error}"
)));
}
}
}
}
for (peer_identity, recipient_spec, recipient_binding) in peer_only_jobs {
if let Some(retiring_comms) = ctx.retiring_comms.as_ref() {
if let Err(error) = self
.notify_peer_retired(
&recipient_spec,
&ctx.agent_identity,
&ctx.entry,
retiring_spec,
retiring_comms,
)
.await
{
if Self::is_peer_destroying_admission_rejection(&error) {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: peer rejected lifecycle notice (already retiring)"
);
} else {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
error = %error,
"dispose_notify_peers: lifecycle notice failed; continuing with critical remote trust cleanup"
);
}
} else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: lifecycle notice sent"
);
}
} else {
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
peer_id = %peer_identity,
"dispose_notify_peers: skipping lifecycle notice because retiring member has no live comms runtime"
);
}
let cleanup_edge = match self.cleanup_retiring_member_machine_wiring_edge(
ctx,
&peer_identity,
"dispose_notify_peers_peer_only_machine_wiring_cleanup",
) {
Ok(edge) => edge,
Err(error) => {
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"failed to reserve machine-wired edge for peer-only trust cleanup of '{peer_identity}': {error}"
)));
}
continue;
}
};
let recipient_error = self
.unwire_peer_only_recipient(
&recipient_spec,
Some(&recipient_binding),
retiring_spec,
std::time::Duration::from_secs(10),
)
.await
.err();
let retiring_error = self
.unwire_retiring_peer_only_side(
&ctx.entry,
retiring_spec,
&recipient_spec,
std::time::Duration::from_secs(10),
)
.await
.err();
if recipient_error.is_some() || retiring_error.is_some() {
let error = match (recipient_error, retiring_error) {
(Some(recipient), Some(retiring)) => format!(
"surviving peer unwire failed: {recipient}; retiring peer unwire failed: {retiring}"
),
(Some(recipient), None) => {
format!("surviving peer unwire failed: {recipient}")
}
(None, Some(retiring)) => {
format!("retiring peer unwire failed: {retiring}")
}
(None, None) => unreachable!("guarded by peer unwire errors"),
};
if let Err(rollback_error) = self.restore_retiring_member_machine_wiring_edge(
ctx,
cleanup_edge,
"dispose_notify_peers_peer_only_machine_wiring_rollback",
) {
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"peer-only unwire failed: {error}; machine wiring rollback failed: {rollback_error}"
)));
}
continue;
}
if topology_error.is_none() {
topology_error = Some(MobError::RetirementTopologyIncomplete(format!(
"peer-only unwire failed for '{peer_identity}': {error}"
)));
}
}
}
match topology_error {
Some(error) => Err(error),
None => Ok(()),
}
}
fn is_peer_destroying_admission_rejection(error: &MobError) -> bool {
matches!(
error,
MobError::CommsError(meerkat_core::comms::SendError::AdmissionDropped {
reason: meerkat_core::comms::AdmissionDropReason::ClassificationRejected
| meerkat_core::comms::AdmissionDropReason::SessionClosed,
})
)
}
pub(super) async fn dispose_archive_session(
&self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
if let Some(session_id) = ctx.entry.member_ref.bridge_session_id()
&& self.retirement_archive_already_complete(session_id).await?
{
tracing::debug!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
session_id = %session_id,
"MobActor::dispose_archive_session accepted existing archive authority"
);
return Ok(());
}
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
member_ref = ?ctx.entry.member_ref,
"MobActor::dispose_archive_session retiring member via provisioner"
);
if let Err(error) = self.provisioner.retire_member(&ctx.entry.member_ref).await {
if matches!(
error,
MobError::SessionError(meerkat_core::service::SessionError::NotFound { .. })
) {
return Ok(());
}
return Err(error);
}
tracing::info!(
mob_id = %self.definition.id,
agent_identity = %ctx.agent_identity,
"MobActor::dispose_archive_session retired member via provisioner"
);
Ok(())
}
async fn observe_member_retirement_archived(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
let session_id_for_journal = ctx
.entry
.member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain);
let terminal_signal = if Self::runtime_binding_for_entry(&ctx.entry).is_some() {
mob_dsl::MobMachineSignal::ObserveRemoteMemberRetirementArchivedAndSupervisorRevoked {
agent_identity: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
}
} else {
mob_dsl::MobMachineSignal::ObserveMemberRetirementArchived {
agent_identity: mob_dsl::AgentIdentity::from_domain(&ctx.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id_for_journal.clone(),
}
};
let prepared = self
.prepare_dsl_signal_transition(terminal_signal, "dispose_member_archive_completed")?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::MemberRetired,
&ctx.entry.agent_identity,
&ctx.entry.agent_runtime_id,
None,
ctx.entry.generation,
session_id_for_journal,
"dispose_member_archive_completed",
)?;
if !self
.retire_event_exists(&ctx.entry.agent_identity, ctx.entry.generation)
.await?
{
self.append_retire_event_for_entry(&ctx.entry).await?;
}
self.commit_prepared_dsl_transition(prepared)
}
async fn dispose_prune_edge_locks(&self, ctx: &DisposalContext) {
self.edge_locks.prune(ctx.agent_identity.as_str()).await;
}
pub(super) async fn dispose_remove_from_roster(&self, ctx: &DisposalContext) {
let mut roster = self.roster.write().await;
roster.remove_member(&ctx.agent_identity);
drop(roster);
self.per_spawn_external_tools
.write()
.await
.remove(&ctx.agent_identity);
self.restore_diagnostics
.write()
.await
.remove(&ctx.agent_identity);
}
async fn handle_complete(&mut self) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_complete preflight")?;
self.ensure_flow_tracker_alignment("handle_complete preflight")
.await?;
self.cancel_all_flow_tasks().await?;
if let Err(error) = self
.notify_orchestrator_lifecycle(format!("Mob '{}' is completing.", self.definition.id))
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"completion proceeding despite orchestrator lifecycle delivery failure"
);
}
self.retire_all_members("complete").await?;
let prepared = self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Complete, "complete_input")
.map_err(|error| {
tracing::debug!(
context = "complete_input",
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(MobState::Completed)
})?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Completed,
"complete_input",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobCompleted,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
self.ensure_pending_spawn_alignment("handle_complete completion")?;
self.ensure_flow_tracker_alignment("handle_complete completion")
.await?;
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
fn remote_destroy_cleanup_deadline(remote_member_count: usize) -> std::time::Duration {
let batches = remote_member_count
.saturating_add(MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS.saturating_sub(1))
/ MAX_PARALLEL_REMOTE_MEMBER_TEARDOWNS.max(1);
let deadline_secs = std::cmp::min(90, 5 + (batches as u64) * 17);
std::time::Duration::from_secs(deadline_secs)
}
fn push_unique_identity(target: &mut Vec<AgentIdentity>, identity: AgentIdentity) {
if !target.iter().any(|existing| existing == &identity) {
target.push(identity);
}
}
fn runtime_binding_for_entry(entry: &RosterEntry) -> Option<crate::RuntimeBinding> {
Self::runtime_binding_for_member_ref(&entry.member_ref)
}
fn runtime_binding_for_member_ref(member_ref: &MemberRef) -> Option<crate::RuntimeBinding> {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id: None,
} => Some(crate::RuntimeBinding::External {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
bootstrap_token: bootstrap_token.clone(),
pubkey: *pubkey,
}),
_ => None,
}
}
fn sanitized_member_ref(member_ref: &MemberRef) -> MemberRef {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id,
..
} => MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
pubkey: *pubkey,
bootstrap_token: bootstrap_token.clone(),
session_id: session_id.clone(),
},
MemberRef::Session { session_id } => MemberRef::Session {
session_id: session_id.clone(),
},
}
}
fn external_binding_overlay_record(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
member_ref: &MemberRef,
) -> Option<crate::store::ExternalBindingOverlayRecord> {
match member_ref {
MemberRef::BackendPeer {
peer_id,
address,
pubkey,
bootstrap_token,
session_id: None,
} => Some(crate::store::ExternalBindingOverlayRecord {
agent_identity: agent_identity.clone(),
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: super::bridge_protocol::canonicalize_bridge_address(address),
pubkey: *pubkey,
bootstrap_token: None,
session_id: None,
}),
bootstrap_token: bootstrap_token.clone(),
status: crate::store::ExternalBindingOverlayStatus::Normalized,
updated_at: chrono::Utc::now(),
}),
_ => None,
}
}
async fn delete_external_binding_overlay_for_member(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<(), MobError> {
self.runtime_metadata
.delete_external_binding_overlay(&self.definition.id, agent_identity, generation)
.await
.map_err(MobError::from)
}
fn remote_runtime_retired_for_entry(&self, entry: &RosterEntry) -> bool {
self.dsl_authority.state().remote_runtime_retired_exact(
&mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
&mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
mob_dsl::Generation::from_domain(entry.generation),
)
}
fn remote_supervisor_revoked_for_entry(&self, entry: &RosterEntry) -> bool {
self.dsl_authority.state().remote_supervisor_revoked_exact(
&mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
&mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
mob_dsl::FenceToken::from_domain(entry.fence_token),
mob_dsl::Generation::from_domain(entry.generation),
)
}
async fn record_remote_member_runtime_retired(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self.remote_runtime_retired_for_entry(entry) {
return Ok(());
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordRemoteMemberRuntimeRetired {
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"record_remote_member_runtime_retired",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::RemoteMemberRuntimeRetired,
&entry.agent_identity,
&entry.agent_runtime_id,
Some(entry.fence_token),
entry.generation,
None,
"record_remote_member_runtime_retired",
)?;
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::RemoteMemberRuntimeRetired {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
fence_token: entry.fence_token,
generation: entry.generation,
},
})
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
async fn record_remote_member_supervisor_revoked(
&mut self,
entry: &RosterEntry,
) -> Result<(), MobError> {
if self.remote_supervisor_revoked_for_entry(entry) {
return Ok(());
}
let prepared = self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::RecordRemoteMemberSupervisorRevoked {
agent_identity: mob_dsl::AgentIdentity::from_domain(&entry.agent_identity),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
fence_token: mob_dsl::FenceToken::from_domain(entry.fence_token),
generation: mob_dsl::Generation::from_domain(entry.generation),
},
"record_remote_member_supervisor_revoked",
)?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::RemoteMemberSupervisorRevoked,
&entry.agent_identity,
&entry.agent_runtime_id,
Some(entry.fence_token),
entry.generation,
None,
"record_remote_member_supervisor_revoked",
)?;
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::RemoteMemberSupervisorRevoked {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
fence_token: entry.fence_token,
generation: entry.generation,
},
})
.await?;
self.commit_prepared_dsl_transition(prepared)?;
Ok(())
}
async fn record_destroy_member_retirement_archived(
&mut self,
ctx: &DisposalContext,
) -> Result<(), MobError> {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&ctx.entry.agent_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&ctx.entry.agent_runtime_id);
let bound_session = self
.dsl_authority
.state()
.member_session_bindings
.get(&dsl_identity)
.cloned();
let pending_retire_session = self
.dsl_authority
.state()
.runtime_retire_pending_sessions
.get(&dsl_runtime_id)
.cloned();
let released_ordinary_retirement =
bound_session.is_none() && pending_retire_session.is_some();
let session_id_for_journal = bound_session.or(pending_retire_session);
let archive_signal = if Self::runtime_binding_for_entry(&ctx.entry).is_some() {
mob_dsl::MobMachineSignal::ObserveRemoteMemberRetirementArchivedAndSupervisorRevoked {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
}
} else if released_ordinary_retirement {
mob_dsl::MobMachineSignal::ObserveMemberRetirementArchived {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id_for_journal.clone(),
}
} else {
mob_dsl::MobMachineSignal::ObserveDestroyMemberRetirementArchived {
agent_identity: dsl_identity,
agent_runtime_id: dsl_runtime_id,
fence_token: mob_dsl::FenceToken::from_domain(ctx.entry.fence_token),
generation: mob_dsl::Generation::from_domain(ctx.entry.generation),
session_id: session_id_for_journal.clone(),
}
};
let prepared =
self.prepare_dsl_signal_transition(archive_signal, "destroy_member_archive_completed")?;
Self::require_member_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::MemberRetired,
&ctx.entry.agent_identity,
&ctx.entry.agent_runtime_id,
None,
ctx.entry.generation,
session_id_for_journal,
"destroy_member_archive_completed",
)?;
if !self
.retire_event_exists(&ctx.entry.agent_identity, ctx.entry.generation)
.await?
{
self.append_retire_event_for_entry(&ctx.entry).await?;
}
self.commit_prepared_dsl_transition(prepared)?;
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_DESTROY_MEMBER_ARCHIVE_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&ctx.entry.agent_identity)
{
target.take();
return Err(MobError::Internal(format!(
"fault-injected cancellation after destroy member archive commit for '{}'",
ctx.entry.agent_identity
)));
}
Ok(())
}
async fn record_destroy_member_retired_event(
&self,
entry: &RosterEntry,
) -> Result<(), MobError> {
let retire_event_already_present = self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await?;
if !retire_event_already_present {
return Err(MobError::Internal(format!(
"destroy cleanup for '{}' reached disposal without a generated durable retire journal event",
entry.agent_identity
)));
}
Ok(())
}
async fn record_destroying_event(&mut self) -> Result<(), MobError> {
let destroying_event_exists = self.destroying_event_exists().await?;
if !destroying_event_exists {
if self.destroy_admitted() {
return Err(MobError::Internal(
"destroy cleanup was admitted without a durable MobDestroying journal event"
.to_string(),
));
}
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyCleanup,
"record_destroying_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Destroying,
"record_destroying_event",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobDestroying,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
} else if !self.destroy_admitted() {
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyCleanup,
"record_destroying_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Destroying,
"record_destroying_event",
)?;
self.commit_prepared_dsl_transition(prepared)?;
}
let _ = self.phase_watch_tx.send(self.state());
Ok(())
}
async fn destroying_event_exists(&self) -> Result<bool, MobError> {
let events = self.events.replay_all().await?;
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |pos| pos + 1);
Ok(events[epoch_start..]
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)))
}
async fn destroy_storage_finalizing_event_exists(&self) -> Result<bool, MobError> {
let events = self.events.replay_all().await?;
let epoch_start = events
.iter()
.rposition(|event| matches!(event.kind, MobEventKind::MobReset))
.map_or(0, |pos| pos + 1);
Ok(events[epoch_start..]
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroyStorageFinalizing)))
}
async fn record_destroy_storage_finalizing_event(&mut self) -> Result<(), MobError> {
if self.destroy_storage_finalizing_event_exists().await? {
return Ok(());
}
let prepared = self.prepare_dsl_signal_transition(
mob_dsl::MobMachineSignal::AdmitDestroyStorageFinalizing,
"record_destroy_storage_finalizing_event",
)?;
Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::DestroyStorageFinalizing,
"record_destroy_storage_finalizing_event",
)?;
let events = self.events.clone();
let mob_id = self.definition.id.clone();
self.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobDestroyStorageFinalizing,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await?;
Ok(())
}
async fn runtime_metadata_snapshot(&self) -> Result<RuntimeMetadataSnapshot, MobError> {
Ok(RuntimeMetadataSnapshot {
supervisor: self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?,
external_binding_overlays: self
.runtime_metadata
.list_external_binding_overlays(&self.definition.id)
.await?,
})
}
async fn restore_runtime_metadata_snapshot(
&mut self,
snapshot: &RuntimeMetadataSnapshot,
) -> Result<(), MobError> {
if let Some(supervisor) = &snapshot.supervisor {
match self
.runtime_metadata
.load_supervisor_authority(&self.definition.id)
.await?
{
Some(current) if current == *supervisor => {}
Some(current) => {
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"supervisor authority changed while restoring metadata for mob '{}': current peer={} epoch={}, snapshot peer={} epoch={}",
self.definition.id,
current.public_peer_id,
current.epoch,
supervisor.public_peer_id,
supervisor.epoch,
),
)));
}
None => {
let prepared = self.prepare_supervisor_authority_persistence(
supervisor.dsl_restore_after_destroy_rollback_input(),
supervisor,
"restore_runtime_metadata_snapshot",
)?;
let inserted = self
.runtime_metadata
.put_supervisor_authority_if_absent(
&self.definition.id,
supervisor,
&prepared.authority,
)
.await?;
if !inserted {
return Err(MobError::from(crate::store::MobStoreError::CasConflict(
format!(
"supervisor authority changed while restoring absent metadata for mob '{}'",
self.definition.id
),
)));
}
self.commit_prepared_dsl_transition(prepared.transition)?;
}
}
}
for overlay in &snapshot.external_binding_overlays {
self.runtime_metadata
.upsert_external_binding_overlay(&self.definition.id, overlay)
.await?;
}
Ok(())
}
async fn incomplete_after_metadata_scrub_error(
&mut self,
mut report: super::handle::MobDestroyReport,
snapshot: &RuntimeMetadataSnapshot,
error: impl std::fmt::Display,
) -> super::handle::MobDestroyError {
report.push_error(error.to_string());
if let Err(restore_error) = self.restore_runtime_metadata_snapshot(snapshot).await {
report.push_error(format!(
"runtime metadata restore failed after incomplete destroy: {restore_error}"
));
}
report.metadata_scrubbed = false;
super::handle::MobDestroyError::Incomplete { report }
}
fn incomplete_destroy_error(
mut report: super::handle::MobDestroyReport,
context: &str,
error: impl std::fmt::Display,
) -> super::handle::MobDestroyError {
report.push_error(format!("{context}: {error}"));
super::handle::MobDestroyError::Incomplete { report }
}
fn expected_revoke_cleanup_failure(error: &MobError) -> Option<ExpectedRevokeCleanupFailure> {
match error {
MobError::BridgeCommandRejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
..
} => Some(ExpectedRevokeCleanupFailure::BridgeRejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
}),
_ => None,
}
}
#[cfg(not(target_arch = "wasm32"))]
async fn destroy_remote_member_for_destroy(
&mut self,
entry: RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
) -> RemoteDestroyOutcome {
let identity = entry.agent_identity.clone();
let agent_identity = entry.agent_identity.clone();
let mut outcome = RemoteDestroyOutcome {
identity: identity.clone(),
force_destroyed: false,
orphaned: false,
errors: Vec::new(),
};
let Some(binding) = Self::runtime_binding_for_entry(&entry) else {
outcome
.errors
.push("remote destroy requested for non peer-only member".to_string());
outcome.orphaned = true;
return outcome;
};
let ctx = self
.disposal_context_from_entry(&agent_identity, &entry, trust_cleanup_plan, false)
.await;
let disposal_already_complete = match self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await
{
Ok(complete) => complete,
Err(error) => {
outcome.errors.push(format!(
"durable retirement completion observation failed: {error}"
));
outcome.orphaned = true;
return outcome;
}
};
if disposal_already_complete {
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx).await;
return outcome;
}
let disposal = self.dispose_member_for_destroy_steps(&ctx).await;
let disposal_failure = disposal
.aborted_at
.as_ref()
.map(|(step, error)| (*step, error.to_string()))
.or_else(|| {
disposal
.skipped
.first()
.map(|(step, error)| (*step, error.to_string()))
});
let mut remote_cleanup_complete = disposal_failure.is_none();
if let Some((failed_step, graceful_error)) = disposal_failure {
if failed_step != DisposalStep::ArchiveSession {
outcome.errors.push(format!(
"graceful retire failed before remote archive at {failed_step}: {graceful_error}"
));
outcome.orphaned = true;
return outcome;
}
match self
.observe_peer_only_binding(&binding, std::time::Duration::from_millis(750))
.await
{
Ok(observation) => match self.observation_is_terminal(&observation) {
Ok(true) => remote_cleanup_complete = true,
Ok(false) => tracing::debug!(
state = %observation.state,
"confirmatory remote observation remained non-terminal before force destroy"
),
Err(error) => tracing::debug!(
%error,
"confirmatory remote observation terminality classification failed before force destroy"
),
},
Err(error) => tracing::debug!(
%error,
"confirmatory remote observation failed before force destroy"
),
}
if !remote_cleanup_complete {
match self
.destroy_peer_only_binding(&binding, std::time::Duration::from_secs(5))
.await
{
Ok(_) => {
remote_cleanup_complete = true;
outcome.force_destroyed = true;
}
Err(error) => outcome.errors.push(format!(
"graceful retire failed at ArchiveSession: {graceful_error}; force destroy failed: {error}"
)),
}
}
}
if !remote_cleanup_complete {
outcome.orphaned = true;
return outcome;
}
if let Err(error) = self.record_remote_member_runtime_retired(&entry).await {
outcome.errors.push(format!(
"durable remote-runtime retirement checkpoint failed: {error}"
));
outcome.orphaned = true;
return outcome;
}
#[cfg(test)]
if let Ok(mut target) = FAIL_AFTER_DESTROY_REMOTE_ARCHIVE_FOR_IDENTITY.lock()
&& target.as_ref() == Some(&entry.agent_identity)
{
target.take();
outcome.errors.push(format!(
"fault-injected cancellation after remote archive for '{}'",
entry.agent_identity
));
outcome.orphaned = true;
return outcome;
}
if let Err(error) = self
.revoke_supervisor_for_retiring_entry(
&entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
{
if let Some(cause) = Self::expected_revoke_cleanup_failure(&error) {
tracing::debug!(
?cause,
error = %error,
"destroy cleanup observed already-revoked supervisor binding"
);
} else {
outcome
.errors
.push(format!("supervisor revoke failed: {error}"));
outcome.orphaned = true;
return outcome;
}
}
if let Err(error) = self.record_destroy_member_retirement_archived(&ctx).await {
outcome
.errors
.push(format!("durable retire completion failed: {error}"));
outcome.orphaned = true;
return outcome;
}
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx).await;
outcome
}
#[cfg(not(target_arch = "wasm32"))]
async fn destroy_remote_members_for_destroy(
&mut self,
remote_entries: Vec<RosterEntry>,
trust_unwire_authority_by_member: &mut BTreeMap<AgentIdentity, RetireTrustCleanupPlan>,
report: &mut super::handle::MobDestroyReport,
) {
if remote_entries.is_empty() {
return;
}
let deadline = Self::remote_destroy_cleanup_deadline(remote_entries.len());
let deadline_at = Instant::now() + deadline;
let mut remaining = VecDeque::from(remote_entries);
while let Some(entry) = remaining.pop_front() {
let identity = entry.agent_identity.clone();
if Instant::now() >= deadline_at {
report.remote_cleanup_deadline_exceeded = true;
Self::push_unique_identity(&mut report.orphaned_remote_members, identity);
for entry in remaining {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
return;
}
let trust_cleanup_plan = trust_unwire_authority_by_member
.remove(&identity)
.unwrap_or_else(RetireTrustCleanupPlan::empty);
let outcome = self
.destroy_remote_member_for_destroy(entry, trust_cleanup_plan)
.await;
if outcome.force_destroyed {
Self::push_unique_identity(
&mut report.force_destroyed_members,
outcome.identity.clone(),
);
}
if outcome.orphaned {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
outcome.identity.clone(),
);
}
for error in outcome.errors {
report.push_error(format!("{}: {error}", outcome.identity));
}
if Instant::now() >= deadline_at && !remaining.is_empty() {
report.remote_cleanup_deadline_exceeded = true;
for entry in remaining {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
return;
}
}
}
#[cfg(target_arch = "wasm32")]
async fn destroy_remote_members_for_destroy(
&self,
remote_entries: Vec<RosterEntry>,
_trust_unwire_authority_by_member: &mut BTreeMap<AgentIdentity, RetireTrustCleanupPlan>,
report: &mut super::handle::MobDestroyReport,
) {
for entry in remote_entries {
Self::push_unique_identity(
&mut report.orphaned_remote_members,
entry.agent_identity.clone(),
);
}
}
async fn dispose_local_member_after_destroy_admission(
&mut self,
entry: RosterEntry,
trust_cleanup_plan: RetireTrustCleanupPlan,
report: &mut super::handle::MobDestroyReport,
) -> Result<(), super::handle::MobDestroyError> {
let ctx = self
.disposal_context_from_entry(&entry.agent_identity, &entry, trust_cleanup_plan, false)
.await;
let disposal_already_complete = match self
.retire_event_exists(&entry.agent_identity, entry.generation)
.await
{
Ok(complete) => complete,
Err(error) => {
report.push_error(format!(
"{}: durable retirement completion observation failed: {error}",
entry.agent_identity
));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
};
if !disposal_already_complete {
let disposal_report = self.dispose_member_for_destroy(&ctx).await;
if let Some(error) = Self::destroy_disposal_failure(&disposal_report) {
report.push_error(format!("{}: {error}", entry.agent_identity));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
}
if let Err(error) = self.record_destroy_member_retired_event(&entry).await {
report.push_error(format!(
"{}: durable retire event append failed: {error}",
entry.agent_identity
));
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
self.dispose_prune_edge_locks(&ctx).await;
self.dispose_remove_from_roster(&ctx).await;
if let Err(error) = self
.delete_external_binding_overlay_for_member(&entry.agent_identity, entry.generation)
.await
{
report.push_error(error.to_string());
return Err(super::handle::MobDestroyError::Incomplete {
report: report.clone(),
});
}
Ok(())
}
async fn handle_destroy(
&mut self,
) -> Result<super::handle::MobDestroyReport, super::handle::MobDestroyError> {
let was_active = self.destroy_cleanup_active;
self.destroy_cleanup_active = true;
let result = self.handle_destroy_inner().await;
self.destroy_cleanup_active = was_active;
result
}
async fn handle_destroy_inner(
&mut self,
) -> Result<super::handle::MobDestroyReport, super::handle::MobDestroyError> {
use super::handle::{MobDestroyError, MobDestroyReport};
let mut report = MobDestroyReport::default();
let destroy_input_needed = self.dsl_state() != crate::runtime::MobState::Destroyed;
self.ensure_pending_spawn_alignment("handle_destroy preflight")
.map_err(|error| {
if self.destroy_admitted() {
Self::incomplete_destroy_error(
report.clone(),
"pending spawn alignment during admitted destroy failed",
error,
)
} else {
MobDestroyError::from(error)
}
})?;
self.ensure_flow_tracker_alignment("handle_destroy preflight")
.await
.map_err(|error| {
if self.destroy_admitted() {
Self::incomplete_destroy_error(
report.clone(),
"flow tracker alignment during admitted destroy failed",
error,
)
} else {
MobDestroyError::from(error)
}
})?;
let entries = {
let roster = self.roster.read().await;
roster.list_all().cloned().collect::<Vec<_>>()
};
let mut trust_cleanup_plan_by_member: BTreeMap<AgentIdentity, RetireTrustCleanupPlan> =
BTreeMap::new();
if destroy_input_needed {
for entry in &entries {
let plan = match self
.member_retire_trust_cleanup_plan(&entry.agent_identity, entry)
.await
{
Ok(plan) => plan,
Err(error) => {
report.push_error(format!(
"{}: destroy retire trust authority failed: {error}",
entry.agent_identity
));
return Err(MobDestroyError::Incomplete { report });
}
};
if plan.has_peers() {
trust_cleanup_plan_by_member.insert(entry.agent_identity.clone(), plan);
}
}
}
if destroy_input_needed && let Err(error) = self.record_destroying_event().await {
report.push_error(format!("destroy marker append failed: {error}"));
return Err(MobDestroyError::Incomplete { report });
}
self.fail_all_pending_spawns("mob is destroying")
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"pending spawn cleanup during destroy failed",
error,
)
})?;
self.cancel_pending_peer_deliveries("mob is destroying")
.await;
if destroy_input_needed && self.has_orchestrator {
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::StopOrchestrator,
"stop_orchestrator_destroy",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"stop orchestrator during destroy failed",
error,
)
})?;
self.apply_dsl_signal(
mob_dsl::MobMachineSignal::DestroyOrchestrator,
"destroy_orchestrator",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy orchestrator during destroy failed",
error,
)
})?;
}
if destroy_input_needed {
for entry in &entries {
if let Err(error) = self.admit_member_retire_for_destroy(entry).await {
report.push_error(format!(
"{}: destroy retire admission failed: {error}",
entry.agent_identity
));
return Err(MobDestroyError::Incomplete { report });
}
}
}
self.cancel_all_flow_tasks().await.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"cancel flow tasks during destroy failed",
error,
)
})?;
if !self.pending_routed_effects.is_empty() {
self.flush_routed_effects().await.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy routed effect dispatch failed",
error,
)
})?;
}
if destroy_input_needed {
if let Err(error) = self
.notify_orchestrator_lifecycle(format!(
"Mob '{}' is destroying.",
self.definition.id
))
.await
{
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"destroy proceeding despite orchestrator lifecycle delivery failure"
);
}
}
let (remote_entries, local_entries): (Vec<_>, Vec<_>) = entries
.into_iter()
.partition(|entry| Self::runtime_binding_for_entry(entry).is_some());
for entry in local_entries {
let plan = trust_cleanup_plan_by_member
.remove(&entry.agent_identity)
.unwrap_or_else(RetireTrustCleanupPlan::empty);
self.dispose_local_member_after_destroy_admission(entry, plan, &mut report)
.await?;
}
self.destroy_remote_members_for_destroy(
remote_entries,
&mut trust_cleanup_plan_by_member,
&mut report,
)
.await;
if report.remote_cleanup_deadline_exceeded
|| !report.orphaned_remote_members.is_empty()
|| !report.errors.is_empty()
{
return Err(MobDestroyError::Incomplete { report });
}
if destroy_input_needed {
self.apply_dsl_input(mob_dsl::MobMachineInput::Destroy, "destroy_input")
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"destroy machine transition failed",
error,
)
})?;
}
self.ensure_pending_spawn_alignment("handle_destroy completion")
.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
self.ensure_flow_tracker_alignment("handle_destroy completion")
.await
.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
if let Err(error) = self.cleanup_namespace().await {
report.push_error(error.to_string());
return Err(MobDestroyError::Incomplete { report });
}
report.namespace_cleaned = true;
self.record_destroy_storage_finalizing_event()
.await
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"record destroy storage finalizing event failed",
error,
)
})?;
let runtime_metadata_snapshot =
self.runtime_metadata_snapshot().await.map_err(|error| {
let mut report = report.clone();
report.push_error(error.to_string());
MobDestroyError::Incomplete { report }
})?;
if let Err(error) = self
.runtime_metadata
.delete_external_binding_overlays(&self.definition.id)
.await
{
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
let supervisor_delete = match runtime_metadata_snapshot.supervisor.as_ref() {
Some(supervisor) => Some(
self.prepare_supervisor_authority_deletion(
supervisor,
"handle_destroy metadata scrub",
)
.map_err(|error| {
Self::incomplete_destroy_error(
report.clone(),
"supervisor authority deletion admission failed",
error,
)
})?,
),
None => None,
};
if let (Some(supervisor), Some(prepared)) = (
runtime_metadata_snapshot.supervisor.as_ref(),
supervisor_delete,
) {
if let Err(error) = self
.runtime_metadata
.delete_supervisor_authority(&self.definition.id, supervisor, &prepared.authority)
.await
.and_then(|deleted| {
if deleted {
Ok(())
} else {
Err(crate::store::MobStoreError::CasConflict(format!(
"supervisor authority changed while deleting metadata for mob '{}'",
self.definition.id
)))
}
})
{
return Err(self
.incomplete_after_metadata_scrub_error(
report,
&runtime_metadata_snapshot,
error,
)
.await);
}
self.commit_prepared_dsl_transition(prepared.transition)?;
}
report.metadata_scrubbed = true;
if let Err(error) = self.events.clear().await {
return Err(self
.incomplete_after_metadata_scrub_error(report, &runtime_metadata_snapshot, error)
.await);
}
report.events_cleared = true;
self.edge_locks.clear().await;
if report.remote_cleanup_deadline_exceeded
|| !report.orphaned_remote_members.is_empty()
|| !report.errors.is_empty()
{
return Err(MobDestroyError::Incomplete { report });
}
Ok(report)
}
async fn handle_rotate_supervisor(
&mut self,
) -> Result<super::handle::SupervisorRotationReport, MobError> {
use super::bridge_protocol::{
BridgeCommand, BridgeSupervisorDelivery, BridgeSupervisorPayload,
BridgeSupervisorRotationObservation, BridgeSupervisorRotationObserve,
BridgeSupervisorRotationState, BridgeSupervisorRotationSubmit,
SupervisorRotationOperationId,
};
if self.state() == MobState::Destroyed {
return Err(self.invalid_transition_to(MobState::Destroyed));
}
self.apply_dsl_input(
mob_dsl::MobMachineInput::AdmitSupervisorRotation,
"handle_rotate_supervisor_admission",
)
.map_err(|error| MobError::MobMachineRejected {
context: "handle_rotate_supervisor_admission",
reason: error.to_string(),
})?;
let loaded = self
.load_supervisor_authority_snapshot()
.await?
.ok_or_else(|| {
MobError::Internal(format!(
"cannot rotate supervisor for mob '{}': missing supervisor runtime metadata",
self.definition.id
))
})?;
let current = loaded.durable;
let stable_current = current.without_pending_rotation();
let existing_pending = current.pending_rotation.clone();
let operation_id = existing_pending
.as_ref()
.and_then(|pending| pending.operation_id)
.unwrap_or_else(SupervisorRotationOperationId::new);
let mut next = existing_pending
.as_ref()
.map(|pending| pending.authority_record())
.unwrap_or_else(|| {
let mut generated = crate::store::SupervisorAuthorityRecord::generate(
super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
);
generated.epoch = stable_current.epoch + 1;
generated
});
next.protocol_version = super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION;
next.pending_rotation = None;
let mut accepted_peer_ids: BTreeSet<String> = existing_pending
.as_ref()
.map(|pending| pending.accepted_peer_ids.iter().cloned().collect())
.unwrap_or_default();
let remote_peers = {
let roster = self.roster.read().await;
roster
.list_all()
.filter_map(Self::runtime_binding_for_entry)
.map(|binding| {
Self::peer_only_spec_for_binding(&binding, "handle_rotate_supervisor")
})
.collect::<Result<Vec<_>, _>>()?
};
let active_peer_ids: BTreeSet<String> = remote_peers
.iter()
.map(|peer| peer.peer_id.to_string())
.collect();
if existing_pending
.as_ref()
.is_some_and(|pending| pending.operation_id.is_none())
{
accepted_peer_ids.retain(|peer_id| active_peer_ids.contains(peer_id));
}
let mut member_targets = existing_pending
.as_ref()
.map(|pending| pending.member_targets.clone())
.unwrap_or_default();
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
let missing_peer_id = match member_targets.entry(peer_id) {
std::collections::btree_map::Entry::Occupied(_) => None,
std::collections::btree_map::Entry::Vacant(entry) => Some(entry.into_key()),
};
if let Some(peer_id) = missing_peer_id {
let target = self
.bridge_supervisor_payload_for_authority_and_recipient(&next, peer)
.await?
.supervisor;
member_targets.entry(peer_id).or_insert(target);
}
}
for (peer_id, target) in &member_targets {
if target.peer_id != next.public_peer_id || target.pubkey != next.public_signing_key() {
return Err(MobError::Internal(format!(
"persisted supervisor rotation target for member '{peer_id}' does not match operation authority {}",
next.public_peer_id
)));
}
}
let mut durable_write_expected = current.clone();
if existing_pending.as_ref().is_none_or(|pending| {
pending.operation_id.is_none()
|| pending.member_targets != member_targets
|| pending
.accepted_peer_ids
.iter()
.cloned()
.collect::<BTreeSet<_>>()
!= accepted_peer_ids
|| !pending
.protocol_version
.same_protocol_as(next.protocol_version)
}) {
let persisted = self
.persist_pending_supervisor_rotation(PendingSupervisorRotationPersistenceRequest {
current: &stable_current,
expected_durable: &durable_write_expected,
pending: &next,
operation_id,
accepted_peer_ids: &accepted_peer_ids,
active_peer_ids: &active_peer_ids,
member_targets: &member_targets,
})
.await?;
let Some(record) = persisted.persisted_record else {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: 0,
rollback_succeeded: false,
pending_authority_recorded: false,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} was not durably recorded before submission"
),
});
};
durable_write_expected = record;
}
self.rotate_supervisor_bridge_to(&stable_current).await?;
let mut submission_diagnostics = BTreeMap::new();
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
if accepted_peer_ids.contains(&peer_id) {
continue;
}
let target = member_targets.get(&peer_id).cloned().ok_or_else(|| {
MobError::Internal(format!(
"supervisor rotation operation {operation_id} has no durable target for peer '{peer_id}'"
))
})?;
let delivery = BridgeSupervisorDelivery::SubmitSupervisorRotation(
BridgeSupervisorRotationSubmit {
operation_id,
target: target.clone(),
target_epoch: next.epoch,
protocol_version: next.protocol_version,
},
);
if let Err(error) = self.supervisor_bridge.trust_recipient(peer).await {
submission_diagnostics.insert(peer_id, error.to_string());
tracing::debug!(%operation_id, peer_id = %peer.peer_id, %error, "supervisor rotation delivery route is not yet available");
continue;
}
let send_result = self
.supervisor_bridge
.send_supervisor_delivery(peer, &delivery)
.await;
if let Err(error) = send_result {
submission_diagnostics.insert(peer_id, error.to_string());
tracing::debug!(%operation_id, peer_id = %peer.peer_id, %error, "one-way supervisor rotation submission remains pending");
}
}
for peer in &remote_peers {
let peer_id = peer.peer_id.to_string();
let accepted_before_observation = accepted_peer_ids.contains(&peer_id);
let expected_target = member_targets.get(&peer_id).cloned().ok_or_else(|| {
MobError::Internal(format!(
"supervisor rotation operation {operation_id} has no durable target for peer '{peer_id}'"
))
})?;
let command =
BridgeCommand::ObserveSupervisorRotation(BridgeSupervisorRotationObserve {
operation_id,
observer: expected_target.clone(),
observer_epoch: next.epoch,
protocol_version: next.protocol_version,
});
let observation_window = if cfg!(test) {
std::time::Duration::from_millis(300)
} else {
std::time::Duration::from_secs(5)
};
let deadline = Instant::now() + observation_window;
let mut last_observation = submission_diagnostics.remove(&peer_id).map_or_else(
|| "operation not yet observable".to_string(),
|error| format!("one-way submission was not delivered: {error}"),
);
let mut legacy_adoption_submitted = false;
let completed = loop {
let now = Instant::now();
if now >= deadline {
break false;
}
let request_timeout = std::cmp::min(
deadline.saturating_duration_since(now),
std::time::Duration::from_millis(500),
);
let value = match self
.supervisor_bridge
.send_bridge_command_as_authority(&next, peer, &command, request_timeout)
.await
{
Ok(value) => value,
Err(error) => {
last_observation =
format!("{last_observation}; observation failed: {error}");
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
};
let observation: BridgeSupervisorRotationObservation =
match super::bridge_protocol::decode_bridge_payload(
&command,
value,
"observe supervisor rotation",
) {
Ok(observation) => observation,
Err(error) => {
last_observation =
format!("{last_observation}; observation decode failed: {error}");
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
};
match observation {
BridgeSupervisorRotationObservation::NotFound {
operation_id: observed_operation_id,
} => {
if observed_operation_id != operation_id {
return Err(MobError::Internal(format!(
"supervisor rotation observation for peer '{peer_id}' returned mismatched not-found operation {observed_operation_id} (expected {operation_id})"
)));
}
if accepted_before_observation && !legacy_adoption_submitted {
let route_refresh =
BridgeCommand::AuthorizeSupervisor(BridgeSupervisorPayload {
supervisor: expected_target.clone(),
epoch: next.epoch,
protocol_version: next.protocol_version,
});
let refresh_timeout = std::cmp::min(
deadline.saturating_duration_since(Instant::now()),
std::time::Duration::from_millis(500),
);
let refresh_value = match self
.supervisor_bridge
.send_bridge_command_as_authority(
&next,
peer,
&route_refresh,
refresh_timeout,
)
.await
{
Ok(value) => value,
Err(error) => {
last_observation = format!(
"legacy next-authority route refresh remains unconfirmed: {error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
};
if let Err(error) = super::bridge_protocol::decode_bridge_ack(
&route_refresh,
refresh_value,
"legacy supervisor route refresh",
) {
last_observation = format!(
"legacy next-authority route refresh was not acknowledged: {error}"
);
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
continue;
}
let delivery = BridgeSupervisorDelivery::SubmitSupervisorRotation(
BridgeSupervisorRotationSubmit {
operation_id,
target: expected_target.clone(),
target_epoch: next.epoch,
protocol_version: next.protocol_version,
},
);
match self
.supervisor_bridge
.send_supervisor_delivery_as_authority(&next, peer, &delivery)
.await
{
Ok(()) => {
legacy_adoption_submitted = true;
last_observation =
"legacy operation adoption submitted".to_string();
}
Err(error) => {
last_observation = format!(
"legacy operation adoption remains pending: {error}"
);
}
}
} else {
last_observation = "operation not found yet".to_string();
}
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Pending { operation, phase },
} => {
if operation.operation_id != operation_id
|| operation.target.target_epoch != next.epoch
|| operation.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched pending receipt for peer '{peer_id}'"
)));
}
last_observation = format!("operation pending in phase {phase:?}");
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Completed { receipt },
} => {
if receipt.operation_id != operation_id
|| receipt.target.target_epoch != next.epoch
|| receipt.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched terminal receipt for peer '{peer_id}'"
)));
}
break true;
}
BridgeSupervisorRotationObservation::Found {
state: BridgeSupervisorRotationState::Rejected { receipt },
} => {
if receipt.operation.operation_id != operation_id
|| receipt.operation.target.target_epoch != next.epoch
|| receipt.operation.target.target != expected_target
{
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned a mismatched rejection receipt for peer '{peer_id}'"
)));
}
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} was rejected by peer '{peer_id}' ({:?}): {}",
receipt.cause, receipt.reason
),
});
}
BridgeSupervisorRotationObservation::Found { state } => {
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned unsupported state {state:?} for peer '{peer_id}'"
)));
}
_ => {
return Err(MobError::Internal(format!(
"supervisor rotation operation {operation_id} returned an unsupported observation for peer '{peer_id}'"
)));
}
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
};
if !completed {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} remains pending for peer '{peer_id}': {last_observation}"
),
});
}
if accepted_peer_ids.insert(peer_id) {
let persisted = self
.persist_pending_supervisor_rotation(
PendingSupervisorRotationPersistenceRequest {
current: &stable_current,
expected_durable: &durable_write_expected,
pending: &next,
operation_id,
accepted_peer_ids: &accepted_peer_ids,
active_peer_ids: &active_peer_ids,
member_targets: &member_targets,
},
)
.await?;
let Some(record) = persisted.persisted_record else {
return Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: false,
pending_authority_recorded: persisted.pending_authority_recorded,
rollback_error: None,
reason: format!(
"supervisor rotation operation {operation_id} completed remotely but its observation checkpoint was not persisted"
),
});
};
durable_write_expected = record;
}
}
let public_peer_id = next.public_peer_id.clone();
match self
.activate_supervisor_authority(
&stable_current,
&durable_write_expected,
operation_id,
&next,
)
.await
{
Ok(()) => Ok(super::handle::SupervisorRotationReport {
previous_epoch: stable_current.epoch,
current_epoch: next.epoch,
public_peer_id,
}),
Err(error) => Err(MobError::SupervisorRotationIncomplete {
previous_epoch: stable_current.epoch,
attempted_epoch: next.epoch,
attempted_public_peer_id: next.public_peer_id.clone(),
rotated_peer_count: accepted_peer_ids.len(),
rollback_succeeded: error.rollback_succeeded,
pending_authority_recorded: error.pending_authority_recorded,
rollback_error: error.rollback_error,
reason: format!(
"supervisor rotation operation {operation_id} completed remotely but local authority activation failed: {}",
error.error
),
}),
}
}
async fn rotate_supervisor_bridge_to(
&self,
authority: &crate::store::SupervisorAuthorityRecord,
) -> Result<(), MobError> {
if !self.supervisor_authority_record_is_machine_authorized(authority) {
return Err(MobError::Internal(format!(
"refusing to rotate supervisor bridge to peer={} epoch={} without generated MobMachine authority",
authority.public_peer_id, authority.epoch
)));
}
let active = self.supervisor_bridge.authority().await;
if active.public_peer_id != authority.public_peer_id
|| active.epoch != authority.epoch
|| active.protocol_version != authority.protocol_version
{
let bridge_authority = self.supervisor_bridge_authority_for_record(authority)?;
let prepared = self
.supervisor_bridge
.prepare_rotation(authority.clone(), &bridge_authority)
.await?;
self.supervisor_bridge
.commit_prepared_rotation(prepared)
.await?;
}
Ok(())
}
async fn persist_pending_supervisor_rotation(
&mut self,
request: PendingSupervisorRotationPersistenceRequest<'_>,
) -> Result<SupervisorPendingRotationPersistence, MobError> {
let PendingSupervisorRotationPersistenceRequest {
current,
expected_durable,
pending,
operation_id,
accepted_peer_ids,
active_peer_ids,
member_targets,
} = request;
let mut record = current.without_pending_rotation();
let pending_rotation = crate::store::SupervisorPendingRotationRecord::from_authority(
pending,
operation_id,
accepted_peer_ids.iter().cloned().collect(),
member_targets.clone(),
);
let record_pending_input =
record.dsl_record_pending_rotation_input(&pending_rotation, active_peer_ids);
record.pending_rotation = Some(pending_rotation);
let prepared = self.prepare_supervisor_authority_persistence(
record_pending_input,
&record,
"persist_pending_supervisor_rotation",
)?;
match self
.runtime_metadata
.compare_and_put_supervisor_authority(
&self.definition.id,
expected_durable,
&record,
&prepared.authority,
)
.await
{
Ok(true) => {
self.commit_prepared_dsl_transition(prepared.transition)?;
Ok(SupervisorPendingRotationPersistence {
pending_authority_recorded: record.pending_rotation.is_some(),
persisted_record: Some(record),
})
}
Ok(false) => {
tracing::warn!(
mob_id = %self.definition.id,
"supervisor authority changed while persisting pending rotation"
);
Ok(SupervisorPendingRotationPersistence {
pending_authority_recorded: false,
persisted_record: None,
})
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
error = %error,
"failed to persist pending supervisor rotation"
);
Ok(SupervisorPendingRotationPersistence {
pending_authority_recorded: expected_durable.pending_rotation.is_some(),
persisted_record: None,
})
}
}
}
async fn activate_supervisor_authority(
&mut self,
current: &crate::store::SupervisorAuthorityRecord,
expected_durable: &crate::store::SupervisorAuthorityRecord,
operation_id: super::bridge_protocol::SupervisorRotationOperationId,
next: &crate::store::SupervisorAuthorityRecord,
) -> Result<(), SupervisorAuthorityActivationError> {
let prepared_commit = self
.prepare_supervisor_authority_persistence(
current.dsl_commit_rotation_input(&operation_id.to_string(), next),
next,
"activate_supervisor_authority",
)
.map_err(|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let prepared_bridge_authority =
crate::store::SupervisorAuthorityBridgeAuthority::from_persistence_authority(
next,
&prepared_commit.authority,
)
.map_err(|error| SupervisorAuthorityActivationError {
error: MobError::from(error),
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let prepared_bridge_rotation = self
.supervisor_bridge
.prepare_rotation(next.clone(), &prepared_bridge_authority)
.await
.map_err(|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
})?;
let previous_private_trust_removal_key = current.public_peer_id.clone();
let session_member_refs = {
let roster = self.roster.read().await;
roster
.list_all()
.filter_map(|entry| match &entry.member_ref {
MemberRef::Session { .. } => Some(entry.member_ref.clone()),
MemberRef::BackendPeer { .. } => None,
})
.collect::<Vec<_>>()
};
for member_ref in session_member_refs {
if let (Some(session_id), Some(comms)) = (
member_ref.bridge_session_id().cloned(),
self.provisioner_comms(&member_ref).await,
) {
let supervisor_spec =
Self::supervisor_spec_for_authority(&self.definition.id, next).map_err(
|error| SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded: true,
rollback_error: None,
},
)?;
match self
.install_supervisor_private_trust_for_session_authority(
&session_id,
&comms,
next,
supervisor_spec,
Some(current),
Some(&previous_private_trust_removal_key),
)
.await
{
Ok(_) => {}
Err(SupervisorPrivateTrustInstallError {
error,
new_trust_cleanup_failed,
}) => {
return Err(Self::supervisor_activation_error(
error,
new_trust_cleanup_failed,
true,
));
}
}
}
}
if let Err(error) = self
.supervisor_bridge
.commit_prepared_rotation(prepared_bridge_rotation)
.await
{
return Err(Self::supervisor_activation_error(error, false, true));
}
match self
.runtime_metadata
.compare_and_put_supervisor_authority(
&self.definition.id,
expected_durable,
next,
&prepared_commit.authority,
)
.await
{
Ok(true) => {}
Ok(false) => {
let cas_error = MobError::from(crate::store::MobStoreError::CasConflict(format!(
"supervisor authority changed before final commit for mob '{}'",
self.definition.id
)));
return Err(Self::supervisor_activation_error(cas_error, false, false));
}
Err(error) => {
return Err(Self::supervisor_activation_error(
MobError::from(error),
false,
true,
));
}
}
if let Err(error) = self.commit_prepared_dsl_transition(prepared_commit.transition) {
return Err(Self::supervisor_activation_error(error, false, false));
}
Ok(())
}
fn supervisor_activation_error(
error: MobError,
new_trust_cleanup_failed: bool,
pending_authority_recorded: bool,
) -> SupervisorAuthorityActivationError {
SupervisorAuthorityActivationError {
error,
rollback_succeeded: false,
pending_authority_recorded,
rollback_error: new_trust_cleanup_failed
.then(|| "supervisor private trust cleanup failed".to_string()),
}
}
async fn fail_reset_to_stopped(&mut self) {
self.provisioner.cancel_all_checkpointers().await;
if let Err(e) =
self.apply_dsl_input(mob_dsl::MobMachineInput::Stop, "fail_reset_to_stopped")
{
tracing::warn!(error = %e, "authority rejected Stop in fail_reset_to_stopped");
}
}
async fn handle_reset(&mut self, prior_state: MobState) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_reset preflight")?;
self.ensure_flow_tracker_alignment("handle_reset preflight")
.await?;
let was_stopped = prior_state == MobState::Stopped;
self.cancel_all_flow_tasks().await?;
if was_stopped {
self.provisioner.rearm_all_checkpointers().await;
}
if let Err(error) = self.retire_all_members("reset").await {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
let prepared = match self
.prepare_dsl_input_transition(mob_dsl::MobMachineInput::Reset, "reset_to_running")
.map_err(|error| {
tracing::debug!(
context = "reset_to_running",
error = %error,
"MobMachine command admission rejected input"
);
self.invalid_transition_to(MobState::Running)
}) {
Ok(prepared) => prepared,
Err(error) => {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
};
if let Err(error) = Self::require_lifecycle_journal_effect(
&prepared.transition,
mob_dsl::MobLifecycleJournalKind::Reset,
"reset_to_running",
) {
if was_stopped {
self.provisioner.cancel_all_checkpointers().await;
}
return Err(error);
}
let events = self.events.clone();
let mob_id = self.definition.id.clone();
if let Err(error) = self
.commit_prepared_dsl_transition_after(prepared, move || async move {
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MobReset,
})
.await
.map_err(MobError::from)?;
Ok(())
})
.await
{
self.fail_reset_to_stopped().await;
return Err(error);
}
self.edge_locks.clear().await;
self.retired_event_index.write().await.clear();
self.ensure_pending_spawn_alignment("handle_reset completion")?;
self.ensure_flow_tracker_alignment("handle_reset completion")
.await?;
Ok(())
}
async fn retire_all_members(&mut self, context: &str) -> Result<(), MobError> {
let prepared_retire_all = self.prepare_command_admission(
mob_dsl::MobMachineInput::RetireAll,
MobState::Running,
context,
)?;
self.commit_prepared_dsl_input(prepared_retire_all)?;
self.ensure_pending_spawn_alignment("retire_all_members preflight")?;
let pending_reason = format!("{context}: draining pending spawns before bulk retirement");
self.fail_all_pending_spawns(&pending_reason).await?;
self.ensure_pending_spawn_alignment("retire_all_members after pending drain")?;
self.cancel_pending_peer_deliveries("all members are retiring")
.await;
let ids = {
let roster = self.roster.read().await;
roster
.list_all()
.map(|entry| entry.agent_identity.clone())
.collect::<Vec<_>>()
};
if ids.is_empty() {
return Ok(());
}
let mut retire_failures: Vec<String> = Vec::new();
for id in ids {
let result = self.retire_one(id).await;
if let Err((id, error)) = result {
let roster_still_contains_member = {
let roster = self.roster.read().await;
roster.get(&id).is_some()
};
if !roster_still_contains_member {
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %id,
error = %error,
"{context}: retire reported cleanup failure after removing member; continuing"
);
continue;
}
tracing::warn!(
mob_id = %self.definition.id,
agent_identity = %id,
error = %error,
"{context}: retire failed for member"
);
retire_failures.push(format!("{id}: {error}"));
}
}
if !retire_failures.is_empty() {
return Err(MobError::Internal(format!(
"{context} aborted: {} member(s) could not be retired: {}",
retire_failures.len(),
retire_failures.join("; ")
)));
}
self.ensure_pending_spawn_alignment("retire_all_members completion")?;
Ok(())
}
async fn retire_one(&mut self, id: AgentIdentity) -> Result<(), (AgentIdentity, MobError)> {
self.handle_retire_inner(&id, true, false)
.await
.map_err(|error| (id, error))
}
async fn handle_submit_work(
&mut self,
payload: Box<super::state::SubmitWorkPayload>,
) -> Result<SubmitWorkDispatchCompletion, MobError> {
let super::state::SubmitWorkPayload {
runtime_id,
fence_token,
work_ref,
content,
origin,
injected_context,
interaction_id,
handling_mode,
render_metadata,
ack_mode,
} = *payload;
tracing::debug!(
agent_identity = %runtime_id.identity,
runtime_id = %runtime_id,
work_ref = %work_ref,
origin = ?origin,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
"handle_submit_work started"
);
self.ensure_pending_spawn_alignment("handle_submit_work preflight")?;
let agent_identity = runtime_id.identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let declared_dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&runtime_id);
let declared_dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
let declared_submit_work_admitted = match self.probe_command_admission(
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: declared_dsl_runtime_id.clone(),
fence_token: declared_dsl_fence_token,
work_id: mob_dsl::WorkId::from_work_ref(&work_ref),
origin: mob_dsl::WorkOrigin::from(origin),
},
MobState::Running,
"submit_work_command_admission",
) {
Ok(()) => true,
Err(error) => {
if self.state() != MobState::Running {
return Err(error);
}
false
}
};
let initial_entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
let initial_entry_present = initial_entry.is_some();
let entry = match initial_entry {
Some(e) => {
self.ensure_member_not_broken(&e.agent_identity).await?;
e
}
None => {
if matches!(origin, WorkOrigin::Internal) {
let current_state = self.state();
return Err(Self::resolve_submit_work_projection_missing_or_rejection(
&mut self.dsl_authority,
declared_submit_work_admitted,
&dsl_identity,
&declared_dsl_runtime_id,
declared_dsl_fence_token,
&runtime_id,
origin,
&agent_identity,
current_state,
));
}
let identity = AgentIdentity::from(agent_identity.as_str());
if let Some(spec) = self.resolve_spawn_policy_via_machine(&identity).await? {
Box::pin(self.spawn_from_policy_inline(&identity, spec, &work_ref, origin))
.await?;
{
let roster = self.roster.read().await;
roster.get(&identity).cloned()
}
.ok_or_else(|| {
MobError::Internal(format!(
"auto-spawned member '{identity}' missing from roster after completion"
))
})?
} else {
let current_state = self.state();
return Err(Self::resolve_submit_work_projection_missing_or_rejection(
&mut self.dsl_authority,
declared_submit_work_admitted,
&dsl_identity,
&declared_dsl_runtime_id,
declared_dsl_fence_token,
&runtime_id,
origin,
&agent_identity,
current_state,
));
}
}
};
let admission_runtime_id = if initial_entry_present {
runtime_id.clone()
} else {
entry.agent_runtime_id.clone()
};
let admission_fence_token = if initial_entry_present {
fence_token
} else {
entry.fence_token
};
if !injected_context.is_empty() {
if handling_mode == meerkat_core::types::HandlingMode::Steer {
return Err(MobError::InjectedContextUndeliverable {
member_id: AgentIdentity::from(agent_identity.as_str()),
reason: "steer dispatch carries no transcript boundary for injected context",
});
}
if entry.runtime_mode == crate::MobRuntimeMode::AutonomousHost {
return Err(MobError::InjectedContextUndeliverable {
member_id: AgentIdentity::from(agent_identity.as_str()),
reason: "autonomous inbox delivery carries no user-channel work boundary",
});
}
}
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&admission_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(admission_fence_token);
let dsl_work_id = mob_dsl::WorkId::from_work_ref(&work_ref);
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let transition = match mob_dsl::MobMachineMutator::apply(
&mut self.dsl_authority,
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: dsl_work_id.clone(),
origin: dsl_origin,
},
) {
Ok(transition) => transition,
Err(_) => {
let current_state = self.state();
return Err(Self::resolve_submit_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&admission_runtime_id,
origin,
&agent_identity,
current_state,
));
}
};
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
let ingress_authority = SubmitWorkIngressAuthority::from_transition(
&transition,
&dsl_runtime_id,
dsl_fence_token,
mob_dsl::Generation::from_domain(admission_runtime_id.generation),
&dsl_work_id,
dsl_origin,
)?;
drop(transition);
let completion = self
.dispatch_member_turn_after_machine_admission(
&entry,
ingress_authority,
SubmitWorkDispatchRequest {
content,
injected_context,
interaction_id,
handling_mode,
render_metadata,
ack_mode,
operation_id: None,
},
)
.await?;
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
work_ref = %work_ref,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
completion = completion.kind(),
"handle_submit_work dispatched after machine admission"
);
Ok(completion)
}
fn spawn_turn_completed_reply(
provisioner: Arc<dyn MobProvisioner>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
reply_tx: oneshot::Sender<Result<(), MobError>>,
) {
tokio::spawn(async move {
let result = provisioner.start_turn(&member_ref, *req).await;
let _ = reply_tx.send(result);
});
}
fn spawn_turn_admission_reply(
provisioner: Arc<dyn MobProvisioner>,
member_ref: MemberRef,
req: Box<meerkat_core::service::StartTurnRequest>,
reply_tx: oneshot::Sender<Result<(), MobError>>,
) {
tokio::spawn(async move {
let result = provisioner.admit_turn(&member_ref, *req).await;
let _ = reply_tx.send(result);
});
}
async fn dispatch_turn_driven_spawn_initial_turn(
&mut self,
agent_identity: &AgentIdentity,
agent_runtime_id: &AgentRuntimeId,
fence_token: FenceToken,
operation_id: &meerkat_core::ops::OperationId,
content: ContentInput,
) -> Result<(), MobError> {
let entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
}
.ok_or_else(|| {
MobError::Internal(format!(
"turn-driven spawn initial SubmitWork for '{agent_identity}' had no roster projection after Spawn admission"
))
})?;
let work_ref = WorkRef::new();
let origin = WorkOrigin::Internal;
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(agent_runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
let dsl_work_id = mob_dsl::WorkId::from_work_ref(&work_ref);
let dsl_origin = mob_dsl::WorkOrigin::from(origin);
let transition = match mob_dsl::MobMachineMutator::apply(
&mut self.dsl_authority,
mob_dsl::MobMachineInput::SubmitWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
work_id: dsl_work_id.clone(),
origin: dsl_origin,
},
) {
Ok(transition) => transition,
Err(_) => {
let current_state = self.state();
return Err(Self::resolve_submit_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
agent_runtime_id,
origin,
agent_identity,
current_state,
));
}
};
if transition.from_phase != transition.to_phase {
let _ = self.phase_watch_tx.send(self.state());
}
self.publish_machine_state_projection();
let ingress_authority = SubmitWorkIngressAuthority::from_transition(
&transition,
&dsl_runtime_id,
dsl_fence_token,
mob_dsl::Generation::from_domain(agent_runtime_id.generation),
&dsl_work_id,
dsl_origin,
)?;
drop(transition);
let completion = self
.dispatch_member_turn_after_machine_admission(
&entry,
ingress_authority,
SubmitWorkDispatchRequest {
content,
injected_context: Vec::new(),
interaction_id: None,
handling_mode: meerkat_core::types::HandlingMode::Queue,
render_metadata: None,
ack_mode: crate::mob_machine::SubmitWorkAckMode::IngressAccepted,
operation_id: Some(operation_id.clone()),
},
)
.await?;
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
completion = completion.kind(),
"dispatch_turn_driven_spawn_initial_turn dispatched after machine admission"
);
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
"dispatch_turn_driven_spawn_initial_turn finishing dispatch"
);
self.finish_submit_work_dispatch(completion).await
}
async fn handle_cancel_all_work(
&mut self,
runtime_id: AgentRuntimeId,
fence_token: FenceToken,
) -> Result<(), MobError> {
let agent_identity = runtime_id.identity.clone();
let domain_identity = AgentIdentity::from(agent_identity.as_str());
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&domain_identity);
let entry = {
let roster = self.roster.read().await;
roster.get(&agent_identity).cloned()
};
let dsl_runtime_id = mob_dsl::AgentRuntimeId::from_domain(&runtime_id);
let dsl_fence_token = mob_dsl::FenceToken::from_domain(fence_token);
let prepared = self
.prepare_dsl_input(
mob_dsl::MobMachineInput::CancelAllWork {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: dsl_fence_token,
},
"handle_cancel_all_work",
)
.map_err(|_| {
let current_state = self.state();
Self::resolve_cancel_all_work_rejection_in_authority(
&mut self.dsl_authority,
&dsl_identity,
&dsl_runtime_id,
dsl_fence_token,
&runtime_id,
&agent_identity,
current_state,
)
})?;
let entry = entry.ok_or_else(|| {
MobError::Internal(format!(
"MobMachine accepted CancelAllWork for '{agent_identity}' but the roster projection has no member entry"
))
})?;
self.commit_prepared_dsl_input(prepared)?;
let machine_member_ref =
self.machine_member_ref_for_behavior(&entry, "cancel all work interrupt")?;
self.provisioner.interrupt_member(&machine_member_ref).await
}
async fn finish_submit_work_dispatch(
&self,
completion: SubmitWorkDispatchCompletion,
) -> Result<(), MobError> {
match completion {
SubmitWorkDispatchCompletion::Completed => {
tracing::debug!("finish_submit_work_dispatch completed without runtime call");
Ok(())
}
SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref,
req,
} => {
tracing::debug!(
member_ref = ?member_ref,
operation_id = ?operation_id,
"finish_submit_work_dispatch admitting turn"
);
let result = if let Some(operation_id) = operation_id.as_ref() {
self.provisioner
.admit_turn_for_operation(&member_ref, operation_id, *req)
.await
} else {
self.provisioner.admit_turn(&member_ref, *req).await
};
tracing::debug!(
member_ref = ?member_ref,
ok = result.is_ok(),
"finish_submit_work_dispatch admitted turn"
);
result
}
SubmitWorkDispatchCompletion::AwaitTurnCompletion { member_ref, req } => {
tracing::debug!(
member_ref = ?member_ref,
"finish_submit_work_dispatch starting turn"
);
let result = self.provisioner.start_turn(&member_ref, *req).await;
tracing::debug!(
member_ref = ?member_ref,
ok = result.is_ok(),
"finish_submit_work_dispatch started turn"
);
result
}
}
}
async fn dispatch_member_turn_after_machine_admission(
&mut self,
entry: &RosterEntry,
ingress_authority: SubmitWorkIngressAuthority,
request: SubmitWorkDispatchRequest,
) -> Result<SubmitWorkDispatchCompletion, MobError> {
let SubmitWorkDispatchRequest {
content,
injected_context,
interaction_id,
handling_mode,
render_metadata,
ack_mode,
operation_id,
} = request;
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_id = %entry.agent_runtime_id,
runtime_mode = ?entry.runtime_mode,
handling_mode = ?handling_mode,
ack_mode = ?ack_mode,
ingress_authority = ingress_authority.variant(),
"dispatch_member_turn_after_machine_admission started"
);
let live_steer_admission = handling_mode == meerkat_core::types::HandlingMode::Steer
&& ack_mode == crate::mob_machine::SubmitWorkAckMode::IngressAccepted;
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission resolving member ref"
);
let machine_member_ref =
self.machine_member_ref_for_behavior(entry, "direct turn delivery")?;
tracing::debug!(
agent_identity = %entry.agent_identity,
member_ref = ?machine_member_ref,
"dispatch_member_turn_after_machine_admission resolved member ref"
);
ingress_authority.verify_member_ref(&machine_member_ref, "direct turn delivery")?;
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission verified ingress authority"
);
if !live_steer_admission
&& let Some(bridge_session_id) = machine_member_ref.bridge_session_id()
{
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission checking live session"
);
match self
.session_service
.has_live_session(bridge_session_id)
.await
{
Ok(true) => {
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission live session exists"
);
}
Ok(false) | Err(meerkat_core::service::SessionError::NotFound { .. }) => {
self.revive_member_live_materialization(
entry,
&machine_member_ref,
bridge_session_id,
)
.await?;
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %bridge_session_id,
"dispatch_member_turn_after_machine_admission revived live session"
);
}
Err(error) => return Err(MobError::SessionError(error)),
}
}
tracing::debug!(
agent_identity = %entry.agent_identity,
runtime_mode = ?entry.runtime_mode,
"dispatch_member_turn_after_machine_admission dispatching runtime mode"
);
match entry.runtime_mode {
crate::MobRuntimeMode::AutonomousHost => {
if ingress_authority.is_peer_runtime() {
return Err(MobError::Internal(format!(
"autonomous direct turn delivery requires generated RequestRuntimeIngress authority for '{}'",
entry.agent_identity
)));
}
let bridge_session_id = machine_member_ref
.bridge_session_id()
.cloned()
.ok_or_else(|| {
MobError::Internal(format!(
"autonomous direct turn delivery requires MobMachine session binding for '{}'",
entry.agent_identity
))
})?;
self.ensure_autonomous_runtime_ready(&entry.agent_identity, &machine_member_ref)
.await?;
if self
.autonomous_steer_requires_admission_barrier(
entry,
&machine_member_ref,
handling_mode,
ack_mode,
)
.await?
{
let req = meerkat_core::service::StartTurnRequest {
injected_context: Vec::new(),
prompt: content,
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::new(
handling_mode,
None,
Vec::new(),
submit_work_turn_metadata(render_metadata, interaction_id),
),
};
return Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref: machine_member_ref,
req: Box::new(req),
});
}
let injector = self
.provisioner
.interaction_event_injector(&bridge_session_id)
.await
.ok_or_else(|| MobError::MissingMemberCapability {
member_id: crate::ids::AgentIdentity::from(entry.agent_identity.as_str()),
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous direct turn delivery",
})?;
let inject_result = match interaction_id {
Some(interaction_id) => injector.inject_with_interaction_id(
interaction_id,
content,
meerkat_core::PlainEventSource::Rpc,
handling_mode,
render_metadata,
),
None => injector.inject(
content,
meerkat_core::PlainEventSource::Rpc,
handling_mode,
render_metadata,
),
};
inject_result.map_err(|error| {
MobError::Internal(format!(
"autonomous dispatch inject failed for '{}': {}",
entry.agent_identity, error
))
})?;
Ok(SubmitWorkDispatchCompletion::Completed)
}
crate::MobRuntimeMode::TurnDriven => {
tracing::debug!(
agent_identity = %entry.agent_identity,
ingress_is_peer_runtime = ingress_authority.is_peer_runtime(),
"dispatch_member_turn_after_machine_admission entering turn-driven dispatch"
);
let machine_member_ref = if ingress_authority.is_peer_runtime() {
self.authorize_peer_only_member_ref_for_behavior(
&machine_member_ref,
"turn-driven direct turn delivery",
)
.await?
} else {
machine_member_ref
};
tracing::debug!(
agent_identity = %entry.agent_identity,
member_ref = ?machine_member_ref,
"dispatch_member_turn_after_machine_admission building turn request"
);
let req = meerkat_core::service::StartTurnRequest {
injected_context,
prompt: content,
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::new(
handling_mode,
None,
Vec::new(),
submit_work_turn_metadata(render_metadata, interaction_id),
),
};
tracing::debug!(
agent_identity = %entry.agent_identity,
ack_mode = ?ack_mode,
"dispatch_member_turn_after_machine_admission built turn request"
);
match ack_mode {
crate::mob_machine::SubmitWorkAckMode::IngressAccepted => {
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxing turn admission request"
);
let req = Box::new(req);
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxed turn admission request"
);
Ok(SubmitWorkDispatchCompletion::AwaitTurnAdmission {
operation_id,
member_ref: machine_member_ref,
req,
})
}
crate::mob_machine::SubmitWorkAckMode::TurnCompleted => {
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxing turn completion request"
);
let req = Box::new(req);
tracing::debug!(
agent_identity = %entry.agent_identity,
"dispatch_member_turn_after_machine_admission boxed turn completion request"
);
Ok(SubmitWorkDispatchCompletion::AwaitTurnCompletion {
member_ref: machine_member_ref,
req,
})
}
}
}
}
}
async fn autonomous_steer_requires_admission_barrier(
&self,
entry: &RosterEntry,
member_ref: &MemberRef,
handling_mode: meerkat_core::types::HandlingMode,
ack_mode: crate::mob_machine::SubmitWorkAckMode,
) -> Result<bool, MobError> {
if handling_mode != meerkat_core::types::HandlingMode::Steer
|| ack_mode != crate::mob_machine::SubmitWorkAckMode::IngressAccepted
{
return Ok(false);
}
#[cfg(feature = "runtime-adapter")]
if let (Some(adapter), Some(session_id)) =
(&self.runtime_adapter, member_ref.bridge_session_id())
{
use meerkat_runtime::service_ext::SessionServiceRuntimeExt as _;
match adapter.runtime_state(session_id).await {
Ok(meerkat_runtime::RuntimeState::Running) => {
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
"active steer admission barrier enabled by running runtime state"
);
return Ok(true);
}
Ok(state) => {
let session_active = self
.provisioner
.is_member_active(member_ref)
.await?
.unwrap_or(false);
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
runtime_state = ?state,
session_active,
"active steer admission barrier checked non-running runtime state"
);
if session_active {
return Ok(true);
}
return Ok(false);
}
Err(error) => {
tracing::debug!(
agent_identity = %entry.agent_identity,
session_id = %session_id,
error = %error,
"runtime state unavailable; requiring autonomous steer admission barrier (fail closed)"
);
return Ok(true);
}
}
}
Ok(false)
}
async fn handle_run_flow(
&mut self,
flow_id: FlowId,
activation_params: serde_json::Value,
scoped_event_tx: Option<mpsc::Sender<meerkat_core::ScopedAgentEvent>>,
) -> Result<RunId, MobError> {
self.ensure_pending_spawn_alignment("handle_run_flow preflight")?;
self.ensure_flow_tracker_alignment("handle_run_flow preflight")
.await?;
let run_id = RunId::new();
self.preview_run_flow_command_admission(&run_id)?;
let config = FlowRunConfig::from_definition(flow_id, &self.definition)?;
let run_flow = MobRun::run_flow_input(&run_id, &config)?;
debug_assert!(matches!(run_flow, mob_dsl::MobMachineInput::RunFlow { .. }));
let prepared_run_flow = self
.prepare_dsl_input(run_flow.clone(), "run_flow")
.map_err(|_| self.invalid_transition_to(MobState::Running))?;
self.create_pending_run(
run_id.clone(),
&config,
prepared_run_flow.authority.state(),
activation_params.clone(),
vec![run_flow],
)
.await
.inspect_err(|error| {
tracing::warn!(
run_id = %run_id,
flow_id = %config.flow_id,
error = %error,
"flow admission run-store create failed before committing MobMachine RunFlow"
);
})?;
self.commit_prepared_dsl_input(prepared_run_flow)?;
if let Err(error) = self.apply_dsl_signal(mob_dsl::MobMachineSignal::StartRun, "start_run")
{
let mut details = Vec::new();
if let Err(rollback_error) = self.apply_dsl_signal(
mob_dsl::MobMachineSignal::CompleteFlow,
"complete_flow_rollback",
) {
details.push(format!(
"RunFlow CompleteFlow rollback failed: {rollback_error}"
));
}
let terminalize_cause = FlowFailureCause::AdmissionFailed {
detail: format!(
"lifecycle StartRun transition failed during flow admission: {error}"
),
};
if let Err(terminalize_error) = self
.terminalize_failed_in_actor(
run_id.clone(),
config.flow_id.clone(),
terminalize_cause,
"run_flow_start_signal_terminalize_failed",
)
.await
{
details.push(format!(
"terminalizing pending run failed: {terminalize_error}"
));
}
let detail_suffix = if details.is_empty() {
String::new()
} else {
format!("; {}", details.join("; "))
};
return Err(MobError::Internal(format!(
"lifecycle StartRun transition failed during flow admission: {error}{detail_suffix}"
)));
}
let cancel_token = tokio_util::sync::CancellationToken::new();
self.run_cancel_tokens.insert(
run_id.clone(),
(cancel_token.clone(), config.flow_id.clone()),
);
if let Some(scoped_event_tx) = scoped_event_tx {
self.flow_streams
.lock()
.await
.insert(run_id.clone(), scoped_event_tx);
}
let engine = self.flow_engine.clone();
let cleanup_tx = self.command_tx.clone();
let flow_engine = self.flow_engine.clone();
let flow_run_id = run_id.clone();
let flow_id_for_task = config.flow_id.clone();
let cleanup_run_id = run_id.clone();
let handle = tokio::spawn(async move {
let run_id_for_execute = flow_run_id.clone();
let execution_cancel = cancel_token.clone();
if let Err(error) = engine
.execute_flow(
run_id_for_execute,
config,
activation_params,
execution_cancel,
)
.await
{
tracing::error!(
run_id = %flow_run_id,
flow_id = %flow_id_for_task,
error = %error,
"flow task execution failed; delegating terminalization to flow-run kernel"
);
if cancel_token.is_cancelled() {
if let Err(finalize_error) = flow_engine
.terminalize_canceled(flow_run_id.clone(), flow_id_for_task)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize canceled run after flow task cancellation"
);
}
} else {
match error {
MobError::RunCanceled(_) => {
if let Err(finalize_error) = flow_engine
.terminalize_canceled(flow_run_id.clone(), flow_id_for_task)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize canceled run after flow task cancellation"
);
}
}
other => {
if let Err(finalize_error) = flow_engine
.terminalize_failed(
flow_run_id.clone(),
flow_id_for_task,
FlowFailureCause::from_step_error(&other),
)
.await
{
tracing::error!(
run_id = %flow_run_id,
error = %finalize_error,
"failed to finalize run after flow task error"
);
}
}
}
}
}
if cleanup_tx
.send(MobCommand::FlowFinished {
run_id: cleanup_run_id,
})
.await
.is_err()
{
tracing::warn!(
run_id = %flow_run_id,
"failed to send FlowFinished cleanup command"
);
}
});
self.run_tasks.insert(run_id.clone(), handle);
self.ensure_flow_tracker_alignment("handle_run_flow completion")
.await?;
Ok(run_id)
}
async fn create_pending_run(
&self,
run_id: RunId,
config: &FlowRunConfig,
machine_state: &mob_dsl::MobMachineState,
activation_params: serde_json::Value,
authority_inputs: Vec<mob_dsl::MobMachineInput>,
) -> Result<RunId, MobError> {
let flow_state =
MobRun::flow_state_for_config_with_authority(&run_id, config, machine_state)?;
let mut run = MobRun::pending_with_run_id(
run_id.clone(),
self.definition.id.clone(),
config.flow_id.clone(),
flow_state,
activation_params,
);
run.append_flow_authority_inputs(authority_inputs)?;
self.run_store.create_run(run).await?;
Ok(run_id)
}
async fn handle_flow_cleanup(
&mut self,
run_id: RunId,
context: &'static str,
) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_flow_cleanup preflight")?;
self.ensure_flow_tracker_alignment("handle_flow_cleanup preflight")
.await?;
let has_task = self.run_tasks.contains_key(&run_id);
let has_token = self.run_cancel_tokens.contains_key(&run_id);
let has_stream = self.flow_streams.lock().await.contains_key(&run_id);
if !has_task && !has_token && !has_stream {
let run_terminal = self
.run_store
.get_run(&run_id)
.await?
.as_ref()
.map(|run| crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status))
.transpose()?
.unwrap_or(false);
if !run_terminal {
return Err(MobError::Internal(format!(
"{context}: received cleanup for run {run_id} with no local trackers before the persisted run reached a terminal status"
)));
}
tracing::debug!(
run_id = %run_id,
context = context,
"flow cleanup command had no local run-tracker entries"
);
return Ok(());
}
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "flow_cleanup")?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "flow_cleanup")?;
let _ = self.run_tasks.remove(&run_id);
let _ = self.run_cancel_tokens.remove(&run_id);
let _ = self.flow_streams.lock().await.remove(&run_id);
self.ensure_flow_tracker_alignment("handle_flow_cleanup completion")
.await?;
Ok(())
}
async fn handle_cancel_flow(&mut self, run_id: RunId) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("handle_cancel_flow preflight")?;
self.apply_command_admission(
mob_dsl::MobMachineInput::CancelFlow {
run_id: mob_dsl::RunId::from(run_id.to_string()),
},
MobState::Running,
"cancel_flow",
)?;
let Some((cancel_token, flow_id)) = self
.run_cancel_tokens
.get(&run_id)
.map(|(token, flow_id)| (token.clone(), flow_id.clone()))
else {
if self.run_tasks.contains_key(&run_id)
|| self.flow_streams.lock().await.contains_key(&run_id)
{
return Err(MobError::Internal(format!(
"handle_cancel_flow: run {run_id} missing cancel token despite live task/stream trackers"
)));
}
self.ensure_flow_tracker_alignment("handle_cancel_flow no-op completion")
.await?;
return Ok(());
};
self.flow_streams.lock().await.remove(&run_id);
cancel_token.cancel();
let Some(mut handle) = self.run_tasks.remove(&run_id) else {
self.flow_engine.cancel_unfinished_steps(&run_id).await?;
self.terminalize_canceled_in_actor(
run_id.clone(),
flow_id,
"cancel_flow_no_handle_terminalize_canceled",
)
.await?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "cancel_flow_no_handle")
.map_err(|error| {
MobError::Internal(format!(
"flow canceled cleanup (no task handle): lifecycle CompleteFlow transition failed for run {run_id}: {error}"
))
})?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "cancel_flow_no_handle")
.map_err(|error| {
MobError::Internal(format!(
"flow canceled cleanup (no task handle): lifecycle FinishRun transition failed for run {run_id}: {error}"
))
})?;
let _ = self.run_cancel_tokens.remove(&run_id);
self.ensure_flow_tracker_alignment("handle_cancel_flow no-task cleanup")
.await?;
return Ok(());
};
let flow_engine = self.flow_engine.clone();
let cleanup_tx = self.command_tx.clone();
let cancel_grace_timeout = self
.definition
.limits
.as_ref()
.and_then(|limits| limits.cancel_grace_timeout_ms)
.map_or_else(
|| std::time::Duration::from_secs(5),
std::time::Duration::from_millis,
);
let cleanup_run_tracker = run_id.clone();
let cleanup_run_id_for_error = run_id.clone();
let cleanup_handle = tokio::spawn(async move {
let cleanup_run_id = run_id.clone();
let completed = tokio::select! {
_ = &mut handle => true,
() = tokio::time::sleep(cancel_grace_timeout) => false,
};
if completed {
let mut terminalized = true;
if let Err(error) = flow_engine.cancel_unfinished_steps(&run_id).await {
terminalized = false;
tracing::error!(
error = %error,
"failed to settle dispatched steps after flow task completion during cancellation"
);
}
if let Err(error) = flow_engine
.terminalize_canceled(run_id.clone(), flow_id)
.await
{
terminalized = false;
tracing::error!(
error = %error,
"failed to apply canceled terminalization after flow task completion"
);
}
if cleanup_tx
.send(MobCommand::FlowCanceledCleanup {
run_id: cleanup_run_id,
terminalized,
})
.await
.is_err()
{
tracing::warn!(
"failed to send FlowCanceledCleanup command after task completion"
);
}
return;
}
handle.abort();
let mut terminalized = true;
if let Err(error) = flow_engine.cancel_unfinished_steps(&run_id).await {
terminalized = false;
tracing::error!(
error = %error,
"failed to settle dispatched steps before flow cancellation terminalization"
);
}
if let Err(error) = flow_engine
.terminalize_canceled(run_id.clone(), flow_id)
.await
{
terminalized = false;
tracing::error!(
error = %error,
"failed flow-run kernel cancellation terminalization"
);
}
if cleanup_tx
.send(MobCommand::FlowCanceledCleanup {
run_id: cleanup_run_id,
terminalized,
})
.await
.is_err()
{
tracing::warn!("failed to send FlowCanceledCleanup command");
}
});
if let Some(replaced) = self.run_tasks.insert(cleanup_run_tracker, cleanup_handle) {
replaced.abort();
return Err(MobError::Internal(format!(
"handle_cancel_flow: duplicate flow cleanup task registration for run {cleanup_run_id_for_error}"
)));
}
self.ensure_flow_tracker_alignment("handle_cancel_flow completion")
.await?;
Ok(())
}
async fn apply_flow_run_command_in_actor(
&mut self,
run_id: &RunId,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<Option<Vec<flow_run::Effect>>, MobError> {
let authority_input = command.authority_input(run_id);
let prepared = self.prepare_dsl_input(authority_input.clone(), context)?;
let machine_state = prepared.authority.state().clone();
let machine_effects = prepared.effects.clone();
let authority =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&authority_input)?;
let effects = if matches!(command, MobMachineFlowRunCommand::StartRun(_)) {
self.flow_engine
.start_run_state_with_machine_state(
run_id,
machine_state,
authority,
machine_effects,
context,
)
.await?
} else {
Some(
self.flow_engine
.apply_command_with_machine_state(
run_id,
command,
machine_state,
authority,
machine_effects,
context,
)
.await?,
)
};
if effects.is_some() {
self.commit_prepared_dsl_input(prepared)?;
}
Ok(effects)
}
async fn commit_flow_run_command_in_actor(
&mut self,
run_id: &RunId,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<Option<Vec<flow_run::Effect>>, MobError> {
self.apply_flow_run_command_in_actor(run_id, command, context)
.await
}
async fn commit_flow_frame_store_plan_in_actor(
&mut self,
run_id: &RunId,
plan: FlowFrameLoopStorePlan,
) -> Result<bool, MobError> {
if !self
.flow_frame_store_plan_expected_matches(run_id, &plan)
.await?
{
return Ok(false);
}
let prepared =
self.prepare_dsl_inputs(plan.machine_inputs(), "flow_frame_loop_store_plan")?;
let authority_inputs = plan.machine_inputs().to_vec();
let won = match &plan {
FlowFrameLoopStorePlan::InsertFrame {
frame_id,
initial_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
None,
initial_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::FrameState {
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
Some(expected_frame),
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteStepAndRecordOutput {
frame_id,
expected_frame,
next_frame,
step_output_key,
step_output,
loop_context,
..
} => self
.run_store
.cas_complete_step_and_record_output_with_authority(
run_id,
frame_id,
expected_frame,
next_frame.clone(),
step_output_key.clone(),
step_output.clone(),
loop_context
.as_ref()
.map(|(loop_id, iteration)| (loop_id, *iteration)),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::GrantNodeSlot {
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_grant_node_slot_with_authority(
run_id,
expected_run_state,
next_run_state.clone(),
frame_id,
expected_frame,
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::StartLoop {
loop_instance_id,
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
initial_loop,
..
} => self
.run_store
.cas_start_loop_with_authority(
run_id,
loop_instance_id,
expected_run_state,
next_run_state.clone(),
frame_id,
expected_frame,
next_frame.clone(),
initial_loop.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::GrantBodyFrameStart {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
initial_frame,
ledger_entry,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_grant_body_frame_start_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
initial_frame.clone(),
ledger_entry.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::RunStateOnly {
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_flow_state_with_authority(
run_id,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::SealFrame {
frame_id,
expected_frame,
next_frame,
..
} => self
.run_store
.cas_frame_state_with_authority(
run_id,
frame_id,
Some(expected_frame),
next_frame.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteBodyFrame {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_complete_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
expected_frame,
next_frame.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::LoopRequestBodyFrame {
loop_instance_id,
expected_loop,
next_loop,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_loop_request_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
FlowFrameLoopStorePlan::CompleteLoop {
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
..
} => self
.run_store
.cas_complete_loop_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop.clone(),
frame_id,
expected_frame,
next_frame.clone(),
expected_run_state,
next_run_state.clone(),
authority_inputs,
)
.await
.map_err(MobError::from)?,
};
if won {
self.commit_prepared_dsl_input(prepared)?;
}
Ok(won)
}
async fn flow_frame_store_plan_expected_matches(
&self,
run_id: &RunId,
plan: &FlowFrameLoopStorePlan,
) -> Result<bool, MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
Ok(match plan {
FlowFrameLoopStorePlan::InsertFrame { frame_id, .. } => {
!run.frames.contains_key(frame_id)
}
FlowFrameLoopStorePlan::FrameState {
frame_id,
expected_frame,
..
}
| FlowFrameLoopStorePlan::CompleteStepAndRecordOutput {
frame_id,
expected_frame,
..
}
| FlowFrameLoopStorePlan::SealFrame {
frame_id,
expected_frame,
..
} => run.frames.get(frame_id) == Some(expected_frame),
FlowFrameLoopStorePlan::GrantNodeSlot {
expected_run_state,
frame_id,
expected_frame,
..
} => {
&run.flow_state == expected_run_state
&& run.frames.get(frame_id) == Some(expected_frame)
}
FlowFrameLoopStorePlan::StartLoop {
loop_instance_id,
expected_run_state,
frame_id,
expected_frame,
..
} => {
&run.flow_state == expected_run_state
&& run.frames.get(frame_id) == Some(expected_frame)
&& !run.loops.contains_key(loop_instance_id)
}
FlowFrameLoopStorePlan::GrantBodyFrameStart {
loop_instance_id,
expected_loop,
frame_id,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
&& !run.frames.contains_key(frame_id)
}
FlowFrameLoopStorePlan::RunStateOnly {
expected_run_state, ..
} => &run.flow_state == expected_run_state,
FlowFrameLoopStorePlan::CompleteBodyFrame {
loop_instance_id,
expected_loop,
frame_id,
expected_frame,
expected_run_state,
..
}
| FlowFrameLoopStorePlan::CompleteLoop {
loop_instance_id,
expected_loop,
frame_id,
expected_frame,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
&& run.frames.get(frame_id) == Some(expected_frame)
}
FlowFrameLoopStorePlan::LoopRequestBodyFrame {
loop_instance_id,
expected_loop,
expected_run_state,
..
} => {
&run.flow_state == expected_run_state
&& run.loops.get(loop_instance_id) == Some(expected_loop)
}
})
}
async fn commit_flow_terminalization_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
target: TerminalizationTarget,
command: MobMachineFlowRunCommand,
context: &'static str,
) -> Result<TerminalizationOutcome, MobError> {
let run = self
.run_store
.get_run(&run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
if crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status)?
&& !matches!(
(&target, &run.status),
(TerminalizationTarget::Canceled, MobRunStatus::Failed)
)
{
return self
.flow_engine
.repair_persisted_terminalization(run_id, flow_id, target)
.await;
}
let authority_input = command.authority_input(&run_id);
let prepared = self.prepare_dsl_input(authority_input.clone(), context)?;
let machine_state = prepared.authority.state().clone();
let machine_effects = prepared.effects.clone();
let authority =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&authority_input)?;
let outcome = self
.flow_engine
.terminalize_with_machine_state(
run_id.clone(),
flow_id,
target.clone(),
command,
machine_state,
authority,
machine_effects,
)
.await;
match outcome {
Ok(TerminalizationOutcome::Transitioned) => {
self.commit_prepared_dsl_input(prepared)?;
Ok(TerminalizationOutcome::Transitioned)
}
Ok(TerminalizationOutcome::Noop) => Ok(TerminalizationOutcome::Noop),
Err(error) => {
if self
.persisted_terminal_status_matches_target(&run_id, &target)
.await?
{
self.commit_prepared_dsl_input(prepared)?;
}
Err(error)
}
}
}
async fn persisted_terminal_status_matches_target(
&self,
run_id: &RunId,
target: &TerminalizationTarget,
) -> Result<bool, MobError> {
Ok(self
.run_store
.get_run(run_id)
.await?
.is_some_and(|run| run.status == target.status()))
}
async fn cancel_unfinished_steps_in_actor(&mut self, run_id: &RunId) -> Result<(), MobError> {
let run = self
.run_store
.get_run(run_id)
.await?
.ok_or_else(|| MobError::RunNotFound(run_id.clone()))?;
for step_id in run.ordered_steps()? {
if run
.flow_state
.step_status
.get(&step_id)
.and_then(|status| *status)
.is_some_and(|status| !matches!(status, flow_run::StepRunStatus::Dispatched))
{
continue;
}
self.apply_flow_run_command_in_actor(
run_id,
MobMachineFlowRunCommand::CancelStep(flow_run::inputs::CancelStep { step_id }),
"actor_cancel_unfinished_step",
)
.await?;
}
Ok(())
}
async fn terminalize_canceled_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
context: &'static str,
) -> Result<(), MobError> {
let _ = self
.commit_flow_terminalization_in_actor(
run_id,
flow_id,
TerminalizationTarget::Canceled,
MobMachineFlowRunCommand::TerminalizeCanceled(
flow_run::inputs::TerminalizeCanceled {},
),
context,
)
.await?;
Ok(())
}
async fn terminalize_failed_in_actor(
&mut self,
run_id: RunId,
flow_id: FlowId,
cause: FlowFailureCause,
context: &'static str,
) -> Result<(), MobError> {
let _ = self
.commit_flow_terminalization_in_actor(
run_id,
flow_id,
TerminalizationTarget::Failed { cause },
MobMachineFlowRunCommand::TerminalizeFailed(flow_run::inputs::TerminalizeFailed {}),
context,
)
.await?;
Ok(())
}
async fn cancel_all_flow_tasks(&mut self) -> Result<(), MobError> {
self.ensure_pending_spawn_alignment("cancel_all_flow_tasks preflight")?;
let tracked_run_ids = self.run_cancel_tokens.keys().cloned().collect::<Vec<_>>();
for run_id in tracked_run_ids {
let Some((token, flow_id)) = self
.run_cancel_tokens
.get(&run_id)
.map(|(token, flow_id)| (token.clone(), flow_id.clone()))
else {
continue;
};
token.cancel();
if let Some(handle) = self.run_tasks.remove(&run_id) {
handle.abort();
}
self.flow_streams.lock().await.remove(&run_id);
self.cancel_unfinished_steps_in_actor(&run_id).await?;
self.terminalize_canceled_in_actor(
run_id.clone(),
flow_id.clone(),
"cancel_all_flow_terminalize_canceled",
)
.await?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::CompleteFlow, "cancel_all_flow")?;
self.apply_dsl_signal(mob_dsl::MobMachineSignal::FinishRun, "cancel_all_flow")?;
let _ = self.run_cancel_tokens.remove(&run_id);
}
self.ensure_flow_tracker_alignment("cancel_all_flow_tasks completion")
.await?;
Ok(())
}
async fn rollback_failed_spawn(
&mut self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
profile_name: &ProfileName,
member_ref: &MemberRef,
successful_wiring_targets: &[AgentIdentity],
planned_wiring_targets: &[AgentIdentity],
) -> Result<(), MobError> {
let spawned_entry = {
let roster = self.roster.read().await;
roster.get(agent_identity).cloned()
};
let retire_event_already_present =
self.retire_event_exists(agent_identity, generation).await?;
if spawned_entry.is_none() && !retire_event_already_present {
return Err(MobError::WiringError(format!(
"spawn rollback requires roster entry for '{agent_identity}' before retiring durable member state"
)));
}
let mut wired_peers = successful_wiring_targets.to_vec();
wired_peers.sort();
wired_peers.dedup();
let mut cleanup_peers = wired_peers.clone();
for peer_id in planned_wiring_targets {
if peer_id != agent_identity && !cleanup_peers.contains(peer_id) {
cleanup_peers.push(peer_id.clone());
}
}
let mut cleanup_handoffs = BTreeMap::new();
if spawned_entry.is_some() {
for peer_member_identity in &cleanup_peers {
let peer_entry = {
let roster = self.roster.read().await;
roster.get(peer_member_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let cleanup_edge = mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(agent_identity),
mob_dsl::AgentIdentity::from_domain(&peer_entry.agent_identity),
);
match self.authorize_member_trust_cleanup(
&cleanup_edge,
"spawn_rollback_trust_cleanup_authority",
) {
Ok(handoff) => {
let retry_handoff = self
.authorize_member_trust_cleanup(
&cleanup_edge,
"spawn_rollback_trust_cleanup_retry_authority",
)
.ok();
cleanup_handoffs
.insert(peer_entry.agent_identity.clone(), (handoff, retry_handoff));
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback trust cleanup skipped without generated unwiring authority"
);
}
}
}
}
let spawned_comms = self.provisioner_comms(member_ref).await;
let mut rollback = LifecycleRollback::new("spawn rollback");
if !wired_peers.is_empty() {
let spawned_entry = spawned_entry.as_ref().ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires roster entry for '{agent_identity}'"
))
})?;
let spawned_sender = self
.sender_runtime_for_entry(spawned_entry)
.await
.ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires sender runtime for '{agent_identity}'"
))
})?;
let spawned_spec = match self
.resolve_wiring_endpoint(spawned_entry, "spawn rollback spawned member")
.await?
{
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
};
let spawned_peer_description = self
.definition
.resolve_profile(&spawned_entry.role, self.realm_profile_store.as_ref())
.await
.map(|p| p.peer_description)
.unwrap_or_default();
for peer_member_identity in &wired_peers {
let peer_spec = {
let roster = self.roster.read().await;
let peer_entry = roster.get(peer_member_identity).cloned().ok_or_else(|| {
MobError::WiringError(format!(
"spawn rollback requires roster entry for wired peer '{peer_member_identity}'"
))
})?;
drop(roster);
match self
.resolve_wiring_endpoint(&peer_entry, "spawn rollback")
.await?
{
WiringEndpoint::Local { spec, .. }
| WiringEndpoint::PeerOnly { spec, .. } => spec,
}
};
if let Err(error) = self
.notify_peer_retired(
&peer_spec,
agent_identity,
spawned_entry,
&spawned_spec,
&spawned_sender,
)
.await
{
return Err(rollback.fail(error).await);
}
rollback.defer(
format!(
"compensating mob.peer_added '{agent_identity}' -> '{peer_member_identity}'"
),
{
let spawned_sender = spawned_sender.clone();
let peer_spec = peer_spec.clone();
let agent_identity = agent_identity.clone();
let role = spawned_entry.role.clone();
let peer_description = spawned_peer_description.clone();
let spawned_spec = spawned_spec.clone();
move || async move {
let peer_route = PeerRoute::with_display_name(
peer_spec.peer_id,
peer_spec.name.clone(),
);
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: agent_identity.as_str().to_string(),
role: Some(role.as_str().to_string()),
description: Some(peer_description),
peer_spec: Some(
super::bridge_protocol::BridgePeerSpec::from(spawned_spec),
),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{agent_identity}': {error}"
))
})?;
let cmd = CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerAdded,
params,
};
spawned_sender.send(cmd).await?;
Ok(())
}
},
);
}
}
if let Some(spawned_entry) = spawned_entry.as_ref()
&& let Ok(spawned_endpoint) = self
.resolve_wiring_endpoint(spawned_entry, "spawn rollback trust cleanup spawned")
.await
{
let (spawned_spec, spawned_comms, spawned_binding) = match spawned_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (spec, Some(comms), None),
WiringEndpoint::PeerOnly { spec, binding } => (spec, None, Some(binding)),
};
for peer_member_identity in &cleanup_peers {
let peer_entry = {
let roster = self.roster.read().await;
roster.get(peer_member_identity).cloned()
};
let Some(peer_entry) = peer_entry else {
continue;
};
let Ok(peer_endpoint) = self
.resolve_wiring_endpoint(&peer_entry, "spawn rollback trust cleanup peer")
.await
else {
continue;
};
let (peer_spec, peer_comms, peer_binding) = match peer_endpoint {
WiringEndpoint::Local { comms, spec, .. } => (spec, Some(comms), None),
WiringEndpoint::PeerOnly { spec, binding } => (spec, None, Some(binding)),
};
let cleanup_handoff = cleanup_handoffs.get(&peer_entry.agent_identity);
if let Some(spawned_comms) = spawned_comms.as_ref() {
let peer_key = Self::trusted_peer_removal_key(&peer_spec);
if let Some((handoff, retry_handoff)) = cleanup_handoff.as_ref() {
let authority =
handoff.unwiring_authority_for(&peer_entry.agent_identity, &peer_key);
if let Ok(authority) = authority {
let removed = self
.apply_trusted_peer_remove(
spawned_comms.as_ref(),
peer_key.clone(),
authority,
)
.await;
if removed.is_err()
&& let Some(retry_handoff) = retry_handoff.as_ref()
&& let Ok(retry_authority) = retry_handoff
.unwiring_authority_for(&peer_entry.agent_identity, &peer_key)
{
let _ = self
.apply_trusted_peer_remove(
spawned_comms.as_ref(),
peer_key.clone(),
retry_authority,
)
.await;
}
}
}
}
if let Some(peer_comms) = peer_comms {
let spawned_key = Self::trusted_peer_removal_key(&spawned_spec);
if let Some((handoff, retry_handoff)) = cleanup_handoff.as_ref() {
let authority =
handoff.unwiring_authority_for(agent_identity, &spawned_key);
if let Ok(authority) = authority {
let removed = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
spawned_key.clone(),
authority,
)
.await;
if removed.is_err()
&& let Some(retry_handoff) = retry_handoff.as_ref()
&& let Ok(retry_authority) = retry_handoff
.unwiring_authority_for(agent_identity, &spawned_key)
{
let _ = self
.apply_trusted_peer_remove(
peer_comms.as_ref(),
spawned_key.clone(),
retry_authority,
)
.await;
}
}
}
}
if spawned_binding.is_some() || peer_binding.is_some() {
if let Err(error) = self.cleanup_member_machine_wiring_edge(
agent_identity,
&peer_entry.agent_identity,
"spawn_rollback_peer_only_machine_wiring_cleanup",
) {
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback could not clean generated peer-only wiring graph"
);
continue;
}
}
if let Some(spawned_binding) = spawned_binding.as_ref()
&& let Err(error) = self
.unwire_peer_only_recipient(
&spawned_spec,
Some(spawned_binding),
&peer_spec,
std::time::Duration::from_secs(2),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback failed to unwire spawned peer-only trust"
);
}
if let Some(peer_binding) = peer_binding.as_ref()
&& let Err(error) = self
.unwire_peer_only_recipient(
&peer_spec,
Some(peer_binding),
&spawned_spec,
std::time::Duration::from_secs(2),
)
.await
{
tracing::warn!(
mob_id = %self.definition.id,
peer = %peer_entry.agent_identity,
%error,
"spawn rollback failed to unwire peer-only trust for spawned member"
);
}
}
}
if !cleanup_peers.is_empty() {
let rollback_inputs = cleanup_peers
.iter()
.filter(|peer_id| *peer_id != agent_identity)
.map(|peer_id| mob_dsl::MobMachineInput::UnwireMembers {
edge: mob_dsl::WiringEdge::new(
mob_dsl::AgentIdentity::from_domain(agent_identity),
mob_dsl::AgentIdentity::from_domain(peer_id),
),
})
.collect::<Vec<_>>();
if !rollback_inputs.is_empty() {
match self.prepare_dsl_inputs(&rollback_inputs, "spawn_rollback_wiring_cleanup") {
Ok(prepared) => {
if let Err(error) = self.commit_prepared_dsl_input(prepared) {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"spawn rollback could not commit generated wiring graph cleanup"
);
}
}
Err(error) => {
tracing::warn!(
mob_id = %self.definition.id,
%error,
"spawn rollback could not clean generated wiring graph"
);
}
}
}
}
if let Some(entry) = spawned_entry.as_ref() {
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&entry.agent_identity);
let detach_session_id = member_ref.bridge_session_id().cloned();
let releasing = member_ref
.bridge_session_id()
.map(mob_dsl::SessionId::from_domain);
let session_id_for_route = releasing.clone();
let retirement_started_journal_kind = if releasing.is_some() {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing
} else {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly
};
let prepared_retire = match self.prepare_dsl_input_transition(
mob_dsl::MobMachineInput::Retire {
mob_id: mob_dsl::MobId::from_domain(&self.definition.id),
agent_runtime_id: mob_dsl::AgentRuntimeId::from_domain(&entry.agent_runtime_id),
agent_identity: dsl_identity.clone(),
generation: mob_dsl::Generation::from_domain(entry.generation),
releasing,
session_id: session_id_for_route.clone(),
},
"rollback_failed_spawn_mark_retiring",
) {
Ok(prepared_retire) => prepared_retire,
Err(error) => {
tracing::warn!(
agent_identity = %agent_identity,
%error,
"spawn rollback could not mark runtime retired in DSL"
);
return Err(rollback.fail(error).await);
}
};
if let Err(error) = Self::require_member_lifecycle_journal_effect(
&prepared_retire.transition,
retirement_started_journal_kind,
&entry.agent_identity,
&entry.agent_runtime_id,
None,
entry.generation,
session_id_for_route.clone(),
"rollback_failed_spawn_mark_retiring",
) {
return Err(rollback.fail(error).await);
}
if !retire_event_already_present
&& let Err(error) = self
.append_retirement_started_event_for_entry(
entry,
retirement_started_journal_kind,
session_id_for_route,
)
.await
{
tracing::warn!(
agent_identity = %agent_identity,
%error,
"spawn rollback could not append durable retired event"
);
return Err(rollback.fail(error).await);
}
let mut detach_obligations =
crate::generated::protocol_mob_destroying_session_ingress::extract_obligations(
&prepared_retire.transition,
);
self.commit_prepared_dsl_transition(prepared_retire)?;
if let (Some(obligation), Some(session_id)) =
(detach_obligations.pop(), detach_session_id)
&& let Err(error) = self
.detach_session_ingress_for_mob_destroy(&session_id, obligation)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self.flush_routed_effects().await {
return Err(rollback.fail(error).await);
}
if let Some(session_id) = member_ref.bridge_session_id() {
self.discard_pending_routed_effects_for_session(session_id);
}
}
let rollback_ctx = DisposalContext {
agent_identity: agent_identity.clone(),
entry: spawned_entry.clone().unwrap_or_else(|| {
let identity = AgentIdentity::from(agent_identity.as_str());
RosterEntry {
agent_identity: identity.clone(),
generation: crate::ids::Generation::INITIAL,
fence_token: crate::ids::FenceToken::new(0),
agent_runtime_id: crate::ids::AgentRuntimeId::initial(identity),
role: profile_name.clone(),
member_ref: member_ref.clone(),
runtime_mode: crate::MobRuntimeMode::TurnDriven,
peer_id: spawned_comms.as_ref().and_then(|c| c.peer_id()),
transport_public_key: spawned_comms.as_ref().and_then(|c| c.public_key()),
wired_to: std::collections::BTreeSet::new(),
external_peer_specs: std::collections::BTreeMap::new(),
labels: std::collections::BTreeMap::new(),
kickoff: None,
effective_profile_override: None,
}
}),
retiring_key: spawned_comms.as_ref().and_then(|c| c.public_key()),
retiring_comms: None,
retiring_spec: None,
preserve_machine_topology: false,
machine_wired_peer_identities: BTreeSet::new(),
trust_unwire_authority_by_peer: BTreeMap::new(),
};
if let Err(error) = self.dispose_archive_session(&rollback_ctx).await {
return Err(rollback.fail(error).await);
}
if let Some(binding) = Self::runtime_binding_for_entry(&rollback_ctx.entry) {
if let Err(error) = self
.record_remote_member_runtime_retired(&rollback_ctx.entry)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self
.revoke_supervisor_for_retiring_entry(
&rollback_ctx.entry,
&binding,
std::time::Duration::from_secs(5),
)
.await
{
return Err(rollback
.fail(MobError::RetirementTopologyIncomplete(format!(
"failed-spawn rollback could not revoke peer-only supervisor authority: {error}"
)))
.await);
}
}
if let Err(error) = self
.delete_external_binding_overlay_for_member(
&rollback_ctx.entry.agent_identity,
rollback_ctx.entry.generation,
)
.await
{
return Err(rollback.fail(error).await);
}
if let Err(error) = self.observe_member_retirement_archived(&rollback_ctx).await {
return Err(rollback.fail(error).await);
}
self.dispose_remove_from_roster(&rollback_ctx).await;
Ok(())
}
fn external_tools_for_profile(
&self,
profile: &crate::profile::Profile,
per_spawn_external_tools: Option<Arc<dyn AgentToolDispatcher>>,
) -> Result<Option<Arc<dyn AgentToolDispatcher>>, MobError> {
let default_tools = self
.default_external_tools_provider
.as_ref()
.and_then(|p| p());
compose_external_tools_for_profile(
profile,
&self.tool_bundles,
self.mob_handle_for_tools(),
default_tools,
per_spawn_external_tools,
None,
)
}
async fn retire_event_exists(
&self,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<bool, MobError> {
let key = Self::retire_event_key(agent_identity, generation);
let index = self.retired_event_index.read().await;
Ok(index.contains(&key))
}
async fn append_retirement_started_event_for_entry(
&self,
entry: &RosterEntry,
journal_kind: mob_dsl::MobLifecycleJournalKind,
session_id: Option<mob_dsl::SessionId>,
) -> Result<(), MobError> {
let parse_session = |session_id: mob_dsl::SessionId| {
SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"generated retirement-start journal carried invalid session id '{}': {error}",
session_id.0
))
})
};
let session_id = session_id.map(parse_session).transpose()?;
let releasing = match journal_kind {
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedReleasing => {
Some(session_id.clone().ok_or_else(|| {
MobError::Internal(
"generated releasing retirement-start journal omitted session_id"
.to_string(),
)
})?)
}
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPreservingBinding => {
if session_id.is_none() {
return Err(MobError::Internal(
"generated preserving retirement-start journal omitted session_id"
.to_string(),
));
}
None
}
mob_dsl::MobLifecycleJournalKind::MemberRetirementStartedPeerOnly => {
if session_id.is_some() {
return Err(MobError::Internal(
"generated peer-only retirement-start journal carried session_id"
.to_string(),
));
}
None
}
other => {
return Err(MobError::Internal(format!(
"generated retirement-start append received non-start journal kind {other:?}"
)));
}
};
let machine_wired_peer_identities =
self.machine_member_wired_peer_identities_for(&entry.agent_identity);
let retiring_peer_endpoint = match self.machine_member_peer_spec_for(
&entry.agent_identity,
"append retirement-start peer endpoint",
)? {
Some(endpoint) => Some(endpoint),
None => match self
.roster_member_peer_spec_for(entry, "append retirement-start peer endpoint")?
{
Some(endpoint) => Some(endpoint),
None => {
self.retained_member_peer_spec_from_wired_peer_trust(
entry,
&machine_wired_peer_identities,
"append retirement-start peer endpoint",
)
.await?
}
},
};
let has_retained_trust_topology = !machine_wired_peer_identities.is_empty()
|| !self
.machine_external_peer_edges_for(&entry.agent_identity)
.is_empty();
if has_retained_trust_topology && retiring_peer_endpoint.is_none() {
return Err(MobError::RetirementTopologyIncomplete(format!(
"retirement-start for '{}' cannot persist wired cleanup authority without its exact peer endpoint",
entry.agent_identity
)));
}
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberRetirementStarted {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id: entry.agent_runtime_id.clone(),
generation: entry.generation,
role: entry.role.clone(),
releasing,
session_id,
retiring_peer_endpoint,
},
})
.await?;
Ok(())
}
async fn append_retire_event_for_entry(&self, entry: &RosterEntry) -> Result<(), MobError> {
self.events
.append(NewMobEvent {
mob_id: self.definition.id.clone(),
timestamp: None,
kind: MobEventKind::MemberRetired {
agent_identity: entry.agent_identity.clone(),
generation: entry.generation,
role: entry.role.clone(),
},
})
.await?;
let key = Self::retire_event_key(&entry.agent_identity, entry.generation);
self.retired_event_index.write().await.insert(key);
Ok(())
}
async fn provisioner_comms(&self, member_ref: &MemberRef) -> Option<Arc<dyn CoreCommsRuntime>> {
self.provisioner.comms_runtime(member_ref).await
}
fn trusted_peer_descriptor_from_machine_endpoint(
endpoint: &mob_dsl::ExternalPeerEndpoint,
) -> Result<TrustedPeerDescriptor, MobError> {
TrustedPeerDescriptor::unsigned_with_pubkey(
endpoint.name.0.clone(),
&endpoint.peer_id.0,
endpoint.signing_key.0,
&endpoint.address.0,
)
.map_err(|error| {
MobError::WiringError(format!(
"MobMachine external peer edge has invalid descriptor '{}': {error}",
endpoint.name.0
))
})
}
fn machine_restore_wiring_plan(
&self,
agent_identity: &AgentIdentity,
) -> Result<RestoreWiringPlan, MobError> {
let local =
mob_dsl::AgentIdentity::from_domain(&AgentIdentity::from(agent_identity.as_str()));
let mut local_peers = Vec::new();
for edge in &self.dsl_authority.state().wiring_edges {
let peer = if edge.a == local {
Some(&edge.b)
} else if edge.b == local {
Some(&edge.a)
} else {
None
};
if let Some(peer) = peer {
local_peers.push(AgentIdentity::from(peer.0.as_str()));
}
}
local_peers.sort();
local_peers.dedup();
let mut external_peers = Vec::new();
for edge in self
.dsl_authority
.state()
.external_peer_edges_by_key
.values()
{
if edge.local == local {
external_peers.push(Self::trusted_peer_descriptor_from_machine_endpoint(
&edge.endpoint,
)?);
}
}
external_peers.sort_by(|a, b| {
a.name
.as_str()
.cmp(b.name.as_str())
.then_with(|| a.peer_id.to_string().cmp(&b.peer_id.to_string()))
});
external_peers.dedup_by(|a, b| a.name == b.name && a.peer_id == b.peer_id);
Ok(RestoreWiringPlan {
local_peers,
external_peers,
})
}
async fn sender_runtime_for_entry(
&self,
entry: &RosterEntry,
) -> Option<Arc<dyn CoreCommsRuntime>> {
if let Some(comms) = self.provisioner_comms(&entry.member_ref).await {
return Some(comms);
}
if matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
) {
return Some(self.supervisor_bridge.runtime_core().await);
}
None
}
fn comms_name_for(&self, entry: &RosterEntry) -> Result<String, MobError> {
render_member_comms_name(
self.definition.id.as_str(),
entry.role.as_str(),
entry.agent_identity.as_str(),
)
}
async fn local_wiring_spec(
&self,
entry: &RosterEntry,
member_ref: &MemberRef,
comms: &Arc<dyn CoreCommsRuntime>,
comms_name: &str,
public_key: &str,
) -> Result<TrustedPeerDescriptor, MobError> {
let mut spec = self
.provisioner
.trusted_peer_spec(member_ref, comms_name, public_key)
.await?;
if let Some(address) = comms.advertised_address() {
spec.address = PeerAddress::parse(&address).map_err(|error| {
MobError::WiringError(format!(
"invalid advertised comms address for '{}': {error}",
entry.agent_identity
))
})?;
}
Ok(spec)
}
async fn resolve_wiring_endpoint(
&self,
entry: &RosterEntry,
context: &'static str,
) -> Result<WiringEndpoint, MobError> {
let comms_name = self.comms_name_for(entry)?;
let member_ref = self.machine_member_ref_for_behavior(entry, context)?;
if let Some(comms) = self.provisioner_comms(&member_ref).await {
let public_key = comms.public_key().ok_or_else(|| {
MobError::WiringError(format!(
"{context} requires public key for '{}'",
entry.agent_identity
))
})?;
let spec = self
.local_wiring_spec(entry, &member_ref, &comms, &comms_name, &public_key)
.await?;
return Ok(WiringEndpoint::Local {
entry: Box::new(entry.clone()),
comms,
spec,
comms_name,
});
}
match &member_ref {
MemberRef::BackendPeer { .. } => {
let binding =
Self::runtime_binding_for_member_ref(&member_ref).ok_or_else(|| {
MobError::WiringError(format!(
"{context} requires external runtime binding for '{}'",
entry.agent_identity
))
})?;
let spec = Self::peer_only_spec_for_binding(&binding, context)?;
Ok(WiringEndpoint::PeerOnly { spec, binding })
}
MemberRef::Session { .. } => Err(MobError::WiringError(format!(
"{context} requires comms runtime for '{}'",
entry.agent_identity
))),
}
}
async fn notify_peer_added(
&self,
sender_comms: &Arc<dyn CoreCommsRuntime>,
recipient_spec: &TrustedPeerDescriptor,
new_peer_id: &AgentIdentity,
new_peer_entry: &RosterEntry,
) -> Result<(), MobError> {
let peer_description = self
.definition
.resolve_profile(&new_peer_entry.role, self.realm_profile_store.as_ref())
.await
.map(|p| p.peer_description)
.unwrap_or_default();
let new_peer_spec = match self
.resolve_wiring_endpoint(new_peer_entry, "notify_peer_added")
.await?
{
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
};
let peer_route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: new_peer_id.as_str().to_string(),
role: Some(new_peer_entry.role.as_str().to_string()),
description: Some(peer_description),
peer_spec: Some(super::bridge_protocol::BridgePeerSpec::from(new_peer_spec)),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{new_peer_id}': {error}"
))
})?;
let cmd = CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerAdded,
params,
};
sender_comms.send(cmd).await?;
Ok(())
}
async fn notify_peer_event(
&self,
intent: &'static str,
recipient_spec: &TrustedPeerDescriptor,
other_peer_id: &AgentIdentity,
other_peer_entry: &RosterEntry,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
let other_peer_spec = match self
.resolve_wiring_endpoint(other_peer_entry, "notify_peer_event")
.await?
{
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
};
self.notify_peer_event_with_spec(
intent,
recipient_spec,
other_peer_id,
other_peer_entry,
&other_peer_spec,
sender_comms,
)
.await
}
async fn notify_peer_event_with_spec(
&self,
intent: &'static str,
recipient_spec: &TrustedPeerDescriptor,
other_peer_id: &AgentIdentity,
other_peer_entry: &RosterEntry,
other_peer_spec: &TrustedPeerDescriptor,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
let peer_route =
PeerRoute::with_display_name(recipient_spec.peer_id, recipient_spec.name.clone());
let params = meerkat_contracts::CommsPeerLifecycleParams {
peer: other_peer_id.as_str().to_string(),
role: Some(other_peer_entry.role.as_str().to_string()),
description: None,
peer_spec: Some(super::bridge_protocol::BridgePeerSpec::from(
other_peer_spec.clone(),
)),
};
let params = serde_json::to_value(¶ms).map_err(|error| {
MobError::WiringError(format!(
"failed to serialize peer lifecycle params for '{other_peer_id}': {error}"
))
})?;
let cmd = match intent {
"mob.peer_retired" => CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerRetired,
params,
},
"mob.peer_unwired" => CommsCommand::PeerLifecycle {
to: peer_route,
kind: PeerLifecycleKind::PeerUnwired,
params,
},
_ => CommsCommand::PeerRequest {
to: peer_route,
intent: intent.to_string(),
params,
blocks: None,
content_taint: None,
handling_mode: meerkat_core::types::HandlingMode::Queue,
stream: meerkat_core::comms::InputStreamMode::None,
},
};
sender_comms.send(cmd).await?;
Ok(())
}
async fn notify_kickoff_event(
&self,
agent_identity: &AgentIdentity,
intent: &'static str,
) -> Result<(), MobError> {
let (entry, wired_peers) = {
let machine_peer_identities = self
.machine_wired_peer_identities_for(&AgentIdentity::from(agent_identity.as_str()));
let roster = self.roster.read().await;
let Some(entry) = roster.get(agent_identity).cloned() else {
return Ok(());
};
let wired_peers: Vec<AgentIdentity> = machine_peer_identities
.iter()
.filter_map(|id| roster.get_by_identity(id).map(|e| e.agent_identity.clone()))
.collect();
(entry, wired_peers)
};
let sender_comms = self.provisioner_comms(&entry.member_ref).await;
let effects = MobRuntimeBridgeAuthority::plan_lifecycle_notice(
sender_comms.is_some()
|| matches!(
entry.member_ref,
MemberRef::BackendPeer {
session_id: None,
..
}
),
&wired_peers,
intent,
);
let Some(sender_comms) = self.sender_runtime_for_entry(&entry).await else {
return Ok(());
};
for effect in effects {
let MobRuntimeBridgeEffect::DeliverLifecycleNotice { peer_id, intent } = effect;
let recipient_entry = {
let roster = self.roster.read().await;
roster.get(&peer_id).cloned()
};
let Some(recipient_entry) = recipient_entry else {
continue;
};
let recipient_spec = match self
.resolve_wiring_endpoint(&recipient_entry, "notify_kickoff_event")
.await?
{
WiringEndpoint::Local { spec, .. } | WiringEndpoint::PeerOnly { spec, .. } => spec,
};
self.notify_peer_event(
intent,
&recipient_spec,
agent_identity,
&entry,
&sender_comms,
)
.await?;
}
Ok(())
}
async fn notify_peer_retired(
&self,
recipient_spec: &TrustedPeerDescriptor,
retired_id: &AgentIdentity,
retired_entry: &RosterEntry,
retired_spec: &TrustedPeerDescriptor,
retiring_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
self.notify_peer_event_with_spec(
"mob.peer_retired",
recipient_spec,
retired_id,
retired_entry,
retired_spec,
retiring_comms,
)
.await
}
async fn notify_peer_unwired(
&self,
recipient_spec: &TrustedPeerDescriptor,
unwired_id: &AgentIdentity,
unwired_entry: &RosterEntry,
sender_comms: &Arc<dyn CoreCommsRuntime>,
) -> Result<(), MobError> {
self.notify_peer_event(
"mob.peer_unwired",
recipient_spec,
unwired_id,
unwired_entry,
sender_comms,
)
.await
}
}
struct ClosedRuntimeEffectRefusal {
kind: crate::error::RuntimeEffectKind,
session_id: SessionId,
refusal_code: String,
reason: String,
broken_member: Option<AgentIdentity>,
}
impl ClosedRuntimeEffectRefusal {
fn into_mob_error(self) -> MobError {
MobError::RuntimeEffectRefused {
kind: self.kind,
session_id: self.session_id,
refusal_code: self.refusal_code,
reason: self.reason,
}
}
}
fn closed_runtime_effect_refusal_from_transition(
transition: &mob_dsl::MobMachineTransition,
) -> Result<ClosedRuntimeEffectRefusal, MobError> {
let mut closed = None;
for effect in transition.effects() {
let candidate = match effect {
mob_dsl::MobMachineEffect::RuntimeBindingRefusalClassified {
agent_identity,
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeBinding,
session_id,
refusal_code,
reason,
Some(AgentIdentity::from(agent_identity.0.as_str())),
)),
mob_dsl::MobMachineEffect::RuntimeIngressRefusalClassified {
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeIngress,
session_id,
refusal_code,
reason,
None,
)),
mob_dsl::MobMachineEffect::RuntimeRetireRefusalClassified {
session_id,
refusal_code,
reason,
..
} => Some((
crate::error::RuntimeEffectKind::RuntimeRetire,
session_id,
refusal_code,
reason,
None,
)),
_ => None,
};
let Some((kind, session_id, refusal_code, reason, broken_member)) = candidate else {
continue;
};
if closed.is_some() {
return Err(MobError::Internal(
"MobMachine refusal feedback emitted multiple runtime refusal classifications"
.to_owned(),
));
}
let session_id = SessionId::parse(&session_id.0).map_err(|error| {
MobError::Internal(format!(
"MobMachine refusal classification carried invalid session id `{}`: {error}",
session_id.0
))
})?;
closed = Some(ClosedRuntimeEffectRefusal {
kind,
session_id,
refusal_code: refusal_code.clone(),
reason: reason.clone(),
broken_member,
});
}
closed.ok_or_else(|| {
MobError::Internal(
"MobMachine refusal feedback emitted no runtime refusal classification".to_owned(),
)
})
}
fn routed_effect_session_scope_dsl(
effect: &mob_dsl::MobMachineEffect,
) -> Option<&mob_dsl::SessionId> {
match effect {
mob_dsl::MobMachineEffect::RequestRuntimeBinding { session_id, .. }
| mob_dsl::MobMachineEffect::RequestRuntimeRetire { session_id, .. }
| mob_dsl::MobMachineEffect::RequestRuntimeDestroy { session_id }
| mob_dsl::MobMachineEffect::RequestRuntimeIngress { session_id, .. } => Some(session_id),
_ => None,
}
}
fn routed_effect_session_scope(effect: &mob_dsl::MobMachineEffect) -> Option<SessionId> {
routed_effect_session_scope_dsl(effect).and_then(|id| SessionId::parse(&id.0).ok())
}
fn revival_error_means_session_already_live(
error: &MobError,
bridge_session_id: &SessionId,
) -> bool {
matches!(
error,
MobError::SessionError(meerkat_core::service::SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
)) if session_id == bridge_session_id
)
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod runtime_observation_tests {
use super::{foreign_runtime_observation, mob_dsl};
#[test]
fn foreign_runtime_observations_are_identified_for_log_downgrade() {
let authority = mob_dsl::MobMachineAuthority::new();
let signal = mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id: mob_dsl::AgentRuntimeId("rt:session:parent".to_string()),
fence_token: mob_dsl::FenceToken(0),
};
assert!(foreign_runtime_observation(authority.state(), &signal).is_some());
}
#[test]
fn live_member_runtime_observations_remain_machine_owned() {
let mut authority = mob_dsl::MobMachineAuthority::new();
let identity = mob_dsl::AgentIdentity("member".to_string());
let runtime_id = mob_dsl::AgentRuntimeId("member:0".to_string());
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: identity.clone(),
profile_name: "test".to_string(),
model: "test-model".to_string(),
profile_material_digest: "test-profile-digest".to_string(),
tool_config_digest: "test-tool-config-digest".to_string(),
skills_digest: "test-skills-digest".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: true,
},
)
.expect("AuthorizeSpawnProfile should seed live runtime ownership");
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::BeginSpawnExec {
agent_identity: identity.clone(),
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: mob_dsl::Generation(0),
profile_material_digest: "test-profile-digest".to_string(),
external_addressable: true,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(mob_dsl::SessionId("member-session".to_string())),
replacing: None,
},
)
.expect("BeginSpawnExec should open the spawn-exec phase");
mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: identity,
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: mob_dsl::Generation(0),
profile_material_digest: "test-profile-digest".to_string(),
external_addressable: true,
runtime_mode: mob_dsl::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(mob_dsl::SessionId("member-session".to_string())),
replacing: None,
},
)
.expect("CommitSpawnMembership should seed live runtime ownership");
let signal = mob_dsl::MobMachineSignal::ObserveRuntimeReady {
agent_runtime_id: runtime_id,
fence_token: mob_dsl::FenceToken(1),
};
assert!(foreign_runtime_observation(authority.state(), &signal).is_none());
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod routed_effect_containment_tests {
use super::*;
#[test]
fn runtime_retire_route_is_deferred_until_correlated_detach_closes() {
let identity = mob_dsl::AgentIdentity::from("worker");
let runtime_id = mob_dsl::AgentRuntimeId::from("worker:1");
let session_id = mob_dsl::SessionId::from("session-worker");
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
state
.runtime_retire_pending_sessions
.insert(runtime_id.clone(), session_id.clone());
state
.pending_session_ingress_detach_runtime_ids
.insert(runtime_id.clone());
let effect = super::super::composition::MobSeamEffect::routed(
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: identity,
agent_runtime_id: runtime_id.clone(),
session_id,
},
)
.expect("runtime retire is a generated routed effect");
assert!(MobActor::runtime_retire_blocked_on_pending_detach(
&state, &effect
));
state
.pending_session_ingress_detach_runtime_ids
.remove(&runtime_id);
assert!(!MobActor::runtime_retire_blocked_on_pending_detach(
&state, &effect
));
}
#[test]
fn actor_boundary_retries_closed_refusals_until_independent_tail_drains() {
let session_id = SessionId::parse("019f0000-0000-7000-8000-000000000001")
.expect("valid test session id");
let closed = |kind| {
Err(MobError::RuntimeEffectRefused {
kind,
session_id: session_id.clone(),
refusal_code: "runtime_refused".to_string(),
reason: "test refusal".to_string(),
})
};
let mut outcomes = VecDeque::from([
closed(crate::error::RuntimeEffectKind::RuntimeBinding),
closed(crate::error::RuntimeEffectKind::RuntimeIngress),
Ok(()),
]);
let mut boundary_attempts = 0;
let mut closed_refusals = 0;
loop {
boundary_attempts += 1;
let outcome = outcomes
.pop_front()
.expect("boundary test must terminate on the drained outcome");
match MobActor::classify_actor_boundary_flush(outcome) {
ActorBoundaryFlushDisposition::Drained => break,
ActorBoundaryFlushDisposition::RetryAfterMachineClosure(_) => {
closed_refusals += 1;
continue;
}
ActorBoundaryFlushDisposition::Fatal(error) => {
panic!("machine-closed tail drain became fatal: {error}")
}
}
}
assert_eq!(boundary_attempts, 3);
assert_eq!(closed_refusals, 2);
assert!(outcomes.is_empty());
assert!(matches!(
MobActor::classify_actor_boundary_flush(Err(MobError::WiringError(
"structural refusal".to_string()
))),
ActorBoundaryFlushDisposition::Fatal(MobError::WiringError(_))
));
}
#[test]
fn every_routed_effect_variant_is_session_scoped() {
let session = mob_dsl::SessionId::from("019f0000-0000-7000-8000-000000000001");
let runtime_id = mob_dsl::AgentRuntimeId("identity:1".to_string());
let routed = [
mob_dsl::MobMachineEffect::RequestRuntimeBinding {
session_id: session.clone(),
agent_identity: mob_dsl::AgentIdentity::from("worker-1"),
agent_runtime_id: runtime_id.clone(),
fence_token: mob_dsl::FenceToken(1),
generation: None,
},
mob_dsl::MobMachineEffect::RequestRuntimeRetire {
agent_identity: mob_dsl::AgentIdentity::from("worker-1"),
agent_runtime_id: runtime_id.clone(),
session_id: session.clone(),
},
mob_dsl::MobMachineEffect::RequestRuntimeDestroy {
session_id: session.clone(),
},
mob_dsl::MobMachineEffect::RequestRuntimeIngress {
session_id: session.clone(),
agent_runtime_id: runtime_id,
fence_token: mob_dsl::FenceToken(1),
generation: None,
work_id: mob_dsl::WorkId::from("work-1"),
origin: mob_dsl::WorkOrigin::External,
},
];
for effect in &routed {
assert_eq!(
routed_effect_session_scope_dsl(effect),
Some(&session),
"routed effect must expose its session scope: {effect:?}"
);
assert!(
routed_effect_session_scope(effect).is_some(),
"session scope must parse to a domain session id"
);
}
}
}
#[cfg(test)]
mod kickoff_tests {
use super::*;
#[test]
fn kickoff_generated_rejection_maps_to_typed_mob_error() {
let mut authority = mob_dsl::MobMachineAuthority::new();
let generated = mob_dsl::MobMachineMutator::apply(
&mut authority,
mob_dsl::MobMachineInput::KickoffMarkStarting {
member_id: mob_dsl::AgentIdentity::from("worker-1"),
},
)
.expect_err("mark-starting without a pending kickoff must be generated rejection");
let error = MobActor::kickoff_rejection_error("test_kickoff", generated);
match error {
MobError::MobMachineRejected { context, reason } => {
assert_eq!(context, "test_kickoff");
assert!(reason.contains("KickoffMarkStarting"));
}
other => panic!("expected typed MobMachineRejected, got {other:?}"),
}
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod member_comms_name_tests {
use super::render_member_comms_name;
use crate::MobError;
#[test]
fn valid_slugs_render_via_single_owner_display() {
let name = render_member_comms_name("mob-1", "worker", "w-a")
.expect("valid identifier-safe slugs should render");
assert_eq!(name, "mob-1/worker/w-a");
}
#[test]
fn invalid_component_fails_closed_with_typed_error() {
let error = render_member_comms_name("mob-1", "wor/ker", "w-a")
.expect_err("a component containing '/' must fail closed");
assert!(
matches!(error, MobError::MemberCommsName(_)),
"comms-name failure must surface the typed owner error, got: {error}"
);
assert!(
!error.to_string().contains("mob-1/wor/ker/w-a"),
"rejected component must not be laundered into a raw routing name: {error}"
);
}
#[test]
fn empty_component_fails_closed() {
let error = render_member_comms_name("mob-1", "", "w-a")
.expect_err("an empty component is not identifier-safe");
assert!(matches!(error, MobError::MemberCommsName(_)));
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod bridge_rejection_tests {
use super::{ExpectedRevokeCleanupFailure, MobActor};
use crate::MobError;
use crate::runtime::bridge_protocol::{
BridgeRejectionCause, SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
decode_legacy_v1_raw_string_rejection,
};
use meerkat_core::comms::{AdmissionDropReason, SendError};
use serde_json::json;
fn known_bridge_rejection_causes() -> &'static [(BridgeRejectionCause, &'static str)] {
&[
(BridgeRejectionCause::NotBound, "not_bound"),
(BridgeRejectionCause::StaleSupervisor, "stale_supervisor"),
(BridgeRejectionCause::SenderMismatch, "sender_mismatch"),
(BridgeRejectionCause::AlreadyBound, "already_bound"),
(
BridgeRejectionCause::InvalidBootstrapToken,
"invalid_bootstrap_token",
),
(
BridgeRejectionCause::UnsupportedProtocolVersion,
"unsupported_protocol_version",
),
(
BridgeRejectionCause::InvalidSupervisorSpec,
"invalid_supervisor_spec",
),
(BridgeRejectionCause::InvalidPeerSpec, "invalid_peer_spec"),
(BridgeRejectionCause::AddressMismatch, "address_mismatch"),
(BridgeRejectionCause::Unsupported, "unsupported"),
(BridgeRejectionCause::Internal, "internal"),
]
}
#[test]
fn actor_decodes_typed_protocol_v2_bridge_rejection() {
let value = json!({
"result": "rejected",
"cause": "stale_supervisor",
"reason": "epoch too low",
});
let rejection =
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value)
.expect("typed rejection should decode");
assert_eq!(
rejection.typed_cause(),
Some(BridgeRejectionCause::StaleSupervisor)
);
assert_eq!(rejection.reason(), "epoch too low");
}
#[test]
fn actor_does_not_promote_raw_string_as_protocol_v2_rejection() {
let value = json!("legacy rejection");
assert!(
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value).is_none()
);
let legacy = decode_legacy_v1_raw_string_rejection(&value)
.expect("legacy raw string should decode only through the explicit v1 helper");
assert_eq!(legacy.typed_cause(), None);
assert!(legacy.is_legacy_v1_raw_string());
}
#[test]
fn actor_bridge_rejection_error_preserves_each_known_typed_cause() {
for (cause, wire_name) in known_bridge_rejection_causes() {
let reason = format!("typed bridge rejection: {wire_name}");
let value = json!({
"result": "rejected",
"cause": wire_name,
"reason": reason,
});
let rejection =
MobActor::bridge_rejection_reply(SUPERVISOR_BRIDGE_PROTOCOL_VERSION, &value)
.expect("typed rejection should decode");
let error = MobActor::bridge_rejection_error(rejection);
match error {
MobError::BridgeCommandRejected {
cause: actual,
reason: actual_reason,
} => {
assert_eq!(actual, *cause);
assert_eq!(actual_reason, reason);
}
other => panic!("expected typed bridge rejection error, got {other:?}"),
}
}
}
#[test]
fn actor_legacy_bridge_rejection_error_stays_untyped() {
let value = json!("legacy rejection");
let legacy = decode_legacy_v1_raw_string_rejection(&value)
.expect("legacy raw string should decode only through the explicit v1 helper");
let error = MobActor::bridge_rejection_error(legacy);
assert!(matches!(
error,
MobError::WiringError(reason) if reason == "legacy rejection"
));
}
#[test]
fn revoke_cleanup_accepts_only_typed_not_bound() {
let not_bound = MobError::BridgeCommandRejected {
cause: BridgeRejectionCause::NotBound,
reason: "no authorized supervisor registered".to_string(),
};
assert_eq!(
MobActor::expected_revoke_cleanup_failure(¬_bound),
Some(ExpectedRevokeCleanupFailure::BridgeRejected {
cause: BridgeRejectionCause::NotBound,
})
);
for unproven_absence in [
MobError::CommsError(SendError::PeerNotFound("peer-1".to_string())),
MobError::CommsError(SendError::PeerOffline),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::UntrustedSender,
}),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::SessionClosed,
}),
] {
assert_eq!(
MobActor::expected_revoke_cleanup_failure(&unproven_absence),
None,
"connectivity failure does not prove supervisor trust was revoked"
);
}
}
#[test]
fn revoke_cleanup_does_not_treat_error_text_as_a_semantic_cause() {
let cases = [
MobError::WiringError("peer offline".to_string()),
MobError::Internal("no authorized supervisor registered".to_string()),
MobError::CommsError(SendError::Validation("peer not found".to_string())),
MobError::CommsError(SendError::AdmissionDropped {
reason: AdmissionDropReason::InboxFull,
}),
MobError::BridgeCommandRejected {
cause: BridgeRejectionCause::SenderMismatch,
reason: "no authorized supervisor registered".to_string(),
},
];
for error in cases {
assert_eq!(MobActor::expected_revoke_cleanup_failure(&error), None);
}
}
#[test]
fn revoke_cleanup_expected_failure_ratchet_rejects_string_parsing() {
let source = include_str!("actor.rs");
let start = source
.find("fn expected_revoke_cleanup_failure")
.expect("expected cleanup classifier exists");
let end = source[start..]
.find("\n async fn destroy_remote_member_for_destroy")
.expect("cleanup classifier precedes remote destroy cleanup");
let body = &source[start..start + end];
for disallowed in [
"to_ascii_lowercase",
"to_lowercase",
"error.to_string()",
".contains(",
] {
assert!(
!body.contains(disallowed),
"expected revoke cleanup classifier must not parse error text with {disallowed}"
);
}
}
#[test]
fn supervisor_activation_failure_is_typed_and_never_reauthorizes_old_trust() {
let source = include_str!("actor.rs");
let start = source
.find("fn supervisor_activation_error")
.expect("supervisor_activation_error exists");
let end = source[start..]
.find("\n /// Cancel checkpointers and transition to Stopped.")
.expect("rollback classifier precedes fail_reset_to_stopped");
let body = &source[start..start + end];
assert!(
body.contains("rollback_error: new_trust_cleanup_failed"),
"failure handling must key on the typed new_trust_cleanup_failed discriminant"
);
assert!(
body.contains("reconstruct old-supervisor trust"),
"activation recovery must explicitly preserve the no-old-reauthorization boundary"
);
assert!(
!body.contains("install_supervisor_private_trust_for_session("),
"activation failure handling must not reinstall old supervisor trust"
);
for disallowed in [
"error_text",
"error.to_string()",
".contains(",
"to_lowercase",
] {
assert!(
!body.contains(disallowed),
"supervisor activation rollback classifier must not parse error text with {disallowed}"
);
}
}
#[test]
fn supervisor_private_trust_install_carries_typed_cleanup_verdict() {
let source = include_str!("actor.rs");
let start = source
.find("async fn install_supervisor_private_trust_for_session_authority")
.expect("install_supervisor_private_trust_for_session_authority exists");
let end = source[start..]
.find("\n async fn stage_supervisor_trust_published_for_private_trust")
.expect("install fn precedes its publish-feedback staging helper");
let body = &source[start..start + end];
assert!(
body.contains(
"Result<SupervisorPrivateTrustInstall, SupervisorPrivateTrustInstallError>"
),
"install must return the typed install error so callers read a structured cleanup verdict"
);
assert!(
body.contains(
"SupervisorPrivateTrustInstallError::with_failed_new_trust_cleanup(error)"
),
"a failed attempted-trust cleanup must be recorded as a typed discriminant"
);
assert!(
body.contains(".is_err()"),
"the cleanup attempt result must be inspected, not discarded"
);
}
}