use super::*;
use crate::definition::{
BackendConfig, CollectionPolicy, ConditionExpr, DependencyMode, DispatchMode, FlowSpec,
FlowStepSpec, LimitsSpec, MobDefinition, OrchestratorConfig, PolicyMode, RoleWiringRule,
SkillSource, SpawnPolicyConfig, StepOutputFormat, TopologyRule, TopologySpec, WiringRules,
};
use crate::event::MobEvent;
use crate::profile::{Profile, ProfileBinding, ToolConfig};
use crate::run::MobRunStatus;
use crate::run::{
FailureLedgerEntry, MobRun, MobRunProvenanceAuthority, StepLedgerEntry, StepRunStatus,
};
use crate::storage::MobStorage;
use crate::store::{
ExternalBindingOverlayRecord, ExternalBindingOverlayStatus, InMemoryMobEventStore,
InMemoryMobIdentityStatusStore, InMemoryMobIdentityStore, InMemoryMobRunStore,
InMemoryMobRuntimeMetadataStore, InMemoryMobSpecStore, MobEventStore, MobIdentityStatusStore,
MobIdentityStore, MobMemberEventCursorRecord, MobRunStore, MobRuntimeMetadataStore,
MobStoreError, RealmProfileStore, SupervisorAuthorityRecord, private, terminal_event_identity,
};
#[cfg(not(target_arch = "wasm32"))]
use crate::store::{SqliteMobEventStore, SqliteMobStores};
use async_trait::async_trait;
use chrono::Utc;
use indexmap::IndexMap;
use meerkat_core::agent::CommsRuntime as CoreCommsRuntime;
use meerkat_core::comms::{
CommsCommand, CommsTrustMutation, CommsTrustMutationAuthority, CommsTrustMutationResult,
PeerCapabilitySet, PeerDirectoryEntry, PeerDirectorySource, PeerId, PeerName, PeerRoute,
PeerSendability, SendError, SendReceipt, TrustedPeerDescriptor,
};
use meerkat_core::error::ToolError;
use meerkat_core::event::{AgentEvent, EventEnvelope, InteractionFailureReason};
use meerkat_core::interaction::InteractionId;
use meerkat_core::ops::ToolDispatchOutcome;
use meerkat_core::service::{
AppendSystemContextRequest, AppendSystemContextResult, CreateSessionRequest,
SessionControlError, SessionError, SessionInfo, SessionQuery, SessionService,
SessionServiceCommsExt, SessionServiceControlExt, SessionServiceHistoryExt, SessionSummary,
SessionUsage, SessionView, StartTurnRequest, TurnToolOverlay,
};
use meerkat_core::types::{
CommsNoticeKind, ContentBlock, ContentInput, HandlingMode, Message, RunResult, SessionId,
SystemNoticeBlock, SystemNoticeDirection, SystemNoticeKind, SystemNoticeMessage, ToolCallView,
ToolDef, ToolProvenance, ToolResult, Usage,
};
use meerkat_core::{
AgentToolDispatcher, AppendSystemContextStatus, EventInjector, EventInjectorError,
PeerCorrelationId, PlainEventSource,
event_injector::{InteractionSubscription, SubscribableInjector},
};
use meerkat_core::{CommsCapabilityError, Provider};
use meerkat_core::{
Session, SessionLlmIdentity, SessionMetadata, SessionTooling, ToolCategoryOverride,
};
use meerkat_machine_schema::catalog::dsl::{
dsl_mob_machine as schema_mob_machine,
mob_machine::{
MobMachineInput as SchemaMobMachineInput,
MobMachineInputVariant as SchemaMobMachineInputVariant,
},
};
use meerkat_machine_schema::{Expr, MachineSchema, TriggerKind, identity::InputVariantId};
use meerkat_session::{SessionAgent, SessionAgentBuilder, SessionSnapshot};
use meerkat_store::{MemoryStore, SessionStore};
use serde::Serialize;
use serde_json::value::RawValue;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::io::Write;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use tempfile::NamedTempFile;
use uuid::Uuid;
type TestMeerkatMachineAuthority = std::sync::Arc<meerkat_runtime::HandleDslAuthority>;
struct TestPeerProjectionAuthorityState {
dsl: TestMeerkatMachineAuthority,
}
static TEST_PEER_PROJECTION_AUTHORITIES: OnceLock<
Mutex<HashMap<String, std::sync::Arc<TestPeerProjectionAuthorityState>>>,
> = OnceLock::new();
fn test_peer_projection_authorities()
-> &'static Mutex<HashMap<String, std::sync::Arc<TestPeerProjectionAuthorityState>>> {
TEST_PEER_PROJECTION_AUTHORITIES.get_or_init(|| Mutex::new(HashMap::new()))
}
fn initialized_test_peer_projection_dsl(session_id: String) -> TestMeerkatMachineAuthority {
let dsl = std::sync::Arc::new(meerkat_runtime::HandleDslAuthority::ephemeral());
dsl.apply_signal(
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineSignal::Initialize,
"test::initialize",
)
.expect("Initialize signal");
dsl.apply_input(
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineInput::RegisterSession {
session_id: meerkat_runtime::meerkat_machine::dsl::SessionId::from(session_id.clone()),
},
"test::register_session",
)
.expect("RegisterSession input");
// One-way supervisor deliveries classify as peer messages, which the
// generated ingress authority admits only after runtime bindings advance
// the test session from Idle to Attached.
dsl.apply_input(
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineInput::PrepareBindings {
agent_runtime_id: meerkat_runtime::meerkat_machine::dsl::AgentRuntimeId::from(format!(
"test-runtime-{session_id}"
)),
fence_token: meerkat_runtime::meerkat_machine::dsl::FenceToken::from(0),
generation: Some(meerkat_runtime::meerkat_machine::dsl::Generation::from(0)),
runtime_epoch_id: None,
session_id: meerkat_runtime::meerkat_machine::dsl::SessionId::from(session_id),
},
"test::prepare_bindings",
)
.expect("PrepareBindings input");
dsl
}
fn test_peer_projection_runtime_key(
runtime: &meerkat_comms::CommsRuntime,
local_peer_id: PeerId,
) -> String {
format!("{local_peer_id}:{runtime:p}")
}
fn register_test_peer_projection_authority(
runtime: &meerkat_comms::CommsRuntime,
dsl: TestMeerkatMachineAuthority,
) {
let local_peer_id = runtime
.peer_id()
.expect("test peer projection runtime peer_id unavailable");
let key = test_peer_projection_runtime_key(runtime, local_peer_id);
let state = std::sync::Arc::new(TestPeerProjectionAuthorityState { dsl });
test_peer_projection_authorities()
.lock()
.expect("test peer projection authority registry")
.insert(key, state);
}
fn test_peer_projection_state_for_runtime(
runtime: &meerkat_comms::CommsRuntime,
local_peer_id: PeerId,
context: &'static str,
) -> std::sync::Arc<TestPeerProjectionAuthorityState> {
let key = test_peer_projection_runtime_key(runtime, local_peer_id);
if let Some(state) = test_peer_projection_authorities()
.lock()
.expect("test peer projection authority registry")
.get(&key)
.cloned()
{
return state;
}
let dsl = initialized_test_peer_projection_dsl(format!("mob-runtime-test-{key}"));
meerkat_runtime::RuntimePeerCommsHandle::install_generated_on(
std::sync::Arc::clone(&dsl),
runtime,
)
.unwrap_or_else(|error| panic!("{context}: install generated peer-comms handle: {error}"));
let state = std::sync::Arc::new(TestPeerProjectionAuthorityState { dsl });
test_peer_projection_authorities()
.lock()
.expect("test peer projection authority registry")
.insert(key, std::sync::Arc::clone(&state));
state
}
impl TestPeerProjectionAuthorityState {
fn add_obligation(
&self,
endpoint: meerkat_runtime::meerkat_machine::dsl::PeerEndpoint,
context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
let snapshot = self.dsl.snapshot_state();
let mut endpoints = snapshot.mob_overlay_peer_endpoints.clone();
let before = endpoints.clone();
let peer_id = endpoint.peer_id.0.clone();
endpoints.retain(|existing| existing.peer_id.0 != peer_id);
endpoints.insert(endpoint);
let changed = endpoints != before;
let epoch = if changed {
snapshot.mob_overlay_epoch.saturating_add(1)
} else {
snapshot.mob_overlay_epoch
};
test_comms_reconcile_obligation_with_dsl(&self.dsl, epoch, endpoints, context)
}
fn remove_obligation(
&self,
peer_id: &str,
context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
let snapshot = self.dsl.snapshot_state();
let mut endpoints = snapshot.mob_overlay_peer_endpoints.clone();
let before = endpoints.len();
endpoints.retain(|endpoint| endpoint.peer_id.0 != peer_id);
let epoch = if endpoints.len() == before {
snapshot.mob_overlay_epoch
} else {
snapshot.mob_overlay_epoch.saturating_add(1)
};
test_comms_reconcile_obligation_with_dsl(&self.dsl, epoch, endpoints, context)
}
}
trait TestPeerProjectionOwnerInstall {
fn test_peer_projection_add_obligation(
&self,
local_peer_id: PeerId,
endpoint: meerkat_runtime::meerkat_machine::dsl::PeerEndpoint,
_context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
let (obligation, _authority) =
test_comms_reconcile_obligation(local_peer_id, BTreeSet::from([endpoint]));
obligation
}
fn test_peer_projection_remove_obligation(
&self,
local_peer_id: PeerId,
_peer_id: &str,
_context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
let (obligation, _authority) =
test_comms_reconcile_obligation(local_peer_id, BTreeSet::new());
obligation
}
}
impl TestPeerProjectionOwnerInstall for meerkat_comms::CommsRuntime {
fn test_peer_projection_add_obligation(
&self,
local_peer_id: PeerId,
endpoint: meerkat_runtime::meerkat_machine::dsl::PeerEndpoint,
context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
test_peer_projection_state_for_runtime(self, local_peer_id, context)
.add_obligation(endpoint, context)
}
fn test_peer_projection_remove_obligation(
&self,
local_peer_id: PeerId,
peer_id: &str,
context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
test_peer_projection_state_for_runtime(self, local_peer_id, context)
.remove_obligation(peer_id, context)
}
}
fn test_comms_reconcile_obligation(
local_peer_id: PeerId,
direct_peer_endpoints: BTreeSet<meerkat_runtime::meerkat_machine::dsl::PeerEndpoint>,
) -> (
meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation,
TestMeerkatMachineAuthority,
) {
let authority =
initialized_test_peer_projection_dsl("mob-runtime-test-comms-reconcile".to_string());
authority
.apply_input(
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineInput::PublishLocalEndpoint {
endpoint: meerkat_runtime::meerkat_machine::dsl::PeerEndpoint::new(
"local",
local_peer_id.to_string(),
"inproc://local",
[0x7f; 32],
),
},
"test::publish_local_endpoint",
)
.expect("PublishLocalEndpoint input");
let obligation = test_comms_reconcile_obligation_with_dsl(
&authority,
1,
direct_peer_endpoints,
"test::apply_mob_peer_overlay",
);
(obligation, authority)
}
fn test_comms_reconcile_obligation_with_dsl(
authority: &TestMeerkatMachineAuthority,
epoch: u64,
direct_peer_endpoints: BTreeSet<meerkat_runtime::meerkat_machine::dsl::PeerEndpoint>,
context: &'static str,
) -> meerkat_runtime::protocol_comms_trust_reconcile::CommsTrustReconcileObligation {
let transition = authority
.apply_input_with_transition(
meerkat_runtime::meerkat_machine::dsl::MeerkatMachineInput::ApplyMobPeerOverlay {
epoch,
endpoints: direct_peer_endpoints,
},
context,
)
.unwrap_or_else(|error| panic!("{context}: ApplyMobPeerOverlay input: {error}"));
let mut obligations =
meerkat_runtime::protocol_comms_trust_reconcile::extract_obligations_with_freshness(
&transition,
authority.peer_projection_freshness_authority(),
);
assert_eq!(
obligations.len(),
1,
"test reconcile effect must produce one generated obligation"
);
obligations.pop().expect("obligation count checked")
}
async fn apply_test_peer_projection_trust<R>(
runtime: &R,
peer: TrustedPeerDescriptor,
context: &'static str,
) where
R: CoreCommsRuntime + TestPeerProjectionOwnerInstall + ?Sized,
{
let peer_id = peer.peer_id.to_string();
remove_test_peer_projection_trust(runtime, &peer_id, context).await;
let endpoint = meerkat_runtime::meerkat_machine::dsl::PeerEndpoint::from(&peer);
let local_peer_id = runtime
.peer_id()
.unwrap_or_else(|| panic!("{context}: runtime peer_id unavailable"));
let obligation =
runtime.test_peer_projection_add_obligation(local_peer_id, endpoint.clone(), context);
CoreCommsRuntime::apply_trust_mutation(
runtime,
CommsTrustMutation::AddTrustedPeer {
authority: meerkat_runtime::protocol_comms_trust_reconcile::authority_for_endpoint(
&obligation,
&endpoint,
)
.expect("generated peer projection add authority"),
peer,
},
)
.await
.unwrap_or_else(|error| panic!("{context}: {error}"));
}
async fn remove_test_peer_projection_trust<R>(runtime: &R, peer_id: &str, context: &'static str)
where
R: CoreCommsRuntime + TestPeerProjectionOwnerInstall + ?Sized,
{
let local_peer_id = runtime
.peer_id()
.unwrap_or_else(|| panic!("{context}: runtime peer_id unavailable"));
let obligation =
runtime.test_peer_projection_remove_obligation(local_peer_id, peer_id, context);
CoreCommsRuntime::apply_trust_mutation(
runtime,
CommsTrustMutation::RemoveTrustedPeer {
authority:
meerkat_runtime::protocol_comms_trust_reconcile::removal_authority_for_peer_id(
&obligation,
peer_id,
)
.expect("generated peer projection remove authority"),
peer_id: peer_id.to_string(),
},
)
.await
.unwrap_or_else(|error| panic!("{context}: {error}"));
}
async fn remove_mob_member_peer_trust_for_test(
handle: &MobHandle,
runtime: &meerkat_comms::CommsRuntime,
local_identity: AgentIdentity,
peer_id: &str,
context: &'static str,
) {
let machine_state = handle
.query_machine_state()
.await
.unwrap_or_else(|error| panic!("{context}: query MobMachine state: {error}"));
let dsl_local = crate::machines::mob_machine::AgentIdentity::from_domain(&local_identity);
let (peer_identity, edge) = machine_state
.wiring_edges
.iter()
.find_map(|edge| {
let peer_identity = if edge.a == dsl_local {
edge.b.clone()
} else if edge.b == dsl_local {
edge.a.clone()
} else {
return None;
};
let candidate_peer_id = machine_state.member_peer_ids.get(&peer_identity)?;
(candidate_peer_id.0 == peer_id).then(|| (peer_identity, edge.clone()))
})
.unwrap_or_else(|| {
let edges = machine_state
.wiring_edges
.iter()
.map(|edge| format!("{} <-> {}", edge.a.0, edge.b.0))
.collect::<Vec<_>>();
panic!(
"{context}: MobMachine has no member peer edge for {local_identity} -> {peer_id}; edges={edges:?}"
)
});
let obligation = handle
.authorize_member_trust_cleanup_for_test(edge)
.await
.unwrap_or_else(|error| panic!("{context}: authorize member trust cleanup: {error}"));
let authority =
crate::generated::protocol_mob_member_trust_unwiring::unwiring_authority_for_identity(
&obligation,
peer_identity.0.as_str(),
peer_id,
)
.unwrap_or_else(|error| panic!("{context}: build member unwiring authority: {error}"));
CoreCommsRuntime::apply_trust_mutation(
runtime,
CommsTrustMutation::RemoveTrustedPeer {
peer_id: peer_id.to_string(),
authority,
},
)
.await
.unwrap_or_else(|error| panic!("{context}: remove generated member trust: {error}"));
}
fn default_supervisor_authority_record() -> SupervisorAuthorityRecord {
SupervisorAuthorityRecord::generate(
super::bridge_protocol::supervisor_bridge_default_protocol_version(),
)
}
async fn persist_supervisor_authority_for_test(
runtime_metadata: &dyn MobRuntimeMetadataStore,
mob_id: &MobId,
record: &SupervisorAuthorityRecord,
) {
let authority = crate::store::supervisor_authority_persistence_authority_for_record(record)
.expect("generated supervisor authority for test metadata");
runtime_metadata
.put_supervisor_authority(mob_id, record, &authority)
.await
.expect("persist supervisor metadata");
}
fn factory_policy_session(mut session: Session, model: String, max_tokens: u32) -> Session {
if session.session_metadata().is_none() {
session
.set_session_metadata(SessionMetadata {
schema_version: meerkat_core::session_metadata_schema_version(),
model,
max_tokens,
structured_output_retries: 2,
provider: Provider::Other,
self_hosted_server_id: None,
provider_params: None,
tooling: SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
})
.expect("test metadata serializes");
}
if session.build_state().is_none() {
session
.set_build_state(meerkat_core::SessionBuildState::default())
.expect("test build state serializes");
}
session
}
fn test_llm_identity(model: impl Into<String>) -> SessionLlmIdentity {
SessionLlmIdentity {
model: model.into(),
provider: Provider::Other,
self_hosted_server_id: None,
provider_params: None,
auth_binding: None,
}
}
fn install_ephemeral_peer_request_response_authority(
runtime: &Arc<meerkat_comms::CommsRuntime>,
session: &str,
) {
let dsl = initialized_test_peer_projection_dsl(session.to_string());
meerkat_runtime::RuntimePeerCommsHandle::install_generated_on(
Arc::clone(&dsl),
runtime.as_ref(),
)
.expect("install generated peer-comms handle");
register_test_peer_projection_authority(runtime.as_ref(), Arc::clone(&dsl));
runtime.install_peer_request_response_authority(
meerkat_comms::PeerRequestResponseAuthority::new(
Arc::new(meerkat_runtime::RuntimePeerInteractionHandle::new(
Arc::clone(&dsl),
)),
Arc::new(meerkat_runtime::RuntimeInteractionStreamHandle::new(dsl)),
),
);
}
fn test_trusted_peer_descriptor(name: &str, address: &str) -> TrustedPeerDescriptor {
let mut pubkey = [0u8; 32];
for (index, byte) in name.bytes().enumerate() {
let slot = index % pubkey.len();
pubkey[slot] = pubkey[slot].wrapping_add(byte).wrapping_add(index as u8);
}
if pubkey == [0u8; 32] {
pubkey[0] = 1;
}
TrustedPeerDescriptor::unsigned_with_pubkey(
name,
PeerId::from_ed25519_pubkey(&pubkey).to_string(),
pubkey,
address,
)
.expect("valid non-zero test trusted peer descriptor")
}
async fn unregister_runtime_session_until_complete(
adapter: &meerkat_runtime::MeerkatMachine,
session_id: &SessionId,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
loop {
match adapter.unregister_session(session_id).await {
Ok(()) => return Ok(()),
Err(meerkat_runtime::RuntimeDriverError::UnregisterInProgress { .. }) => {}
Err(error) => return Err(error),
}
}
}
async fn install_machine_peer_request_response_authority(
adapter: &meerkat_runtime::MeerkatMachine,
runtime: &Arc<meerkat_comms::CommsRuntime>,
session_id: &SessionId,
) -> Result<(), SessionError> {
let bindings = adapter
.prepare_local_session_bindings(session_id.clone())
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to prepare comms lifecycle authority: {err}"
)))
})?;
bindings
.install_peer_comms_on(runtime.as_ref())
.expect("install generated peer-comms handle");
runtime.install_peer_request_response_authority(
meerkat_comms::PeerRequestResponseAuthority::new(
Arc::clone(bindings.peer_interaction()),
Arc::clone(bindings.interaction_stream()),
),
);
Ok(())
}
// -----------------------------------------------------------------------
// Mock CommsRuntime
// -----------------------------------------------------------------------
#[derive(Clone, Copy, Debug, Default)]
struct MockCommsBehavior {
missing_public_key: bool,
fail_add_trust: bool,
reject_private_publish_ack_after_insert: bool,
fail_remove_trust: bool,
fail_remove_trust_once: bool,
fail_send_peer_added: bool,
fail_send_peer_retired: bool,
fail_send_peer_unwired: bool,
peer_lifecycle_delay_ms: u64,
peer_message_delay_ms: u64,
}
struct MockCommsRuntime {
session_id: SessionId,
runtime_adapter: Option<Arc<meerkat_runtime::MeerkatMachine>>,
default_name: String,
default_address: String,
default_peer_id: PeerId,
default_public_key: String,
default_public_key_bytes: [u8; 32],
behavior: std::sync::RwLock<MockCommsBehavior>,
// Exact predecessor fence for the successor-supervisor add fault. A
// generic next-call counter can be consumed by unrelated trust traffic.
private_add_failure_replacing_peer_id: std::sync::Mutex<Option<String>>,
private_ack_rejections_remaining: std::sync::Mutex<usize>,
remove_failures_remaining: std::sync::Mutex<usize>,
trusted_peers: RwLock<HashMap<String, TrustedPeerDescriptor>>,
trusted_peer_sources:
RwLock<HashMap<String, meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind>>,
sent_intents: RwLock<Vec<String>>,
peer_lifecycle_in_flight: AtomicU64,
peer_lifecycle_max_in_flight: AtomicU64,
inbox_notify: Arc<tokio::sync::Notify>,
mob_machine_trust_owner: std::sync::RwLock<Option<Arc<dyn std::any::Any + Send + Sync>>>,
outbound_content_taint: std::sync::RwLock<Option<meerkat_core::comms::SenderContentTaint>>,
}
impl MockCommsRuntime {
fn new(
name: &str,
behavior: MockCommsBehavior,
session_id: SessionId,
runtime_adapter: Option<Arc<meerkat_runtime::MeerkatMachine>>,
) -> Self {
let mut key_bytes = [0u8; 32];
for (index, byte) in name.as_bytes().iter().copied().enumerate() {
let slot = index % key_bytes.len();
key_bytes[slot] = key_bytes[slot]
.wrapping_add(byte)
.rotate_left((index % 8) as u32);
}
if key_bytes == [0u8; 32] {
key_bytes[0] = 1;
}
let public_key = meerkat_comms::PubKey::new(key_bytes);
Self {
session_id,
runtime_adapter,
default_name: name.to_string(),
default_address: format!("inproc://{name}"),
default_peer_id: public_key.to_peer_id(),
default_public_key: public_key.to_pubkey_string(),
default_public_key_bytes: key_bytes,
behavior: std::sync::RwLock::new(behavior),
private_add_failure_replacing_peer_id: std::sync::Mutex::new(None),
private_ack_rejections_remaining: std::sync::Mutex::new(usize::from(
behavior.reject_private_publish_ack_after_insert,
)),
remove_failures_remaining: std::sync::Mutex::new(usize::from(
behavior.fail_remove_trust_once,
)),
trusted_peers: RwLock::new(HashMap::new()),
trusted_peer_sources: RwLock::new(HashMap::new()),
sent_intents: RwLock::new(Vec::new()),
peer_lifecycle_in_flight: AtomicU64::new(0),
peer_lifecycle_max_in_flight: AtomicU64::new(0),
inbox_notify: Arc::new(tokio::sync::Notify::new()),
mob_machine_trust_owner: std::sync::RwLock::new(None),
outbound_content_taint: std::sync::RwLock::new(None),
}
}
fn clear_public_key(&self) {
let mut behavior = self
.behavior
.write()
.expect("poisoned behavior lock in mock runtime");
behavior.missing_public_key = true;
}
fn set_behavior(&self, behavior: MockCommsBehavior) {
{
let mut current = self
.behavior
.write()
.expect("poisoned behavior lock in mock runtime");
*current = behavior;
}
let mut remaining = self
.remove_failures_remaining
.lock()
.expect("poisoned remove_failures_remaining lock in mock runtime");
*remaining = usize::from(behavior.fail_remove_trust_once);
let mut private_ack_remaining = self
.private_ack_rejections_remaining
.lock()
.expect("poisoned private_ack_rejections_remaining lock in mock runtime");
*private_ack_remaining = usize::from(behavior.reject_private_publish_ack_after_insert);
}
fn fail_next_supervisor_private_add_after_insert_replacing(&self, peer_id: &str) {
*self
.private_add_failure_replacing_peer_id
.lock()
.expect("poisoned private_add_failure_replacing_peer_id lock in mock runtime") =
Some(peer_id.to_string());
}
fn clear_mob_machine_trust_owner(&self) {
*self
.mob_machine_trust_owner
.write()
.expect("poisoned mob_machine_trust_owner lock in mock runtime") = None;
}
async fn trusted_peer_names(&self) -> Vec<String> {
let mut names = self
.trusted_peers
.read()
.await
.values()
.map(|peer| peer.name.as_str().to_string())
.collect::<Vec<_>>();
names.sort();
names
}
#[allow(dead_code)]
async fn trusted_peer_addresses(&self) -> Vec<String> {
let mut addresses = self
.trusted_peers
.read()
.await
.values()
.map(|peer| peer.address.to_string())
.collect::<Vec<_>>();
addresses.sort();
addresses
}
async fn sent_intents(&self) -> Vec<String> {
self.sent_intents.read().await.clone()
}
fn max_concurrent_peer_lifecycle_sends(&self) -> u64 {
self.peer_lifecycle_max_in_flight.load(Ordering::Relaxed)
}
fn validate_mob_trust_authority_owner(
&self,
authority: &CommsTrustMutationAuthority,
) -> Result<(), SendError> {
if !authority.is_mob_machine_source() {
return Ok(());
}
let expected = self
.mob_machine_trust_owner
.read()
.expect("poisoned mob_machine_trust_owner lock in mock runtime");
authority
.validate_raw_source_owner_token(expected.as_ref())
.map_err(SendError::Validation)
}
}
impl TestPeerProjectionOwnerInstall for MockCommsRuntime {}
// Trust-map helpers formerly on the CommsRuntime trait; the trait methods were
// deleted (trust mutation flows through apply_trust_mutation), but the mock's
// internal trust map operations and direct test callers keep these inherent.
impl MockCommsRuntime {
async fn add_trusted_peer(
&self,
peer: TrustedPeerDescriptor,
source: meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind,
) -> Result<(), SendError> {
if self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime")
.fail_add_trust
{
return Err(SendError::Unsupported(
"mock add_trusted_peer failure".to_string(),
));
}
TrustedPeerDescriptor::validate_pubkey_for_peer_id(peer.peer_id, &peer.pubkey)
.map_err(SendError::Validation)?;
let peer_id = peer.peer_id.to_string();
let mut peers = self.trusted_peers.write().await;
peers.insert(peer_id.clone(), peer);
drop(peers);
self.trusted_peer_sources
.write()
.await
.insert(peer_id, source);
Ok(())
}
async fn remove_trusted_peer(&self, peer_id: &str) -> Result<bool, SendError> {
if self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime")
.fail_remove_trust
{
return Err(SendError::Unsupported(
"mock remove_trusted_peer failure".to_string(),
));
}
{
let mut remaining = self
.remove_failures_remaining
.lock()
.expect("poisoned remove_failures_remaining lock in mock runtime");
if *remaining > 0 {
*remaining -= 1;
return Err(SendError::Unsupported(
"mock remove_trusted_peer failure (one-shot)".to_string(),
));
}
}
let mut peers = self.trusted_peers.write().await;
let removed = peers.remove(peer_id).is_some();
drop(peers);
self.trusted_peer_sources.write().await.remove(peer_id);
Ok(removed)
}
async fn add_private_trusted_peer_with_source(
&self,
peer: TrustedPeerDescriptor,
source: meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind,
) -> Result<(), SendError> {
let added_peer_id = peer.peer_id.as_str().to_owned();
self.add_trusted_peer(peer, source).await?;
if let Some(adapter) = self.runtime_adapter.as_ref()
&& let meerkat_runtime::meerkat_machine::SupervisorBinding::Bound {
peer_id, epoch, ..
} = adapter.supervisor_binding(&self.session_id).await
&& peer_id == added_peer_id
{
let fail_private_add_after_insert = {
let mut replacing_peer_id = self
.private_add_failure_replacing_peer_id
.lock()
.expect("poisoned private_add_failure_replacing_peer_id lock in mock runtime");
let should_fail = replacing_peer_id
.as_deref()
.is_some_and(|previous_peer_id| previous_peer_id != added_peer_id);
if should_fail {
*replacing_peer_id = None;
}
should_fail
};
if fail_private_add_after_insert {
return Err(SendError::Unsupported(
"mock add_private_trusted_peer failure after insert".to_string(),
));
}
let should_try_reject_private_publish_ack = {
let remaining = self
.private_ack_rejections_remaining
.lock()
.expect("poisoned private_ack_rejections_remaining lock in mock runtime");
*remaining > 0
};
if should_try_reject_private_publish_ack
&& adapter
.stage_supervisor_trust_publish_failed(
&self.session_id,
peer_id,
epoch,
"mock consumed pending supervisor publication before success ack"
.to_string(),
)
.await
.is_ok()
{
let mut remaining = self
.private_ack_rejections_remaining
.lock()
.expect("poisoned private_ack_rejections_remaining lock in mock runtime");
if *remaining > 0 {
*remaining -= 1;
}
}
}
Ok(())
}
}
#[async_trait]
impl CoreCommsRuntime for MockCommsRuntime {
fn set_outbound_content_taint(
&self,
taint: Option<meerkat_core::comms::SenderContentTaint>,
) -> Result<(), meerkat_core::comms::SendError> {
*self
.outbound_content_taint
.write()
.expect("poisoned taint lock in mock runtime") = taint;
Ok(())
}
fn peer_id(&self) -> Option<PeerId> {
let behavior = self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime");
if behavior.missing_public_key {
None
} else {
Some(self.default_peer_id)
}
}
fn public_key(&self) -> Option<String> {
let behavior = self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime");
if behavior.missing_public_key {
None
} else {
Some(self.default_public_key.clone())
}
}
fn public_key_bytes(&self) -> Option<[u8; 32]> {
let behavior = self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime");
if behavior.missing_public_key {
None
} else {
Some(self.default_public_key_bytes)
}
}
fn comms_name(&self) -> Option<String> {
Some(self.default_name.clone())
}
fn advertised_address(&self) -> Option<String> {
Some(self.default_address.clone())
}
async fn apply_trust_mutation(
&self,
mutation: CommsTrustMutation,
) -> Result<CommsTrustMutationResult, SendError> {
match mutation {
CommsTrustMutation::AddTrustedPeer { peer, authority } => {
let source = authority.trust_row_owner_kind();
self.validate_mob_trust_authority_owner(&authority)?;
authority
.validate_public_add(self.peer_id(), &peer)
.map_err(SendError::Validation)?;
let created = !self
.trusted_peers
.read()
.await
.contains_key(&peer.peer_id.to_string());
self.add_trusted_peer(peer, source).await?;
Ok(CommsTrustMutationResult::Added { created })
}
CommsTrustMutation::RemoveTrustedPeer { peer_id, authority } => {
self.validate_mob_trust_authority_owner(&authority)?;
let parsed_peer_id = PeerId::parse(&peer_id)
.map_err(|error| SendError::Validation(error.to_string()))?;
authority
.validate_public_remove(self.peer_id(), parsed_peer_id)
.map_err(SendError::Validation)?;
let removed = self.remove_trusted_peer(&peer_id).await?;
Ok(CommsTrustMutationResult::Removed { removed })
}
CommsTrustMutation::AddPrivateTrustedPeer { peer, authority } => {
let source = authority.trust_row_owner_kind();
self.validate_mob_trust_authority_owner(&authority)?;
authority
.validate_private_add(self.peer_id(), &peer)
.map_err(SendError::Validation)?;
let created = !self
.trusted_peers
.read()
.await
.contains_key(&peer.peer_id.to_string());
self.add_private_trusted_peer_with_source(peer, source)
.await?;
Ok(CommsTrustMutationResult::Added { created })
}
CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority } => {
self.validate_mob_trust_authority_owner(&authority)?;
let parsed_peer_id = PeerId::parse(&peer_id)
.map_err(|error| SendError::Validation(error.to_string()))?;
authority
.validate_private_remove(self.peer_id(), parsed_peer_id)
.map_err(SendError::Validation)?;
let removed = self.remove_private_trusted_peer(&peer_id).await?;
Ok(CommsTrustMutationResult::Removed { removed })
}
}
}
async fn install_generated_mob_trust_owner(
&self,
owner: Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
let mut expected = self
.mob_machine_trust_owner
.write()
.expect("poisoned mob_machine_trust_owner lock in mock runtime");
if let Some(existing) = expected.as_ref() {
if Arc::ptr_eq(existing, &owner) {
return Ok(());
}
return Err(SendError::Validation(
"target runtime is already bound to a different generated MobMachine trust owner"
.to_string(),
));
}
*expected = Some(owner);
Ok(())
}
async fn validate_recovered_generated_mob_trust_owner(
&self,
owner: Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
let expected = self
.mob_machine_trust_owner
.read()
.expect("poisoned mob_machine_trust_owner lock in mock runtime");
if let Some(existing) = expected.as_ref()
&& !Arc::ptr_eq(existing, &owner)
{
return Err(SendError::Validation(
"target runtime is already bound to a different generated MobMachine trust owner"
.to_string(),
));
}
Ok(())
}
async fn install_recovered_generated_mob_trust_owner(
&self,
owner: Arc<dyn std::any::Any + Send + Sync>,
) -> Result<(), SendError> {
let mut expected = self
.mob_machine_trust_owner
.write()
.expect("poisoned mob_machine_trust_owner lock in mock runtime");
if let Some(existing) = expected.as_ref() {
if Arc::ptr_eq(existing, &owner) {
return Ok(());
}
return Err(SendError::Validation(
"target runtime is already bound to a different generated MobMachine trust owner"
.to_string(),
));
}
*expected = Some(owner);
Ok(())
}
async fn add_private_trusted_peer(&self, peer: TrustedPeerDescriptor) -> Result<(), SendError> {
self.add_private_trusted_peer_with_source(
peer,
meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind::MeerkatMachineSupervisorPublish,
)
.await
}
async fn remove_private_trusted_peer(&self, peer_id: &str) -> Result<bool, SendError> {
self.remove_trusted_peer(peer_id).await
}
async fn send(&self, cmd: CommsCommand) -> Result<SendReceipt, SendError> {
match cmd {
CommsCommand::PeerLifecycle { to, kind, .. } => {
let behavior = *self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime");
if behavior.peer_lifecycle_delay_ms > 0 {
let in_flight = self
.peer_lifecycle_in_flight
.fetch_add(1, Ordering::Relaxed)
+ 1;
self.peer_lifecycle_max_in_flight
.fetch_max(in_flight, Ordering::Relaxed);
tokio::time::sleep(Duration::from_millis(behavior.peer_lifecycle_delay_ms))
.await;
self.peer_lifecycle_in_flight
.fetch_sub(1, Ordering::Relaxed);
}
match kind {
meerkat_core::comms::PeerLifecycleKind::PeerAdded
if behavior.fail_send_peer_added =>
{
return Err(SendError::Unsupported(
"mock mob.peer_added notification failure".to_string(),
));
}
meerkat_core::comms::PeerLifecycleKind::PeerRetired
if behavior.fail_send_peer_retired =>
{
return Err(SendError::Unsupported(
"mock mob.peer_retired notification failure".to_string(),
));
}
meerkat_core::comms::PeerLifecycleKind::PeerUnwired
if behavior.fail_send_peer_unwired =>
{
return Err(SendError::Unsupported(
"mock mob.peer_unwired notification failure".to_string(),
));
}
_ => {}
}
let trusted = self.trusted_peers.read().await;
let peer_id = to.peer_id.as_str();
if !trusted.contains_key(&peer_id) {
return Err(SendError::PeerNotFound(to.label()));
}
drop(trusted);
self.sent_intents
.write()
.await
.push(kind.as_str().to_string());
Ok(SendReceipt::PeerLifecycleSent {
envelope_id: uuid::Uuid::new_v4(),
})
}
CommsCommand::PeerRequest { to, intent, .. } => {
let behavior = *self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime");
if behavior.peer_lifecycle_delay_ms > 0 && intent.starts_with("mob.peer_") {
let in_flight = self
.peer_lifecycle_in_flight
.fetch_add(1, Ordering::Relaxed)
+ 1;
self.peer_lifecycle_max_in_flight
.fetch_max(in_flight, Ordering::Relaxed);
tokio::time::sleep(Duration::from_millis(behavior.peer_lifecycle_delay_ms))
.await;
self.peer_lifecycle_in_flight
.fetch_sub(1, Ordering::Relaxed);
}
if intent == "mob.peer_added" && behavior.fail_send_peer_added {
return Err(SendError::Unsupported(
"mock mob.peer_added notification failure".to_string(),
));
}
if intent == "mob.peer_retired" && behavior.fail_send_peer_retired {
return Err(SendError::Unsupported(
"mock mob.peer_retired notification failure".to_string(),
));
}
if intent == "mob.peer_unwired" && behavior.fail_send_peer_unwired {
return Err(SendError::Unsupported(
"mock mob.peer_unwired notification failure".to_string(),
));
}
let trusted = self.trusted_peers.read().await;
let peer_id = to.peer_id.as_str();
if !trusted.contains_key(&peer_id) {
return Err(SendError::PeerNotFound(to.label()));
}
drop(trusted);
self.sent_intents.write().await.push(intent);
Ok(SendReceipt::PeerRequestSent {
envelope_id: uuid::Uuid::new_v4(),
interaction_id: InteractionId(uuid::Uuid::new_v4()),
stream_reserved: false,
})
}
CommsCommand::PeerMessage { to, .. } => {
{
let trusted = self.trusted_peers.read().await;
let peer_id = to.peer_id.as_str();
if !trusted.contains_key(&peer_id) {
return Err(SendError::PeerNotFound(to.label()));
}
}
let delay_ms = self
.behavior
.read()
.expect("poisoned behavior lock in mock runtime")
.peer_message_delay_ms;
self.sent_intents
.write()
.await
.push("peer_message_started".to_string());
if delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
}
self.sent_intents
.write()
.await
.push("peer_message".to_string());
Ok(SendReceipt::PeerMessageSent {
envelope_id: uuid::Uuid::new_v4(),
delivery: meerkat_core::comms::PeerDeliveryOutcome::HandedOff,
})
}
unsupported => Err(SendError::Unsupported(format!(
"mock send unsupported command kind '{}'",
unsupported.command_kind()
))),
}
}
async fn peers(&self) -> Vec<PeerDirectoryEntry> {
let trusted = self.trusted_peers.read().await;
trusted
.iter()
.filter_map(|(peer_id, peer)| {
let peer_id = meerkat_core::comms::PeerId::parse(peer_id).ok()?;
let name = peer.name.clone();
Some(PeerDirectoryEntry {
name,
peer_id,
address: peer.address.clone(),
source: PeerDirectorySource::Trusted,
sendable_kinds: PeerSendability::directory_defaults(),
capabilities: PeerCapabilitySet::default(),
meta: meerkat_core::PeerMeta::default(),
})
})
.collect()
}
async fn trusted_peer_projection_snapshot_for_source(
&self,
source_kind: meerkat_core::comms::GeneratedCommsTrustAuthoritySourceKind,
) -> Result<Vec<TrustedPeerDescriptor>, CommsCapabilityError> {
let sources = self.trusted_peer_sources.read().await.clone();
Ok(self
.trusted_peers
.read()
.await
.iter()
.filter(|(peer_id, _)| sources.get(*peer_id) == Some(&source_kind))
.map(|(_, peer)| peer.clone())
.collect())
}
async fn drain_messages(&self) -> Vec<String> {
Vec::new()
}
fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
self.inbox_notify.clone()
}
}
// -----------------------------------------------------------------------
// Mock SessionService
// -----------------------------------------------------------------------
fn mock_run_result(session_id: SessionId, text: String) -> RunResult {
RunResult {
text,
session_id,
usage: Usage::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: None,
}
}
fn begin_test_runtime_actor_materialization(
req: &CreateSessionRequest,
) -> Result<Option<meerkat_runtime::RuntimeActorMaterializationPermit>, SessionError> {
let Some(bindings) = req
.build
.as_ref()
.and_then(|build| match &build.runtime_build_mode {
meerkat_core::RuntimeBuildMode::SessionOwned(bindings) => Some(bindings),
meerkat_core::RuntimeBuildMode::StandaloneEphemeral => None,
})
else {
return Ok(None);
};
match meerkat_runtime::begin_session_runtime_actor_materialization(bindings) {
Ok(permit) => Ok(Some(permit)),
Err(meerkat_runtime::RuntimeActorMaterializationError::RegistrationClosed) => {
Err(SessionError::NotFound {
id: bindings.session_id().clone(),
})
}
Err(meerkat_runtime::RuntimeActorMaterializationError::InvalidAuthority(reason)) => Err(
SessionError::Agent(meerkat_core::error::AgentError::InternalError(reason)),
),
}
}
fn requested_test_session_id(req: &CreateSessionRequest) -> SessionId {
req.build
.as_ref()
.and_then(|build| match &build.runtime_build_mode {
meerkat_core::RuntimeBuildMode::SessionOwned(bindings) => {
Some(bindings.session_id().clone())
}
meerkat_core::RuntimeBuildMode::StandaloneEphemeral => build
.resume_session
.as_ref()
.map(|session| session.id().clone()),
})
.unwrap_or_default()
}
fn test_actor_transient_turn_context_state() -> meerkat_core::TransientTurnContextStateHandle {
meerkat_core::TransientTurnContextStateHandle::new()
}
fn install_session_owned_peer_request_response_authority(
req: &CreateSessionRequest,
runtime: &Arc<meerkat_comms::CommsRuntime>,
) -> Result<bool, SessionError> {
let Some(bindings) = req
.build
.as_ref()
.and_then(|build| match &build.runtime_build_mode {
meerkat_core::RuntimeBuildMode::SessionOwned(bindings) => Some(bindings),
meerkat_core::RuntimeBuildMode::StandaloneEphemeral => None,
})
else {
return Ok(false);
};
bindings
.install_peer_comms_on(runtime.as_ref())
.map_err(|error| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to install exact peer-comms authority: {error}"
)))
})?;
runtime.install_peer_request_response_authority(
meerkat_comms::PeerRequestResponseAuthority::new(
Arc::clone(bindings.peer_interaction()),
Arc::clone(bindings.interaction_stream()),
),
);
Ok(true)
}
#[derive(Clone, Debug)]
struct CreateSessionRecord {
initial_turn: meerkat_core::service::InitialTurnPolicy,
comms_name: Option<String>,
peer_meta_labels: BTreeMap<String, String>,
runtime_build_mode_session_owned: bool,
budget_limits: Option<meerkat_core::BudgetLimits>,
model: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum RuntimeBoundaryAcknowledgement {
WholeBlob {
session_id: SessionId,
store_revision: u64,
blob_sha256: String,
},
HeadCanonical {
session_id: SessionId,
head_token: String,
},
Provisional {
session_id: SessionId,
store_revision: u64,
authority_token: String,
},
}
struct AtomicInFlightGuard<'a>(&'a AtomicU64);
impl Drop for AtomicInFlightGuard<'_> {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::Relaxed);
}
}
/// A mock session service that creates sessions with mock comms runtimes.
struct MockSessionService {
sessions: RwLock<HashMap<SessionId, Arc<MockCommsRuntime>>>,
actor_registry: meerkat_session::LiveSessionActorRegistry,
live_session_data: RwLock<HashMap<SessionId, Session>>,
persisted_sessions: RwLock<HashMap<SessionId, Session>>,
/// Test-only physical-head bodies installed by the explicit operational
/// resume preparation seam.
prepared_resume_sessions: RwLock<HashMap<SessionId, Session>>,
resume_prepare_calls: AtomicU64,
keep_alive_notifiers: RwLock<HashMap<SessionId, Arc<tokio::sync::Notify>>>,
session_comms_names: RwLock<HashMap<SessionId, String>>,
/// Optional test-only key seed for the next materialization of a durable
/// session. The public comms name/address stay canonical; only the
/// incarnation's signing identity rotates. This lets recovery tests model
/// an exact replacement actor without asking the runtime to adopt an
/// unattached, surface-inserted actor.
comms_identity_seeds: RwLock<HashMap<SessionId, String>>,
runtime_adapter: Mutex<Option<Arc<meerkat_runtime::MeerkatMachine>>>,
session_counter: AtomicU64,
/// Records (session_id, prompt) for each create_session call.
prompts: RwLock<Vec<(SessionId, String)>>,
create_requests: RwLock<Vec<CreateSessionRecord>>,
/// Records whether each create_session had external_tools configured.
create_with_external_tools: RwLock<Vec<bool>>,
/// Records the declarative MCP server names per create_session call,
/// in call order.
mcp_server_names: RwLock<Vec<(SessionId, Vec<String>)>>,
/// External tool dispatcher captured per session.
external_tools_by_session: RwLock<HashMap<SessionId, Arc<dyn AgentToolDispatcher>>>,
/// Optional per-comms-name behavior overrides.
comms_behaviors: RwLock<HashMap<String, MockCommsBehavior>>,
/// Sessions whose comms runtime should appear unavailable.
missing_comms_sessions: RwLock<HashSet<SessionId>>,
/// Successful comms-runtime observations remaining before every runtime
/// becomes unavailable. Test-only seam separating builder reconciliation
/// from a later actor-owned operational startup failure.
comms_runtime_failure_after_successes_enabled: AtomicBool,
comms_runtime_successes_before_missing: AtomicUsize,
comms_runtime_observations: AtomicUsize,
/// Sessions whose durable archive commit should fail before runtime
/// retirement begins.
archive_precommit_fail_sessions: RwLock<HashSet<SessionId>>,
/// Sessions whose archive() call should fail.
archive_fail_sessions: RwLock<HashSet<SessionId>>,
archive_calls: RwLock<HashMap<SessionId, u64>>,
archive_not_found_sessions: RwLock<HashSet<SessionId>>,
/// Sessions whose durable document belongs to another host. The live
/// actor may still be attached locally, but this mock archive authority
/// must not claim ownership of its durable lifecycle.
archive_authority_unknown_sessions: RwLock<HashSet<SessionId>>,
/// Comms names whose created sessions should fail archive().
archive_fail_comms_names: RwLock<HashSet<String>>,
/// Sent intents for sessions that were archived and removed.
archived_sent_intents: RwLock<HashMap<SessionId, Vec<String>>>,
/// Fanout concurrency observed by sessions that were archived and removed.
archived_peer_lifecycle_max_in_flight: RwLock<HashMap<SessionId, u64>>,
/// Sessions that were archived. Mirrors the real persistent-service
/// contract: `load_persisted_session` returns `None` for archived
/// sessions, so archive stays terminal (no revival from an archived
/// snapshot).
archived_session_ids: RwLock<HashSet<SessionId>>,
/// Sessions whose create_session (re-materialization) should fail.
create_fail_sessions: RwLock<HashSet<SessionId>>,
/// Optional custom failure detail per failing session (presence implies
/// failure). Lets tests model provider-specific failure prose such as the
/// rpc-gateway's "callback transport closed".
create_fail_details: RwLock<HashMap<SessionId, String>>,
/// Timestamps of every refused create_session attempt, per session, so
/// retry-schedule tests can assert real attempt spacing.
create_fail_attempts: RwLock<HashMap<SessionId, Vec<std::time::Instant>>>,
/// Sessions whose create_session should materialize live state but return
/// the already-active comms identity error observed in the #37 deliver race.
create_identity_already_active_sessions: RwLock<HashSet<SessionId>>,
/// Sessions whose create_session should fail with the "Session identity
/// already active" error WITHOUT registering a live session — the
/// production-faithful shape of the user-reported revival failure, where the
/// inproc comms-claim acquire fails inside `build_agent` (before the live
/// handle is registered) because a not-yet-dropped prior runtime still holds
/// the process-scoped claim, so `has_live_session` reads false at the
/// recovery re-check. Distinct from
/// `create_identity_already_active_sessions`, which leaves the session live
/// (the idempotent-success case).
create_identity_already_active_no_live_sessions: RwLock<HashSet<SessionId>>,
fail_start_turn: std::sync::atomic::AtomicBool,
fail_inject: std::sync::atomic::AtomicBool,
disable_interaction_event_injector: std::sync::atomic::AtomicBool,
start_turn_calls: AtomicU64,
keep_alive_start_turn_calls: AtomicU64,
keep_alive_turns_complete_immediately: std::sync::atomic::AtomicBool,
start_turn_prompts: RwLock<Vec<(SessionId, String)>>,
/// Ordered ordinary System content received at the member boundary.
start_turn_system_prompts: RwLock<Vec<(SessionId, Vec<String>)>>,
/// Per turn request: the user-channel shape the runner would materialize
/// — `(is_injected_context, text)` pairs in transcript order, derived
/// from the request's active lowering carrier (typed appends in runtime
/// mode, the `injected_context` field + prompt in direct mode).
start_turn_user_channel: RwLock<Vec<(SessionId, Vec<(bool, String)>)>>,
/// Per turn request: the typed runtime turn metadata carrier, so tests
/// can assert transcript identity stamping on the delivery path.
start_turn_metadata: RwLock<
Vec<(
SessionId,
Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
)>,
>,
/// Interaction ids received through the SubscribableInjector's
/// caller-chosen-id entry point (autonomous submit-work delivery).
injected_interaction_ids: Arc<std::sync::Mutex<Vec<InteractionId>>>,
keep_alive_prompts: RwLock<Vec<(SessionId, String)>>,
interrupt_calls: AtomicU64,
cancel_after_boundary_supported: AtomicBool,
cancel_after_boundary_calls: AtomicU64,
inject_calls: Arc<AtomicU64>,
flow_turn_in_flight: Arc<AtomicU64>,
flow_turn_max_in_flight: Arc<AtomicU64>,
execution_snapshot_delay_ms: AtomicU64,
execution_snapshot_started: tokio::sync::Notify,
create_session_delay_ms: AtomicU64,
load_persisted_session_delay_ms: AtomicU64,
load_persisted_session_started: tokio::sync::Notify,
load_persisted_session_calls: AtomicU64,
load_persisted_session_in_flight: AtomicU64,
load_persisted_session_max_in_flight: AtomicU64,
create_session_in_flight: AtomicU64,
create_session_max_in_flight: AtomicU64,
archive_delay_ms: AtomicU64,
start_turn_delay_ms: AtomicU64,
start_turn_interrupts: RwLock<HashMap<SessionId, tokio::sync::watch::Sender<u64>>>,
inject_delay_ms: AtomicU64,
flow_turn_delay_ms: AtomicU64,
flow_turn_never_terminal: std::sync::atomic::AtomicBool,
flow_turn_fail: std::sync::atomic::AtomicBool,
flow_turn_fail_sessions: RwLock<HashSet<SessionId>>,
flow_turn_completed_result: RwLock<String>,
flow_turn_overlays: RwLock<Vec<(SessionId, Option<meerkat_core::service::TurnToolOverlay>)>>,
/// Number of exact actor-discard calls that should fail before mutation.
/// Models a retryable service cleanup outage after machine unregister has
/// already retained attachment-local cleanup authority.
discard_actor_failures_remaining: AtomicU64,
/// Session IDs for which `subscribe_session_events` must return an error.
subscribe_fail_sessions: RwLock<HashSet<SessionId>>,
/// Session IDs for which `read()`/`list()` should report `is_active=true`.
///
/// Mirrors real SessionService semantics: `is_active` tracks in-flight
/// turn execution (Admitted/Running/Completing), not session presence.
/// A created-but-idle session is not active. Tests that exercise
/// in-flight-turn semantics can flip this via
/// [`Self::mark_session_active`]. Without this, the disposal-time poll
/// loop in [`stop_autonomous_member`] spins the full 40×25ms=1s grace
/// window on every retire/complete/reset because the mock was reporting
/// an always-`true` flag that the real service never would.
active_sessions: RwLock<HashSet<SessionId>>,
runtime_boundary_acknowledgements: RwLock<Vec<RuntimeBoundaryAcknowledgement>>,
}
impl MockSessionService {
fn new() -> Self {
Self {
sessions: RwLock::new(HashMap::new()),
actor_registry: meerkat_session::LiveSessionActorRegistry::default(),
live_session_data: RwLock::new(HashMap::new()),
persisted_sessions: RwLock::new(HashMap::new()),
prepared_resume_sessions: RwLock::new(HashMap::new()),
resume_prepare_calls: AtomicU64::new(0),
keep_alive_notifiers: RwLock::new(HashMap::new()),
session_comms_names: RwLock::new(HashMap::new()),
comms_identity_seeds: RwLock::new(HashMap::new()),
runtime_adapter: Mutex::new(None),
session_counter: AtomicU64::new(0),
prompts: RwLock::new(Vec::new()),
create_requests: RwLock::new(Vec::new()),
create_with_external_tools: RwLock::new(Vec::new()),
mcp_server_names: RwLock::new(Vec::new()),
external_tools_by_session: RwLock::new(HashMap::new()),
comms_behaviors: RwLock::new(HashMap::new()),
missing_comms_sessions: RwLock::new(HashSet::new()),
comms_runtime_failure_after_successes_enabled: AtomicBool::new(false),
comms_runtime_successes_before_missing: AtomicUsize::new(0),
comms_runtime_observations: AtomicUsize::new(0),
archive_precommit_fail_sessions: RwLock::new(HashSet::new()),
archive_fail_sessions: RwLock::new(HashSet::new()),
archive_calls: RwLock::new(HashMap::new()),
archive_not_found_sessions: RwLock::new(HashSet::new()),
archive_authority_unknown_sessions: RwLock::new(HashSet::new()),
archive_fail_comms_names: RwLock::new(HashSet::new()),
archived_sent_intents: RwLock::new(HashMap::new()),
archived_peer_lifecycle_max_in_flight: RwLock::new(HashMap::new()),
archived_session_ids: RwLock::new(HashSet::new()),
create_fail_sessions: RwLock::new(HashSet::new()),
create_fail_details: RwLock::new(HashMap::new()),
create_fail_attempts: RwLock::new(HashMap::new()),
create_identity_already_active_sessions: RwLock::new(HashSet::new()),
create_identity_already_active_no_live_sessions: RwLock::new(HashSet::new()),
fail_start_turn: std::sync::atomic::AtomicBool::new(false),
fail_inject: std::sync::atomic::AtomicBool::new(false),
disable_interaction_event_injector: std::sync::atomic::AtomicBool::new(false),
start_turn_calls: AtomicU64::new(0),
keep_alive_start_turn_calls: AtomicU64::new(0),
keep_alive_turns_complete_immediately: std::sync::atomic::AtomicBool::new(false),
start_turn_prompts: RwLock::new(Vec::new()),
start_turn_system_prompts: RwLock::new(Vec::new()),
start_turn_user_channel: RwLock::new(Vec::new()),
start_turn_metadata: RwLock::new(Vec::new()),
injected_interaction_ids: Arc::new(std::sync::Mutex::new(Vec::new())),
keep_alive_prompts: RwLock::new(Vec::new()),
interrupt_calls: AtomicU64::new(0),
cancel_after_boundary_supported: AtomicBool::new(true),
cancel_after_boundary_calls: AtomicU64::new(0),
inject_calls: Arc::new(AtomicU64::new(0)),
flow_turn_in_flight: Arc::new(AtomicU64::new(0)),
flow_turn_max_in_flight: Arc::new(AtomicU64::new(0)),
execution_snapshot_delay_ms: AtomicU64::new(0),
execution_snapshot_started: tokio::sync::Notify::new(),
create_session_delay_ms: AtomicU64::new(0),
load_persisted_session_delay_ms: AtomicU64::new(0),
load_persisted_session_started: tokio::sync::Notify::new(),
load_persisted_session_calls: AtomicU64::new(0),
load_persisted_session_in_flight: AtomicU64::new(0),
load_persisted_session_max_in_flight: AtomicU64::new(0),
create_session_in_flight: AtomicU64::new(0),
create_session_max_in_flight: AtomicU64::new(0),
archive_delay_ms: AtomicU64::new(0),
start_turn_delay_ms: AtomicU64::new(0),
start_turn_interrupts: RwLock::new(HashMap::new()),
inject_delay_ms: AtomicU64::new(0),
flow_turn_delay_ms: AtomicU64::new(0),
flow_turn_never_terminal: std::sync::atomic::AtomicBool::new(false),
flow_turn_fail: std::sync::atomic::AtomicBool::new(false),
flow_turn_fail_sessions: RwLock::new(HashSet::new()),
flow_turn_completed_result: RwLock::new("\"Turn completed\"".to_string()),
flow_turn_overlays: RwLock::new(Vec::new()),
discard_actor_failures_remaining: AtomicU64::new(0),
subscribe_fail_sessions: RwLock::new(HashSet::new()),
active_sessions: RwLock::new(HashSet::new()),
runtime_boundary_acknowledgements: RwLock::new(Vec::new()),
}
}
async fn fail_subscribe_for_session(&self, session_id: SessionId) {
self.subscribe_fail_sessions
.write()
.await
.insert(session_id);
}
async fn active_session_count(&self) -> usize {
self.sessions.read().await.len()
}
async fn session_id_for_comms_name(&self, comms_name: &str) -> Option<SessionId> {
self.session_comms_names
.read()
.await
.iter()
.find_map(|(session_id, name)| (name == comms_name).then(|| session_id.clone()))
}
async fn live_session_clone(&self, session_id: &SessionId) -> Option<Session> {
self.live_session_data.read().await.get(session_id).cloned()
}
async fn persisted_session_clone(&self, session_id: &SessionId) -> Option<Session> {
self.persisted_sessions
.read()
.await
.get(session_id)
.cloned()
}
async fn replace_live_session(&self, session: Session) {
let session_id = session.id().clone();
self.live_session_data
.write()
.await
.insert(session_id.clone(), session.clone());
self.persisted_sessions
.write()
.await
.insert(session_id, session);
}
async fn delete_persisted_session(&self, session_id: &SessionId) {
self.persisted_sessions.write().await.remove(session_id);
}
async fn stage_prepared_resume_session(&self, session: Session) {
self.prepared_resume_sessions
.write()
.await
.insert(session.id().clone(), session);
}
/// Construct a process-restart-faithful service around the same durable
/// session state. Live runtimes and their generated trust projections are
/// deliberately not copied: a cold service restart must rebuild those
/// from durable owners rather than inheriting warm in-memory evidence.
async fn cold_restart_preserving_durable_state(&self) -> Self {
let restarted = Self::new();
let persisted_sessions = self.persisted_sessions.read().await.clone();
let persisted_ids = persisted_sessions.keys().cloned().collect::<HashSet<_>>();
*restarted.persisted_sessions.write().await = persisted_sessions;
*restarted.session_comms_names.write().await = self
.session_comms_names
.read()
.await
.iter()
.filter(|(session_id, _)| persisted_ids.contains(*session_id))
.map(|(session_id, name)| (session_id.clone(), name.clone()))
.collect();
restarted
}
fn enable_runtime_adapter(&self) -> Arc<meerkat_runtime::MeerkatMachine> {
let mut guard = self.runtime_adapter.lock().expect("runtime_adapter mutex");
if let Some(adapter) = guard.as_ref() {
return adapter.clone();
}
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
*guard = Some(adapter.clone());
adapter
}
fn set_runtime_adapter(&self, adapter: Arc<meerkat_runtime::MeerkatMachine>) {
*self.runtime_adapter.lock().expect("runtime_adapter mutex") = Some(adapter);
}
async fn set_comms_identity_seed(&self, session_id: &SessionId, seed: impl Into<String>) {
self.comms_identity_seeds
.write()
.await
.insert(session_id.clone(), seed.into());
}
fn disable_interaction_event_injector(&self) {
self.disable_interaction_event_injector
.store(true, Ordering::Relaxed);
}
/// Get recorded prompts for inspection in tests.
async fn recorded_prompts(&self) -> Vec<(SessionId, String)> {
self.prompts.read().await.clone()
}
async fn recorded_create_requests(&self) -> Vec<CreateSessionRecord> {
self.create_requests.read().await.clone()
}
async fn recorded_start_turn_prompts(&self) -> Vec<(SessionId, String)> {
self.start_turn_prompts.read().await.clone()
}
async fn recorded_start_turn_system_prompts(&self) -> Vec<(SessionId, Vec<String>)> {
self.start_turn_system_prompts.read().await.clone()
}
async fn recorded_start_turn_user_channel(&self) -> Vec<(SessionId, Vec<(bool, String)>)> {
self.start_turn_user_channel.read().await.clone()
}
async fn recorded_start_turn_metadata(
&self,
) -> Vec<(
SessionId,
Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>,
)> {
self.start_turn_metadata.read().await.clone()
}
fn recorded_injected_interaction_ids(&self) -> Vec<InteractionId> {
self.injected_interaction_ids
.lock()
.expect("injected interaction id mutex")
.clone()
}
async fn recorded_external_tools_flags(&self) -> Vec<bool> {
self.create_with_external_tools.read().await.clone()
}
async fn recorded_mcp_server_names(&self) -> Vec<(SessionId, Vec<String>)> {
self.mcp_server_names.read().await.clone()
}
async fn external_tool_names(&self, session_id: &SessionId) -> Vec<String> {
let tools = self.external_tools_by_session.read().await;
tools
.get(session_id)
.map(|dispatcher| {
dispatcher
.tools()
.iter()
.map(|tool| tool.name.to_string())
.collect()
})
.unwrap_or_default()
}
async fn dispatch_external_tool(
&self,
session_id: &SessionId,
tool_name: &str,
args: serde_json::Value,
) -> Result<ToolResult, ToolError> {
self.dispatch_external_tool_outcome(session_id, tool_name, args)
.await
.map(|outcome| outcome.result)
}
async fn dispatch_external_tool_outcome(
&self,
session_id: &SessionId,
tool_name: &str,
args: serde_json::Value,
) -> Result<ToolDispatchOutcome, ToolError> {
let dispatcher = {
let tools = self.external_tools_by_session.read().await;
tools.get(session_id).cloned()
}
.ok_or_else(|| ToolError::not_found(tool_name))?;
let raw = RawValue::from_string(args.to_string())
.map_err(|error| ToolError::execution_failed(format!("raw args: {error}")))?;
dispatcher
.dispatch(ToolCallView {
id: "tool-1",
name: tool_name,
args: &raw,
})
.await
}
async fn set_comms_behavior(&self, comms_name: &str, behavior: MockCommsBehavior) {
self.comms_behaviors
.write()
.await
.insert(comms_name.to_string(), behavior);
let session_ids = {
let names = self.session_comms_names.read().await;
names
.iter()
.filter_map(|(session_id, name)| {
if name == comms_name {
Some(session_id.clone())
} else {
None
}
})
.collect::<Vec<_>>()
};
if session_ids.is_empty() {
return;
}
let sessions = self.sessions.read().await;
for session_id in session_ids {
if let Some(runtime) = sessions.get(&session_id) {
runtime.set_behavior(behavior);
}
}
}
async fn set_missing_comms_runtime(&self, session_id: &SessionId) {
self.missing_comms_sessions
.write()
.await
.insert(session_id.clone());
}
fn set_comms_runtime_missing_after_successes(&self, successful_observations: usize) {
self.comms_runtime_successes_before_missing
.store(successful_observations, Ordering::Release);
self.comms_runtime_failure_after_successes_enabled
.store(true, Ordering::Release);
}
async fn set_archive_failure(&self, session_id: &SessionId) {
self.archive_fail_sessions
.write()
.await
.insert(session_id.clone());
}
async fn set_archive_precommit_failure(&self, session_id: &SessionId) {
self.archive_precommit_fail_sessions
.write()
.await
.insert(session_id.clone());
}
async fn archive_call_count(&self, session_id: &SessionId) -> u64 {
self.archive_calls
.read()
.await
.get(session_id)
.copied()
.unwrap_or(0)
}
/// Mirror PersistentSessionService for a session absent from the durable
/// store: archive returns `SessionError::NotFound`.
async fn set_archive_not_found(&self, session_id: &SessionId) {
self.archive_not_found_sessions
.write()
.await
.insert(session_id.clone());
}
async fn set_archive_authority_unknown(&self, session_id: &SessionId) {
self.archive_authority_unknown_sessions
.write()
.await
.insert(session_id.clone());
}
/// Make any (re-)materialization of `session_id` via create_session fail.
async fn set_create_session_failure(&self, session_id: &SessionId) {
self.create_fail_sessions
.write()
.await
.insert(session_id.clone());
}
/// Make any (re-)materialization of `session_id` via create_session fail
/// with the given detail prose (e.g. the rpc-gateway's "callback
/// transport closed").
async fn set_create_session_failure_detail(&self, session_id: &SessionId, detail: &str) {
self.create_fail_sessions
.write()
.await
.insert(session_id.clone());
self.create_fail_details
.write()
.await
.insert(session_id.clone(), detail.to_string());
}
/// Timestamps of every refused create_session attempt for `session_id`,
/// in attempt order.
async fn create_fail_attempt_instants(
&self,
session_id: &SessionId,
) -> Vec<std::time::Instant> {
self.create_fail_attempts
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn set_create_session_identity_already_active(&self, session_id: &SessionId) {
self.create_identity_already_active_sessions
.write()
.await
.insert(session_id.clone());
}
/// Inject the production-faithful revival failure: create_session fails with
/// the typed `SessionIdentityInUse` cause and does NOT register a live
/// session, so `has_live_session` returns false at the actor's recovery
/// re-check (the leaked-claim-without-live-session terminal path the user hit).
async fn set_create_session_identity_already_active_without_live_session(
&self,
session_id: &SessionId,
) {
self.create_identity_already_active_no_live_sessions
.write()
.await
.insert(session_id.clone());
}
async fn clear_create_session_failure(&self, session_id: &SessionId) {
self.create_fail_sessions.write().await.remove(session_id);
self.create_fail_details.write().await.remove(session_id);
}
async fn clear_archive_failure(&self, session_id: &SessionId) {
self.archive_fail_sessions.write().await.remove(session_id);
}
async fn clear_archive_precommit_failure(&self, session_id: &SessionId) {
self.archive_precommit_fail_sessions
.write()
.await
.remove(session_id);
}
async fn set_archive_failure_for_comms_name(&self, comms_name: &str) {
self.archive_fail_comms_names
.write()
.await
.insert(comms_name.to_string());
}
async fn clear_public_key(&self, session_id: &SessionId) {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
if let Some(runtime) = runtime {
runtime.clear_public_key();
}
}
fn set_fail_start_turn(&self, enabled: bool) {
self.fail_start_turn
.store(enabled, std::sync::atomic::Ordering::Relaxed);
}
#[allow(dead_code)]
fn set_fail_inject(&self, enabled: bool) {
self.fail_inject
.store(enabled, std::sync::atomic::Ordering::Relaxed);
}
fn start_turn_call_count(&self) -> u64 {
self.start_turn_calls.load(Ordering::Acquire)
}
fn keep_alive_start_turn_call_count(&self) -> u64 {
self.keep_alive_start_turn_calls.load(Ordering::Relaxed)
}
fn set_keep_alive_turns_complete_immediately(&self, enabled: bool) {
self.keep_alive_turns_complete_immediately
.store(enabled, Ordering::Relaxed);
}
/// True iff the session was terminally archived (data-loss). Distinct from
/// `has_live_session`: a `discard_live_session` drops the live in-memory
/// copy while leaving the durable session recoverable, which is NOT
/// archival.
async fn test_is_session_archived(&self, id: &SessionId) -> bool {
self.archived_session_ids.read().await.contains(id)
}
fn interrupt_call_count(&self) -> u64 {
self.interrupt_calls.load(Ordering::Relaxed)
}
fn cancel_after_boundary_call_count(&self) -> u64 {
self.cancel_after_boundary_calls.load(Ordering::Relaxed)
}
fn set_cancel_after_boundary_supported(&self, enabled: bool) {
self.cancel_after_boundary_supported
.store(enabled, Ordering::Relaxed);
}
fn inject_call_count(&self) -> u64 {
self.inject_calls.load(Ordering::Relaxed)
}
fn max_concurrent_flow_turns(&self) -> u64 {
self.flow_turn_max_in_flight.load(Ordering::Relaxed)
}
fn set_create_session_delay_ms(&self, delay_ms: u64) {
self.create_session_delay_ms
.store(delay_ms, Ordering::Relaxed);
}
fn set_load_persisted_session_delay_ms(&self, delay_ms: u64) {
self.load_persisted_session_delay_ms
.store(delay_ms, Ordering::Relaxed);
}
fn set_execution_snapshot_delay_ms(&self, delay_ms: u64) {
self.execution_snapshot_delay_ms
.store(delay_ms, Ordering::Relaxed);
}
async fn wait_for_execution_snapshot(&self) {
self.execution_snapshot_started.notified().await;
}
async fn wait_for_load_persisted_session(&self) {
self.load_persisted_session_started.notified().await;
}
fn max_concurrent_load_persisted_session_calls(&self) -> u64 {
self.load_persisted_session_max_in_flight
.load(Ordering::Relaxed)
}
fn max_concurrent_create_session_calls(&self) -> u64 {
self.create_session_max_in_flight.load(Ordering::Relaxed)
}
fn set_archive_delay_ms(&self, delay_ms: u64) {
self.archive_delay_ms.store(delay_ms, Ordering::Relaxed);
}
fn fail_next_exact_actor_discard(&self) {
self.discard_actor_failures_remaining
.store(1, Ordering::Relaxed);
}
fn set_start_turn_delay_ms(&self, delay_ms: u64) {
self.start_turn_delay_ms
.store(delay_ms, std::sync::atomic::Ordering::Relaxed);
}
fn set_inject_delay_ms(&self, delay_ms: u64) {
self.inject_delay_ms
.store(delay_ms, std::sync::atomic::Ordering::Relaxed);
}
fn set_flow_turn_delay_ms(&self, delay_ms: u64) {
self.flow_turn_delay_ms
.store(delay_ms, std::sync::atomic::Ordering::Relaxed);
}
fn set_flow_turn_never_terminal(&self, enabled: bool) {
self.flow_turn_never_terminal
.store(enabled, std::sync::atomic::Ordering::Relaxed);
}
fn set_flow_turn_fail(&self, enabled: bool) {
self.flow_turn_fail
.store(enabled, std::sync::atomic::Ordering::Relaxed);
}
async fn set_flow_turn_fail_for_session(&self, session_id: &SessionId, enabled: bool) {
let mut set = self.flow_turn_fail_sessions.write().await;
if enabled {
set.insert(session_id.clone());
} else {
set.remove(session_id);
}
}
async fn set_flow_turn_completed_result(&self, result: impl Into<String>) {
*self.flow_turn_completed_result.write().await = result.into();
}
async fn recorded_flow_turn_overlays(
&self,
) -> Vec<(SessionId, Option<meerkat_core::service::TurnToolOverlay>)> {
self.flow_turn_overlays.read().await.clone()
}
async fn trusted_peer_names(&self, session_id: &SessionId) -> Vec<String> {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
match runtime {
Some(runtime) => runtime.trusted_peer_names().await,
None => Vec::new(),
}
}
#[allow(dead_code)]
async fn trusted_peer_addresses(&self, session_id: &SessionId) -> Vec<String> {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
match runtime {
Some(runtime) => runtime.trusted_peer_addresses().await,
None => Vec::new(),
}
}
async fn sent_intents(&self, session_id: &SessionId) -> Vec<String> {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
match runtime {
Some(runtime) => runtime.sent_intents().await,
None => self
.archived_sent_intents
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default(),
}
}
async fn outbound_content_taint(
&self,
session_id: &SessionId,
) -> Option<meerkat_core::comms::SenderContentTaint> {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
runtime.and_then(|runtime| {
*runtime
.outbound_content_taint
.read()
.expect("poisoned taint lock in mock runtime")
})
}
async fn max_concurrent_peer_lifecycle_sends(&self, session_id: &SessionId) -> u64 {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
match runtime {
Some(runtime) => runtime.max_concurrent_peer_lifecycle_sends(),
None => self
.archived_peer_lifecycle_max_in_flight
.read()
.await
.get(session_id)
.copied()
.unwrap_or_default(),
}
}
async fn force_remove_trust(&self, from_session: &SessionId, to_session: &SessionId) {
let (from_runtime, to_runtime) = {
let sessions = self.sessions.read().await;
(
sessions.get(from_session).cloned(),
sessions.get(to_session).cloned(),
)
};
if let (Some(from_runtime), Some(to_runtime)) = (from_runtime, to_runtime)
&& let Some(to_key) = to_runtime.peer_id()
{
remove_test_peer_projection_trust(
from_runtime.as_ref(),
&to_key.to_string(),
"force remove trusted peer",
)
.await;
}
}
async fn force_remove_trust_by_peer_id(&self, session_id: &SessionId, peer_id: &str) {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
if let Some(runtime) = runtime {
remove_test_peer_projection_trust(
runtime.as_ref(),
peer_id,
"force remove trusted peer by peer id",
)
.await;
}
}
async fn clear_mob_machine_trust_owner(&self, session_id: &SessionId) {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
if let Some(runtime) = runtime {
runtime.clear_mob_machine_trust_owner();
}
}
async fn force_add_trust_from_spec(&self, session_id: &SessionId, spec: TrustedPeerDescriptor) {
let runtime = {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
};
if let Some(runtime) = runtime {
apply_test_peer_projection_trust(runtime.as_ref(), spec, "force add trusted peer")
.await;
}
}
}
impl MockSessionService {
async fn create_session_with_actor_witness_slot(
&self,
req: CreateSessionRequest,
actor_witness_slot: Option<&meerkat_session::LiveSessionActorWitnessSlot>,
) -> Result<RunResult, SessionError> {
let in_flight = self
.create_session_in_flight
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1);
loop {
let observed_max = self.create_session_max_in_flight.load(Ordering::Relaxed);
if in_flight <= observed_max {
break;
}
if self
.create_session_max_in_flight
.compare_exchange(
observed_max,
in_flight,
Ordering::Relaxed,
Ordering::Relaxed,
)
.is_ok()
{
break;
}
}
let create_delay_ms = self.create_session_delay_ms.load(Ordering::Relaxed);
if create_delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(create_delay_ms)).await;
}
let mut session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
let session_id = session.id().clone();
if self.create_fail_sessions.read().await.contains(&session_id) {
self.create_fail_attempts
.write()
.await
.entry(session_id.clone())
.or_default()
.push(std::time::Instant::now());
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
let detail = self
.create_fail_details
.read()
.await
.get(&session_id)
.cloned()
.unwrap_or_else(|| "mock create_session failure".to_string());
return Err(SessionError::Store(Box::new(std::io::Error::other(detail))));
}
// Production-faithful "already active" failure: the inproc comms-claim
// acquire fails INSIDE build_agent, before any live session handle is
// registered, so this returns the typed build-time cause WITHOUT
// inserting into self.sessions — leaving has_live_session false at the
// actor's recovery re-check (the leaked-claim-without-live-session
// terminal path).
if self
.create_identity_already_active_no_live_sessions
.read()
.await
.contains(&session_id)
{
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
return Err(SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
));
}
let actor_materialization_permit = match begin_test_runtime_actor_materialization(&req) {
Ok(permit) => permit,
Err(error) => {
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
return Err(error);
}
};
let n = self.session_counter.fetch_add(1, Ordering::Relaxed);
// Extract comms_name from build options for mock runtime naming
let comms_name = req
.build
.as_ref()
.and_then(|b| b.comms_name.clone())
.unwrap_or_else(|| format!("mock-session-{n}"));
let behavior = self
.comms_behaviors
.read()
.await
.get(&comms_name)
.copied()
.unwrap_or_default();
let runtime_adapter = self
.runtime_adapter
.lock()
.expect("runtime_adapter mutex")
.clone();
let comms_identity_seed = self
.comms_identity_seeds
.write()
.await
.remove(&session_id)
.unwrap_or_else(|| comms_name.clone());
let mut comms = MockCommsRuntime::new(
&comms_identity_seed,
behavior,
session_id.clone(),
runtime_adapter,
);
comms.default_name.clone_from(&comms_name);
comms.default_address = format!("inproc://{comms_name}");
let comms = Arc::new(comms);
let transient_turn_context_state = meerkat_core::TransientTurnContextStateHandle::new();
let local_actor_witness_slot = meerkat_session::LiveSessionActorWitnessSlot::default();
let actor_witness_slot = actor_witness_slot.unwrap_or(&local_actor_witness_slot);
let mut sessions = self.sessions.write().await;
if sessions.contains_key(&session_id) {
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
return Err(SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
));
}
let replaced = sessions.insert(session_id.clone(), comms);
debug_assert!(replaced.is_none());
let actor_witness = self
.actor_registry
.insert_and_publish(
actor_witness_slot,
session_id.clone(),
transient_turn_context_state,
)
.inspect_err(|_| {
sessions.remove(&session_id);
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
})?;
debug_assert_eq!(
self.actor_registry.current(&session_id).as_ref(),
Some(&actor_witness)
);
drop(sessions);
self.start_turn_interrupts
.write()
.await
.entry(session_id.clone())
.or_insert_with(|| tokio::sync::watch::channel(0).0);
if let Some(system_prompt) = req.system_prompt.as_set_prompt() {
session.append_system_message(system_prompt.to_string());
}
if session.session_metadata().is_none() {
let build = req.build.as_ref();
let metadata = SessionMetadata {
schema_version: meerkat_core::session_metadata_schema_version(),
model: req.model.clone(),
max_tokens: req.max_tokens.unwrap_or(4096),
structured_output_retries: 2,
provider: build
.and_then(|b| b.provider)
.unwrap_or(Provider::Anthropic),
self_hosted_server_id: None,
provider_params: build.and_then(|b| b.provider_params.clone()),
tooling: SessionTooling {
builtins: build
.map(|b| b.override_builtins)
.unwrap_or(ToolCategoryOverride::from_effective(true)),
shell: build
.map(|b| b.override_shell)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
comms: ToolCategoryOverride::from_effective(
build.and_then(|b| b.comms_name.as_ref()).is_some(),
),
mob: build
.map(|b| b.override_mob)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
memory: build
.map(|b| b.override_memory)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
schedule: build
.map(|b| b.override_schedule)
.unwrap_or(ToolCategoryOverride::Inherit),
workgraph: build
.map(|b| b.override_workgraph)
.unwrap_or(ToolCategoryOverride::Inherit),
image_generation: build
.map(|b| b.override_image_generation)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
web_search: build
.map(|b| b.override_web_search)
.unwrap_or(ToolCategoryOverride::Inherit),
tool_access_policy: build.and_then(|b| b.tool_access_policy.clone()),
active_skills: build.and_then(|b| b.preload_skills.clone()),
},
keep_alive: build.map(|b| b.keep_alive).unwrap_or(false),
comms_name: build.and_then(|b| b.comms_name.clone()),
peer_meta: build.and_then(|b| b.peer_meta.clone()),
realm_id: build.and_then(|b| b.realm_id.clone()),
instance_id: build.and_then(|b| b.instance_id.clone()),
backend: build.and_then(|b| b.backend.map(|k| k.as_str().to_string())),
config_generation: build.and_then(|b| b.config_generation),
auth_binding: None,
// Mirror the real producer: carry the typed durable member
// binding so persisted-session ownership routing recognizes
// mob members exactly as production does.
mob_member_binding: build.and_then(|b| b.mob_member_binding.clone()),
};
session
.set_session_metadata(metadata)
.expect("mock session metadata should serialize");
}
self.live_session_data
.write()
.await
.insert(session_id.clone(), session.clone());
self.persisted_sessions
.write()
.await
.insert(session_id.clone(), session);
let is_keep_alive = req.build.as_ref().map(|b| b.keep_alive).unwrap_or(false);
if is_keep_alive {
self.keep_alive_notifiers
.write()
.await
.insert(session_id.clone(), Arc::new(tokio::sync::Notify::new()));
}
self.session_comms_names
.write()
.await
.insert(session_id.clone(), comms_name.clone());
if self
.archive_fail_comms_names
.read()
.await
.contains(&comms_name)
{
self.archive_fail_sessions
.write()
.await
.insert(session_id.clone());
}
let peer_meta_labels = req
.build
.as_ref()
.and_then(|build| build.peer_meta.as_ref())
.map(|meta| meta.labels.clone())
.unwrap_or_default();
self.create_requests
.write()
.await
.push(CreateSessionRecord {
initial_turn: req.initial_turn,
comms_name: req
.build
.as_ref()
.and_then(|build| build.comms_name.clone()),
peer_meta_labels,
runtime_build_mode_session_owned: req.build.as_ref().is_some_and(|build| {
matches!(
build.runtime_build_mode,
meerkat_core::runtime_epoch::RuntimeBuildMode::SessionOwned(_)
)
}),
budget_limits: req
.build
.as_ref()
.and_then(|build| build.budget_limits.clone()),
model: req.model.clone(),
});
let mcp_server_names: Vec<String> = req
.build
.as_ref()
.map(|build| {
build
.mcp_servers
.iter()
.map(|server| server.name.clone())
.collect()
})
.unwrap_or_default();
self.mcp_server_names
.write()
.await
.push((session_id.clone(), mcp_server_names));
// Record the prompt for test inspection
self.prompts
.write()
.await
.push((session_id.clone(), req.prompt.text_content()));
let external_tools = req
.build
.as_ref()
.and_then(|b| b.external_tools.as_ref())
.cloned();
self.create_with_external_tools
.write()
.await
.push(external_tools.is_some());
if let Some(dispatcher) = external_tools {
self.external_tools_by_session
.write()
.await
.insert(session_id.clone(), dispatcher);
}
if self
.create_identity_already_active_sessions
.read()
.await
.contains(&session_id)
{
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
return Err(SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
));
}
if let Some(permit) = actor_materialization_permit
&& let Err(error) = permit.commit()
{
let cleanup = self
.discard_live_session_actor_under_runtime_turn_boundary(&actor_witness)
.await;
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(match cleanup {
Ok(removed) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor removed: {removed}"
),
Err(cleanup_error) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor cleanup also failed: {cleanup_error}"
),
}),
));
}
self.create_session_in_flight
.fetch_sub(1, Ordering::Relaxed);
Ok(mock_run_result(session_id, "Session created".to_string()))
}
}
#[async_trait]
impl SessionService for MockSessionService {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, None).await
}
async fn start_turn(
&self,
id: &SessionId,
req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
let mut interrupt_rx = self
.start_turn_interrupts
.read()
.await
.get(id)
.map(tokio::sync::watch::Sender::subscribe);
self.flow_turn_overlays
.write()
.await
.push((id.clone(), req.runtime.turn_tool_overlay.clone()));
self.start_turn_prompts
.write()
.await
.push((id.clone(), req.prompt.text_content()));
let mut system_prompts = req
.runtime
.turn_metadata
.as_ref()
.map(|metadata| metadata.system_prompts.clone())
.unwrap_or_default();
system_prompts.extend(req.system_prompt.iter().cloned());
self.start_turn_system_prompts
.write()
.await
.push((id.clone(), system_prompts));
let user_channel: Vec<(bool, String)> = if req.runtime.typed_turn_appends.is_empty() {
req.injected_context
.iter()
.map(|entry| (true, entry.text_content()))
.chain(std::iter::once((false, req.prompt.text_content())))
.collect()
} else {
use meerkat_core::lifecycle::run_primitive::ConversationAppendRole;
req.runtime
.typed_turn_appends
.iter()
.filter_map(|append| match append.role {
ConversationAppendRole::InjectedContext => {
Some((true, append.content.render_text()))
}
ConversationAppendRole::User => Some((false, append.content.render_text())),
_ => None,
})
.collect()
};
self.start_turn_user_channel
.write()
.await
.push((id.clone(), user_channel));
self.start_turn_metadata
.write()
.await
.push((id.clone(), req.runtime.turn_metadata.clone()));
// Publish the observable call count only after every request record is
// committed. Tests use this counter as the readiness signal before
// reading those async record buffers.
self.start_turn_calls.fetch_add(1, Ordering::Release);
// Determine keep-alive by checking if a notifier was registered for this session
// (created in create_session when build.keep_alive is true).
let is_keep_alive = self.keep_alive_notifiers.read().await.contains_key(id);
if is_keep_alive {
self.keep_alive_start_turn_calls
.fetch_add(1, Ordering::Relaxed);
self.keep_alive_prompts
.write()
.await
.push((id.clone(), req.prompt.text_content()));
}
let start_turn_delay = self.start_turn_delay_ms.load(Ordering::Relaxed);
if start_turn_delay > 0 {
match interrupt_rx.as_mut() {
Some(interrupt_rx) => {
tokio::select! {
() = tokio::time::sleep(std::time::Duration::from_millis(start_turn_delay)) => {}
changed = interrupt_rx.changed() => {
if changed.is_ok() {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::Cancelled,
));
}
}
}
}
None => {
tokio::time::sleep(std::time::Duration::from_millis(start_turn_delay)).await;
}
}
}
if self
.fail_start_turn
.load(std::sync::atomic::Ordering::Relaxed)
{
return Err(SessionError::Store(Box::new(std::io::Error::other(
"mock start_turn failure",
))));
}
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if is_keep_alive {
let complete_immediately = self
.keep_alive_turns_complete_immediately
.load(Ordering::Relaxed);
if complete_immediately {
return Ok(mock_run_result(
id.clone(),
"Autonomous kickoff completed".to_string(),
));
}
let notifier = self
.keep_alive_notifiers
.read()
.await
.get(id)
.cloned()
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
notifier.notified().await;
return Ok(mock_run_result(
id.clone(),
"Host loop interrupted".to_string(),
));
}
if let Some(event_tx) = req.event_tx {
let delay_ms = self
.flow_turn_delay_ms
.load(std::sync::atomic::Ordering::Relaxed);
let never_terminal = self
.flow_turn_never_terminal
.load(std::sync::atomic::Ordering::Relaxed);
let fail_flow_turn_global = self
.flow_turn_fail
.load(std::sync::atomic::Ordering::Relaxed);
let fail_flow_turn_session = self.flow_turn_fail_sessions.read().await.contains(id);
let completed_result = self.flow_turn_completed_result.read().await.clone();
let session_id = id.clone();
tokio::spawn(async move {
if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
if never_terminal {
tokio::time::sleep(std::time::Duration::from_secs(60)).await;
return;
}
if fail_flow_turn_global || fail_flow_turn_session {
let _ = event_tx
.send(EventEnvelope::new_session(
session_id.clone(),
1,
None,
AgentEvent::RunFailed {
session_id,
terminal_cause_kind: None,
error_report: meerkat_core::event::AgentErrorReport {
class: meerkat_core::event::AgentErrorClass::Internal,
reason: None,
message: "mock flow turn failure".to_string(),
},
},
))
.await;
} else {
let _ = event_tx
.send(EventEnvelope::new_session(
session_id.clone(),
1,
None,
AgentEvent::RunCompleted {
session_id,
result: completed_result,
structured_output: None,
extraction_required: false,
usage: Usage::default(),
terminal_cause_kind: None,
},
))
.await;
}
});
}
Ok(mock_run_result(id.clone(), "Turn completed".to_string()))
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
self.interrupt_calls.fetch_add(1, Ordering::Relaxed);
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(interrupt_tx) = self.start_turn_interrupts.read().await.get(id) {
interrupt_tx.send_modify(|generation| *generation = generation.saturating_add(1));
}
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
}
// Real SessionService flips is_active to false once the turn/keep-alive
// loop unwinds after interrupt. Mirror that here so the disposal-time
// poll loop in `stop_autonomous_member` doesn't spin to its 1s cap.
self.active_sessions.write().await.remove(id);
Ok(())
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
self.cancel_after_boundary_calls
.fetch_add(1, Ordering::Relaxed);
if !self.cancel_after_boundary_supported.load(Ordering::Relaxed) {
return Err(SessionError::Unsupported(
"cancel_after_boundary".to_string(),
));
}
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
return Ok(());
}
Err(SessionError::NotRunning { id: id.clone() })
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
let session = self
.live_session_clone(id)
.await
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
let metadata = session.session_metadata();
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
let is_active = self.active_sessions.read().await.contains(id);
Ok(SessionView {
state: SessionInfo {
session_id: id.clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
is_active,
model: metadata
.as_ref()
.map(|meta| meta.model.clone())
.unwrap_or_else(|| "claude-sonnet-4-5".to_string()),
provider: metadata
.as_ref()
.map(|meta| meta.provider)
.unwrap_or(Provider::Anthropic),
last_assistant_text: session.last_assistant_text(),
labels: Default::default(),
},
billing: SessionUsage {
total_tokens: session.total_tokens(),
usage: session.total_usage(),
},
})
}
async fn list(&self, _query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
let sessions = self.live_session_data.read().await;
let active = self.active_sessions.read().await;
Ok(sessions
.values()
.map(|session| SessionSummary {
session_id: session.id().clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
is_active: active.contains(session.id()),
labels: Default::default(),
})
.collect())
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
Ok(self.sessions.read().await.contains_key(id))
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
*self
.archive_calls
.write()
.await
.entry(id.clone())
.or_default() += 1;
if self.archive_not_found_sessions.read().await.contains(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
if self.archive_fail_sessions.read().await.contains(id) {
return Err(SessionError::Store(Box::new(std::io::Error::other(
"mock archive failure",
))));
}
let archive_delay_ms = self.archive_delay_ms.load(Ordering::Relaxed);
if archive_delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(archive_delay_ms)).await;
}
self.archived_session_ids.write().await.insert(id.clone());
let runtime = {
let mut sessions = self.sessions.write().await;
let runtime = sessions.remove(id);
if runtime.is_some() {
self.actor_registry.remove_current(id);
}
runtime
};
if let Some(runtime) = runtime {
self.session_comms_names.write().await.remove(id);
self.external_tools_by_session.write().await.remove(id);
self.live_session_data.write().await.remove(id);
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(id) {
notifier.notify_waiters();
}
let intents = runtime.sent_intents().await;
let peer_lifecycle_max_in_flight = runtime.max_concurrent_peer_lifecycle_sends();
self.archived_sent_intents
.write()
.await
.insert(id.clone(), intents);
self.archived_peer_lifecycle_max_in_flight
.write()
.await
.insert(id.clone(), peer_lifecycle_max_in_flight);
}
Ok(())
}
}
#[cfg(feature = "runtime-adapter")]
async fn retire_test_runtime_archive(
adapter: &meerkat_runtime::MeerkatMachine,
session_id: &SessionId,
session_present: bool,
) -> Result<(), SessionError> {
if !session_present && !adapter.contains_session(session_id).await {
return Ok(());
}
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(session_id);
if matches!(
meerkat_runtime::RuntimeControlPlane::runtime_state(adapter, &runtime_id).await,
Ok(meerkat_runtime::RuntimeState::Retired)
) {
return Ok(());
}
match meerkat_runtime::RuntimeControlPlane::retire(adapter, &runtime_id).await {
Ok(_) => Ok(()),
// An already-absent runtime has no live obligations to retire. Never
// synthesize a SessionId-only registration here: runtime-backed test
// fixtures must be materialized through the exact attachment
// transaction, and archive must not manufacture a torn registration.
Err(meerkat_runtime::RuntimeControlPlaneError::NotFound(_)) => Ok(()),
Err(error) => Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(format!(
"test machine archive retire failed: {error}"
)),
)),
}
}
struct FlowTurnInFlightGuard {
active_turns: Arc<AtomicU64>,
}
impl FlowTurnInFlightGuard {
fn enter(active_turns: Arc<AtomicU64>, max_active_turns: Arc<AtomicU64>) -> Self {
let active = active_turns.fetch_add(1, Ordering::Relaxed) + 1;
loop {
let observed = max_active_turns.load(Ordering::Relaxed);
if active <= observed {
break;
}
if max_active_turns
.compare_exchange(observed, active, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
break;
}
}
Self { active_turns }
}
}
impl Drop for FlowTurnInFlightGuard {
fn drop(&mut self) {
self.active_turns.fetch_sub(1, Ordering::Relaxed);
}
}
struct CountingInjector {
calls: Arc<AtomicU64>,
active_turns: Arc<AtomicU64>,
max_active_turns: Arc<AtomicU64>,
inject_delay_ms: u64,
delay_ms: u64,
never_terminal: bool,
fail: bool,
fail_inject: bool,
completed_result: String,
interaction_ids: Arc<std::sync::Mutex<Vec<InteractionId>>>,
}
impl EventInjector for CountingInjector {
fn inject(
&self,
_content: meerkat_core::types::ContentInput,
_source: PlainEventSource,
_handling_mode: meerkat_core::types::HandlingMode,
_render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), EventInjectorError> {
if self.fail_inject {
return Err(EventInjectorError::Closed);
}
if self.inject_delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(self.inject_delay_ms));
}
self.calls.fetch_add(1, Ordering::Relaxed);
Ok(())
}
}
impl SubscribableInjector for CountingInjector {
fn inject_with_delivery_identity(
&self,
_input_identity: meerkat_core::service::StartTurnInputIdentity,
_objective_id: Option<meerkat_core::interaction::ObjectiveId>,
_content: meerkat_core::types::ContentInput,
_source: PlainEventSource,
_handling_mode: meerkat_core::types::HandlingMode,
_render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), EventInjectorError> {
Err(EventInjectorError::Closed)
}
fn inject_with_subscription(
&self,
body: meerkat_core::types::ContentInput,
source: PlainEventSource,
handling_mode: meerkat_core::types::HandlingMode,
render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<InteractionSubscription, EventInjectorError> {
self.inject(body, source, handling_mode, render_metadata)?;
let (tx, rx) = tokio::sync::mpsc::channel(1);
let interaction_id = InteractionId(uuid::Uuid::new_v4());
let delay_ms = self.delay_ms;
let never_terminal = self.never_terminal;
let fail = self.fail;
let completed_result = self.completed_result.clone();
let active_turns = self.active_turns.clone();
let max_active_turns = self.max_active_turns.clone();
let interaction_id_for_task = interaction_id;
tokio::spawn(async move {
let _active = FlowTurnInFlightGuard::enter(active_turns, max_active_turns);
if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
if never_terminal {
tokio::time::sleep(std::time::Duration::from_secs(60)).await;
return;
}
let event = if fail {
AgentEvent::InteractionFailed {
interaction_id: interaction_id_for_task,
reason: InteractionFailureReason::abandoned("mock flow turn failure"),
}
} else {
AgentEvent::InteractionComplete {
interaction_id: interaction_id_for_task,
result: completed_result,
structured_output: None,
}
};
let _ = tx.send(event).await;
});
Ok(InteractionSubscription {
id: interaction_id,
events: rx,
})
}
fn inject_with_interaction_id(
&self,
interaction_id: InteractionId,
body: meerkat_core::types::ContentInput,
source: PlainEventSource,
handling_mode: meerkat_core::types::HandlingMode,
render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), EventInjectorError> {
self.inject(body, source, handling_mode, render_metadata)?;
self.interaction_ids
.lock()
.expect("interaction id mutex")
.push(interaction_id);
Ok(())
}
}
#[async_trait]
impl SessionServiceCommsExt for MockSessionService {
async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CoreCommsRuntime>> {
self.comms_runtime_observations
.fetch_add(1, Ordering::Relaxed);
if self
.missing_comms_sessions
.read()
.await
.contains(session_id)
{
return None;
}
let unavailable_after_reconciliation = self
.comms_runtime_failure_after_successes_enabled
.load(Ordering::Acquire)
&& self
.comms_runtime_successes_before_missing
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
remaining.checked_sub(1)
})
.is_err();
if unavailable_after_reconciliation {
return None;
}
let sessions = self.sessions.read().await;
sessions
.get(session_id)
.map(|comms| Arc::clone(comms) as Arc<dyn CoreCommsRuntime>)
}
async fn event_injector(&self, session_id: &SessionId) -> Option<Arc<dyn EventInjector>> {
if !self.sessions.read().await.contains_key(session_id) {
return None;
}
let delay_ms = self.flow_turn_delay_ms.load(Ordering::Relaxed);
let inject_delay_ms = self.inject_delay_ms.load(Ordering::Relaxed);
let never_terminal = self
.flow_turn_never_terminal
.load(std::sync::atomic::Ordering::Relaxed);
let fail = self
.flow_turn_fail
.load(std::sync::atomic::Ordering::Relaxed)
|| self
.flow_turn_fail_sessions
.read()
.await
.contains(session_id);
let completed_result = self.flow_turn_completed_result.read().await.clone();
let fail_inject = self.fail_inject.load(std::sync::atomic::Ordering::Relaxed);
Some(Arc::new(CountingInjector {
calls: self.inject_calls.clone(),
active_turns: self.flow_turn_in_flight.clone(),
max_active_turns: self.flow_turn_max_in_flight.clone(),
inject_delay_ms,
delay_ms,
never_terminal,
fail,
fail_inject,
completed_result,
interaction_ids: self.injected_interaction_ids.clone(),
}))
}
async fn interaction_event_injector(
&self,
session_id: &SessionId,
) -> Option<Arc<dyn SubscribableInjector>> {
if self
.disable_interaction_event_injector
.load(std::sync::atomic::Ordering::Relaxed)
{
return None;
}
if !self.sessions.read().await.contains_key(session_id) {
return None;
}
let delay_ms = self.flow_turn_delay_ms.load(Ordering::Relaxed);
let inject_delay_ms = self.inject_delay_ms.load(Ordering::Relaxed);
let never_terminal = self
.flow_turn_never_terminal
.load(std::sync::atomic::Ordering::Relaxed);
let fail = self
.flow_turn_fail
.load(std::sync::atomic::Ordering::Relaxed)
|| self
.flow_turn_fail_sessions
.read()
.await
.contains(session_id);
let completed_result = self.flow_turn_completed_result.read().await.clone();
let fail_inject = self.fail_inject.load(std::sync::atomic::Ordering::Relaxed);
Some(Arc::new(CountingInjector {
calls: self.inject_calls.clone(),
active_turns: self.flow_turn_in_flight.clone(),
max_active_turns: self.flow_turn_max_in_flight.clone(),
inject_delay_ms,
delay_ms,
never_terminal,
fail,
fail_inject,
completed_result,
interaction_ids: self.injected_interaction_ids.clone(),
}))
}
}
#[async_trait]
impl meerkat_core::service::SessionServiceHistoryExt for MockSessionService {
async fn read_history(
&self,
id: &SessionId,
query: meerkat_core::service::SessionHistoryQuery,
) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
let session = self
.live_session_clone(id)
.await
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
Ok(meerkat_core::service::SessionHistoryPage::from_messages(
id.clone(),
session.messages(),
query,
))
}
}
#[async_trait]
impl SessionServiceControlExt for MockSessionService {
async fn append_system_context(
&self,
id: &SessionId,
_req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() }.into());
}
Ok(AppendSystemContextResult {
status: AppendSystemContextStatus::Applied,
})
}
}
#[async_trait]
impl MobSessionService for MockSessionService {
async fn prepare_session_for_resume(&self, session_id: &SessionId) -> Result<(), SessionError> {
self.resume_prepare_calls.fetch_add(1, Ordering::Relaxed);
if let Some(prepared) = self
.prepared_resume_sessions
.write()
.await
.remove(session_id)
{
self.persisted_sessions
.write()
.await
.insert(session_id.clone(), prepared);
}
Ok(())
}
/// Test double: nothing durable survives archive here, so the two-read
/// composition is the exact truth — `ArchivedNotRevivable` cannot exist.
async fn load_session_for_resume(
&self,
session_id: &SessionId,
) -> Result<ResumeSessionLoad, SessionError> {
if let Some(session) = self.load_persisted_session(session_id).await? {
return Ok(ResumeSessionLoad::Active(Box::new(session)));
}
if let Some(session) = self.load_revivable_retired_session(session_id).await? {
return Ok(ResumeSessionLoad::Revivable(Box::new(session)));
}
Ok(ResumeSessionLoad::Absent)
}
async fn create_session_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
) -> Result<RunResult, SessionError> {
SessionService::create_session(self, req).await
}
async fn create_session_with_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, Some(actor_witness_slot))
.await
}
async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.archive_with_mob_lifecycle_authority(session_id).await
}
async fn discard_live_session_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.discard_live_session(session_id).await
}
async fn subscribe_session_events(
&self,
session_id: &SessionId,
) -> Result<meerkat_core::EventStream, meerkat_core::StreamError> {
if self
.subscribe_fail_sessions
.read()
.await
.contains(session_id)
{
return Err(meerkat_core::StreamError::Internal(format!(
"simulated subscribe failure for session {session_id}"
)));
}
if !self.sessions.read().await.contains_key(session_id) {
return Err(meerkat_core::StreamError::NotFound(format!(
"session {session_id}"
)));
}
Ok(Box::pin(
futures::stream::empty::<EventEnvelope<AgentEvent>>(),
))
}
fn supports_persistent_sessions(&self) -> bool {
true
}
fn supports_runtime_turn_apply(&self) -> bool {
true
}
async fn live_session_actor_registered(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
Ok(self.actor_registry.contains(session_id))
}
fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
self.runtime_adapter
.lock()
.expect("runtime_adapter mutex")
.clone()
}
async fn interrupt_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::interrupt(self, session_id).await
}
async fn cancel_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_expected_run_id: &meerkat_core::RunId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn cancel_current_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn execution_snapshot(
&self,
_session_id: &SessionId,
) -> Result<Option<meerkat_core::agent::AgentExecutionSnapshot>, SessionError> {
self.execution_snapshot_started.notify_one();
let delay_ms = self.execution_snapshot_delay_ms.load(Ordering::Relaxed);
if delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
}
Ok(None)
}
async fn archive_with_mob_lifecycle_authority(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
// PersistentSessionService commits the durable archive document before
// retiring the runtime. Keep this fault distinct from the existing
// post-retire archive projection fault: both partial-order shapes are
// intentional test contracts.
if self
.archive_precommit_fail_sessions
.read()
.await
.contains(session_id)
{
*self
.archive_calls
.write()
.await
.entry(session_id.clone())
.or_default() += 1;
return Err(SessionError::Store(Box::new(std::io::Error::other(
"mock archive precommit failure",
))));
}
if let Some(adapter) = self.runtime_adapter() {
let session_present = self.sessions.read().await.contains_key(session_id);
retire_test_runtime_archive(adapter.as_ref(), session_id, session_present).await?;
}
SessionService::archive(self, session_id).await
}
async fn session_belongs_to_mob(&self, session_id: &SessionId, mob_id: &MobId) -> bool {
if self
.session_comms_names
.read()
.await
.get(session_id)
.is_some_and(|name| name.starts_with(&format!("{mob_id}/")))
{
return true;
}
self.persisted_sessions
.read()
.await
.get(session_id)
.and_then(Session::session_metadata)
.is_some_and(|metadata| {
metadata
.comms_name
.as_deref()
.is_some_and(|name| name.starts_with(&format!("{mob_id}/")))
})
}
async fn load_persisted_session(
&self,
session_id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.load_persisted_session_started.notify_one();
self.load_persisted_session_calls
.fetch_add(1, Ordering::Relaxed);
let in_flight = self
.load_persisted_session_in_flight
.fetch_add(1, Ordering::Relaxed)
+ 1;
self.load_persisted_session_max_in_flight
.fetch_max(in_flight, Ordering::Relaxed);
let _in_flight_guard = AtomicInFlightGuard(&self.load_persisted_session_in_flight);
let delay_ms = self.load_persisted_session_delay_ms.load(Ordering::Relaxed);
if delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
}
// Mirror PersistentSessionService: archived sessions have no loadable
// durable snapshot (archive is terminal).
let persisted = if self.archived_session_ids.read().await.contains(session_id) {
None
} else {
self.persisted_session_clone(session_id).await
};
Ok(persisted)
}
async fn session_known_to_archive_authority(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
if self
.archive_authority_unknown_sessions
.read()
.await
.contains(session_id)
{
return Ok(false);
}
// Mirror PersistentSessionService: any session with a record —
// live, persisted, or already archived — is authority-owned; only a
// session this service never saw is host-owned.
if self.archived_session_ids.read().await.contains(session_id) {
return Ok(true);
}
if self.sessions.read().await.contains_key(session_id) {
return Ok(true);
}
Ok(self.persisted_session_clone(session_id).await.is_some())
}
async fn apply_runtime_turn(
&self,
session_id: &SessionId,
run_id: meerkat_core::RunId,
req: StartTurnRequest,
boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
contributing_input_ids: Vec<meerkat_core::InputId>,
) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
<Self as SessionService>::start_turn(self, session_id, req).await?;
Ok(
meerkat_core::lifecycle::core_executor::CoreApplyOutput::with_untyped_snapshot(
meerkat_core::lifecycle::run_receipt::RunBoundaryReceiptDraft {
run_id,
boundary,
contributing_input_ids,
conversation_digest: None,
message_count: 0,
},
None,
None,
),
)
}
async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
&self,
session_id: &SessionId,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
let acknowledgement = match authority {
meerkat_core::CommittedSessionBoundaryAuthority::WholeBlob {
session_id: authority_session_id,
committed_store_revision,
committed_blob_sha256,
} if authority_session_id == session_id => RuntimeBoundaryAcknowledgement::WholeBlob {
session_id: session_id.clone(),
store_revision: *committed_store_revision,
blob_sha256: committed_blob_sha256.clone(),
},
meerkat_core::CommittedSessionBoundaryAuthority::HeadCanonical {
session_id: authority_session_id,
committed_head_token,
} if authority_session_id == session_id => {
RuntimeBoundaryAcknowledgement::HeadCanonical {
session_id: session_id.clone(),
head_token: committed_head_token.clone(),
}
}
meerkat_core::CommittedSessionBoundaryAuthority::Provisional {
session_id: authority_session_id,
committed_store_revision,
committed_authority_token,
} if authority_session_id == session_id => {
RuntimeBoundaryAcknowledgement::Provisional {
session_id: session_id.clone(),
store_revision: *committed_store_revision,
authority_token: committed_authority_token.clone(),
}
}
_ => {
return Err(SessionError::Unsupported(
"boundary acknowledgement session identity mismatch".to_string(),
));
}
};
self.runtime_boundary_acknowledgements
.write()
.await
.push(acknowledgement);
Ok(())
}
async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
{
let mut sessions = self.sessions.write().await;
self.actor_registry.remove_current(session_id);
sessions.remove(session_id);
}
self.live_session_data.write().await.remove(session_id);
self.keep_alive_notifiers.write().await.remove(session_id);
self.start_turn_interrupts.write().await.remove(session_id);
self.session_comms_names.write().await.remove(session_id);
self.external_tools_by_session
.write()
.await
.remove(session_id);
Ok(())
}
async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
if self
.discard_actor_failures_remaining
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| {
(remaining > 0).then(|| remaining - 1)
})
.is_ok()
{
return Err(SessionError::Store(Box::new(std::io::Error::other(
"synthetic transient exact actor discard failure",
))));
}
let mut sessions = self.sessions.write().await;
if self.actor_registry.current(witness.session_id()).as_ref() != Some(witness)
|| !sessions.contains_key(witness.session_id())
{
return Ok(false);
}
let registry_removed = self.actor_registry.compare_remove(witness);
debug_assert!(registry_removed);
sessions.remove(witness.session_id());
// This mock has no independent B implementation. Keep its actor-map
// guard until every attachment-local side table is cleared so a
// replacement cannot publish between actor removal and cleanup.
let session_id = witness.session_id();
self.live_session_data.write().await.remove(session_id);
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(session_id) {
notifier.notify_waiters();
}
self.start_turn_interrupts.write().await.remove(session_id);
self.session_comms_names.write().await.remove(session_id);
self.external_tools_by_session
.write()
.await
.remove(session_id);
drop(sessions);
Ok(true)
}
}
struct FaultInjectedMobEventStore {
events: RwLock<Vec<MobEvent>>,
event_tx: tokio::sync::broadcast::Sender<MobEvent>,
external_deliveries: InMemoryMobEventStore,
fail_on_kind: RwLock<HashSet<&'static str>>,
fail_after_append_on_kind: RwLock<HashSet<&'static str>>,
fail_next_external_delivery_claim: AtomicBool,
fail_external_delivery_load_after_claim: AtomicBool,
fail_next_external_delivery_load: AtomicBool,
fail_clear: AtomicBool,
poll_calls: AtomicU64,
replay_calls: AtomicU64,
}
/// SQLite-backed store that can faithfully model a committed append whose
/// acknowledgement is lost at the caller boundary. Delegating every storage
/// operation keeps the pagination behavior under test SQLite-native.
#[cfg(not(target_arch = "wasm32"))]
struct SqliteWriteThenErrorMobEventStore {
inner: SqliteMobEventStore,
fail_next_members_unwired_after_write: AtomicBool,
}
#[cfg(not(target_arch = "wasm32"))]
impl SqliteWriteThenErrorMobEventStore {
fn new(inner: SqliteMobEventStore) -> Self {
Self {
inner,
fail_next_members_unwired_after_write: AtomicBool::new(false),
}
}
fn fail_next_members_unwired_after_write(&self) {
self.fail_next_members_unwired_after_write
.store(true, Ordering::SeqCst);
}
}
#[cfg(not(target_arch = "wasm32"))]
impl private::MobEventStoreSealed for SqliteWriteThenErrorMobEventStore {}
#[cfg(not(target_arch = "wasm32"))]
#[async_trait]
impl MobEventStore for SqliteWriteThenErrorMobEventStore {
async fn append(&self, event: NewMobEvent) -> Result<MobEvent, MobStoreError> {
let is_members_unwired = matches!(&event.kind, MobEventKind::MembersUnwired { .. });
let stored = self.inner.append(event).await?;
if is_members_unwired
&& self
.fail_next_members_unwired_after_write
.swap(false, Ordering::SeqCst)
{
return Err(MobStoreError::Internal(
"fault-injected SQLite wrote-then-error MembersUnwired append".to_string(),
));
}
Ok(stored)
}
async fn append_terminal_event_if_absent(
&self,
event: NewMobEvent,
) -> Result<Option<MobEvent>, MobStoreError> {
self.inner.append_terminal_event_if_absent(event).await
}
async fn append_batch(&self, events: Vec<NewMobEvent>) -> Result<Vec<MobEvent>, MobStoreError> {
self.inner.append_batch(events).await
}
async fn poll(&self, after_cursor: u64, limit: usize) -> Result<Vec<MobEvent>, MobStoreError> {
self.inner.poll(after_cursor, limit).await
}
async fn replay_all(&self) -> Result<Vec<MobEvent>, MobStoreError> {
self.inner.replay_all().await
}
async fn latest_cursor(&self) -> Result<u64, MobStoreError> {
self.inner.latest_cursor().await
}
fn subscribe(&self) -> Result<crate::store::MobEventReceiver, MobStoreError> {
self.inner.subscribe()
}
async fn clear(&self) -> Result<(), MobStoreError> {
self.inner.clear().await
}
async fn prune(&self, older_than: chrono::DateTime<Utc>) -> Result<u64, MobStoreError> {
self.inner.prune(older_than).await
}
}
impl FaultInjectedMobEventStore {
fn new() -> Self {
let (event_tx, _event_rx) = tokio::sync::broadcast::channel(4096);
Self {
events: RwLock::new(Vec::new()),
event_tx,
external_deliveries: InMemoryMobEventStore::new(),
fail_on_kind: RwLock::new(HashSet::new()),
fail_after_append_on_kind: RwLock::new(HashSet::new()),
fail_next_external_delivery_claim: AtomicBool::new(false),
fail_external_delivery_load_after_claim: AtomicBool::new(false),
fail_next_external_delivery_load: AtomicBool::new(false),
fail_clear: AtomicBool::new(false),
poll_calls: AtomicU64::new(0),
replay_calls: AtomicU64::new(0),
}
}
async fn fail_appends_for(&self, kind: &'static str) {
self.fail_on_kind.write().await.insert(kind);
}
async fn fail_after_append_for(&self, kind: &'static str) {
self.fail_after_append_on_kind.write().await.insert(kind);
}
fn fail_clear(&self) {
self.fail_clear.store(true, Ordering::Relaxed);
}
fn allow_clear(&self) {
self.fail_clear.store(false, Ordering::Relaxed);
}
async fn allow_appends_for(&self, kind: &'static str) {
self.fail_on_kind.write().await.remove(kind);
}
fn fail_next_external_delivery_claim(&self) {
self.fail_next_external_delivery_claim
.store(true, Ordering::SeqCst);
}
fn fail_next_external_delivery_claim_and_reread(&self) {
self.fail_next_external_delivery_claim();
self.fail_external_delivery_load_after_claim
.store(true, Ordering::SeqCst);
}
fn replay_calls(&self) -> u64 {
self.replay_calls.load(Ordering::Relaxed)
}
fn poll_calls(&self) -> u64 {
self.poll_calls.load(Ordering::Relaxed)
}
fn kind_label(kind: &MobEventKind) -> &'static str {
match kind {
MobEventKind::MobCreated { .. } => "MobCreated",
MobEventKind::MobCompleted => "MobCompleted",
MobEventKind::MobDestroying => "MobDestroying",
MobEventKind::MobDestroyStorageFinalizing => "MobDestroyStorageFinalizing",
MobEventKind::MobReset => "MobReset",
MobEventKind::MobOwnerBridgeSessionBound { .. } => "MobOwnerBridgeSessionBound",
MobEventKind::MemberSpawned(..) => "MemberSpawned",
MobEventKind::MemberRetirementStarted { .. } => "MemberRetirementStarted",
MobEventKind::RemoteMemberRuntimeRetired { .. } => "RemoteMemberRuntimeRetired",
MobEventKind::RemoteMemberSupervisorRevoked { .. } => "RemoteMemberSupervisorRevoked",
MobEventKind::MemberRetired { .. } => "MemberRetired",
MobEventKind::RespawnTopologyAbandoned { .. } => "RespawnTopologyAbandoned",
MobEventKind::MemberReset { .. } => "MemberReset",
MobEventKind::MemberSessionBindingRecovered(..) => "MemberSessionBindingRecovered",
MobEventKind::MemberKickoffUpdated { .. } => "MemberKickoffUpdated",
MobEventKind::ObjectiveOwnerBound { .. } => "ObjectiveOwnerBound",
MobEventKind::ObjectiveConcluded { .. } => "ObjectiveConcluded",
MobEventKind::MembersWired { .. } => "MembersWired",
MobEventKind::MembersWiredBatch { .. } => "MembersWiredBatch",
MobEventKind::MembersUnwired { .. } => "MembersUnwired",
MobEventKind::ExternalPeerWired { .. } => "ExternalPeerWired",
MobEventKind::ExternalPeerUnwired { .. } => "ExternalPeerUnwired",
MobEventKind::FlowStarted { .. } => "FlowStarted",
MobEventKind::FlowCompleted { .. } => "FlowCompleted",
MobEventKind::FlowFailed { .. } => "FlowFailed",
MobEventKind::FlowCanceled { .. } => "FlowCanceled",
MobEventKind::StepDispatched { .. } => "StepDispatched",
MobEventKind::StepTargetCompleted { .. } => "StepTargetCompleted",
MobEventKind::StepTargetFailed { .. } => "StepTargetFailed",
MobEventKind::RemoteTurnObligationRecorded { .. } => "RemoteTurnObligationRecorded",
MobEventKind::RemoteTurnOutcomeResolved { .. } => "RemoteTurnOutcomeResolved",
MobEventKind::RemoteTurnOutcomeAcknowledged { .. } => "RemoteTurnOutcomeAcknowledged",
MobEventKind::RemoteTurnOutcomeDisposed { .. } => "RemoteTurnOutcomeDisposed",
MobEventKind::PlacedCompletionLifecycleQuiesceStarted { .. } => {
"PlacedCompletionLifecycleQuiesceStarted"
}
MobEventKind::MobStopped => "MobStopped",
MobEventKind::PlacedCompletionLifecycleQuiesceEnded { .. } => {
"PlacedCompletionLifecycleQuiesceEnded"
}
MobEventKind::PlacedCompletionObligationRecorded { .. } => {
"PlacedCompletionObligationRecorded"
}
MobEventKind::PlacedCompletionCancellationRequested { .. } => {
"PlacedCompletionCancellationRequested"
}
MobEventKind::PlacedCompletionOutcomeResolved { .. } => {
"PlacedCompletionOutcomeResolved"
}
MobEventKind::PlacedCompletionOutcomeClosed { .. } => "PlacedCompletionOutcomeClosed",
MobEventKind::PlacedCompletionOutcomeAcknowledged { .. } => {
"PlacedCompletionOutcomeAcknowledged"
}
MobEventKind::PlacedCompletionOutcomeDisposed { .. } => {
"PlacedCompletionOutcomeDisposed"
}
MobEventKind::PlacedKickoffObligationRecorded { .. } => {
"PlacedKickoffObligationRecorded"
}
MobEventKind::PlacedKickoffOutcomeResolved { .. } => "PlacedKickoffOutcomeResolved",
MobEventKind::PlacedKickoffRejectedNoEffect { .. } => "PlacedKickoffRejectedNoEffect",
MobEventKind::PlacedKickoffOutcomeAcknowledged { .. } => {
"PlacedKickoffOutcomeAcknowledged"
}
MobEventKind::PlacedKickoffOutcomeDisposed { .. } => "PlacedKickoffOutcomeDisposed",
MobEventKind::RemoteMemberReleaseConfirmed { .. } => "RemoteMemberReleaseConfirmed",
MobEventKind::RemoteHostBindStarted { .. } => "RemoteHostBindStarted",
MobEventKind::RemoteHostBindConfirmed { .. } => "RemoteHostBindConfirmed",
MobEventKind::RemoteHostBindCompleted { .. } => "RemoteHostBindCompleted",
MobEventKind::RemoteHostBindAbortedNoEffect { .. } => "RemoteHostBindAbortedNoEffect",
MobEventKind::RemoteHostRevokeStarted { .. } => "RemoteHostRevokeStarted",
MobEventKind::RemoteHostRevokeConfirmed { .. } => "RemoteHostRevokeConfirmed",
MobEventKind::RemoteHostRevokeCompleted { .. } => "RemoteHostRevokeCompleted",
MobEventKind::StepCompleted { .. } => "StepCompleted",
MobEventKind::StepFailed { .. } => "StepFailed",
MobEventKind::StepSkipped { .. } => "StepSkipped",
MobEventKind::TopologyViolation { .. } => "TopologyViolation",
MobEventKind::SupervisorEscalation { .. } => "SupervisorEscalation",
MobEventKind::OperatorActionRecorded { .. } => "OperatorActionRecorded",
}
}
}
impl private::MobEventStoreSealed for FaultInjectedMobEventStore {}
#[async_trait]
impl MobEventStore for FaultInjectedMobEventStore {
async fn begin_external_delivery(
&self,
intent: &crate::store::MobExternalDeliveryIntent,
) -> Result<crate::store::MobExternalDeliveryBeginOutcome, MobStoreError> {
self.external_deliveries
.begin_external_delivery(intent)
.await
}
async fn complete_external_delivery(
&self,
intent: &crate::store::MobExternalDeliveryIntent,
terminal: &crate::store::MobExternalDeliveryTerminal,
) -> Result<crate::store::MobExternalDeliveryCompleteOutcome, MobStoreError> {
self.external_deliveries
.complete_external_delivery(intent, terminal)
.await
}
async fn claim_external_delivery_realization(
&self,
commit: &crate::store::MobExternalDeliveryRealizationCommit,
) -> Result<crate::store::MobExternalDeliveryClaimOutcome, MobStoreError> {
if self
.fail_next_external_delivery_claim
.swap(false, Ordering::SeqCst)
{
if self
.fail_external_delivery_load_after_claim
.swap(false, Ordering::SeqCst)
{
self.fail_next_external_delivery_load
.store(true, Ordering::SeqCst);
}
return Err(MobStoreError::Internal(
"fault-injected external-delivery realization claim failure".to_string(),
));
}
self.external_deliveries
.claim_external_delivery_realization(commit)
.await
}
async fn complete_external_delivery_realization(
&self,
commit: &crate::store::MobExternalDeliveryRealizerCompletionCommit,
) -> Result<crate::store::MobExternalDeliveryCompleteOutcome, MobStoreError> {
self.external_deliveries
.complete_external_delivery_realization(commit)
.await
}
async fn abandon_external_delivery(
&self,
intent: &crate::store::MobExternalDeliveryIntent,
terminal: &crate::store::MobExternalDeliveryTerminal,
) -> Result<crate::store::MobExternalDeliveryAbandonOutcome, MobStoreError> {
self.external_deliveries
.abandon_external_delivery(intent, terminal)
.await
}
async fn resolve_external_delivery_after_realizer_fenced(
&self,
commit: &crate::store::MobExternalDeliveryFencedResolutionCommit,
) -> Result<crate::store::MobExternalDeliveryAbandonOutcome, MobStoreError> {
self.external_deliveries
.resolve_external_delivery_after_realizer_fenced(commit)
.await
}
async fn schedule_external_delivery_repair(
&self,
intent: &crate::store::MobExternalDeliveryIntent,
) -> Result<crate::store::MobExternalDeliveryRepairOutcome, MobStoreError> {
self.external_deliveries
.schedule_external_delivery_repair(intent)
.await
}
async fn load_external_delivery(
&self,
mob_id: &MobId,
idempotency_key: &str,
) -> Result<Option<crate::store::MobExternalDeliveryRecord>, MobStoreError> {
if self
.fail_next_external_delivery_load
.swap(false, Ordering::SeqCst)
{
return Err(MobStoreError::Internal(
"fault-injected external-delivery exact reread failure".to_string(),
));
}
self.external_deliveries
.load_external_delivery(mob_id, idempotency_key)
.await
}
async fn append(&self, event: NewMobEvent) -> Result<MobEvent, MobStoreError> {
let kind_label = Self::kind_label(&event.kind);
if self.fail_on_kind.read().await.contains(kind_label) {
return Err(MobStoreError::Internal(format!(
"fault-injected append failure for {kind_label}"
)));
}
let mut events = self.events.write().await;
let cursor = events.len() as u64 + 1;
let stored = MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: event.mob_id,
kind: event.kind,
};
events.push(stored.clone());
drop(events);
let _ = self.event_tx.send(stored.clone());
if self
.fail_after_append_on_kind
.read()
.await
.contains(kind_label)
{
return Err(MobStoreError::Internal(format!(
"fault-injected lost append acknowledgement for {kind_label}"
)));
}
Ok(stored)
}
async fn append_terminal_event_if_absent(
&self,
event: NewMobEvent,
) -> Result<Option<MobEvent>, MobStoreError> {
let Some((run_id, flow_id)) = terminal_event_identity(&event.kind) else {
return Err(MobStoreError::Internal(
"append_terminal_event_if_absent requires a terminal flow event".to_string(),
));
};
let run_id = run_id.clone();
let flow_id = flow_id.clone();
let mob_id = event.mob_id.clone();
let mut events = self.events.write().await;
if events.iter().any(|existing| {
existing.mob_id == mob_id
&& terminal_event_identity(&existing.kind).is_some_and(
|(existing_run_id, existing_flow_id)| {
existing_run_id == &run_id && existing_flow_id == &flow_id
},
)
}) {
return Ok(None);
}
let kind_label = Self::kind_label(&event.kind);
if self.fail_on_kind.read().await.contains(kind_label) {
return Err(MobStoreError::Internal(format!(
"fault-injected append failure for {kind_label}"
)));
}
let cursor = events.len() as u64 + 1;
let stored = MobEvent {
cursor,
timestamp: Utc::now(),
mob_id: event.mob_id,
kind: event.kind,
};
events.push(stored.clone());
drop(events);
let _ = self.event_tx.send(stored.clone());
Ok(Some(stored))
}
async fn poll(&self, after_cursor: u64, limit: usize) -> Result<Vec<MobEvent>, MobStoreError> {
self.poll_calls.fetch_add(1, Ordering::Relaxed);
let events = self.events.read().await;
Ok(events
.iter()
.filter(|e| e.cursor > after_cursor)
.take(limit)
.cloned()
.collect())
}
async fn replay_all(&self) -> Result<Vec<MobEvent>, MobStoreError> {
self.replay_calls.fetch_add(1, Ordering::Relaxed);
Ok(self.events.read().await.clone())
}
async fn latest_cursor(&self) -> Result<u64, MobStoreError> {
Ok(self
.events
.read()
.await
.last()
.map_or(0, |event| event.cursor))
}
fn subscribe(&self) -> Result<crate::store::MobEventReceiver, MobStoreError> {
Ok(self.event_tx.subscribe())
}
async fn append_batch(&self, events: Vec<NewMobEvent>) -> Result<Vec<MobEvent>, MobStoreError> {
let mut results = Vec::with_capacity(events.len());
for event in events {
results.push(self.append(event).await?);
}
Ok(results)
}
async fn clear(&self) -> Result<(), MobStoreError> {
if self.fail_clear.load(Ordering::Relaxed) {
return Err(MobStoreError::Internal(
"fault-injected clear failure".to_string(),
));
}
self.events.write().await.clear();
self.external_deliveries.clear().await?;
Ok(())
}
}
struct RecordingRunStore {
inner: InMemoryMobRunStore,
fail_create_run_once: AtomicBool,
snapshot_cas_history: RwLock<Vec<(RunId, MobRunStatus, MobRunStatus)>>,
}
impl RecordingRunStore {
fn new() -> Self {
Self {
inner: InMemoryMobRunStore::new(),
fail_create_run_once: AtomicBool::new(false),
snapshot_cas_history: RwLock::new(Vec::new()),
}
}
fn fail_next_create_run(&self) {
self.fail_create_run_once.store(true, Ordering::Relaxed);
}
async fn snapshot_cas_history(&self) -> Vec<(RunId, MobRunStatus, MobRunStatus)> {
self.snapshot_cas_history.read().await.clone()
}
}
#[async_trait]
impl MobRunStore for RecordingRunStore {
async fn create_run(&self, run: MobRun) -> Result<(), MobStoreError> {
if self.fail_create_run_once.swap(false, Ordering::Relaxed) {
return Err(MobStoreError::Internal(
"fault-injected create_run failure".to_string(),
));
}
self.inner.create_run(run).await
}
async fn get_run(&self, run_id: &RunId) -> Result<Option<MobRun>, MobStoreError> {
self.inner.get_run(run_id).await
}
async fn list_runs(
&self,
mob_id: &MobId,
flow_id: Option<&crate::FlowId>,
) -> Result<Vec<MobRun>, MobStoreError> {
self.inner.list_runs(mob_id, flow_id).await
}
async fn append_step_entry_with_authority(
&self,
run_id: &RunId,
entry: StepLedgerEntry,
authority: MobRunProvenanceAuthority,
) -> Result<(), MobStoreError> {
self.inner
.append_step_entry_with_authority(run_id, entry, authority)
.await
}
async fn append_step_entry_if_absent_with_authority(
&self,
run_id: &RunId,
entry: StepLedgerEntry,
authority: MobRunProvenanceAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.append_step_entry_if_absent_with_authority(run_id, entry, authority)
.await
}
async fn append_failure_entry_with_authority(
&self,
run_id: &RunId,
entry: FailureLedgerEntry,
authority: MobRunProvenanceAuthority,
) -> Result<(), MobStoreError> {
self.inner
.append_failure_entry_with_authority(run_id, entry, authority)
.await
}
async fn cas_flow_state_with_authority(
&self,
run_id: &RunId,
expected: &crate::run::flow_run::State,
next: &crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_flow_state_with_authority(run_id, expected, next, authority_inputs)
.await
}
async fn cas_run_snapshot_with_authority(
&self,
run_id: &RunId,
expected_status: MobRunStatus,
expected_flow_state: &crate::run::flow_run::State,
next_status: MobRunStatus,
next_flow_state: &crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.snapshot_cas_history.write().await.push((
run_id.clone(),
expected_status.clone(),
next_status.clone(),
));
self.inner
.cas_run_snapshot_with_authority(
run_id,
expected_status,
expected_flow_state,
next_status,
next_flow_state,
authority_inputs,
)
.await
}
async fn cas_frame_state_with_authority(
&self,
run_id: &RunId,
frame_id: &crate::ids::FrameId,
expected: Option<&crate::run::FrameSnapshot>,
next: crate::run::FrameSnapshot,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_frame_state_with_authority(run_id, frame_id, expected, next, authority_inputs)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_grant_node_slot_with_authority(
&self,
run_id: &RunId,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
frame_id: &crate::ids::FrameId,
expected_frame: &crate::run::FrameSnapshot,
next_frame: crate::run::FrameSnapshot,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_grant_node_slot_with_authority(
run_id,
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_complete_step_and_record_output_with_authority(
&self,
run_id: &RunId,
frame_id: &crate::ids::FrameId,
expected_frame: &crate::run::FrameSnapshot,
next_frame: crate::run::FrameSnapshot,
step_output_key: String,
step_output: serde_json::Value,
loop_context: Option<(&crate::ids::LoopId, u64)>,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_complete_step_and_record_output_with_authority(
run_id,
frame_id,
expected_frame,
next_frame,
step_output_key,
step_output,
loop_context,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_start_loop_with_authority(
&self,
run_id: &RunId,
loop_instance_id: &crate::ids::LoopInstanceId,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
frame_id: &crate::ids::FrameId,
expected_frame: &crate::run::FrameSnapshot,
next_frame: crate::run::FrameSnapshot,
initial_loop: crate::run::LoopSnapshot,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_start_loop_with_authority(
run_id,
loop_instance_id,
expected_run_state,
next_run_state,
frame_id,
expected_frame,
next_frame,
initial_loop,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_loop_request_body_frame_with_authority(
&self,
run_id: &RunId,
loop_instance_id: &crate::ids::LoopInstanceId,
expected_loop: &crate::run::LoopSnapshot,
next_loop: crate::run::LoopSnapshot,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_loop_request_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_grant_body_frame_start_with_authority(
&self,
run_id: &RunId,
loop_instance_id: &crate::ids::LoopInstanceId,
expected_loop: &crate::run::LoopSnapshot,
next_loop: crate::run::LoopSnapshot,
frame_id: &crate::ids::FrameId,
initial_frame: crate::run::FrameSnapshot,
ledger_entry: crate::run::LoopIterationLedgerEntry,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_grant_body_frame_start_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop,
frame_id,
initial_frame,
ledger_entry,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_complete_body_frame_with_authority(
&self,
run_id: &RunId,
loop_instance_id: &crate::ids::LoopInstanceId,
expected_loop: &crate::run::LoopSnapshot,
next_loop: crate::run::LoopSnapshot,
frame_id: &crate::ids::FrameId,
expected_frame: &crate::run::FrameSnapshot,
next_frame: crate::run::FrameSnapshot,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_complete_body_frame_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn cas_complete_loop_with_authority(
&self,
run_id: &RunId,
loop_instance_id: &crate::ids::LoopInstanceId,
expected_loop: &crate::run::LoopSnapshot,
next_loop: crate::run::LoopSnapshot,
frame_id: &crate::ids::FrameId,
expected_frame: &crate::run::FrameSnapshot,
next_frame: crate::run::FrameSnapshot,
expected_run_state: &crate::run::flow_run::State,
next_run_state: crate::run::flow_run::State,
authority_inputs: Vec<crate::machines::mob_machine::MobMachineInput>,
) -> Result<bool, MobStoreError> {
self.inner
.cas_complete_loop_with_authority(
run_id,
loop_instance_id,
expected_loop,
next_loop,
frame_id,
expected_frame,
next_frame,
expected_run_state,
next_run_state,
authority_inputs,
)
.await
}
}
struct FaultInjectedRuntimeMetadataStore {
inner: InMemoryMobRuntimeMetadataStore,
fail_load_supervisor: AtomicBool,
fail_next_load_supervisor: AtomicBool,
fail_put_supervisor_countdown: AtomicUsize,
lose_compare_supervisor_completion_countdown: AtomicUsize,
fail_reread_after_lost_supervisor_completion: AtomicBool,
conflict_compare_supervisor_countdown: AtomicUsize,
fail_put_host_authority: AtomicBool,
fail_list_overlays: AtomicBool,
fail_upsert_overlay: AtomicBool,
fail_delete_overlay: AtomicBool,
}
impl FaultInjectedRuntimeMetadataStore {
fn new() -> Self {
Self {
inner: InMemoryMobRuntimeMetadataStore::new(),
fail_load_supervisor: AtomicBool::new(false),
fail_next_load_supervisor: AtomicBool::new(false),
fail_put_supervisor_countdown: AtomicUsize::new(0),
lose_compare_supervisor_completion_countdown: AtomicUsize::new(0),
fail_reread_after_lost_supervisor_completion: AtomicBool::new(false),
conflict_compare_supervisor_countdown: AtomicUsize::new(0),
fail_put_host_authority: AtomicBool::new(false),
fail_list_overlays: AtomicBool::new(false),
fail_upsert_overlay: AtomicBool::new(false),
fail_delete_overlay: AtomicBool::new(false),
}
}
fn set_fail_load_supervisor(&self, fail: bool) {
self.fail_load_supervisor.store(fail, Ordering::Relaxed);
}
fn fail_next_put_supervisor(&self) {
self.fail_nth_put_supervisor(1);
}
fn fail_nth_put_supervisor(&self, n: usize) {
assert!(n > 0, "failure countdown must target a future write");
self.fail_put_supervisor_countdown
.store(n, Ordering::Relaxed);
}
fn lose_nth_compare_supervisor_completion(&self, n: usize) {
self.configure_lost_compare_supervisor_completion(n, false);
}
fn lose_nth_compare_supervisor_completion_with_unreadable_reread(&self, n: usize) {
self.configure_lost_compare_supervisor_completion(n, true);
}
fn configure_lost_compare_supervisor_completion(&self, n: usize, fail_reread: bool) {
assert!(
n > 0,
"completion-loss countdown must target a future successful compare"
);
self.lose_compare_supervisor_completion_countdown
.store(n, Ordering::Relaxed);
self.fail_reread_after_lost_supervisor_completion
.store(fail_reread, Ordering::Relaxed);
}
fn conflict_next_compare_supervisor(&self) {
self.conflict_nth_compare_supervisor(1);
}
fn conflict_nth_compare_supervisor(&self, n: usize) {
assert!(n > 0, "conflict countdown must target a future compare");
self.conflict_compare_supervisor_countdown
.store(n, Ordering::Relaxed);
}
fn fail_next_upsert_overlay(&self) {
self.fail_upsert_overlay.store(true, Ordering::Relaxed);
}
fn fail_next_delete_overlay(&self) {
self.fail_delete_overlay.store(true, Ordering::Relaxed);
}
fn fail_next_put_host_authority(&self) {
self.fail_put_host_authority.store(true, Ordering::Relaxed);
}
fn maybe_fail_supervisor_write(&self) -> Result<(), MobStoreError> {
let mut remaining = self.fail_put_supervisor_countdown.load(Ordering::Relaxed);
while remaining > 0 {
match self.fail_put_supervisor_countdown.compare_exchange(
remaining,
remaining - 1,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
if remaining == 1 {
return Err(MobStoreError::WriteFailed(
"fault-injected runtime metadata supervisor write failure".to_string(),
));
}
break;
}
Err(actual) => remaining = actual,
}
}
Ok(())
}
fn maybe_conflict_supervisor_compare(&self) -> bool {
let mut remaining = self
.conflict_compare_supervisor_countdown
.load(Ordering::Relaxed);
while remaining > 0 {
match self.conflict_compare_supervisor_countdown.compare_exchange(
remaining,
remaining - 1,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => return remaining == 1,
Err(actual) => remaining = actual,
}
}
false
}
fn maybe_lose_supervisor_compare_completion(&self) -> bool {
let mut remaining = self
.lose_compare_supervisor_completion_countdown
.load(Ordering::Relaxed);
while remaining > 0 {
match self
.lose_compare_supervisor_completion_countdown
.compare_exchange(
remaining,
remaining - 1,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => return remaining == 1,
Err(actual) => remaining = actual,
}
}
false
}
fn competing_supervisor_authority(
expected: &SupervisorAuthorityRecord,
) -> Result<
(
SupervisorAuthorityRecord,
crate::store::SupervisorAuthorityPersistenceAuthority,
),
MobStoreError,
> {
let stable_expected = expected.without_pending_rotation();
let mut competing = SupervisorAuthorityRecord::generate(stable_expected.protocol_version);
competing.epoch = stable_expected.epoch + 1;
let mut authority = crate::machines::mob_machine::MobMachineAuthority::recover_from_state(
crate::machines::mob_machine::MobMachineState::default(),
)
.map_err(|error| {
MobStoreError::Internal(format!(
"generated supervisor conflict authority could not initialize: {error}"
))
})?;
authority
.apply_signal(stable_expected.dsl_recover_signal())
.map_err(|error| {
MobStoreError::Internal(format!(
"generated supervisor conflict authority rejected expected recovery: {error}"
))
})?;
let operation_id =
meerkat_contracts::wire::supervisor_bridge::SupervisorRotationOperationId::new();
let pending = crate::store::SupervisorPendingRotationRecord::from_authority(
&competing,
operation_id,
Vec::new(),
BTreeMap::new(),
);
crate::machines::mob_machine::MobMachineMutator::apply(
&mut authority,
stable_expected.dsl_record_pending_rotation_input(&pending, &BTreeSet::new()),
)
.map_err(|error| {
MobStoreError::Internal(format!(
"generated supervisor conflict authority rejected pending operation: {error}"
))
})?;
let transition = crate::machines::mob_machine::MobMachineMutator::apply(
&mut authority,
stable_expected.dsl_commit_rotation_input(&operation_id.to_string(), &competing),
)
.map_err(|error| {
MobStoreError::Internal(format!(
"generated supervisor conflict authority rejected commit: {error}"
))
})?;
let persistence_authority =
crate::store::SupervisorAuthorityPersistenceAuthority::from_transition(
&competing,
&transition,
)?;
Ok((competing, persistence_authority))
}
}
#[async_trait]
impl MobRuntimeMetadataStore for FaultInjectedRuntimeMetadataStore {
async fn begin_member_operator_request(
&self,
mob_id: &MobId,
record: &crate::store::MobMemberOperatorRequestRecord,
) -> Result<crate::store::MobMemberOperatorRequestBegin, MobStoreError> {
self.inner
.begin_member_operator_request(mob_id, record)
.await
}
async fn load_member_operator_request(
&self,
mob_id: &MobId,
key: &crate::store::MobMemberOperatorRequestKey,
) -> Result<Option<crate::store::MobMemberOperatorRequestRecord>, MobStoreError> {
self.inner.load_member_operator_request(mob_id, key).await
}
async fn compare_and_put_member_operator_request(
&self,
mob_id: &MobId,
expected: &crate::store::MobMemberOperatorRequestRecord,
record: &crate::store::MobMemberOperatorRequestRecord,
) -> Result<bool, MobStoreError> {
self.inner
.compare_and_put_member_operator_request(mob_id, expected, record)
.await
}
async fn list_member_operator_requests(
&self,
mob_id: &MobId,
) -> Result<Vec<crate::store::MobMemberOperatorRequestRecord>, MobStoreError> {
self.inner.list_member_operator_requests(mob_id).await
}
async fn prune_stale_member_operator_requests(
&self,
mob_id: &MobId,
authority: &crate::store::MobMemberOperatorPruneAuthority,
) -> Result<u64, MobStoreError> {
self.inner
.prune_stale_member_operator_requests(mob_id, authority)
.await
}
async fn delete_member_operator_requests(&self, mob_id: &MobId) -> Result<u64, MobStoreError> {
self.inner.delete_member_operator_requests(mob_id).await
}
async fn load_placed_spawn(
&self,
mob_id: &MobId,
agent_identity: &str,
) -> Result<Option<crate::store::MobPlacedSpawnCarrierRecord>, MobStoreError> {
self.inner.load_placed_spawn(mob_id, agent_identity).await
}
async fn list_placed_spawns(
&self,
mob_id: &MobId,
) -> Result<Vec<crate::store::MobPlacedSpawnCarrierRecord>, MobStoreError> {
self.inner.list_placed_spawns(mob_id).await
}
async fn begin_placed_spawn_if_absent(
&self,
mob_id: &MobId,
record: &crate::store::MobPlacedSpawnCarrierRecord,
authority: &crate::store::MobPlacedSpawnPendingPersistenceAuthority,
) -> Result<crate::store::BeginPlacedSpawnResult, MobStoreError> {
self.inner
.begin_placed_spawn_if_absent(mob_id, record, authority)
.await
}
async fn compare_and_commit_placed_spawn(
&self,
mob_id: &MobId,
expected_pending: &crate::store::MobPlacedSpawnCarrierRecord,
committed: &crate::store::MobPlacedSpawnCarrierRecord,
authority: &crate::store::MobPlacedSpawnCommitPersistenceAuthority,
) -> Result<crate::store::CommitPlacedSpawnResult, MobStoreError> {
self.inner
.compare_and_commit_placed_spawn(mob_id, expected_pending, committed, authority)
.await
}
async fn compare_and_delete_placed_spawn(
&self,
mob_id: &MobId,
expected: &crate::store::MobPlacedSpawnCarrierRecord,
authority: &crate::store::MobPlacedSpawnCleanupAuthority,
) -> Result<crate::store::DeletePlacedSpawnResult, MobStoreError> {
self.inner
.compare_and_delete_placed_spawn(mob_id, expected, authority)
.await
}
async fn list_mob_operator_grants(
&self,
mob_id: &MobId,
) -> Result<Vec<crate::store::MobOperatorGrantRecord>, MobStoreError> {
self.inner.list_mob_operator_grants(mob_id).await
}
async fn put_mob_operator_grant(
&self,
mob_id: &MobId,
record: &crate::store::MobOperatorGrantRecord,
authority: &crate::store::MobOperatorGrantPersistenceAuthority,
) -> Result<(), MobStoreError> {
self.inner
.put_mob_operator_grant(mob_id, record, authority)
.await
}
async fn delete_mob_operator_grant(
&self,
mob_id: &MobId,
principal: &str,
authority: &crate::store::MobOperatorGrantDeletionAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.delete_mob_operator_grant(mob_id, principal, authority)
.await
}
async fn delete_mob_operator_grants(&self, mob_id: &MobId) -> Result<u64, MobStoreError> {
self.inner.delete_mob_operator_grants(mob_id).await
}
async fn load_supervisor_authority(
&self,
mob_id: &MobId,
) -> Result<Option<SupervisorAuthorityRecord>, MobStoreError> {
if self
.fail_next_load_supervisor
.swap(false, Ordering::Relaxed)
|| self.fail_load_supervisor.load(Ordering::Relaxed)
{
return Err(MobStoreError::ReadFailed(
"fault-injected runtime metadata load failure".to_string(),
));
}
self.inner.load_supervisor_authority(mob_id).await
}
async fn put_supervisor_authority(
&self,
mob_id: &MobId,
record: &SupervisorAuthorityRecord,
authority: &crate::store::SupervisorAuthorityPersistenceAuthority,
) -> Result<(), MobStoreError> {
self.maybe_fail_supervisor_write()?;
self.inner
.put_supervisor_authority(mob_id, record, authority)
.await
}
async fn compare_and_put_supervisor_authority(
&self,
mob_id: &MobId,
expected: &SupervisorAuthorityRecord,
record: &SupervisorAuthorityRecord,
authority: &crate::store::SupervisorAuthorityPersistenceAuthority,
) -> Result<bool, MobStoreError> {
self.maybe_fail_supervisor_write()?;
if self.maybe_conflict_supervisor_compare() {
let (competing, competing_authority) = Self::competing_supervisor_authority(expected)?;
self.inner
.put_supervisor_authority(mob_id, &competing, &competing_authority)
.await?;
return Ok(false);
}
let changed = self
.inner
.compare_and_put_supervisor_authority(mob_id, expected, record, authority)
.await?;
if changed && self.maybe_lose_supervisor_compare_completion() {
if self
.fail_reread_after_lost_supervisor_completion
.swap(false, Ordering::Relaxed)
{
self.fail_next_load_supervisor
.store(true, Ordering::Relaxed);
}
return Err(MobStoreError::WriteFailed(
"fault-injected supervisor CAS completion loss after durable commit".to_string(),
));
}
Ok(changed)
}
async fn put_supervisor_authority_if_absent(
&self,
mob_id: &MobId,
record: &SupervisorAuthorityRecord,
authority: &crate::store::SupervisorAuthorityPersistenceAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.put_supervisor_authority_if_absent(mob_id, record, authority)
.await
}
async fn delete_supervisor_authority(
&self,
mob_id: &MobId,
expected: &SupervisorAuthorityRecord,
authority: &crate::store::SupervisorAuthorityDeletionAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.delete_supervisor_authority(mob_id, expected, authority)
.await
}
async fn load_mob_host_authority(
&self,
mob_id: &MobId,
host_id: &str,
) -> Result<Option<crate::store::MobHostAuthorityRecord>, MobStoreError> {
self.inner.load_mob_host_authority(mob_id, host_id).await
}
async fn list_mob_host_authorities(
&self,
mob_id: &MobId,
) -> Result<Vec<crate::store::MobHostAuthorityRecord>, MobStoreError> {
self.inner.list_mob_host_authorities(mob_id).await
}
async fn put_mob_host_authority(
&self,
mob_id: &MobId,
record: &crate::store::MobHostAuthorityRecord,
authority: &crate::store::MobHostAuthorityPersistenceAuthority,
) -> Result<(), MobStoreError> {
if self.fail_put_host_authority.swap(false, Ordering::Relaxed) {
return Err(MobStoreError::WriteFailed(
"fault-injected host authority write failure".to_string(),
));
}
self.inner
.put_mob_host_authority(mob_id, record, authority)
.await
}
async fn put_mob_host_authority_if_absent(
&self,
mob_id: &MobId,
record: &crate::store::MobHostAuthorityRecord,
authority: &crate::store::MobHostAuthorityPersistenceAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.put_mob_host_authority_if_absent(mob_id, record, authority)
.await
}
async fn compare_and_put_mob_host_authority(
&self,
mob_id: &MobId,
expected: &crate::store::MobHostAuthorityRecord,
record: &crate::store::MobHostAuthorityRecord,
authority: &crate::store::MobHostAuthorityPersistenceAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.compare_and_put_mob_host_authority(mob_id, expected, record, authority)
.await
}
async fn delete_mob_host_authority(
&self,
mob_id: &MobId,
expected: &crate::store::MobHostAuthorityRecord,
authority: &crate::store::MobHostAuthorityDeletionAuthority,
) -> Result<bool, MobStoreError> {
self.inner
.delete_mob_host_authority(mob_id, expected, authority)
.await
}
async fn put_mob_host_binding_generation_highwater(
&self,
mob_id: &MobId,
expected: &crate::store::MobHostAuthorityRecord,
authority: &crate::store::MobHostAuthorityDeletionAuthority,
) -> Result<(), MobStoreError> {
self.inner
.put_mob_host_binding_generation_highwater(mob_id, expected, authority)
.await
}
async fn list_mob_host_binding_generation_highwaters(
&self,
mob_id: &MobId,
) -> Result<Vec<(String, u64)>, MobStoreError> {
self.inner
.list_mob_host_binding_generation_highwaters(mob_id)
.await
}
async fn list_external_binding_overlays(
&self,
mob_id: &MobId,
) -> Result<Vec<ExternalBindingOverlayRecord>, MobStoreError> {
if self.fail_list_overlays.load(Ordering::Relaxed) {
return Err(MobStoreError::ReadFailed(
"fault-injected runtime overlay load failure".to_string(),
));
}
self.inner.list_external_binding_overlays(mob_id).await
}
async fn put_external_binding_overlay_if_absent(
&self,
mob_id: &MobId,
record: &ExternalBindingOverlayRecord,
) -> Result<bool, MobStoreError> {
self.inner
.put_external_binding_overlay_if_absent(mob_id, record)
.await
}
async fn upsert_external_binding_overlay(
&self,
mob_id: &MobId,
record: &ExternalBindingOverlayRecord,
) -> Result<(), MobStoreError> {
if self.fail_upsert_overlay.swap(false, Ordering::Relaxed) {
return Err(MobStoreError::WriteFailed(
"fault-injected runtime overlay upsert failure".to_string(),
));
}
self.inner
.upsert_external_binding_overlay(mob_id, record)
.await
}
async fn delete_external_binding_overlay(
&self,
mob_id: &MobId,
agent_identity: &AgentIdentity,
generation: crate::ids::Generation,
) -> Result<(), MobStoreError> {
if self.fail_delete_overlay.swap(false, Ordering::Relaxed) {
return Err(MobStoreError::WriteFailed(
"fault-injected runtime overlay delete failure".to_string(),
));
}
self.inner
.delete_external_binding_overlay(mob_id, agent_identity, generation)
.await
}
async fn delete_external_binding_overlays(&self, mob_id: &MobId) -> Result<(), MobStoreError> {
self.inner.delete_external_binding_overlays(mob_id).await
}
async fn list_member_live_cleanup_records(
&self,
mob_id: &MobId,
) -> Result<Vec<crate::store::MobMemberLiveCleanupRecord>, MobStoreError> {
self.inner.list_member_live_cleanup_records(mob_id).await
}
async fn put_member_live_cleanup_record_if_absent(
&self,
mob_id: &MobId,
record: &crate::store::MobMemberLiveCleanupRecord,
) -> Result<bool, MobStoreError> {
self.inner
.put_member_live_cleanup_record_if_absent(mob_id, record)
.await
}
async fn delete_member_live_cleanup_record(
&self,
mob_id: &MobId,
expected: &crate::store::MobMemberLiveCleanupRecord,
) -> Result<bool, MobStoreError> {
self.inner
.delete_member_live_cleanup_record(mob_id, expected)
.await
}
}
// -----------------------------------------------------------------------
// Helpers
// -----------------------------------------------------------------------
fn sample_definition() -> MobDefinition {
let mut profiles = BTreeMap::new();
profiles.insert(
ProfileName::from("lead"),
ProfileBinding::Inline(Box::new(Profile {
model: "claude-opus-4-8".into(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: vec![],
tools: ToolConfig {
builtins: true,
shell: false,
comms: true,
memory: false,
workgraph: false,
mob: true,
schedule: false,
image_generation: false,
mcp: vec![],
mcp_servers: vec![],
rust_bundles: vec![],
},
peer_description: "The lead".into(),
external_addressable: true,
backend: None,
runtime_mode: crate::MobRuntimeMode::AutonomousHost,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
profiles.insert(
ProfileName::from("worker"),
ProfileBinding::Inline(Box::new(Profile {
model: "claude-sonnet-4-5".into(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: vec![],
tools: ToolConfig {
comms: true,
..ToolConfig::default()
},
peer_description: "A worker".into(),
external_addressable: false,
backend: None,
runtime_mode: crate::MobRuntimeMode::AutonomousHost,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
let mut definition = MobDefinition::explicit("test-mob");
definition.orchestrator = Some(OrchestratorConfig {
profile: ProfileName::from("lead"),
});
definition.profiles = profiles;
definition
}
fn sample_definition_with_auto_wire() -> MobDefinition {
let mut def = sample_definition();
def.wiring.auto_wire_orchestrator = true;
def
}
fn sample_definition_with_role_wiring() -> MobDefinition {
let mut def = sample_definition();
def.wiring.role_wiring = vec![RoleWiringRule {
a: ProfileName::from("worker"),
b: ProfileName::from("worker"),
}];
def
}
fn sample_definition_with_cross_role_wiring() -> MobDefinition {
let mut def = sample_definition();
def.wiring.role_wiring = vec![RoleWiringRule {
a: ProfileName::from("lead"),
b: ProfileName::from("worker"),
}];
def
}
fn sample_definition_with_overlapping_orchestrator_and_role_wiring() -> MobDefinition {
let mut def = sample_definition_with_cross_role_wiring();
def.wiring.auto_wire_orchestrator = true;
def
}
fn sample_definition_with_tool_bundle(bundle_name: &str) -> MobDefinition {
let mut def = sample_definition();
let worker = def
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|b| b.as_inline_mut())
.expect("worker profile exists");
worker.tools.rust_bundles = vec![bundle_name.to_string()];
def
}
fn sample_definition_with_mob_tools() -> MobDefinition {
let mut def = sample_definition();
let worker = def
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|b| b.as_inline_mut())
.expect("worker profile exists");
worker.tools.mob = true;
worker.tools.comms = true;
def
}
fn generated_mob_operator_authority_with_scope(
mob_id: &str,
) -> meerkat_core::service::MobToolAuthorityContext {
let authority = meerkat_runtime::mob_operator_authority::create_only_mob_operator_authority()
.expect("generated create-only mob authority should be accepted");
let authority =
meerkat_runtime::mob_operator_authority::set_create_authority(&authority, false)
.expect("generated mob authority should disable create scope");
meerkat_runtime::mob_operator_authority::grant_manage_mob(&authority, mob_id)
.expect("generated mob authority should grant managed mob scope")
}
fn generated_mob_operator_authority_with_spawn_profile(
mob_id: &str,
profile: &str,
) -> meerkat_core::service::MobToolAuthorityContext {
// Create-only authority with NO manage scope on the target mob, granted a
// single spawnable profile in that mob. This yields
// `spawn_profile_scope_present == true` (spawn scope non-empty) without
// `can_manage_mob`, so current-mob management tools stay denied while the
// coarse spawn-tool gate must admit (the machine composes the disjunction).
let authority = meerkat_runtime::mob_operator_authority::create_only_mob_operator_authority()
.expect("generated create-only mob authority should be accepted");
let authority =
meerkat_runtime::mob_operator_authority::set_create_authority(&authority, false)
.expect("generated mob authority should disable create scope");
meerkat_runtime::mob_operator_authority::grant_spawn_profile_in_mob(&authority, mob_id, profile)
.expect("generated mob authority should grant a spawnable profile")
}
fn sample_definition_with_external_backend() -> MobDefinition {
let mut def = sample_definition();
def.backend.external = Some(crate::definition::ExternalBackendConfig {
address_base: "https://backend.example.invalid/mesh".to_string(),
supervisor_bridge: None,
});
def
}
fn sample_definition_without_mob_flags() -> MobDefinition {
let mut def = sample_definition();
let lead = def
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile exists")
.as_inline_mut()
.unwrap();
lead.tools.mob = false;
def
}
fn flow_step(role: impl Into<crate::ids::ProfileName>, message: &str) -> FlowStepSpec {
FlowStepSpec {
role: role.into(),
message: meerkat_core::types::ContentInput::from(message),
depends_on: Vec::new(),
dispatch_mode: DispatchMode::FanOut,
collection_policy: CollectionPolicy::All,
condition: None,
timeout_ms: None,
expected_schema_ref: None,
branch: None,
depends_on_mode: DependencyMode::All,
allowed_tools: None,
blocked_tools: None,
output_format: Some(StepOutputFormat::Json),
}
}
fn flow_id(value: &str) -> crate::FlowId {
crate::FlowId::from(value)
}
fn step_id(value: &str) -> crate::StepId {
crate::StepId::from(value)
}
fn sample_definition_with_single_step_flow(
timeout_ms: u64,
max_orphaned_turns: u32,
) -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut step = flow_step("worker", "Execute step");
step.timeout_ms = Some(timeout_ms);
steps.insert(step_id("start"), step);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("demo"),
FlowSpec::new(Some("single step demo flow".to_string()), steps, None),
);
def.flows = flows;
def.limits = Some(LimitsSpec {
max_flow_duration_ms: None,
max_step_retries: None,
max_orphaned_turns: Some(max_orphaned_turns),
cancel_grace_timeout_ms: None,
..Default::default()
});
def
}
fn with_cancel_grace_timeout(
mut def: MobDefinition,
cancel_grace_timeout_ms: u64,
) -> MobDefinition {
let limits = def.limits.get_or_insert(LimitsSpec {
max_flow_duration_ms: None,
max_step_retries: None,
max_orphaned_turns: None,
cancel_grace_timeout_ms: None,
..Default::default()
});
limits.cancel_grace_timeout_ms = Some(cancel_grace_timeout_ms);
def
}
fn sample_definition_with_branch_flow() -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
steps.insert(step_id("start"), flow_step("worker", "Start"));
let mut fix_critical = flow_step("worker", "Fix critical");
fix_critical.depends_on = vec![step_id("start")];
fix_critical.condition = Some(ConditionExpr::Eq {
path: "params.severity".to_string(),
value: serde_json::json!("critical"),
});
fix_critical.branch = Some(crate::ids::BranchId::from("repair"));
steps.insert(step_id("fix_critical"), fix_critical);
let mut fix_minor = flow_step("worker", "Fix minor");
fix_minor.depends_on = vec![step_id("start")];
fix_minor.condition = Some(ConditionExpr::Eq {
path: "params.severity".to_string(),
value: serde_json::json!("minor"),
});
fix_minor.branch = Some(crate::ids::BranchId::from("repair"));
steps.insert(step_id("fix_minor"), fix_minor);
let mut summarize = flow_step("worker", "Summarize {{steps.fix_critical}}");
summarize.depends_on = vec![step_id("fix_critical"), step_id("fix_minor")];
summarize.depends_on_mode = DependencyMode::Any;
steps.insert(step_id("summarize"), summarize);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("branching"),
FlowSpec::new(Some("branch winner flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_strict_topology_deny() -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
def.topology = Some(TopologySpec {
mode: PolicyMode::Strict,
rules: vec![TopologyRule {
from_role: ProfileName::from("lead"),
to_role: ProfileName::from("worker"),
allowed: false,
}],
});
def
}
fn sample_definition_with_advisory_topology_deny() -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
def.topology = Some(TopologySpec {
mode: PolicyMode::Advisory,
rules: vec![TopologyRule {
from_role: ProfileName::from("lead"),
to_role: ProfileName::from("worker"),
allowed: false,
}],
});
def
}
fn sample_definition_with_strict_topology_wildcard_deny() -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
def.topology = Some(TopologySpec {
mode: PolicyMode::Strict,
rules: vec![TopologyRule {
from_role: ProfileName::from("*"),
to_role: ProfileName::from("worker"),
allowed: false,
}],
});
def
}
fn sample_definition_with_advisory_topology_wildcard_deny() -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
def.topology = Some(TopologySpec {
mode: PolicyMode::Advisory,
rules: vec![TopologyRule {
from_role: ProfileName::from("*"),
to_role: ProfileName::from("worker"),
allowed: false,
}],
});
def
}
fn sample_definition_with_collection_policy(policy: CollectionPolicy) -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut step = flow_step("worker", "Collect results");
step.dispatch_mode = DispatchMode::FanOut;
step.collection_policy = policy;
step.timeout_ms = Some(500);
steps.insert(step_id("collect"), step);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("collect"),
FlowSpec::new(Some("collection policy flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_dispatch_mode(mode: DispatchMode) -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut step = flow_step("worker", "Dispatch mode");
step.dispatch_mode = mode;
step.collection_policy = CollectionPolicy::Any;
step.timeout_ms = Some(500);
steps.insert(step_id("dispatch"), step);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("dispatch"),
FlowSpec::new(Some("dispatch mode flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_dispatch_mode_and_policy(
mode: DispatchMode,
policy: CollectionPolicy,
) -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut step = flow_step("worker", "Dispatch mode");
step.dispatch_mode = mode;
step.collection_policy = policy;
step.timeout_ms = Some(500);
steps.insert(step_id("dispatch"), step);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("dispatch"),
FlowSpec::new(Some("dispatch mode flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_retry_flow(max_step_retries: u32) -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(250, 8);
def.limits = Some(LimitsSpec {
max_flow_duration_ms: None,
max_step_retries: Some(max_step_retries),
max_orphaned_turns: Some(8),
cancel_grace_timeout_ms: None,
..Default::default()
});
def
}
fn sample_definition_with_branch_fallback_flow() -> MobDefinition {
let mut def = sample_definition();
let worker_template = def
.profiles
.get(&ProfileName::from("worker"))
.cloned()
.expect("worker profile exists");
def.profiles
.insert(ProfileName::from("worker_fail"), worker_template.clone());
def.profiles
.insert(ProfileName::from("worker_ok"), worker_template);
let mut steps = IndexMap::new();
let mut start = flow_step("worker_ok", "Start");
start.timeout_ms = Some(500);
steps.insert(step_id("start"), start);
let mut first = flow_step("worker_fail", "First branch candidate");
first.depends_on = vec![step_id("start")];
first.condition = Some(ConditionExpr::Eq {
path: "params.try_fallback".to_string(),
value: serde_json::json!(true),
});
// Keep the fallback fixture independent of the production default turn
// deadline. A failed candidate is the subject of this test; waiting the
// full default deadline makes loaded CI spend most of its lane budget on
// an intentionally unsuccessful branch.
first.timeout_ms = Some(500);
first.branch = Some(crate::ids::BranchId::from("repair"));
steps.insert(step_id("candidate_first"), first);
let mut second = flow_step("worker_ok", "Second branch candidate");
second.depends_on = vec![step_id("start")];
second.condition = Some(ConditionExpr::Eq {
path: "params.try_fallback".to_string(),
value: serde_json::json!(true),
});
second.timeout_ms = Some(500);
second.branch = Some(crate::ids::BranchId::from("repair"));
steps.insert(step_id("candidate_second"), second);
let mut join = flow_step("worker_ok", "Join");
join.depends_on = vec![step_id("candidate_first"), step_id("candidate_second")];
join.depends_on_mode = DependencyMode::Any;
join.timeout_ms = Some(500);
steps.insert(step_id("join"), join);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("branch_fallback"),
FlowSpec::new(Some("branch fallback flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_template_message(template: &str) -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
let step = def
.flows
.get_mut(&flow_id("demo"))
.and_then(|flow| flow.steps.get_mut(&step_id("start")))
.expect("demo.start step exists");
step.message = template.to_string().into();
def
}
fn sample_definition_with_shared_path_resolution_flow() -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut start = flow_step("worker", "Start");
start.dispatch_mode = DispatchMode::OneToOne;
start.collection_policy = CollectionPolicy::Any;
steps.insert(step_id("start"), start);
let mut follow = flow_step("worker", "Follow {{steps.start.output}}");
follow.depends_on = vec![step_id("start")];
follow.condition = Some(ConditionExpr::Eq {
path: "steps.start".to_string(),
value: serde_json::json!("Turn completed"),
});
steps.insert(step_id("follow"), follow);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("shared_paths"),
FlowSpec::new(Some("shared path resolver flow".to_string()), steps, None),
);
def.flows = flows;
def
}
fn sample_definition_with_schema_ref(schema_ref: &str) -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
let step = def
.flows
.get_mut(&flow_id("demo"))
.expect("demo flow exists")
.steps
.get_mut(&step_id("start"))
.expect("start step exists");
step.expected_schema_ref =
Some(crate::definition::FlowSchemaRef::parse(schema_ref).expect("valid schema ref"));
def
}
fn sample_definition_with_supervisor_threshold(threshold: u32) -> MobDefinition {
let mut def = sample_definition_with_single_step_flow(500, 8);
def.supervisor = Some(crate::definition::SupervisorSpec {
role: ProfileName::from("lead"),
escalation_threshold: threshold,
escalation_turn_timeout_ms: None,
});
def
}
fn sample_definition_with_two_step_flow(timeout_ms: u64) -> MobDefinition {
let mut def = sample_definition();
let mut steps = IndexMap::new();
let mut first = flow_step("worker", "First");
first.timeout_ms = Some(timeout_ms);
steps.insert(step_id("first"), first);
let mut second = flow_step("worker", "Second");
second.depends_on = vec![step_id("first")];
second.timeout_ms = Some(timeout_ms);
steps.insert(step_id("second"), second);
let mut flows = BTreeMap::new();
flows.insert(
flow_id("two_step"),
FlowSpec::new(Some("cooperative cancel flow".to_string()), steps, None),
);
def.flows = flows;
def
}
async fn wait_for_run_terminal(
handle: &MobHandle,
run_id: &RunId,
timeout: Duration,
) -> crate::run::MobRun {
let timeout = timeout.max(Duration::from_secs(60));
let deadline = Instant::now() + timeout;
loop {
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow_status should succeed")
.expect("run should exist");
if crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status)
.expect("MobMachine terminality classifier should accept run status")
{
return run;
}
assert!(
Instant::now() < deadline,
"timed out waiting for run {run_id} terminal state; last status: {:?}",
run.status
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
async fn wait_for_run_terminal_in_store(
run_store: &dyn MobRunStore,
run_id: &RunId,
timeout: Duration,
) -> crate::run::MobRun {
let deadline = Instant::now() + timeout;
loop {
let run = run_store
.get_run(run_id)
.await
.expect("run-store read should succeed")
.expect("run should exist");
if crate::run::mob_machine_run_status_is_terminal(&run.run_id, &run.status)
.expect("MobMachine terminality classifier should accept run status")
{
return run;
}
assert!(
Instant::now() < deadline,
"timed out waiting for run {run_id} terminal state in durable store; last status: {:?}",
run.status
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
async fn wait_for_actor_fail_stop(handle: &MobHandle) {
tokio::time::timeout(Duration::from_secs(2), async {
while !handle.command_tx.is_closed() {
tokio::task::yield_now().await;
}
})
.await
.expect("ambiguous terminal persistence should fail-stop the live actor");
}
/// Build a test RuntimeBinding::External for a given member identity.
fn test_external_binding(agent_identity: &str) -> crate::RuntimeBinding {
let bootstrap_token = format!("bootstrap-{agent_identity}");
crate::RuntimeBinding::External {
peer_id: format!("ed25519:test-key:{agent_identity}"),
address: format!("tcp://test.invalid/{agent_identity}"),
bootstrap_token: Some(bootstrap_token.into()),
pubkey: [7u8; 32],
}
}
async fn spawn_spec_with_test_ops_owner(
handle: &MobHandle,
spec: SpawnMemberSpec,
) -> Result<super::handle::SpawnResult, MobError> {
let identity = spec.identity.clone();
let owner_context = handle.create_generated_ops_owner_context_for_test().await?;
handle
.spawn_spec_receipt_with_owner_context(spec, owner_context)
.await?;
let entry = handle.get_member(&identity).await?.ok_or_else(|| {
MobError::Internal(format!(
"spawn succeeded but roster entry missing for '{identity}'"
))
})?;
Ok(super::handle::SpawnResult {
agent_identity: entry.agent_identity,
agent_runtime_id: entry.agent_runtime_id,
fence_token: entry.fence_token,
})
}
fn canonical_external_address(address: &str) -> &str {
address.split('?').next().unwrap_or(address)
}
#[derive(Clone)]
struct HarnessSupervisorState {
supervisor: meerkat_core::comms::TrustedPeerDescriptor,
epoch: u64,
}
struct LiveExternalPeerHarness {
binding: crate::RuntimeBinding,
runtime: Arc<meerkat_comms::CommsRuntime>,
bind_count: Arc<std::sync::atomic::AtomicUsize>,
interrupt_count: Arc<std::sync::atomic::AtomicUsize>,
bind_protocol_versions: Arc<RwLock<Vec<super::bridge_protocol::BridgeProtocolVersion>>>,
delivered_inputs: Arc<RwLock<Vec<String>>>,
hard_cancel_reasons: Arc<RwLock<Vec<String>>>,
received_intents: Arc<RwLock<Vec<String>>>,
supervisor_addresses: Arc<RwLock<Vec<String>>>,
rotation_operation_ids: Arc<RwLock<Vec<super::bridge_protocol::SupervisorRotationOperationId>>>,
suppress_rotation_observations: Arc<AtomicBool>,
drop_next_attempted_rotation_observation_response: Arc<AtomicBool>,
drop_next_authorize_response: Arc<AtomicBool>,
reject_next_rotation: Arc<AtomicBool>,
supervisor_state: Arc<RwLock<Option<HarnessSupervisorState>>>,
bind_peer_id_override: Arc<RwLock<Option<String>>>,
fail_authorize_countdown: Arc<AtomicUsize>,
fail_next_bind: Arc<AtomicBool>,
fail_next_retire: Arc<AtomicBool>,
fail_next_revoke: Arc<AtomicBool>,
fail_next_unwire: Arc<AtomicBool>,
/// When set, the next `BindMember` reply is well-formed JSON that is
/// neither a `BridgeReply` payload nor a typed rejection, forcing the
/// supervisor-side decode path to fail after trust was installed.
garble_next_bind: Arc<AtomicBool>,
retire_count: Arc<AtomicUsize>,
task: tokio::task::JoinHandle<()>,
}
impl LiveExternalPeerHarness {
fn binding(&self) -> crate::RuntimeBinding {
self.binding.clone()
}
async fn trusted_peer_names(&self) -> Vec<String> {
self.runtime
.peers()
.await
.into_iter()
.map(|entry| entry.name.as_str().to_string())
.collect()
}
async fn trusted_peer_routes(&self) -> Vec<(String, String, String)> {
self.runtime
.peers()
.await
.into_iter()
.map(|entry| {
(
entry.name.as_str().to_string(),
entry.peer_id.to_string(),
entry.address.to_string(),
)
})
.collect()
}
async fn remove_trusted_peer(&self, peer_id: &str) {
remove_test_peer_projection_trust(
self.runtime.as_ref(),
peer_id,
"remove trusted peer from harness runtime",
)
.await;
}
async fn remove_trusted_peer_descriptor(
runtime: &Arc<meerkat_comms::CommsRuntime>,
peer: &meerkat_core::comms::TrustedPeerDescriptor,
) {
remove_test_peer_projection_trust(
runtime.as_ref(),
&peer.peer_id.to_string(),
"remove trusted peer from harness runtime",
)
.await;
}
fn bind_count(&self) -> usize {
self.bind_count.load(Ordering::Relaxed)
}
fn interrupt_count(&self) -> usize {
self.interrupt_count.load(Ordering::Relaxed)
}
async fn bind_protocol_versions(&self) -> Vec<super::bridge_protocol::BridgeProtocolVersion> {
self.bind_protocol_versions.read().await.clone()
}
async fn delivered_input_ids(&self) -> Vec<String> {
self.delivered_inputs.read().await.clone()
}
async fn hard_cancel_reasons(&self) -> Vec<String> {
self.hard_cancel_reasons.read().await.clone()
}
async fn received_intents(&self) -> Vec<String> {
self.received_intents.read().await.clone()
}
async fn supervisor_addresses(&self) -> Vec<String> {
self.supervisor_addresses.read().await.clone()
}
async fn rotation_operation_ids(
&self,
) -> Vec<super::bridge_protocol::SupervisorRotationOperationId> {
self.rotation_operation_ids.read().await.clone()
}
fn suppress_rotation_observations(&self, suppress: bool) {
self.suppress_rotation_observations
.store(suppress, Ordering::Relaxed);
}
fn drop_next_attempted_rotation_observation_response(&self) {
assert!(
!self
.drop_next_attempted_rotation_observation_response
.swap(true, Ordering::Relaxed),
"attempted-authority observation response drop is already armed"
);
}
fn attempted_rotation_observation_response_drop_is_armed(&self) -> bool {
self.drop_next_attempted_rotation_observation_response
.load(Ordering::Relaxed)
}
fn drop_next_authorize_response(&self) {
assert!(
!self
.drop_next_authorize_response
.swap(true, Ordering::Relaxed),
"authorize response drop is already armed"
);
}
fn authorize_response_drop_is_armed(&self) -> bool {
self.drop_next_authorize_response.load(Ordering::Relaxed)
}
fn reject_next_rotation(&self) {
self.reject_next_rotation.store(true, Ordering::Relaxed);
}
async fn authorized_supervisor_peer_id(&self) -> Option<String> {
self.supervisor_state
.read()
.await
.as_ref()
.map(|state| state.supervisor.peer_id.to_string())
}
async fn authorized_supervisor_spec(&self) -> Option<super::bridge_protocol::BridgePeerSpec> {
self.supervisor_state
.read()
.await
.as_ref()
.map(|state| state.supervisor.clone().into())
}
async fn install_legacy_rotated_supervisor(
&self,
target: super::bridge_protocol::BridgePeerSpec,
epoch: u64,
) {
let target = meerkat_core::comms::TrustedPeerDescriptor::try_from(target)
.expect("valid legacy rotation target");
apply_test_peer_projection_trust(
self.runtime.as_ref(),
target.clone(),
"install legacy rotated supervisor trust",
)
.await;
let previous = self
.supervisor_state
.write()
.await
.replace(HarnessSupervisorState {
supervisor: target,
epoch,
});
if let Some(previous) = previous {
remove_test_peer_projection_trust(
self.runtime.as_ref(),
&previous.supervisor.peer_id.to_string(),
"fence old supervisor for legacy rotation",
)
.await;
}
}
async fn forget_supervisor(&self) {
let _ = self.supervisor_state.write().await.take();
}
fn fail_next_authorize(&self) {
self.fail_nth_authorize(1);
}
fn fail_nth_authorize(&self, n: usize) {
assert!(
n > 0,
"authorize failure countdown must target a future command"
);
self.fail_authorize_countdown.store(n, Ordering::Relaxed);
}
fn fail_next_bind(&self) {
self.fail_next_bind.store(true, Ordering::Relaxed);
}
fn fail_next_retire(&self) {
self.fail_next_retire.store(true, Ordering::Relaxed);
}
fn fail_next_revoke(&self) {
self.fail_next_revoke.store(true, Ordering::Relaxed);
}
fn fail_next_unwire(&self) {
self.fail_next_unwire.store(true, Ordering::Relaxed);
}
fn garble_next_bind_reply(&self) {
self.garble_next_bind.store(true, Ordering::Relaxed);
}
fn retire_count(&self) -> usize {
self.retire_count.load(Ordering::Relaxed)
}
async fn override_next_bind_peer_id(&self, peer_id: String) {
*self.bind_peer_id_override.write().await = Some(peer_id);
}
}
fn consume_countdown_failure(countdown: &AtomicUsize) -> bool {
let mut remaining = countdown.load(Ordering::Relaxed);
while remaining > 0 {
match countdown.compare_exchange(
remaining,
remaining - 1,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => return remaining == 1,
Err(actual) => remaining = actual,
}
}
false
}
impl Drop for LiveExternalPeerHarness {
fn drop(&mut self) {
self.task.abort();
}
}
async fn spawn_live_external_peer(peer_name: &str) -> LiveExternalPeerHarness {
spawn_live_external_peer_with_transport(peer_name, false).await
}
#[cfg(not(target_arch = "wasm32"))]
async fn spawn_live_external_tcp_peer(peer_name: &str) -> LiveExternalPeerHarness {
spawn_live_external_peer_with_transport(peer_name, true).await
}
async fn spawn_live_external_peer_with_transport(
peer_name: &str,
listen_tcp: bool,
) -> LiveExternalPeerHarness {
let peer_name = peer_name.to_string();
let mut runtime = meerkat_comms::CommsRuntime::inproc_only(&peer_name)
.expect("create live external peer runtime");
#[cfg(not(target_arch = "wasm32"))]
if listen_tcp {
runtime
.set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
.expect("configure live external peer TCP listener");
runtime
.start_listeners()
.await
.expect("start live external peer TCP listener");
}
#[cfg(target_arch = "wasm32")]
let _ = listen_tcp;
let runtime_address = runtime.advertised_address();
let runtime = Arc::new(runtime);
install_ephemeral_peer_request_response_authority(
&runtime,
&format!("live-external-peer-{peer_name}"),
);
let bootstrap_token = runtime.bridge_bootstrap_token().to_string();
let peer_pubkey = *runtime.public_key().as_bytes();
let binding = crate::RuntimeBinding::External {
peer_id: runtime.public_key().to_peer_id().to_string(),
address: runtime_address.clone(),
bootstrap_token: Some(bootstrap_token.into()),
pubkey: peer_pubkey,
};
let responder_runtime = runtime.clone();
let bind_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let responder_bind_count = bind_count.clone();
let interrupt_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let responder_interrupt_count = interrupt_count.clone();
let bind_protocol_versions = Arc::new(RwLock::new(Vec::new()));
let responder_bind_protocol_versions = bind_protocol_versions.clone();
let delivered_inputs = Arc::new(RwLock::new(Vec::new()));
let responder_delivered_inputs = delivered_inputs.clone();
let hard_cancel_reasons = Arc::new(RwLock::new(Vec::new()));
let responder_hard_cancel_reasons = hard_cancel_reasons.clone();
let delivery_responses = Arc::new(RwLock::new(HashMap::new()));
let responder_delivery_responses = delivery_responses.clone();
let received_intents = Arc::new(RwLock::new(Vec::new()));
let responder_received_intents = received_intents.clone();
let supervisor_addresses = Arc::new(RwLock::new(Vec::new()));
let responder_supervisor_addresses = supervisor_addresses.clone();
let rotation_operation_ids = Arc::new(RwLock::new(Vec::new()));
let responder_rotation_operation_ids = rotation_operation_ids.clone();
let suppress_rotation_observations = Arc::new(AtomicBool::new(false));
let responder_suppress_rotation_observations = suppress_rotation_observations.clone();
let drop_next_attempted_rotation_observation_response = Arc::new(AtomicBool::new(false));
let responder_drop_next_attempted_rotation_observation_response =
drop_next_attempted_rotation_observation_response.clone();
let drop_next_authorize_response = Arc::new(AtomicBool::new(false));
let responder_drop_next_authorize_response = drop_next_authorize_response.clone();
let reject_next_rotation = Arc::new(AtomicBool::new(false));
let responder_reject_next_rotation = reject_next_rotation.clone();
let supervisor_state: Arc<RwLock<Option<HarnessSupervisorState>>> = Arc::new(RwLock::new(None));
let responder_supervisor_state = supervisor_state.clone();
let rotation_operations: Arc<
RwLock<
HashMap<
super::bridge_protocol::SupervisorRotationOperationId,
super::bridge_protocol::BridgeSupervisorRotationState,
>,
>,
> = Arc::new(RwLock::new(HashMap::new()));
let responder_rotation_operations = rotation_operations.clone();
let bind_peer_id_override: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
let responder_bind_peer_id_override = bind_peer_id_override.clone();
let fail_authorize_countdown = Arc::new(AtomicUsize::new(0));
let responder_fail_authorize_countdown = fail_authorize_countdown.clone();
let fail_next_bind = Arc::new(AtomicBool::new(false));
let responder_fail_next_bind = fail_next_bind.clone();
let fail_next_retire = Arc::new(AtomicBool::new(false));
let responder_fail_next_retire = fail_next_retire.clone();
let fail_next_revoke = Arc::new(AtomicBool::new(false));
let responder_fail_next_revoke = fail_next_revoke.clone();
let fail_next_unwire = Arc::new(AtomicBool::new(false));
let responder_fail_next_unwire = fail_next_unwire.clone();
let garble_next_bind = Arc::new(AtomicBool::new(false));
let responder_garble_next_bind = garble_next_bind.clone();
let retire_count = Arc::new(AtomicUsize::new(0));
let responder_retire_count = retire_count.clone();
let responder_runtime_address = runtime_address.clone();
let task = tokio::spawn(async move {
let mut state = super::bridge_protocol::BridgeMemberRuntimeState::Idle;
let inbox_notify = responder_runtime.inbox_notify();
let mut reply_tasks = tokio::task::JoinSet::new();
loop {
while let Some(result) = reply_tasks.try_join_next() {
result.expect("live external peer reply task should not panic");
}
let notified = inbox_notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let candidates = responder_runtime.drain_peer_input_candidates().await;
if candidates.is_empty() {
if responder_runtime.dismiss_received() {
break;
}
if reply_tasks.is_empty() {
(&mut notified).await;
} else {
tokio::select! {
() = &mut notified => {}
result = reply_tasks.join_next() => {
result
.expect("non-empty live external peer reply task set")
.expect("live external peer reply task should not panic");
}
}
}
continue;
}
for candidate in candidates {
match &candidate.interaction.content {
meerkat_core::interaction::InteractionContent::Message { body, .. } => {
let delivery = serde_json::from_str::<
super::bridge_protocol::BridgeSupervisorDelivery,
>(body);
if let Ok(
super::bridge_protocol::BridgeSupervisorDelivery::SubmitSupervisorRotation(
submission,
),
) = delivery
{
responder_rotation_operation_ids
.write()
.await
.push(submission.operation_id);
if let Some(existing) = responder_rotation_operations
.read()
.await
.get(&submission.operation_id)
.cloned()
{
let existing_operation = match &existing {
super::bridge_protocol::BridgeSupervisorRotationState::Pending {
operation,
..
} => operation,
super::bridge_protocol::BridgeSupervisorRotationState::Completed {
receipt,
} => receipt,
super::bridge_protocol::BridgeSupervisorRotationState::Rejected {
receipt,
} => &receipt.operation,
_ => panic!("unsupported rotation state in test harness"),
};
assert_eq!(
existing_operation.target.target, submission.target,
"exact operation retry must preserve the target"
);
assert_eq!(
existing_operation.target.target_epoch,
submission.target_epoch,
"exact operation retry must preserve the target epoch"
);
responder_runtime
.mark_interaction_complete(candidate.interaction.id.0);
continue;
}
if consume_countdown_failure(
responder_fail_authorize_countdown.as_ref(),
) {
responder_runtime
.mark_interaction_complete(candidate.interaction.id.0);
continue;
}
let operation =
super::bridge_protocol::BridgeSupervisorRotationOperationReceipt {
operation_id: submission.operation_id,
target:
super::bridge_protocol::BridgeSupervisorRotationTargetReceipt {
target: submission.target.clone(),
target_epoch: submission.target_epoch,
},
};
if responder_reject_next_rotation.swap(false, Ordering::Relaxed) {
responder_rotation_operations.write().await.insert(
submission.operation_id,
super::bridge_protocol::BridgeSupervisorRotationState::Rejected {
receipt: super::bridge_protocol::BridgeSupervisorRotationRejectionReceipt {
operation,
cause: super::bridge_protocol::BridgeSupervisorRotationRejectionCause::Internal,
reason: "injected terminal rotation rejection".to_string(),
},
},
);
responder_runtime
.mark_interaction_complete(candidate.interaction.id.0);
continue;
}
let target =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
submission.target.clone(),
)
.expect("valid rotation target");
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
target.clone(),
"publish rotated supervisor trust",
)
.await;
let previous = responder_supervisor_state.write().await.replace(
HarnessSupervisorState {
supervisor: target.clone(),
epoch: submission.target_epoch,
},
);
responder_rotation_operations.write().await.insert(
submission.operation_id,
super::bridge_protocol::BridgeSupervisorRotationState::Completed {
receipt: operation,
},
);
if let Some(previous) = previous {
remove_test_peer_projection_trust(
responder_runtime.as_ref(),
&previous.supervisor.peer_id.to_string(),
"revoke previous supervisor after operation commit",
)
.await;
}
}
responder_runtime.mark_interaction_complete(candidate.interaction.id.0);
}
meerkat_core::interaction::InteractionContent::Request {
intent,
params,
..
} => {
responder_received_intents
.write()
.await
.push(intent.clone());
responder_runtime
.peer_interaction_handle()
.expect("live external peer semantic authority")
.request_received(
PeerCorrelationId::from_uuid(candidate.interaction.id.0),
candidate.interaction.handling_mode,
)
.expect("record inbound peer request before response");
let bridge_parse: Result<super::bridge_protocol::BridgeCommand, _> =
serde_json::from_value(params.clone());
let is_rotation_observation = matches!(
&bridge_parse,
Ok(super::bridge_protocol::BridgeCommand::ObserveSupervisorRotation(_))
);
let drop_authorize_response = matches!(
&bridge_parse,
Ok(super::bridge_protocol::BridgeCommand::AuthorizeSupervisor(
_
))
) && responder_drop_next_authorize_response
.swap(false, Ordering::Relaxed);
let use_correlated_bridge_reply = bridge_parse.is_ok()
&& candidate
.ingress
.declared_reply_endpoint
.as_ref()
.is_some_and(|endpoint| {
endpoint.transport() == meerkat_core::comms::PeerTransport::Tcp
});
let is_attempted_rotation_observer = match &bridge_parse {
Ok(
super::bridge_protocol::BridgeCommand::ObserveSupervisorRotation(
payload,
),
) => responder_rotation_operations
.read()
.await
.get(&payload.operation_id)
.is_some_and(|state| {
let target = match state {
super::bridge_protocol::BridgeSupervisorRotationState::Pending {
operation,
..
} => &operation.target,
super::bridge_protocol::BridgeSupervisorRotationState::Completed {
receipt,
} => &receipt.target,
super::bridge_protocol::BridgeSupervisorRotationState::Rejected {
receipt,
} => &receipt.operation.target,
_ => return false,
};
target.target.peer_id == payload.observer.peer_id
&& target.target.pubkey == payload.observer.pubkey
&& target.target_epoch == payload.observer_epoch
}),
_ => false,
};
if is_rotation_observation
&& (responder_suppress_rotation_observations.load(Ordering::Relaxed)
|| (is_attempted_rotation_observer
&& responder_drop_next_attempted_rotation_observation_response
.swap(false, Ordering::Relaxed)))
{
responder_runtime.mark_interaction_complete(candidate.interaction.id.0);
continue;
}
let to = if use_correlated_bridge_reply {
// Production keeps bridge callbacks
// correlation-scoped: it stages only the
// authenticated TCP callback carried by each signed
// request and never mints durable trust for a probe.
stage_candidate_sender_for_correlated_reply(
responder_runtime.as_ref(),
&candidate,
)
.await
} else {
trust_candidate_sender_for_reply(responder_runtime.as_ref(), &candidate)
.await
};
let mut remove_supervisors_after_response = Vec::new();
let response = if let Ok(command) = bridge_parse {
match command {
super::bridge_protocol::BridgeCommand::BindMember(payload) => {
responder_bind_protocol_versions
.write()
.await
.push(payload.protocol_version);
if responder_fail_next_bind.swap(false, Ordering::Relaxed) {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Internal,
reason: "bind member failed: injected test failure"
.to_string(),
},
)
.expect("bind rejection")
} else {
let garble_reply = responder_garble_next_bind
.swap(false, Ordering::Relaxed);
// Mirror generated BindMember admission: unbound state
// allows bootstrap; bound state only accepts an exact
// idempotent retry from the same supervisor at the same
// epoch from the same sender. Anything else is a typed
// rejection — mob-level tests must see the same rejection
// vocabulary as production or the gate is invisible.
let sender_label = candidate.interaction.from.clone();
let sender_peer_id = candidate
.ingress
.route
.as_ref()
.map(|route| route.peer_id.to_string())
.or_else(|| {
candidate
.interaction
.from_route
.as_ref()
.map(std::string::ToString::to_string)
});
let mut state_guard =
responder_supervisor_state.write().await;
let gate_result: Result<bool, (super::bridge_protocol::BridgeRejectionCause, String)> = match state_guard.as_ref() {
None => {
if sender_peer_id.as_deref()
!= Some(payload.supervisor.peer_id.as_str())
{
Err((
super::bridge_protocol::BridgeRejectionCause::SenderMismatch,
format!(
"bind member failed: request sender '{sender_label}' does not match supervisor '{}'",
payload.supervisor.peer_id
),
))
} else {
Ok(true) // bootstrap: bind fresh
}
}
Some(current) => {
if payload.supervisor.peer_id
!= current.supervisor.peer_id.to_string()
{
Err((
super::bridge_protocol::BridgeRejectionCause::AlreadyBound,
format!(
"bind member failed: supervisor already bound as '{}'; use authorize_supervisor to rotate",
current.supervisor.peer_id
),
))
} else if payload.epoch != current.epoch {
Err((
super::bridge_protocol::BridgeRejectionCause::AlreadyBound,
format!(
"bind member failed: epoch {} does not match bound supervisor epoch {}; use authorize_supervisor to rotate",
payload.epoch, current.epoch
),
))
} else {
let current_peer_id =
current.supervisor.peer_id.to_string();
if sender_peer_id.as_deref()
!= Some(current_peer_id.as_str())
{
Err((
super::bridge_protocol::BridgeRejectionCause::SenderMismatch,
format!(
"bind member failed: request sender '{sender_label}' does not match authorized supervisor '{}'",
current.supervisor.peer_id
),
))
} else {
Ok(false) // idempotent ack, no mutation
}
}
}
};
match gate_result {
Err((cause, reason)) => serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause,
reason,
},
)
.expect("bind rejection"),
Ok(mutate) => {
if mutate {
responder_bind_count
.fetch_add(1, Ordering::Relaxed);
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor.clone(),
)
.expect("valid supervisor spec");
responder_supervisor_addresses
.write()
.await
.push(supervisor_spec.address.to_string());
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
supervisor_spec,
"bind supervisor",
)
.await;
*state_guard = Some(HarnessSupervisorState {
supervisor:
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor.clone(),
)
.expect("valid supervisor spec"),
epoch: payload.epoch,
});
}
if garble_reply {
// The remote bind has already committed, but
// the terminal reply cannot authenticate that
// outcome. This is the fast equivalent of a
// post-send response timeout.
serde_json::json!({
"r044": "garbled bind reply after remote commit"
})
} else {
serde_json::to_value(
super::bridge_protocol::BridgeReply::BindMember(
super::bridge_protocol::BridgeBindResponse {
peer_id: responder_bind_peer_id_override
.write()
.await
.take()
.unwrap_or_else(|| {
responder_runtime
.public_key()
.to_peer_id()
.to_string()
}),
address: responder_runtime_address.clone(),
capabilities:
super::bridge_protocol::BridgeCapabilities {
deliver_member_input: true,
observe_member: true,
interrupt_member: true,
hard_cancel_member: false,
tracked_input_cancel: false,
retire_member: true,
destroy_member: true,
wire_member: true,
unwire_member: true,
..super::bridge_protocol::BridgeCapabilities::default()
},
},
),
)
.expect("bind response")
}
}
}
}
}
super::bridge_protocol::BridgeCommand::AuthorizeSupervisor(
payload,
) => {
let sender_label = candidate.interaction.from.clone();
let sender_peer_id = candidate
.ingress
.route
.as_ref()
.map(|route| route.peer_id.to_string())
.or_else(|| {
candidate
.interaction
.from_route
.as_ref()
.map(std::string::ToString::to_string)
});
if consume_countdown_failure(
&responder_fail_authorize_countdown,
) {
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor.clone(),
)
.expect("valid supervisor spec");
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
supervisor_spec,
"trust supervisor for rejection",
)
.await;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Internal,
reason: "authorize supervisor failed: injected test failure".to_string(),
},
)
.expect("authorize rejection")
} else {
let mut guard = responder_supervisor_state.write().await;
if let Some(current) = guard.as_ref() {
if payload.epoch < current.epoch {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: format!(
"authorize supervisor failed: stale supervisor epoch {} (current {})",
payload.epoch, current.epoch
),
},
)
.expect("authorize stale rejection")
} else {
let current_peer_id =
current.supervisor.peer_id.to_string();
if sender_peer_id.as_deref()
!= Some(current_peer_id.as_str())
{
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::SenderMismatch,
reason: format!(
"authorize supervisor failed: request sender '{sender_label}' does not match authorized supervisor '{}'",
current.supervisor.peer_id
),
},
)
.expect("authorize sender mismatch rejection")
} else {
if payload.epoch > current.epoch
|| payload.supervisor.peer_id
!= current.supervisor.peer_id.to_string()
{
remove_supervisors_after_response.push(
meerkat_comms::PubKey::new(
current.supervisor.pubkey,
),
);
}
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor.clone(),
)
.expect("valid supervisor spec");
responder_supervisor_addresses
.write()
.await
.push(supervisor_spec.address.to_string());
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
supervisor_spec,
"authorize supervisor",
)
.await;
*guard = Some(HarnessSupervisorState {
supervisor:
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor,
)
.expect("valid supervisor spec"),
epoch: payload.epoch,
});
serde_json::to_value(
super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
),
)
.expect("authorize ack")
}
}
} else {
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor.clone(),
)
.expect("valid supervisor spec");
responder_supervisor_addresses
.write()
.await
.push(supervisor_spec.address.to_string());
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
supervisor_spec,
"trust supervisor for initial rejection",
)
.await;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
reason: "authorize supervisor failed: use bind_member to establish initial supervisor authority".to_string(),
},
)
.expect("authorize initial rejection")
}
}
}
super::bridge_protocol::BridgeCommand::RevokeSupervisor(
payload,
) => {
if responder_fail_next_revoke.swap(false, Ordering::Relaxed) {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Internal,
reason: "revoke supervisor failed: injected test failure"
.to_string(),
},
)
.expect("revoke rejection")
} else if responder_supervisor_state.read().await.is_none() {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
reason: "revoke supervisor found no active binding"
.to_string(),
},
)
.expect("already-revoked rejection")
} else {
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor,
)
.expect("valid supervisor spec");
remove_supervisors_after_response.push(
meerkat_comms::PubKey::new(supervisor_spec.pubkey),
);
*responder_supervisor_state.write().await = None;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
),
)
.expect("revoke ack")
}
}
super::bridge_protocol::BridgeCommand::InterruptMember(_) => {
responder_interrupt_count.fetch_add(1, Ordering::Relaxed);
serde_json::to_value(super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
))
.expect("interrupt ack")
}
super::bridge_protocol::BridgeCommand::HardCancelMember(
payload,
) => {
responder_hard_cancel_reasons
.write()
.await
.push(payload.reason);
serde_json::to_value(super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
))
.expect("hard-cancel ack")
}
super::bridge_protocol::BridgeCommand::DeliverMemberInput(
payload,
) => {
let current = responder_supervisor_state
.read()
.await
.clone()
.expect("delivery supervisor state");
if payload.epoch != current.epoch {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: format!(
"stale supervisor epoch {} (current {})",
payload.epoch, current.epoch
),
},
)
.expect("stale delivery rejection")
} else if payload.supervisor.peer_id
!= current.supervisor.peer_id.to_string()
{
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: format!(
"stale supervisor peer '{}' (current '{}')",
payload.supervisor.peer_id,
current.supervisor.peer_id
),
},
)
.expect("stale supervisor rejection")
} else {
let mut responses = responder_delivery_responses.write().await;
if let Some(existing) =
responses.get(&payload.input_id).cloned()
{
serde_json::to_value(
super::bridge_protocol::BridgeReply::Delivery(existing),
)
.expect("existing delivery response")
} else {
let supervisor_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.supervisor,
)
.expect("valid supervisor spec");
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
supervisor_spec,
"refresh supervisor for delivery",
)
.await;
responder_delivered_inputs
.write()
.await
.push(payload.input_id.clone());
let response =
super::bridge_protocol::BridgeDeliveryResponse {
input_id: payload.input_id.clone(),
canonical_input_id: Some(
Uuid::new_v5(
&Uuid::NAMESPACE_URL,
payload.input_id.as_bytes(),
)
.to_string(),
),
outcome: super::bridge_protocol::BridgeDeliveryOutcome::Accepted,
};
responses
.insert(payload.input_id.clone(), response.clone());
serde_json::to_value(
super::bridge_protocol::BridgeReply::Delivery(response),
)
.expect("delivery response")
}
}
}
super::bridge_protocol::BridgeCommand::WireMember(payload) => {
let peer_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.peer_spec,
)
.expect("valid peer spec");
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
peer_spec,
"wire trust",
)
.await;
serde_json::to_value(super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
))
.expect("wire ack")
}
super::bridge_protocol::BridgeCommand::UnwireMember(payload) => {
if responder_fail_next_unwire.swap(false, Ordering::Relaxed) {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Internal,
reason: "unwire member failed: injected test failure"
.to_string(),
},
)
.expect("unwire rejection")
} else {
let peer_spec =
meerkat_core::comms::TrustedPeerDescriptor::try_from(
payload.peer_spec,
)
.expect("valid peer spec");
remove_test_peer_projection_trust(
responder_runtime.as_ref(),
&peer_spec.peer_id.to_string(),
"unwire trust",
)
.await;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Ack(
super::bridge_protocol::BridgeAck { ok: true },
),
)
.expect("unwire ack")
}
}
super::bridge_protocol::BridgeCommand::RetireMember(_) => {
responder_retire_count.fetch_add(1, Ordering::Relaxed);
if responder_fail_next_retire.swap(false, Ordering::Relaxed) {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Internal,
reason: "retire member failed: injected cleanup uncertainty".to_string(),
},
)
.expect("retire rejection")
} else {
state = super::bridge_protocol::BridgeMemberRuntimeState::Retired;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Retire(
super::bridge_protocol::BridgeRetireResponse {
inputs_abandoned: 0,
inputs_pending_drain: 0,
},
),
)
.expect("retire response")
}
}
super::bridge_protocol::BridgeCommand::DestroyMember(_) => {
state =
super::bridge_protocol::BridgeMemberRuntimeState::Destroyed;
serde_json::to_value(
super::bridge_protocol::BridgeReply::Destroy(
super::bridge_protocol::BridgeDestroyResponse {
inputs_abandoned: 0,
},
),
)
.expect("destroy response")
}
super::bridge_protocol::BridgeCommand::ObserveMember(_) => {
serde_json::to_value(
super::bridge_protocol::BridgeReply::Observation(
super::bridge_protocol::BridgeObservationResponse::new(
state,
Some(matches!(
state,
super::bridge_protocol::BridgeMemberRuntimeState::Idle
| super::bridge_protocol::BridgeMemberRuntimeState::Attached
| super::bridge_protocol::BridgeMemberRuntimeState::Running
)),
None,
Some(
super::bridge_protocol::BridgePeerConnectivity::Reachable,
),
None,
Utc::now().to_rfc3339(),
),
),
)
.expect("observe response")
}
super::bridge_protocol::BridgeCommand::ObserveSupervisorRotation(
payload,
) => {
let known_state = responder_rotation_operations
.read()
.await
.get(&payload.operation_id)
.cloned();
let retained_current =
responder_supervisor_state.read().await.clone();
let retained_current_matches = retained_current
.as_ref()
.is_some_and(|current| {
super::bridge_protocol::BridgePeerSpec::from(
current.supervisor.clone(),
) == payload.observer
&& current.epoch == payload.observer_epoch
});
let observer_authorized = known_state.as_ref().is_some_and(
|state| match state {
super::bridge_protocol::BridgeSupervisorRotationState::Pending {
operation,
..
} => {
retained_current_matches
|| (operation.target.target == payload.observer
&& operation.target.target_epoch
== payload.observer_epoch)
}
super::bridge_protocol::BridgeSupervisorRotationState::Completed {
receipt,
} => {
retained_current_matches
|| (receipt.target.target == payload.observer
&& receipt.target.target_epoch
== payload.observer_epoch)
}
super::bridge_protocol::BridgeSupervisorRotationState::Rejected {
..
} => retained_current_matches,
_ => false,
},
);
let observation = if observer_authorized {
super::bridge_protocol::BridgeSupervisorRotationObservation::Found {
state: known_state.expect("authorized known rotation"),
}
} else {
super::bridge_protocol::BridgeSupervisorRotationObservation::NotFound {
operation_id: payload.operation_id,
}
};
serde_json::to_value(
super::bridge_protocol::BridgeReply::SupervisorRotation(
observation,
),
)
.expect("rotation observation response")
}
_ => serde_json::json!({
"error": "unsupported bridge command in test harness"
}),
}
} else {
// Non-bridge lifecycle messages
match intent.as_str() {
"mob.peer_added" => {
// K15: lifecycle params are the typed wire
// contract; `peer_spec` is a `BridgePeerSpec`.
let lifecycle: meerkat_contracts::CommsPeerLifecycleParams =
serde_json::from_value(params.clone())
.expect("typed peer_added lifecycle params");
let peer_spec: meerkat_core::comms::TrustedPeerDescriptor =
lifecycle
.peer_spec
.expect("peer_added peer_spec")
.try_into()
.expect("peer_added trusted peer spec");
apply_test_peer_projection_trust(
responder_runtime.as_ref(),
peer_spec,
"peer_added trust",
)
.await;
serde_json::json!({ "ok": true })
}
"mob.peer_unwired" | "mob.peer_retired" => {
let lifecycle: meerkat_contracts::CommsPeerLifecycleParams =
serde_json::from_value(params.clone())
.expect("typed peer lifecycle params");
let peer_spec: meerkat_core::comms::TrustedPeerDescriptor =
lifecycle
.peer_spec
.expect("peer lifecycle peer_spec")
.try_into()
.expect("peer lifecycle trusted peer spec");
remove_test_peer_projection_trust(
responder_runtime.as_ref(),
&peer_spec.peer_id.to_string(),
"peer lifecycle remove trust",
)
.await;
serde_json::json!({ "ok": true })
}
_ => serde_json::json!({ "ok": true }),
}
};
if drop_authorize_response {
// Simulate a post-commit response loss: the peer
// applied the authorization, but the caller must
// converge by retrying the exact command.
responder_runtime.mark_interaction_complete(candidate.interaction.id.0);
for supervisor_pubkey in remove_supervisors_after_response {
remove_test_peer_projection_trust(
responder_runtime.as_ref(),
&supervisor_pubkey.to_peer_id().to_string(),
"remove rotated supervisor trust after dropped response",
)
.await;
}
continue;
}
if use_correlated_bridge_reply {
// Match the production bridge responder: the
// authenticated callback is one-shot and remains
// live only for this request, so consume it with
// an inline response rather than deferring the
// send behind a spawned retry task.
let interaction_id = candidate.interaction.id;
match responder_runtime
.send(CommsCommand::PeerResponse {
objective_id: None,
content_taint: None,
to,
in_reply_to: interaction_id,
status: meerkat_core::interaction::ResponseStatus::Completed,
result: response,
blocks: None,
handling_mode: None,
})
.await
{
Ok(_) => {}
Err(error @ SendError::Transport(_)) => {
// A Response has no transport ACK. The
// one-shot probe may retire immediately after
// admitting it and reset this connection while
// the sender is still finishing its write. The
// production responder treats that outcome as
// ambiguous and continues; the harness must do
// the same so a retry can establish terminality.
tracing::warn!(
%interaction_id,
%error,
"correlated rotation response send was not confirmed"
);
}
Err(error) => {
panic!(
"send correlated rotation observation response failed: {error}"
);
}
}
responder_runtime.mark_interaction_complete(interaction_id.0);
for supervisor_pubkey in remove_supervisors_after_response {
remove_test_peer_projection_trust(
responder_runtime.as_ref(),
&supervisor_pubkey.to_peer_id().to_string(),
"remove rotated supervisor trust",
)
.await;
}
continue;
}
// `trust_candidate_sender_for_reply` records trust for
// the requester, but registration propagates
// asynchronously. A short-lived alternate-authority
// probe can disappear before that route lands. Retry
// the response in an owned task so one dead probe can
// never head-of-line block later exact-operation
// submissions/observations in this single harness
// responder. The JoinSet is dropped with the harness,
// aborting any remaining dead-probe retries.
let reply_runtime = Arc::clone(&responder_runtime);
let interaction_id = candidate.interaction.id;
reply_tasks.spawn(async move {
let send_deadline =
tokio::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
match reply_runtime
.send(CommsCommand::PeerResponse {
objective_id: None,
content_taint: None,
to: to.clone(),
in_reply_to: interaction_id,
status:
meerkat_core::interaction::ResponseStatus::Completed,
result: response.clone(),
blocks: None,
handling_mode: None,
})
.await
{
Ok(_) => break,
Err(meerkat_core::comms::SendError::PeerNotFound(_))
if tokio::time::Instant::now() < send_deadline =>
{
tokio::time::sleep(std::time::Duration::from_millis(25))
.await;
}
Err(error) => {
panic!("send live external peer response failed: {error}");
}
}
}
reply_runtime.mark_interaction_complete(interaction_id.0);
for supervisor_pubkey in remove_supervisors_after_response {
remove_test_peer_projection_trust(
reply_runtime.as_ref(),
&supervisor_pubkey.to_peer_id().to_string(),
"remove rotated supervisor trust",
)
.await;
}
});
}
_ => {
responder_runtime.mark_interaction_complete(candidate.interaction.id.0);
}
}
}
}
});
LiveExternalPeerHarness {
binding,
runtime,
bind_count,
interrupt_count,
bind_protocol_versions,
delivered_inputs,
hard_cancel_reasons,
received_intents,
supervisor_addresses,
rotation_operation_ids,
suppress_rotation_observations,
drop_next_attempted_rotation_observation_response,
drop_next_authorize_response,
reject_next_rotation,
supervisor_state,
bind_peer_id_override,
fail_authorize_countdown,
fail_next_bind,
fail_next_retire,
fail_next_revoke,
fail_next_unwire,
garble_next_bind,
retire_count,
task,
}
}
/// Create a MobHandle using the mock session service.
async fn create_test_mob(definition: MobDefinition) -> (MobHandle, Arc<MockSessionService>) {
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
#[tokio::test]
async fn actor_startup_prunes_crash_recovered_stale_member_operator_rows() {
let definition = with_unique_mob_id(sample_definition(), "operator-startup-prune");
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let stale = crate::store::MobMemberOperatorRequestRecord::pending(
crate::store::MobMemberOperatorRequestKey::new(
"member-from-crashed-incarnation",
7,
11,
"host-from-crashed-incarnation",
3,
"session-from-crashed-incarnation",
"request-from-crashed-incarnation",
),
"0".repeat(64),
);
runtime_metadata
.begin_member_operator_request(&mob_id, &stale)
.await
.expect("seed crash-recovered stale member-operator row");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob over crash-recovered operator ledger");
handle
.query_machine_state()
.await
.expect("startup barrier after operator ledger recovery prune");
assert!(
runtime_metadata
.list_member_operator_requests(&mob_id)
.await
.expect("list operator ledger after actor startup")
.is_empty(),
"actor startup must converge stale durable negative memory before serving commands"
);
}
/// Deterministic member-operator race seam. Pending is persisted by the real
/// in-memory ledger, then execution pauses immediately before routing a real
/// actor command through an exact fenced handle.
struct OperatorExecutionRaceSeams {
handle: MobHandle,
ledger: Arc<InMemoryMobRuntimeMetadataStore>,
mob_id: MobId,
authority: crate::store::MobMemberOperatorAuthorityRecord,
execution_entered: tokio::sync::Notify,
execution_release: tokio::sync::Notify,
operator_effects: AtomicUsize,
typed_stale_errors: AtomicUsize,
}
#[async_trait]
impl super::upcall_responder::UpcallServeSeams for OperatorExecutionRaceSeams {
async fn admit(
&self,
_request: &super::handle::MemberOperatorAdmissionRequest,
) -> Result<super::handle::MemberOperatorAdmissionVerdict, MobError> {
Ok(super::handle::MemberOperatorAdmissionVerdict::Admitted)
}
async fn prune_stale_requests(&self) -> Result<u64, MobError> {
Ok(0)
}
async fn load_authority_record(
&self,
_agent_identity: &AgentIdentity,
requester_generation: u64,
) -> Result<Option<crate::store::MobMemberOperatorAuthorityRecord>, MobStoreError> {
Ok((self.authority.generation == requester_generation).then(|| self.authority.clone()))
}
async fn begin_request(
&self,
record: &crate::store::MobMemberOperatorRequestRecord,
) -> Result<crate::store::MobMemberOperatorRequestBegin, MobStoreError> {
self.ledger
.begin_member_operator_request(&self.mob_id, record)
.await
}
async fn terminalize_request(
&self,
expected: &crate::store::MobMemberOperatorRequestRecord,
record: &crate::store::MobMemberOperatorRequestRecord,
) -> Result<bool, MobStoreError> {
self.ledger
.compare_and_put_member_operator_request(&self.mob_id, expected, record)
.await
}
async fn load_request(
&self,
record: &crate::store::MobMemberOperatorRequestRecord,
) -> Result<Option<crate::store::MobMemberOperatorRequestRecord>, MobStoreError> {
self.ledger
.load_member_operator_request(&self.mob_id, &record.key())
.await
}
async fn execute(
&self,
pending: &crate::store::MobMemberOperatorRequestRecord,
_record: &crate::store::MobMemberOperatorAuthorityRecord,
_request_id: &str,
_op: super::bridge_protocol::MemberOperatorOp,
) -> Result<super::member_upcall::UpcallToolOutcome, super::upcall_responder::UpcallExecuteError>
{
self.execution_entered.notify_one();
self.execution_release.notified().await;
let fenced = self.handle.clone().with_command_authority(
crate::control_policy::CommandAuthority::remote_member_operator(
crate::control_policy::MemberOperatorExecutionFence {
agent_identity: pending.agent_identity.clone(),
host_id: pending.host_id.clone(),
host_binding_generation: pending.host_binding_generation,
requester_member_session_id: pending.member_session_id.clone(),
generation: pending.generation,
fence_token: pending.fence_token,
},
),
);
let outcome = async {
fenced.validate_command_authority().await?;
fenced
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::BeginHostBind {
host_id: crate::machines::mob_machine::HostId::from(
"operator-race-side-effect-host",
),
expected_endpoint: crate::machines::mob_machine::PeerAddress::from(
"tcp://operator-race-side-effect-host",
),
binding_generation: 1,
},
)
.await
}
.await;
match outcome {
Ok(effects) => {
assert!(effects.iter().any(|effect| matches!(
effect,
crate::machines::mob_machine::MobMachineEffect::RequestHostBind {
host_id,
binding_generation: 1,
..
} if host_id.as_str() == "operator-race-side-effect-host"
)));
self.operator_effects.fetch_add(1, Ordering::SeqCst);
Ok(super::member_upcall::UpcallToolOutcome::Ok {
content: "{\"effect\":true}".to_string(),
is_error: false,
})
}
Err(error) => {
if matches!(error, MobError::StaleMemberOperatorAuthority { .. }) {
self.typed_stale_errors.fetch_add(1, Ordering::SeqCst);
}
Ok(super::member_upcall::UpcallToolOutcome::from_tool_error(
&ToolError::execution_failed(error.to_string()),
))
}
}
}
}
async fn apply_operator_race_machine_input(
handle: &MobHandle,
input: crate::machines::mob_machine::MobMachineInput,
) {
handle
.apply_machine_input_effects(input)
.await
.expect("operator execution-fence race setup transition");
}
async fn seed_operator_execution_residency(handle: &MobHandle) {
use crate::machines::mob_machine as dsl;
let identity = dsl::AgentIdentity::from("member-a");
let host = dsl::HostId::from("host-a");
let runtime_id = dsl::AgentRuntimeId::from("member-a:0");
let spawn_id = dsl::PlacedSpawnId("operator-race-spawn".to_string());
let profile_digest = "operator-race-profile-digest".to_string();
let spec_digest = "operator-race-spec-digest".to_string();
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::BindOwnerBridgeSession {
bridge_session_id: dsl::SessionId::from("operator-race-owner-session"),
destroy_on_owner_archive: false,
implicit_delegation_mob: false,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::BeginHostBind {
host_id: host.clone(),
expected_endpoint: dsl::PeerAddress::from("tcp://host-a"),
binding_generation: 1,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::CommitHostBind {
host_id: host.clone(),
pubkey: dsl::PeerSigningKey([9; 32]),
endpoint: dsl::PeerAddress::from("tcp://host-a"),
epoch: 1,
binding_generation: 1,
protocol_min: 4,
protocol_max: 4,
engine_version: "operator-race-host".to_string(),
durable_sessions: true,
autonomous_members: true,
hard_cancel_member: true,
tracked_input_cancel: true,
memory_store: true,
mcp: true,
resolvable_providers: BTreeSet::new(),
approval_forwarding: false,
live_endpoint: None,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: identity.clone(),
profile_name: "worker".to_string(),
model: "test-model".to_string(),
profile_material_digest: profile_digest.clone(),
tool_config_digest: "operator-race-tool-digest".to_string(),
skills_digest: "operator-race-skills-digest".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: false,
resolved_spec_digest: Some(spec_digest.clone()),
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::BeginSpawnExec {
agent_identity: identity.clone(),
agent_runtime_id: runtime_id.clone(),
fence_token: dsl::FenceToken(1),
generation: dsl::Generation(0),
profile_material_digest: profile_digest.clone(),
external_addressable: false,
runtime_mode: dsl::SpawnPolicyRuntimeMode::TurnDriven,
bridge_session_id: None,
replacing: None,
placement: Some(host.clone()),
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: false,
default_llm_client_override_present: false,
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: Some(spawn_id.clone()),
placed_provision_operation_id: Some("operator-race-provision".to_string()),
placed_operation_owner_session_id: Some(dsl::SessionId::from(
"operator-race-owner-session",
)),
effective_profile_override_present: false,
effective_model_override_present: false,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::CommitSpawnMembership {
agent_identity: identity.clone(),
agent_runtime_id: runtime_id,
fence_token: dsl::FenceToken(1),
generation: dsl::Generation(0),
profile_material_digest: profile_digest,
external_addressable: false,
runtime_mode: dsl::SpawnPolicyRuntimeMode::TurnDriven,
bridge_session_id: Some(dsl::SessionId::from("member-session-a")),
replacing: None,
member_peer_endpoint: Some(dsl::MemberPeerEndpoint {
name: dsl::PeerName::from("member-a"),
peer_id: dsl::PeerId::from("peer-member-a"),
address: dsl::PeerAddress::from("tcp://member-a"),
signing_key: dsl::PeerSigningKey([3; 32]),
}),
spec_digest_echo: Some(spec_digest),
ack_engine_version: Some("operator-race-host".to_string()),
placed_spawn_id: Some(spawn_id),
provision_operation_id: Some("operator-race-provision".to_string()),
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::CommitSpawnActivation {
agent_identity: identity,
},
)
.await;
}
async fn promote_operator_execution_residency_to_g2(handle: &MobHandle) {
use crate::machines::mob_machine as dsl;
let identity = dsl::AgentIdentity::from("member-a");
let host = dsl::HostId::from("host-a");
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::RevokeHost {
host_id: host.clone(),
binding_generation: 1,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::BeginHostBind {
host_id: host.clone(),
expected_endpoint: dsl::PeerAddress::from("tcp://host-a"),
binding_generation: 2,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::CommitHostBind {
host_id: host.clone(),
pubkey: dsl::PeerSigningKey([9; 32]),
endpoint: dsl::PeerAddress::from("tcp://host-a"),
epoch: 1,
binding_generation: 2,
protocol_min: 4,
protocol_max: 4,
engine_version: "operator-race-host".to_string(),
durable_sessions: true,
autonomous_members: true,
hard_cancel_member: true,
tracked_input_cancel: true,
memory_store: true,
mcp: true,
resolvable_providers: BTreeSet::new(),
approval_forwarding: false,
live_endpoint: None,
},
)
.await;
apply_operator_race_machine_input(
handle,
dsl::MobMachineInput::PromoteCommittedPlacedSpawnCarrierBinding {
spawn_id: dsl::PlacedSpawnId("operator-race-spawn".to_string()),
agent_identity: identity,
generation: dsl::Generation(0),
fence_token: dsl::FenceToken(1),
host_id: host,
expected_host_binding_generation: 1,
host_binding_generation: 2,
},
)
.await;
}
#[tokio::test]
async fn pending_member_operator_request_is_revalidated_at_actor_dispatch_after_host_promotion() {
use crate::runtime::bridge_protocol::{
BridgeMemberOperatorPayload, BridgeProtocolVersion, MemberOperatorOp, MemberOperatorOutcome,
};
let definition = with_unique_mob_id(sample_definition(), "operator-execution-fence-race");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let ledger = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
ledger.clone(),
);
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create operator execution-fence race mob");
seed_operator_execution_residency(&handle).await;
let seams = Arc::new(OperatorExecutionRaceSeams {
mob_id: handle.mob_id().clone(),
handle: handle.clone(),
ledger,
authority: crate::store::MobMemberOperatorAuthorityRecord {
agent_identity: "member-a".to_string(),
generation: 0,
can_create_mobs: false,
can_mutate_profiles: false,
can_run_adaptive_packs: false,
managed_mob_scope: BTreeSet::new(),
spawn_profile_scope: BTreeMap::new(),
},
execution_entered: tokio::sync::Notify::new(),
execution_release: tokio::sync::Notify::new(),
operator_effects: AtomicUsize::new(0),
typed_stale_errors: AtomicUsize::new(0),
});
let g1 = BridgeMemberOperatorPayload {
agent_identity: "member-a".to_string(),
requester_generation: 0,
requester_fence_token: 1,
requester_host_id: "host-a".to_string(),
requester_host_binding_generation: 1,
requester_member_session_id: "member-session-a".to_string(),
request_id: "same-request-id".to_string(),
op: MemberOperatorOp::ListMembers,
protocol_version: BridgeProtocolVersion::V4,
};
let sender = PeerId::from_ed25519_pubkey(&[3; 32]);
let stale = tokio::spawn({
let seams = Arc::clone(&seams);
let g1 = g1.clone();
async move {
super::upcall_responder::serve_member_operator_request(seams.as_ref(), &g1, sender)
.await
}
});
seams.execution_entered.notified().await;
let pending_rows = seams
.ledger
.list_member_operator_requests(&seams.mob_id)
.await
.expect("list G1 Pending before host promotion");
assert_eq!(pending_rows.len(), 1);
assert!(matches!(
pending_rows[0].state,
crate::store::MobMemberOperatorRequestState::Pending
));
promote_operator_execution_residency_to_g2(&handle).await;
let promoted = handle
.query_machine_state()
.await
.expect("query promoted operator residency");
assert_eq!(
promoted.current_placed_spawn_host_binding_generations.get(
&crate::machines::mob_machine::AgentIdentity::from("member-a")
),
Some(&2),
);
seams.execution_release.notify_one();
let stale_reply = stale.await.expect("join stale G1 request");
assert!(matches!(
stale_reply.outcome,
MemberOperatorOutcome::Completed { .. }
));
assert_eq!(
seams.typed_stale_errors.load(Ordering::SeqCst),
1,
"actor dispatch must return the typed stale-member-operator error"
);
assert_eq!(
seams.operator_effects.load(Ordering::SeqCst),
0,
"the stale G1 Pending owner must have zero operator effects"
);
let after_stale = handle
.query_machine_state()
.await
.expect("query state after stale G1 actor dispatch");
assert!(
!after_stale
.host_bind_phase
.contains_key(&crate::machines::mob_machine::HostId::from(
"operator-race-side-effect-host"
)),
"the stale command must be rejected before its machine mutation"
);
// Session identity is an independent fence from host generation. Keep
// every G2 atom current except the session and prove the same actor gate
// rejects before a second potential mutation.
let stale_session_handle = handle.clone().with_command_authority(
crate::control_policy::CommandAuthority::remote_member_operator(
crate::control_policy::MemberOperatorExecutionFence {
agent_identity: "member-a".to_string(),
host_id: "host-a".to_string(),
host_binding_generation: 2,
requester_member_session_id: "stale-member-session".to_string(),
generation: 0,
fence_token: 1,
},
),
);
let stale_session_error = stale_session_handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::BeginHostBind {
host_id: crate::machines::mob_machine::HostId::from(
"operator-race-stale-session-host",
),
expected_endpoint: crate::machines::mob_machine::PeerAddress::from(
"tcp://operator-race-stale-session-host",
),
binding_generation: 1,
},
)
.await
.expect_err("a stale member session must fail actor execution admission");
assert!(matches!(
stale_session_error,
MobError::StaleMemberOperatorAuthority { .. }
));
assert!(
!handle
.query_machine_state()
.await
.expect("query after stale-session actor dispatch")
.host_bind_phase
.contains_key(&crate::machines::mob_machine::HostId::from(
"operator-race-stale-session-host"
)),
"the stale-session command must have zero machine effects"
);
let mut g2 = g1.clone();
g2.requester_host_binding_generation = 2;
// Store one permit so the current G2 request passes the reusable race gate
// as soon as it reaches actor dispatch.
seams.execution_release.notify_one();
let g2_reply = super::upcall_responder::serve_member_operator_request(
seams.as_ref(),
&g2,
PeerId::from_ed25519_pubkey(&[3; 32]),
)
.await;
assert!(matches!(
g2_reply.outcome,
MemberOperatorOutcome::Completed { .. }
));
assert_eq!(seams.typed_stale_errors.load(Ordering::SeqCst), 1);
assert_eq!(
seams.operator_effects.load(Ordering::SeqCst),
1,
"the exact G2 residency may execute once"
);
let after_g2 = handle
.query_machine_state()
.await
.expect("query state after current G2 actor dispatch");
assert_eq!(
after_g2
.host_bind_phase
.get(&crate::machines::mob_machine::HostId::from(
"operator-race-side-effect-host"
)),
Some(&crate::machines::mob_machine::HostBindPhase::Requested),
);
let g2_replay = super::upcall_responder::serve_member_operator_request(
seams.as_ref(),
&g2,
PeerId::from_ed25519_pubkey(&[3; 32]),
)
.await;
assert_eq!(
g2_replay, g2_reply,
"the exact G2 duplicate must replay its immutable terminal"
);
assert_eq!(
seams.operator_effects.load(Ordering::SeqCst),
1,
"an exact G2 duplicate must not re-execute the operator effect"
);
let rows = seams
.ledger
.list_member_operator_requests(&seams.mob_id)
.await
.expect("list G1/G2 operator request rows");
assert_eq!(rows.len(), 2);
assert!(rows.iter().all(|row| row.request_id == g1.request_id));
assert_eq!(
rows.iter()
.map(|row| row.host_binding_generation)
.collect::<BTreeSet<_>>(),
BTreeSet::from([1, 2]),
"same request_id under G1 and G2 must remain distinct durable identities"
);
assert_eq!(
handle
.prune_stale_member_operator_requests()
.await
.expect("actor-authorized operator ledger prune"),
1,
"the actor must prune only the superseded G1 residency"
);
let current_rows = seams
.ledger
.list_member_operator_requests(&seams.mob_id)
.await
.expect("list current operator request rows after actor prune");
assert_eq!(current_rows.len(), 1);
assert_eq!(current_rows[0].host_binding_generation, 2);
assert!(matches!(
current_rows[0].state,
crate::store::MobMemberOperatorRequestState::Terminal { .. }
));
}
#[tokio::test]
async fn member_operator_projection_is_revalidated_after_result_capture() {
use crate::runtime::bridge_protocol::MemberOperatorOp;
use crate::runtime::member_upcall::{UpcallToolErrorClass, UpcallToolOutcome};
use crate::runtime::upcall_responder::{
MobUpcallSeams, ProjectionPostValidationGate, UpcallServeSeams,
};
use crate::store::{
MobMemberOperatorAuthorityRecord, MobMemberOperatorRequestRecord, member_operator_op_digest,
};
let definition = with_unique_mob_id(sample_definition(), "operator-projection-fence-race");
let (handle, _service) = create_test_mob(definition).await;
seed_operator_execution_residency(&handle).await;
let projection_captured = Arc::new(tokio::sync::Notify::new());
let release_validation = Arc::new(tokio::sync::Notify::new());
let metadata_store: Arc<dyn MobRuntimeMetadataStore> =
Arc::new(InMemoryMobRuntimeMetadataStore::new());
let seams = Arc::new(
MobUpcallSeams::new(handle.clone(), metadata_store).with_projection_post_validation_gate(
ProjectionPostValidationGate {
projection_captured: Arc::clone(&projection_captured),
release_validation: Arc::clone(&release_validation),
},
),
);
let op = MemberOperatorOp::ListMembers;
let pending = MobMemberOperatorRequestRecord::pending(
crate::store::MobMemberOperatorRequestKey::new(
"member-a",
0,
1,
"host-a",
1,
"member-session-a",
"projection-race-request",
),
member_operator_op_digest(&op).expect("digest projection-race operation"),
);
let authority = MobMemberOperatorAuthorityRecord {
agent_identity: "member-a".to_string(),
generation: 0,
can_create_mobs: false,
can_mutate_profiles: false,
can_run_adaptive_packs: false,
managed_mob_scope: BTreeSet::from([handle.mob_id().as_str().to_string()]),
spawn_profile_scope: BTreeMap::new(),
};
let projection = tokio::spawn({
let seams = Arc::clone(&seams);
let pending = pending.clone();
let authority = authority.clone();
let op = op.clone();
async move {
UpcallServeSeams::execute(
seams.as_ref(),
&pending,
&authority,
"projection-race-request",
op,
)
.await
.expect("member-operator projection execution")
}
});
tokio::time::timeout(Duration::from_secs(5), projection_captured.notified())
.await
.expect("ListMembers must capture a successful G1 projection before the race gate");
promote_operator_execution_residency_to_g2(&handle).await;
release_validation.notify_one();
match projection.await.expect("join fenced projection") {
UpcallToolOutcome::ToolError(error) => {
assert_eq!(error.class, UpcallToolErrorClass::ExecutionFailed);
assert_eq!(
error.data.as_ref(),
Some(&serde_json::json!({
"code": "STALE_FENCE",
"details": {},
})),
"the member-visible tool error must retain the typed stale-fence code",
);
assert!(
error
.message
.contains("stale member-operator execution authority"),
"the post-projection actor check must preserve the typed stale cause: {}",
error.message,
);
}
outcome => panic!(
"a G1 projection captured before G2 promotion must not return its stale result: {outcome:?}"
),
}
}
async fn try_create_test_mob_without_adapter(
definition: MobDefinition,
) -> Result<(MobHandle, Arc<MockSessionService>), MobError> {
let service = Arc::new(MockSessionService::new());
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await?;
Ok((handle, service))
}
async fn create_test_mob_with_runtime_adapter(
definition: MobDefinition,
) -> (MobHandle, Arc<MockSessionService>) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
async fn create_test_mob_with_events(
definition: MobDefinition,
events: Arc<dyn MobEventStore>,
) -> (MobHandle, Arc<MockSessionService>) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::with_events(events);
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
async fn create_test_mob_with_recoverable_fault_events(
definition: MobDefinition,
events: Arc<FaultInjectedMobEventStore>,
) -> (
MobHandle,
Arc<MockSessionService>,
Arc<InMemoryMobRunStore>,
Arc<InMemoryMobSpecStore>,
Arc<InMemoryMobRuntimeMetadataStore>,
Arc<InMemoryMobIdentityStore>,
Arc<InMemoryMobIdentityStatusStore>,
) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let runs = Arc::new(InMemoryMobRunStore::new());
let specs = Arc::new(InMemoryMobSpecStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let identity = Arc::new(InMemoryMobIdentityStore::new());
let identity_status = Arc::new(InMemoryMobIdentityStatusStore::new());
let storage = MobStorage::custom_with_runtime_metadata(
events,
runs.clone(),
specs.clone(),
runtime_metadata.clone(),
identity.clone(),
identity_status.clone(),
);
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create recoverable fault-injected mob");
(
handle,
service,
runs,
specs,
runtime_metadata,
identity,
identity_status,
)
}
async fn cold_resume_after_terminal_append_fail_stop(
events: Arc<FaultInjectedMobEventStore>,
runs: Arc<InMemoryMobRunStore>,
specs: Arc<InMemoryMobSpecStore>,
runtime_metadata: Arc<InMemoryMobRuntimeMetadataStore>,
identity: Arc<InMemoryMobIdentityStore>,
identity_status: Arc<InMemoryMobIdentityStatusStore>,
service: Arc<MockSessionService>,
) -> MobHandle {
MobBuilder::for_resume(MobStorage::custom_with_runtime_metadata(
events,
runs,
specs,
runtime_metadata,
identity,
identity_status,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("cold recovery should reconcile the terminal run and restart the actor")
}
async fn create_test_mob_with_run_store(
definition: MobDefinition,
run_store: Arc<dyn MobRunStore>,
) -> (MobHandle, Arc<MockSessionService>) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage {
events: Arc::new(InMemoryMobEventStore::new()),
runs: run_store,
specs: Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata: Arc::new(InMemoryMobRuntimeMetadataStore::new()),
identity: Arc::new(InMemoryMobIdentityStore::new()),
identity_member: None,
identity_status: Arc::new(InMemoryMobIdentityStatusStore::new()),
realm_profiles: None,
};
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
struct UnusedFlowTurnExecutor;
#[async_trait]
impl FlowTurnExecutor for UnusedFlowTurnExecutor {
async fn dispatch(
&self,
_run_id: &RunId,
_step_id: &crate::StepId,
_target: &AgentIdentity,
_message: ContentInput,
_turn_tool_overlay: Option<TurnToolOverlay>,
) -> Result<FlowTurnTicket, MobError> {
Err(MobError::Internal(
"unused flow turn executor dispatch called".into(),
))
}
async fn await_terminal(
&self,
_ticket: FlowTurnTicket,
_timeout: Duration,
) -> Result<FlowTurnOutcome, MobError> {
Err(MobError::Internal(
"unused flow turn executor await_terminal called".into(),
))
}
async fn on_timeout(&self, _ticket: FlowTurnTicket) -> Result<TimeoutDisposition, MobError> {
Err(MobError::Internal(
"unused flow turn executor on_timeout called".into(),
))
}
}
async fn seed_test_run_in_mob_machine(handle: &MobHandle, run_id: &RunId) {
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::CreateRunSeed {
run_id: crate::machines::mob_machine::RunId::from(run_id.to_string()),
step_ids: Default::default(),
ordered_steps: Vec::new(),
step_status: Default::default(),
output_recorded: Default::default(),
step_condition_results: Default::default(),
step_has_conditions: Default::default(),
step_dependencies: Default::default(),
step_dependency_modes: Default::default(),
step_branches: Default::default(),
step_collection_policies: Default::default(),
step_quorum_thresholds: Default::default(),
step_target_counts: Default::default(),
step_target_success_counts: Default::default(),
step_target_terminal_failure_counts: Default::default(),
escalation_threshold: 0,
max_step_retries: 0,
max_active_nodes: 0,
max_active_frames: 0,
max_frame_depth: 0,
},
)
.await
.expect("seed run in MobMachine");
}
fn authority_backed_empty_test_run(mob_id: &str, flow_id: &str) -> MobRun {
MobRun::authority_backed_for_steps(
crate::ids::RunId::new(),
crate::ids::MobId::from(mob_id),
FlowId::from(flow_id),
std::iter::empty::<crate::ids::StepId>(),
MobRunStatus::Pending,
serde_json::json!({}),
)
.expect("authority-backed empty flow run")
}
fn adaptive_test_limits() -> AdaptiveRunLimits {
AdaptiveRunLimits {
max_depth: 3,
max_total_decisions: 5,
max_repair_attempts: 2,
max_layer_failures: 2,
max_attempts_per_layer: 2,
max_members_per_layer: 4,
max_total_spawned_members: 8,
max_active_members: 4,
max_retained_layer_mobs: 1,
max_aggregate_tokens: 10_000,
max_aggregate_tool_calls: 100,
allowed_model_classes: BTreeSet::from(["standard".to_string()]),
allowed_tool_classes: BTreeSet::from(["readonly".to_string()]),
allowed_skill_identities: BTreeSet::from(["audit".to_string()]),
allowed_auth_binding_refs: BTreeSet::from(["default".to_string()]),
deadline_ms: 10_000,
}
}
fn adaptive_test_operation_deadline() -> crate::adaptive::AdaptiveOperationDeadline {
crate::adaptive::AdaptiveOperationDeadline::from_policy(
&crate::adaptive::AdaptivePolicy {
limits: crate::adaptive::AdaptiveLimitRecord {
max_depth: 3,
max_total_decisions: 5,
max_repair_attempts: 2,
max_layer_failures: 2,
max_attempts_per_layer: 2,
max_members_per_layer: 4,
max_total_spawned_members: 8,
max_active_members: 4,
max_retained_layer_mobs: 1,
max_wall_clock_ms: 60_000,
max_aggregate_tokens: 10_000,
max_aggregate_tool_calls: 100,
},
..crate::adaptive::AdaptivePolicy::default()
},
1_000,
)
}
fn adaptive_admission_request(layer_id: &str, member_count: u64) -> AdaptiveLayerAdmissionRequest {
AdaptiveLayerAdmissionRequest {
layer_id: layer_id.to_string(),
attempt: 1,
plan_digest: format!("sha256:{layer_id}-plan"),
child_mob_id: format!("adaptive-test-{layer_id}-a1"),
member_count,
token_reservation: 100,
tool_call_reservation: 3,
used_model_classes: BTreeSet::from(["standard".to_string()]),
used_tool_classes: BTreeSet::from(["readonly".to_string()]),
used_skill_identities: BTreeSet::from(["audit".to_string()]),
used_auth_binding_refs: BTreeSet::from(["default".to_string()]),
observed_at_ms: 100,
}
}
#[tokio::test]
async fn adaptive_public_seam_drives_layer_to_collecting_then_completed() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let capability = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-test-run".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize adaptive run");
assert_eq!(capability.adaptive_run_id(), "adaptive-test-run");
let layer_id = "adaptive-test-run-layer-one";
handle
.record_adaptive_planning_decision(&capability, AdaptivePlanningDecisionKind::RunLayer)
.await
.expect("record planning decision");
let admission = handle
.resolve_adaptive_layer_admission(&capability, adaptive_admission_request(layer_id, 2))
.await
.expect("resolve layer admission");
assert_eq!(admission, AdaptiveLayerAdmission::Allowed);
handle
.record_adaptive_layer_provisioned(
&capability,
AdaptiveLayerAttempt {
layer_id: layer_id.to_string(),
attempt: 1,
},
)
.await
.expect("record provisioned");
let mut child_run = authority_backed_empty_test_run("adaptive-child", "layer-flow");
child_run.status = MobRunStatus::Completed;
handle
.record_adaptive_layer_run_started(
&capability,
AdaptiveLayerRunStart {
layer_id: layer_id.to_string(),
attempt: 1,
child_run_id: child_run.run_id.clone(),
},
)
.await
.expect("record run started");
handle
.ingest_adaptive_layer_terminal(
&capability,
AdaptiveLayerAttempt {
layer_id: layer_id.to_string(),
attempt: 1,
},
&child_run,
)
.await
.expect("ingest terminal child run");
let collecting = handle
.adaptive_run_snapshot(&capability)
.await
.expect("snapshot");
assert_eq!(collecting.total_decisions, 1);
assert_eq!(collecting.depth, 1);
assert_eq!(collecting.total_spawned_members, 2);
assert_eq!(collecting.active_members, 2);
assert_eq!(collecting.aggregate_token_reserved, 100);
assert_eq!(collecting.aggregate_tool_call_reserved, 3);
assert_eq!(
collecting.layers[layer_id].plan_digest.as_deref(),
Some("sha256:adaptive-test-run-layer-one-plan")
);
assert_eq!(
collecting.layers[layer_id].child_mob_id.as_deref(),
Some("adaptive-test-adaptive-test-run-layer-one-a1")
);
assert_eq!(
collecting.layers[layer_id].phase,
AdaptiveLayerPhaseView::Collecting,
"terminal ingest must not skip result validation"
);
handle
.record_adaptive_layer_result_validated(
&capability,
AdaptiveLayerResultDigest {
layer_id: layer_id.to_string(),
attempt: 1,
result_digest: "sha256:test-result".to_string(),
},
)
.await
.expect("record result");
handle
.record_adaptive_layer_mob_destroyed(
&capability,
AdaptiveLayerAttempt {
layer_id: layer_id.to_string(),
attempt: 1,
},
)
.await
.expect("record layer mob destroyed");
handle
.record_adaptive_layer_mob_destroyed(
&capability,
AdaptiveLayerAttempt {
layer_id: layer_id.to_string(),
attempt: 1,
},
)
.await
.expect("duplicate destroyed feedback must be idempotent");
let cleanup_replayed = handle
.adaptive_run_snapshot(&capability)
.await
.expect("snapshot after duplicate cleanup feedback");
assert_eq!(
cleanup_replayed.active_members, 0,
"duplicate destroyed feedback must not debit active members twice"
);
handle
.resolve_adaptive_finish(&capability, "sha256:final-result")
.await
.expect("finish run");
let finished = handle
.adaptive_run_snapshot(&capability)
.await
.expect("finished snapshot");
assert_eq!(finished.phase, Some(AdaptiveRunPhaseView::Finished));
assert_eq!(
finished.stop_reason,
Some(AdaptiveStopReasonView::FinishDecision)
);
assert_eq!(
finished.layers[layer_id].phase,
AdaptiveLayerPhaseView::Completed
);
assert_eq!(
finished.layers[layer_id].result_digest.as_deref(),
Some("sha256:test-result")
);
assert_eq!(
finished.active_members, 0,
"cleanup observation must release active-member occupancy"
);
}
#[tokio::test]
async fn adaptive_public_seam_returns_kernel_admission_denial_for_member_limit() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let capability = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-limit-run".to_string(),
limits: AdaptiveRunLimits {
max_members_per_layer: 1,
..adaptive_test_limits()
},
})
.await
.expect("initialize adaptive run");
let admission = handle
.resolve_adaptive_layer_admission(&capability, adaptive_admission_request("too-wide", 2))
.await
.expect("resolve denied admission");
assert_eq!(admission, AdaptiveLayerAdmission::Denied);
let snapshot = handle
.adaptive_run_snapshot(&capability)
.await
.expect("snapshot");
assert_eq!(snapshot.depth, 0, "denied admission must not consume depth");
assert_eq!(
snapshot.total_spawned_members, 0,
"denied admission must not debit spawned members"
);
assert_eq!(
snapshot.repair_attempts, 1,
"kernel owns admission-rejection repair accounting"
);
assert!(
!snapshot.layers.contains_key("too-wide"),
"denied layer must not be recorded as admitted"
);
}
#[tokio::test]
async fn adaptive_public_seam_requests_host_cancel_through_mob_machine() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let capability = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-cancel-run".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize adaptive run");
handle
.request_adaptive_cancel(&capability)
.await
.expect("request adaptive cancel");
let snapshot = handle
.adaptive_run_snapshot(&capability)
.await
.expect("snapshot");
assert_eq!(snapshot.phase, Some(AdaptiveRunPhaseView::Canceled));
assert_eq!(
snapshot.stop_reason,
Some(AdaptiveStopReasonView::HostCancel)
);
}
#[tokio::test]
async fn adaptive_driver_run_lease_cancels_before_layer_and_releases_run_slot() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let driver = crate::adaptive::AdaptiveDriver::new(handle.clone());
let adaptive_run_id = crate::adaptive::AdaptiveRunId::new("adaptive-run-lease-cancel")
.expect("valid leased adaptive run id");
let initialization = crate::adaptive::AdaptiveKernel::initialize_run(
&driver,
&adaptive_run_id,
&crate::adaptive::AdaptivePolicy {
limits: crate::adaptive::AdaptiveLimitRecord {
max_depth: 3,
max_total_decisions: 5,
max_repair_attempts: 2,
max_layer_failures: 2,
max_attempts_per_layer: 2,
max_members_per_layer: 4,
max_total_spawned_members: 8,
max_active_members: 4,
max_retained_layer_mobs: 1,
max_wall_clock_ms: 10_000,
max_aggregate_tokens: 10_000,
max_aggregate_tool_calls: 100,
},
allowed_model_classes: BTreeSet::from(["standard".to_string()]),
allowed_tool_classes: BTreeSet::from(["readonly".to_string()]),
allowed_skill_classes: BTreeSet::from(["audit".to_string()]),
allowed_auth_bindings: BTreeSet::from(["default".to_string()]),
..crate::adaptive::AdaptivePolicy::default()
},
0,
);
let (capability, run_lease) = initialization
.resolve()
.await
.expect("initialize leased adaptive run");
drop(run_lease);
let snapshot = tokio::time::timeout(Duration::from_secs(3), async {
loop {
let snapshot = handle
.adaptive_run_snapshot(&capability)
.await
.expect("read leased-run cancellation snapshot");
if snapshot.phase == Some(AdaptiveRunPhaseView::Canceled) {
break snapshot;
}
tokio::task::yield_now().await;
}
})
.await
.expect("prestarted run guardian must close the canceled run");
assert_eq!(
snapshot.stop_reason,
Some(AdaptiveStopReasonView::HostCancel)
);
let state = handle
.query_machine_state()
.await
.expect("read machine after leased-run cancellation");
assert_eq!(
state.adaptive_active_run, None,
"typed host cancellation must release the canonical adaptive-run slot"
);
let next = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-run-after-lease-cancel".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("a second adaptive run must initialize after cancellation");
handle
.request_adaptive_cancel(&next)
.await
.expect("close second adaptive test run");
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn adaptive_loop_real_planning_error_cancels_run_and_allows_next_initialization() {
let (handle, service) = create_test_mob(sample_definition()).await;
let adaptive_run_id = crate::adaptive::AdaptiveRunId::new("adaptive-real-plan-error")
.expect("valid adaptive run id");
let driver = crate::adaptive::AdaptiveDriver::new(handle.clone());
let mut runtime = crate::runtime::mobpack_execution::PackAdaptiveRuntime {
control_mob: handle.clone(),
session_service: service,
active_planning_run: None,
layer_created_probe: None,
};
let limits = crate::adaptive::AdaptiveLimitRecord {
max_depth: 3,
max_total_decisions: 5,
max_repair_attempts: 2,
max_layer_failures: 2,
max_attempts_per_layer: 2,
max_members_per_layer: 4,
max_total_spawned_members: 8,
max_active_members: 4,
max_retained_layer_mobs: 1,
max_wall_clock_ms: 10_000,
max_aggregate_tokens: 10_000,
max_aggregate_tool_calls: 100,
};
let error = crate::adaptive::run_adaptive_loop(
&driver,
&mut runtime,
crate::adaptive::AdaptiveRunRequest {
adaptive_run_id: adaptive_run_id.clone(),
policy: crate::adaptive::AdaptivePolicy {
limits,
..crate::adaptive::AdaptivePolicy::default()
},
compile_context: crate::adaptive::CompileContext {
adaptive_run_id: adaptive_run_id.clone(),
attempt: 1,
schema_registry: crate::adaptive::SchemaRegistry::default(),
profile_templates: BTreeMap::new(),
previous_layer_result: None,
},
objective: "Exercise a real planning failure.".to_string(),
started_at_ms: 1_000,
},
)
.await
.expect_err("missing plan flow must fail through the real runtime");
assert!(error.to_string().contains("flow not found"));
let snapshot = handle
.adaptive_run_snapshot_by_id(adaptive_run_id.as_str())
.await
.expect("read real planning-error snapshot");
assert_eq!(snapshot.phase, Some(AdaptiveRunPhaseView::Canceled));
assert_eq!(
snapshot.stop_reason,
Some(AdaptiveStopReasonView::HostCancel)
);
assert_eq!(
handle
.query_machine_state()
.await
.expect("read real planning-error machine state")
.adaptive_active_run,
None
);
let next = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-after-real-plan-error".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("planning-error cancellation must release the next run slot");
handle
.request_adaptive_cancel(&next)
.await
.expect("close next adaptive run");
}
#[tokio::test]
async fn adaptive_proven_absent_layer_cleanup_releases_admission_reservation() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let capability = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-absent-layer".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize absent-layer adaptive run");
let layer_id = "adaptive-absent-layer-create-failed";
handle
.record_adaptive_planning_decision(&capability, AdaptivePlanningDecisionKind::RunLayer)
.await
.expect("record absent-layer planning decision");
assert_eq!(
handle
.resolve_adaptive_layer_admission(&capability, adaptive_admission_request(layer_id, 2),)
.await
.expect("admit layer before create failure"),
AdaptiveLayerAdmission::Allowed
);
handle
.record_adaptive_layer_setup_fault(
&capability,
AdaptiveLayerSetupFaultObservation {
layer_id: layer_id.to_string(),
attempt: 1,
fault: AdaptiveLayerSetupFault::MobCreateFailed,
spawned_members: 0,
requested_members: 2,
},
)
.await
.expect("record handle-less mob create failure");
handle
.record_adaptive_layer_mob_destroyed(
&capability,
AdaptiveLayerAttempt {
layer_id: layer_id.to_string(),
attempt: 1,
},
)
.await
.expect("record proven physical absence through cleanup authority");
handle
.request_adaptive_cancel(&capability)
.await
.expect("cancel failed adaptive run");
let snapshot = handle
.adaptive_run_snapshot(&capability)
.await
.expect("read absent-layer cleanup snapshot");
assert_eq!(snapshot.phase, Some(AdaptiveRunPhaseView::Canceled));
assert_eq!(
snapshot.active_members, 0,
"proven physical absence must release admitted active-member accounting"
);
let state = handle
.query_machine_state()
.await
.expect("read absent-layer machine state");
assert_eq!(state.adaptive_active_run, None);
assert_eq!(
state
.adaptive_layer_disposition
.get(&crate::machines::mob_machine::AdaptiveLayerId::from(
layer_id
)),
Some(&crate::machines::mob_machine::AdaptiveLayerDispositionKind::Destroyed)
);
}
#[tokio::test]
async fn adaptive_run_snapshot_filters_layers_by_scoped_run_id() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let first = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "run-a".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize first adaptive run");
handle
.resolve_adaptive_layer_admission(&first, adaptive_admission_request("run-a-layer", 1))
.await
.expect("admit first layer");
handle
.record_adaptive_layer_setup_fault(
&first,
AdaptiveLayerSetupFaultObservation {
layer_id: "run-a-layer".to_string(),
attempt: 1,
fault: AdaptiveLayerSetupFault::SpawnFailed,
spawned_members: 0,
requested_members: 1,
},
)
.await
.expect("settle first layer setup failure");
handle
.record_adaptive_layer_mob_destroyed(
&first,
AdaptiveLayerAttempt {
layer_id: "run-a-layer".to_string(),
attempt: 1,
},
)
.await
.expect("record first layer cleanup");
handle
.resolve_adaptive_finish(&first, "sha256:first")
.await
.expect("finish first run");
let second = handle
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "run-b".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize second adaptive run");
handle
.resolve_adaptive_layer_admission(&second, adaptive_admission_request("run-b-layer", 1))
.await
.expect("admit second layer");
let first_snapshot = handle
.adaptive_run_snapshot_by_id("run-a")
.await
.expect("first snapshot");
assert!(first_snapshot.layers.contains_key("run-a-layer"));
assert!(
!first_snapshot.layers.contains_key("run-b-layer"),
"run snapshots must not project layer ledger entries from another adaptive run"
);
let second_snapshot = handle
.adaptive_run_snapshot_by_id("run-b")
.await
.expect("second snapshot");
assert!(second_snapshot.layers.contains_key("run-b-layer"));
assert!(!second_snapshot.layers.contains_key("run-a-layer"));
}
async fn seed_test_loop_in_mob_machine(
handle: &MobHandle,
loop_instance_id: &crate::ids::LoopInstanceId,
parent_frame_id: &crate::ids::FrameId,
parent_node_id: &crate::ids::FlowNodeId,
loop_id: &crate::ids::LoopId,
depth: u32,
max_iterations: u64,
) {
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::CreateLoopSeed {
loop_instance_id: crate::machines::mob_machine::LoopInstanceId::from(
loop_instance_id.as_str(),
),
parent_frame_id: crate::machines::mob_machine::FrameId::from(
parent_frame_id.as_str(),
),
parent_node_id: crate::machines::mob_machine::FlowNodeId::from(
parent_node_id.as_str(),
),
loop_id: crate::machines::mob_machine::LoopId::from(loop_id.as_str()),
depth,
max_iterations,
},
)
.await
.expect("seed loop in MobMachine");
}
async fn seed_test_body_frame_in_mob_machine(
handle: &MobHandle,
run_id: &RunId,
frame_id: &crate::ids::FrameId,
loop_instance_id: &crate::ids::LoopInstanceId,
iteration: u32,
) {
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::CreateFrameSeed {
run_id: crate::machines::mob_machine::RunId::from(run_id.to_string()),
frame_id: crate::machines::mob_machine::FrameId::from(frame_id.as_str()),
frame_scope: crate::machines::mob_machine::FrameScope::Body,
loop_instance_id: Some(crate::machines::mob_machine::LoopInstanceId::from(
loop_instance_id.as_str(),
)),
iteration,
tracked_nodes: Default::default(),
ordered_nodes: Vec::new(),
node_kind: Default::default(),
node_dependencies: Default::default(),
node_dependency_modes: Default::default(),
node_branches: Default::default(),
node_step_ids: Default::default(),
node_loop_ids: Default::default(),
node_status: Default::default(),
ready_queue: Vec::new(),
output_recorded: Default::default(),
node_condition_results: Default::default(),
last_admitted_node: None,
},
)
.await
.expect("seed body frame in MobMachine");
}
fn test_comms_name(profile: &str, agent_identity: &str) -> String {
format!("test-mob/{profile}/{agent_identity}")
}
fn test_comms_name_for(mob_id: &MobId, profile: &str, agent_identity: &str) -> String {
format!("{mob_id}/{profile}/{agent_identity}")
}
struct SpawnProvisionedCommandDelayGuard;
impl SpawnProvisionedCommandDelayGuard {
fn set(delay_ms: u64) -> Self {
super::actor::SPAWN_PROVISIONED_COMMAND_DELAY_MS.store(delay_ms, Ordering::Relaxed);
Self
}
}
impl Drop for SpawnProvisionedCommandDelayGuard {
fn drop(&mut self) {
super::actor::SPAWN_PROVISIONED_COMMAND_DELAY_MS.store(0, Ordering::Relaxed);
}
}
async fn wait_for_session_id_for_comms_name(
service: &MockSessionService,
comms_name: &str,
) -> SessionId {
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let Some(session_id) = service.session_id_for_comms_name(comms_name).await {
break session_id;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap_or_else(|_| panic!("timed out waiting for session {comms_name}"))
}
async fn test_peer_route<C>(comms: &C, name: &str) -> PeerRoute
where
C: CoreCommsRuntime + ?Sized,
{
let peers = comms.peers().await;
let entry = peers
.iter()
.find(|entry| entry.name.as_str() == name)
.unwrap_or_else(|| panic!("peer route not found for {name}; peers={peers:?}"));
PeerRoute::with_display_name(entry.peer_id, entry.name.clone())
}
fn test_trusted_peer_route(comms: &meerkat_comms::CommsRuntime, name: &str) -> PeerRoute {
let trusted = comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
if let Some(peer) = trusted_peers.iter().find(|peer| peer.name.as_str() == name) {
return PeerRoute::with_display_name(peer.pubkey.to_peer_id(), peer.name.clone());
}
let namespace = comms.inproc_namespace().unwrap_or("");
let inproc_peers = meerkat_comms::InprocRegistry::global().peers_in_namespace(namespace);
if let Some(peer) = inproc_peers.iter().find(|peer| peer.name == name) {
return PeerRoute::with_display_name(
peer.pubkey.to_peer_id(),
PeerName::new(peer.name.clone()).expect("valid inproc peer name"),
);
}
let trusted = comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
panic!("trusted or inproc peer route not found for {name}; trusted={trusted_peers:?}");
}
async fn trust_candidate_sender_for_reply(
comms: &meerkat_comms::CommsRuntime,
candidate: &meerkat_core::interaction::PeerInputCandidate,
) -> PeerRoute {
let route = candidate
.ingress
.route
.clone()
.or_else(|| {
candidate.interaction.from_route.map(|peer_id| {
PeerRoute::with_display_name(
peer_id,
PeerName::new(candidate.interaction.from.clone())
.expect("candidate sender display name should be valid"),
)
})
})
.expect("peer request candidate must carry a typed reply route");
let display_name = route
.display_name
.clone()
.or_else(|| candidate.ingress.display_name.clone())
.expect("inproc reply route must carry the sender display name");
let signing_pubkey = candidate
.ingress
.signing_pubkey
.expect("peer request candidate must carry the sender signing pubkey");
let descriptor = TrustedPeerDescriptor::unsigned_with_pubkey(
display_name.as_str(),
route.peer_id.to_string(),
signing_pubkey,
format!("inproc://{}", display_name.as_str()),
)
.expect("typed ingress sender should convert to a reply trusted peer");
apply_test_peer_projection_trust(
comms,
descriptor,
"trust typed ingress sender for bridge reply",
)
.await;
route
}
async fn stage_candidate_sender_for_correlated_reply(
comms: &meerkat_comms::CommsRuntime,
candidate: &meerkat_core::interaction::PeerInputCandidate,
) -> PeerRoute {
let route = candidate
.ingress
.route
.clone()
.or_else(|| {
candidate
.interaction
.from_route
.map(meerkat_core::PeerRoute::new)
})
.expect("peer request candidate must carry a typed reply route");
if let (Some(sender_peer_id), Some(signing_pubkey), Some(reply_endpoint)) = (
candidate.ingress.canonical_peer_id,
candidate.ingress.signing_pubkey,
candidate.ingress.declared_reply_endpoint.clone(),
) {
match CoreCommsRuntime::stage_correlated_reply_endpoint(
comms,
sender_peer_id,
candidate.interaction.id,
signing_pubkey,
reply_endpoint,
)
.await
{
Ok(()) | Err(SendError::Unsupported(_)) => {}
Err(error) => {
panic!("stage authenticated correlated bridge reply endpoint failed: {error}")
}
}
}
route
}
fn with_unique_mob_id(mut definition: MobDefinition, label: &str) -> MobDefinition {
let suffix = uuid::Uuid::new_v4().simple().to_string();
definition.id = MobId::from(format!("test-mob-{label}-{suffix}"));
definition
}
struct EchoBundleDispatcher;
#[async_trait]
impl AgentToolDispatcher for EchoBundleDispatcher {
fn tools(&self) -> Arc<[Arc<meerkat_core::ToolDef>]> {
vec![Arc::new(meerkat_core::ToolDef {
name: "bundle_echo".into(),
description: "Echo input value".to_string(),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: None,
})]
.into()
}
async fn dispatch(
&self,
call: meerkat_core::ToolCallView<'_>,
) -> Result<meerkat_core::ToolDispatchOutcome, meerkat_core::ToolError> {
if call.name != "bundle_echo" {
return Err(meerkat_core::ToolError::not_found(call.name));
}
let args: serde_json::Value = serde_json::from_str(call.args.get())
.map_err(|e| meerkat_core::ToolError::invalid_arguments(call.name, e.to_string()))?;
let echoed = args
.get("value")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
Ok(meerkat_core::ToolResult::new(
call.id.to_string(),
serde_json::json!({ "echo": echoed }).to_string(),
false,
)
.into())
}
}
struct PersistentMockAgent {
session_id: SessionId,
llm_identity: SessionLlmIdentity,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle,
}
#[async_trait]
impl SessionAgent for PersistentMockAgent {
async fn run_with_events(
&mut self,
_prompt: meerkat_core::types::ContentInput,
_event_tx: tokio::sync::mpsc::Sender<AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
Ok(mock_run_result(
self.session_id.clone(),
"Persistent mock turn".to_string(),
))
}
fn set_skill_references(&mut self, _refs: Option<Vec<meerkat_core::skills::SkillKey>>) {}
fn set_turn_tool_overlay(
&mut self,
_overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
Ok(())
}
fn cancel(&mut self) {}
fn observed_session_tail(&self) -> meerkat_core::pending_continuation::ObservedSessionTailKind {
meerkat_core::pending_continuation::ObservedSessionTailKind::Empty
}
fn hot_swap_llm_identity(
&mut self,
_client: std::sync::Arc<dyn meerkat_core::AgentLlmClient>,
_identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
Ok(())
}
fn session_id(&self) -> SessionId {
self.session_id.clone()
}
fn snapshot(&self) -> SessionSnapshot {
SessionSnapshot {
created_at: SystemTime::now(),
updated_at: SystemTime::now(),
message_count: 0,
total_tokens: 0,
usage: Usage::default(),
last_assistant_text: None,
}
}
fn session_clone(&self) -> Result<Session, meerkat_core::error::AgentError> {
Ok(Session::with_id(self.session_id.clone()))
}
fn session_transcript_authority(
&self,
) -> Result<
meerkat_session::ephemeral::SessionTranscriptAuthoritySnapshot,
meerkat_core::error::AgentError,
> {
let session = self.session_clone()?;
meerkat_session::ephemeral::SessionTranscriptAuthoritySnapshot::from_session(&session)
}
fn durable_llm_identity(&self) -> Option<SessionLlmIdentity> {
Some(self.llm_identity.clone())
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.transient_turn_context_state.clone()
}
}
struct PersistentMockBuilder;
#[async_trait]
impl SessionAgentBuilder for PersistentMockBuilder {
type Agent = PersistentMockAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session_id = requested_test_session_id(req);
Ok(PersistentMockAgent {
session_id,
llm_identity: test_llm_identity(req.model.clone()),
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle::new(),
})
}
}
struct PersistedListingSessionService {
inner: Arc<MockSessionService>,
}
impl PersistedListingSessionService {
fn new(inner: Arc<MockSessionService>) -> Self {
Self { inner }
}
}
#[async_trait]
impl SessionService for PersistedListingSessionService {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
self.inner.create_session(req).await
}
async fn start_turn(
&self,
id: &SessionId,
req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
self.inner.start_turn(id, req).await
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.interrupt(id).await
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.cancel_after_boundary(id).await
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
self.inner.read(id).await
}
async fn list(&self, _query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
let live_sessions = self.inner.live_session_data.read().await;
let mut summaries = live_sessions
.values()
.map(|session| SessionSummary {
session_id: session.id().clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
is_active: true,
labels: Default::default(),
})
.collect::<Vec<_>>();
let live_ids = summaries
.iter()
.map(|summary| summary.session_id.clone())
.collect::<HashSet<_>>();
drop(live_sessions);
let persisted = self.inner.persisted_sessions.read().await;
for session in persisted.values() {
if live_ids.contains(session.id()) {
continue;
}
summaries.push(SessionSummary {
session_id: session.id().clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
is_active: false,
labels: Default::default(),
});
}
Ok(summaries)
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
self.inner.has_live_session(id).await
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.archive(id).await
}
async fn subscribe_session_events(&self, id: &SessionId) -> Result<EventStream, StreamError> {
SessionService::subscribe_session_events(&*self.inner, id).await
}
}
#[async_trait]
impl SessionServiceCommsExt for PersistedListingSessionService {
async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CoreCommsRuntime>> {
self.inner.comms_runtime(session_id).await
}
async fn event_injector(&self, session_id: &SessionId) -> Option<Arc<dyn EventInjector>> {
self.inner.event_injector(session_id).await
}
async fn interaction_event_injector(
&self,
session_id: &SessionId,
) -> Option<Arc<dyn SubscribableInjector>> {
self.inner.interaction_event_injector(session_id).await
}
}
#[async_trait]
impl SessionServiceHistoryExt for PersistedListingSessionService {
async fn read_history(
&self,
id: &SessionId,
query: meerkat_core::service::SessionHistoryQuery,
) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
self.inner.read_history(id, query).await
}
}
#[async_trait]
impl SessionServiceControlExt for PersistedListingSessionService {
async fn append_system_context(
&self,
id: &SessionId,
req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
self.inner.append_system_context(id, req).await
}
}
#[async_trait]
impl MobSessionService for PersistedListingSessionService {
async fn prepare_session_for_resume(&self, session_id: &SessionId) -> Result<(), SessionError> {
self.inner.prepare_session_for_resume(session_id).await
}
async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
&self,
session_id: &SessionId,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
self.inner
.acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
session_id, authority,
)
.await
}
/// Test double: nothing durable survives archive here, so the two-read
/// composition is the exact truth — `ArchivedNotRevivable` cannot exist.
async fn load_session_for_resume(
&self,
session_id: &SessionId,
) -> Result<ResumeSessionLoad, SessionError> {
if let Some(session) = self.load_persisted_session(session_id).await? {
return Ok(ResumeSessionLoad::Active(Box::new(session)));
}
if let Some(session) = self.load_revivable_retired_session(session_id).await? {
return Ok(ResumeSessionLoad::Revivable(Box::new(session)));
}
Ok(ResumeSessionLoad::Absent)
}
async fn create_session_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_under_runtime_turn_boundary(req)
.await
}
async fn create_session_with_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_with_actor_witness_under_runtime_turn_boundary(req, actor_witness_slot)
.await
}
#[cfg(feature = "runtime-adapter")]
async fn create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
req,
authorization,
actor_witness_slot,
)
.await
}
async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.inner
.archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(session_id)
.await
}
async fn discard_live_session_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.inner
.discard_live_session_under_runtime_turn_boundary(session_id)
.await
}
async fn subscribe_session_events(
&self,
session_id: &SessionId,
) -> Result<EventStream, StreamError> {
SessionService::subscribe_session_events(&*self.inner, session_id).await
}
fn supports_persistent_sessions(&self) -> bool {
self.inner.supports_persistent_sessions()
}
fn supports_runtime_turn_apply(&self) -> bool {
self.inner.supports_runtime_turn_apply()
}
async fn live_session_actor_registered(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
self.inner.live_session_actor_registered(session_id).await
}
fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
self.inner.runtime_adapter()
}
async fn interrupt_with_machine_authority(
&self,
session_id: &SessionId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.interrupt_with_machine_authority(session_id, authority)
.await
}
async fn cancel_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
expected_run_id: &meerkat_core::RunId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.cancel_after_boundary_with_machine_authority(session_id, expected_run_id, authority)
.await
}
async fn cancel_current_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.cancel_current_after_boundary_with_machine_authority(session_id, authority)
.await
}
async fn session_belongs_to_mob(&self, session_id: &SessionId, mob_id: &MobId) -> bool {
self.inner.session_belongs_to_mob(session_id, mob_id).await
}
async fn load_persisted_session(
&self,
session_id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.inner.load_persisted_session(session_id).await
}
async fn apply_runtime_turn(
&self,
session_id: &SessionId,
run_id: meerkat_core::RunId,
req: StartTurnRequest,
boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
contributing_input_ids: Vec<meerkat_core::InputId>,
) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
self.inner
.apply_runtime_turn(session_id, run_id, req, boundary, contributing_input_ids)
.await
}
async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
self.inner.discard_live_session(session_id).await
}
async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
self.inner
.discard_live_session_actor_under_runtime_turn_boundary(witness)
.await
}
async fn cancel_all_checkpointers(&self) {
self.inner.cancel_all_checkpointers().await;
}
async fn rearm_all_checkpointers(&self) {
self.inner.rearm_all_checkpointers().await;
}
}
struct InactiveReadSessionService {
inner: Arc<MockSessionService>,
}
impl InactiveReadSessionService {
fn new(inner: Arc<MockSessionService>) -> Self {
Self { inner }
}
}
#[async_trait]
impl SessionService for InactiveReadSessionService {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
self.inner.create_session(req).await
}
async fn start_turn(
&self,
id: &SessionId,
req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
self.inner.start_turn(id, req).await
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.interrupt(id).await
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.cancel_after_boundary(id).await
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
let session = self
.inner
.live_session_clone(id)
.await
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
let metadata = session.session_metadata();
if !self.inner.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
Ok(SessionView {
state: SessionInfo {
session_id: id.clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
is_active: false,
model: metadata
.as_ref()
.map(|meta| meta.model.clone())
.unwrap_or_else(|| "claude-sonnet-4-5".to_string()),
provider: metadata
.as_ref()
.map(|meta| meta.provider)
.unwrap_or(Provider::Anthropic),
last_assistant_text: session.last_assistant_text(),
labels: Default::default(),
},
billing: SessionUsage {
total_tokens: session.total_tokens(),
usage: session.total_usage(),
},
})
}
async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
self.inner.list(query).await
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
self.inner.archive(id).await
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
self.inner.has_live_session(id).await
}
async fn subscribe_session_events(&self, id: &SessionId) -> Result<EventStream, StreamError> {
SessionService::subscribe_session_events(&*self.inner, id).await
}
}
#[async_trait]
impl SessionServiceCommsExt for InactiveReadSessionService {
async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CoreCommsRuntime>> {
self.inner.comms_runtime(session_id).await
}
async fn event_injector(&self, session_id: &SessionId) -> Option<Arc<dyn EventInjector>> {
self.inner.event_injector(session_id).await
}
async fn interaction_event_injector(
&self,
session_id: &SessionId,
) -> Option<Arc<dyn SubscribableInjector>> {
self.inner.interaction_event_injector(session_id).await
}
}
#[async_trait]
impl SessionServiceHistoryExt for InactiveReadSessionService {
async fn read_history(
&self,
id: &SessionId,
query: meerkat_core::service::SessionHistoryQuery,
) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
self.inner.read_history(id, query).await
}
}
#[async_trait]
impl SessionServiceControlExt for InactiveReadSessionService {
async fn append_system_context(
&self,
id: &SessionId,
req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
self.inner.append_system_context(id, req).await
}
}
#[async_trait]
impl MobSessionService for InactiveReadSessionService {
async fn prepare_session_for_resume(&self, session_id: &SessionId) -> Result<(), SessionError> {
self.inner.prepare_session_for_resume(session_id).await
}
async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
&self,
session_id: &SessionId,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
self.inner
.acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
session_id, authority,
)
.await
}
/// Test double: nothing durable survives archive here, so the two-read
/// composition is the exact truth — `ArchivedNotRevivable` cannot exist.
async fn load_session_for_resume(
&self,
session_id: &SessionId,
) -> Result<ResumeSessionLoad, SessionError> {
if let Some(session) = self.load_persisted_session(session_id).await? {
return Ok(ResumeSessionLoad::Active(Box::new(session)));
}
if let Some(session) = self.load_revivable_retired_session(session_id).await? {
return Ok(ResumeSessionLoad::Revivable(Box::new(session)));
}
Ok(ResumeSessionLoad::Absent)
}
async fn create_session_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_under_runtime_turn_boundary(req)
.await
}
async fn create_session_with_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_with_actor_witness_under_runtime_turn_boundary(req, actor_witness_slot)
.await
}
#[cfg(feature = "runtime-adapter")]
async fn create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
authorization: meerkat_runtime::ArchivedSessionActorMaterializationAuthorization,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.inner
.create_session_with_machine_archived_resume_authority_and_actor_witness_under_runtime_turn_boundary(
req,
authorization,
actor_witness_slot,
)
.await
}
async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.inner
.archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(session_id)
.await
}
async fn discard_live_session_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.inner
.discard_live_session_under_runtime_turn_boundary(session_id)
.await
}
async fn subscribe_session_events(
&self,
session_id: &SessionId,
) -> Result<EventStream, StreamError> {
SessionService::subscribe_session_events(&*self.inner, session_id).await
}
fn supports_persistent_sessions(&self) -> bool {
self.inner.supports_persistent_sessions()
}
fn supports_runtime_turn_apply(&self) -> bool {
self.inner.supports_runtime_turn_apply()
}
async fn live_session_actor_registered(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
self.inner.live_session_actor_registered(session_id).await
}
fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
self.inner.runtime_adapter()
}
async fn interrupt_with_machine_authority(
&self,
session_id: &SessionId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.interrupt_with_machine_authority(session_id, authority)
.await
}
async fn cancel_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
expected_run_id: &meerkat_core::RunId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.cancel_after_boundary_with_machine_authority(session_id, expected_run_id, authority)
.await
}
async fn cancel_current_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.cancel_current_after_boundary_with_machine_authority(session_id, authority)
.await
}
async fn session_belongs_to_mob(&self, session_id: &SessionId, mob_id: &MobId) -> bool {
self.inner.session_belongs_to_mob(session_id, mob_id).await
}
async fn load_persisted_session(
&self,
session_id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.inner.load_persisted_session(session_id).await
}
async fn apply_runtime_turn(
&self,
session_id: &SessionId,
run_id: meerkat_core::RunId,
req: StartTurnRequest,
boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
contributing_input_ids: Vec<meerkat_core::InputId>,
) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
self.inner
.apply_runtime_turn(session_id, run_id, req, boundary, contributing_input_ids)
.await
}
async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
self.inner.discard_live_session(session_id).await
}
async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
self.inner
.discard_live_session_actor_under_runtime_turn_boundary(witness)
.await
}
async fn cancel_all_checkpointers(&self) {
self.inner.cancel_all_checkpointers().await;
}
async fn rearm_all_checkpointers(&self) {
self.inner.rearm_all_checkpointers().await;
}
}
async fn create_test_mob_with_persistent_service(definition: MobDefinition) -> MobHandle {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> =
Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
PersistentMockBuilder,
16,
store,
runtime_store,
blob_store,
));
MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.create()
.await
.expect("create mob with persistent session service")
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_persistent_resume_classifies_lifecycle_only_session_as_absent() {
let session_store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_dyn: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
PersistentMockBuilder,
16,
session_store,
runtime_store_dyn.clone(),
blob_store.clone(),
));
let session_id = SessionId::new();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let machine = meerkat_runtime::MeerkatMachine::persistent(runtime_store_dyn, blob_store);
machine
.register_session(session_id.clone())
.await
.expect("seed lifecycle authority without a committed session body");
assert!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read lifecycle-only runtime state")
.is_some(),
"fixture must retain lifecycle authority while the session body is absent"
);
assert!(
meerkat_runtime::RuntimeStore::load_session_snapshot(runtime_store.as_ref(), &runtime_id)
.await
.expect("read lifecycle-only session body")
.is_none(),
"fixture must not accidentally persist a session body"
);
let loaded = service
.load_session_for_resume(&session_id)
.await
.expect("lifecycle-only resume classification");
assert!(
matches!(loaded, ResumeSessionLoad::Absent),
"a never-persisted session body must remain fresh-eligible even when lifecycle authority exists"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn archived_document_without_runtime_record_is_revivable() {
let session_store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_dyn: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
OverlayProbeSessionAgentBuilder {
provider_visible_tools: Arc::new(Mutex::new(Vec::new())),
provider_turn_overlays: Arc::new(Mutex::new(Vec::new())),
},
16,
session_store,
runtime_store_dyn,
blob_store,
));
let mut session = Session::new();
session.push(Message::User(meerkat_core::UserMessage::text(
"retained pre-runtime transcript".to_string(),
)));
session
.set_lifecycle_terminal(meerkat_core::SessionLifecycleTerminal::Archived)
.expect("seed archived document authority");
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let session_bytes = serde_json::to_vec(&session).expect("serialize archived document");
meerkat_runtime::RuntimeStore::commit_session_snapshot(
runtime_store.as_ref(),
&runtime_id,
meerkat_runtime::SerializedSessionSnapshot {
session_snapshot: session_bytes.into(),
},
)
.await
.expect("seed imported archived document without runtime state");
assert!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read absent runtime state")
.is_none(),
"fixture must model an imported document with no runtime record"
);
let loaded = service
.load_session_for_resume(&session_id)
.await
.expect("classify imported archived document");
let ResumeSessionLoad::Revivable(revivable) = loaded else {
panic!("archived document with no competing runtime authority must be revivable");
};
assert_eq!(revivable.messages(), session.messages());
assert_eq!(
revivable.lifecycle_terminal(),
Some(meerkat_core::SessionLifecycleTerminal::Archived),
"classification must preserve the terminal for the machine-owned revive transition"
);
let adapter = service
.runtime_adapter()
.expect("persistent service runtime adapter");
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let receipt = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "resume imported archived document".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(*revivable),
comms_name: Some("test-mob/worker/imported-archive".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("imported-archive-worker"),
peer_name: "imported-archive-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(session_id.clone()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect("materialize imported archived document through the real resume transaction");
let runtime_state_after_resume =
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("inspect revived runtime state");
assert!(
adapter.contains_session(&session_id).await,
"the revived actor must remain registered with the runtime adapter; runtime state: {runtime_state_after_resume:?}"
);
assert!(
service
.has_live_session(&session_id)
.await
.expect("inspect revived actor"),
"the product resume path must reconstruct a live actor; runtime state: {runtime_state_after_resume:?}"
);
SessionService::read(service.as_ref(), &session_id)
.await
.expect("revived imported session must be publicly readable");
let listed = SessionService::list(service.as_ref(), SessionQuery::default())
.await
.expect("list revived imported session");
assert!(
listed
.iter()
.any(|summary| summary.session_id == session_id),
"a live runtime must override its imported archived catalog projection"
);
let resumed = service
.load_authoritative_session(&session_id)
.await
.expect("load resumed authority")
.expect("resumed session remains durable");
assert_eq!(
resumed.lifecycle_terminal(),
Some(meerkat_core::SessionLifecycleTerminal::Active),
"the generated revival verdict must be realized in durable document authority"
);
assert!(
resumed.messages().iter().any(|message| {
matches!(
message,
Message::User(user) if user.text_content() == "retained pre-runtime transcript"
)
}),
"revival must preserve the imported transcript"
);
provisioner
.retire_member(&receipt.member_ref)
.await
.expect("retire revived imported session");
}
fn overlay_probe_visible_tools(overlay: Option<&TurnToolOverlay>) -> Vec<meerkat_core::ToolName> {
let blocked = overlay
.and_then(|overlay| overlay.blocked_tools.as_ref())
.map(|tools| {
tools
.iter()
.map(meerkat_core::ToolName::as_str)
.collect::<HashSet<_>>()
})
.unwrap_or_default();
let candidates = overlay
.and_then(|overlay| overlay.allowed_tools.clone())
.unwrap_or_else(|| vec!["alpha".into(), "beta".into()]);
candidates
.into_iter()
.filter(|tool| !blocked.contains(tool.as_str()))
.collect()
}
struct OverlayProbeSessionAgent {
session: Session,
provider_visible_tools: Arc<Mutex<Vec<Vec<meerkat_core::ToolName>>>>,
provider_turn_overlays: Arc<Mutex<Vec<Option<TurnToolOverlay>>>>,
turn_tool_overlay: Option<TurnToolOverlay>,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle,
}
#[async_trait]
impl SessionAgent for OverlayProbeSessionAgent {
async fn run_with_events(
&mut self,
_prompt: meerkat_core::types::ContentInput,
event_tx: tokio::sync::mpsc::Sender<AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
let visible_tools = overlay_probe_visible_tools(self.turn_tool_overlay.as_ref());
self.provider_visible_tools
.lock()
.expect("provider_visible_tools lock poisoned")
.push(visible_tools);
self.provider_turn_overlays
.lock()
.expect("provider_turn_overlays lock poisoned")
.push(self.turn_tool_overlay.clone());
let session_id = self.session.id().clone();
let result = mock_run_result(session_id.clone(), "{}".to_string());
let _ = event_tx
.send(AgentEvent::RunCompleted {
session_id,
result: result.text.clone(),
structured_output: result.structured_output.clone(),
extraction_required: false,
usage: result.usage.clone(),
terminal_cause_kind: None,
})
.await;
Ok(result)
}
async fn run_turn_with_events(
&mut self,
input: meerkat_session::ephemeral::SessionAgentTurnInput,
event_tx: tokio::sync::mpsc::Sender<AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
if input.handling_mode != HandlingMode::Queue {
return Err(meerkat_core::error::AgentError::ConfigError(format!(
"unexpected overlay-probe handling mode: {:?}",
input.handling_mode,
)));
}
if input.render_metadata.is_some() {
return Err(meerkat_core::error::AgentError::ConfigError(
"overlay probe does not consume render metadata".to_string(),
));
}
self.run_with_events(input.prompt, event_tx).await
}
fn set_skill_references(&mut self, _refs: Option<Vec<meerkat_core::skills::SkillKey>>) {}
fn set_turn_tool_overlay(
&mut self,
overlay: Option<TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
self.turn_tool_overlay = overlay;
Ok(())
}
fn cancel(&mut self) {}
fn observed_session_tail(&self) -> meerkat_core::pending_continuation::ObservedSessionTailKind {
meerkat_core::pending_continuation::observe_session_tail(self.session.messages())
}
fn hot_swap_llm_identity(
&mut self,
_client: std::sync::Arc<dyn meerkat_core::AgentLlmClient>,
_identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
Ok(())
}
fn session_id(&self) -> SessionId {
self.session.id().clone()
}
fn snapshot(&self) -> SessionSnapshot {
SessionSnapshot {
created_at: self.session.created_at(),
updated_at: self.session.updated_at(),
message_count: self.session.messages().len(),
total_tokens: self.session.total_tokens(),
usage: self.session.total_usage(),
last_assistant_text: self.session.last_assistant_text(),
}
}
fn session_clone(&self) -> Result<Session, meerkat_core::error::AgentError> {
Ok(self.session.clone())
}
fn session_transcript_authority(
&self,
) -> Result<
meerkat_session::ephemeral::SessionTranscriptAuthoritySnapshot,
meerkat_core::error::AgentError,
> {
meerkat_session::ephemeral::SessionTranscriptAuthoritySnapshot::from_session(&self.session)
}
#[cfg(not(target_arch = "wasm32"))]
fn sync_session_from_durable_snapshot(
&mut self,
session: Session,
) -> Result<(), meerkat_core::error::AgentError> {
if session.id() != self.session.id() {
return Err(meerkat_core::error::AgentError::InternalError(format!(
"durable overlay-probe session {} does not match live session {}",
session.id(),
self.session.id()
)));
}
self.session = session;
Ok(())
}
fn durable_llm_identity(&self) -> Option<SessionLlmIdentity> {
self.session
.session_metadata()
.map(|metadata| metadata.llm_identity())
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.transient_turn_context_state.clone()
}
}
struct OverlayProbeSessionAgentBuilder {
provider_visible_tools: Arc<std::sync::Mutex<Vec<Vec<meerkat_core::ToolName>>>>,
provider_turn_overlays: Arc<std::sync::Mutex<Vec<Option<TurnToolOverlay>>>>,
}
#[async_trait]
impl SessionAgentBuilder for OverlayProbeSessionAgentBuilder {
type Agent = OverlayProbeSessionAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
let session =
factory_policy_session(session, req.model.clone(), req.max_tokens.unwrap_or(1024));
Ok(OverlayProbeSessionAgent {
session,
provider_visible_tools: Arc::clone(&self.provider_visible_tools),
provider_turn_overlays: Arc::clone(&self.provider_turn_overlays),
turn_tool_overlay: None,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle::new(),
})
}
}
async fn create_test_mob_with_overlay_probe_service(
definition: MobDefinition,
) -> (
MobHandle,
Arc<std::sync::Mutex<Vec<Vec<meerkat_core::ToolName>>>>,
) {
let (handle, provider_visible_tools, _) =
create_test_mob_with_overlay_probe_service_and_workgraph(definition, None).await;
(handle, provider_visible_tools)
}
async fn create_test_mob_with_overlay_probe_service_and_workgraph(
definition: MobDefinition,
workgraph_service: Option<meerkat::WorkGraphService>,
) -> (
MobHandle,
Arc<std::sync::Mutex<Vec<Vec<meerkat_core::ToolName>>>>,
Arc<std::sync::Mutex<Vec<Option<TurnToolOverlay>>>>,
) {
let provider_visible_tools = Arc::new(std::sync::Mutex::new(
Vec::<Vec<meerkat_core::ToolName>>::new(),
));
let provider_turn_overlays =
Arc::new(std::sync::Mutex::new(Vec::<Option<TurnToolOverlay>>::new()));
let session_service = Arc::new(meerkat_session::EphemeralSessionService::new(
OverlayProbeSessionAgentBuilder {
provider_visible_tools: Arc::clone(&provider_visible_tools),
provider_turn_overlays: Arc::clone(&provider_turn_overlays),
},
16,
));
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(session_service)
.with_workgraph_service(workgraph_service)
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create mob with overlay probe session service");
(handle, provider_visible_tools, provider_turn_overlays)
}
// -----------------------------------------------------------------------
// P1-T03: MobBuilder + MobHandle + MobActor wiring
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_mob_create_returns_handle() {
let (handle, _service) = create_test_mob(sample_definition()).await;
assert_eq!(handle.status().await.unwrap(), MobState::Running);
assert_eq!(handle.mob_id().as_str(), "test-mob");
}
#[tokio::test]
async fn test_mob_builder_runs_with_persistent_storage_bundle() {
let dir = tempfile::tempdir().expect("tempdir");
let db_path = dir.path().join("mob.db");
let storage = MobStorage::persistent(&db_path).expect("construct persistent storage");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition_with_single_step_flow(500, 8), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob with persistent storage");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(15)).await;
assert_eq!(
terminal.status,
MobRunStatus::Completed,
"flow should complete for runtime overlay probe: {terminal:?}"
);
}
#[tokio::test]
async fn test_mob_builder_from_mobpack_creates_valid_mob() {
let mut definition = sample_definition();
definition.skills.insert(
"review".to_string(),
SkillSource::Path {
path: "skills/review.md".to_string(),
},
);
let packed_skills = BTreeMap::from([(
"skills/review.md".to_string(),
b"You are a reviewer.".to_vec(),
)]);
let mock_service = MockSessionService::new();
let _ = mock_service.enable_runtime_adapter();
let session_service: Arc<dyn crate::runtime::MobSessionService> = Arc::new(mock_service);
let storage = MobStorage::in_memory();
let handle = MobBuilder::from_mobpack(definition.clone(), packed_skills, storage)
.expect("from_mobpack should construct builder")
.with_session_service(session_service)
.allow_ephemeral_sessions(true)
.create()
.await
.expect("mob should create");
assert_eq!(handle.definition().id, definition.id);
match &handle.definition().skills["review"] {
SkillSource::Inline { content } => assert_eq!(content, "You are a reviewer."),
other => panic!("expected inlined skill, got {other:?}"),
}
}
#[tokio::test]
async fn test_mob_builder_create_rejects_mismatched_spec_store_definition() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let definition = sample_definition();
let mut mismatched = definition.clone();
mismatched
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile exists")
.as_inline_mut()
.unwrap()
.peer_description = "mismatched worker profile".to_string();
storage
.specs
.put_spec(&definition.id, &mismatched, None)
.await
.expect("preseed mismatched spec");
let error = match MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
{
Ok(_) => panic!("create should fail when persisted spec mismatches runtime definition"),
Err(error) => error,
};
assert!(
error
.to_string()
.contains("persisted spec store definition does not match"),
"error should explain spec-store mismatch"
);
}
#[tokio::test]
async fn test_mob_builder_persists_spec_and_resume_requires_consistency() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let definition = sample_definition();
let mob_id = definition.id.clone();
let storage_for_resume = MobStorage {
events: storage.events.clone(),
runs: storage.runs.clone(),
specs: storage.specs.clone(),
runtime_metadata: storage.runtime_metadata.clone(),
identity: storage.identity.clone(),
identity_member: storage.identity_member.clone(),
identity_status: storage.identity_status.clone(),
realm_profiles: storage.realm_profiles.clone(),
};
let _handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let (persisted_definition, _revision) = storage_for_resume
.specs
.get_spec(&mob_id)
.await
.expect("get persisted spec")
.expect("spec should be persisted on create");
assert_eq!(
persisted_definition, definition,
"create should persist runtime definition into spec store"
);
let mut mismatched = definition.clone();
mismatched
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile exists")
.as_inline_mut()
.unwrap()
.peer_description = "resume mismatch".to_string();
let _ = storage_for_resume
.specs
.put_spec(&mob_id, &mismatched, None)
.await
.expect("overwrite persisted spec with mismatch");
let error = match MobBuilder::for_resume(storage_for_resume)
.with_session_service(service)
.resume()
.await
{
Ok(_) => panic!("resume should fail when persisted spec mismatches runtime definition"),
Err(error) => error,
};
assert!(
error
.to_string()
.contains("persisted spec store definition does not match"),
"resume should enforce spec-store consistency"
);
}
#[tokio::test]
async fn test_resume_spec_consistency_ignores_legacy_owner_projection_fields() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let definition = sample_definition();
let mut legacy_definition =
serde_json::to_value(definition.clone()).expect("serialize definition");
legacy_definition["owner_bridge_session_id"] = serde_json::json!(SessionId::new().to_string());
legacy_definition["session_cleanup_policy"] = serde_json::json!("destroy_on_owner_archive");
legacy_definition["is_implicit"] = serde_json::json!(true);
let legacy_definition: MobDefinition =
serde_json::from_value(legacy_definition).expect("legacy definition");
storage
.specs
.put_spec(&definition.id, &definition, None)
.await
.expect("preseed canonical spec");
storage
.events
.append(NewMobEvent {
mob_id: definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(legacy_definition),
},
})
.await
.expect("append legacy projection event");
let resumed = MobBuilder::for_resume(storage)
.with_session_service(service)
.resume()
.await
.expect("resume must ignore compatibility-only definition differences");
assert_eq!(
resumed.owner_bridge_session_lifecycle_authority(),
None,
"legacy projection fields must not recover owner lifecycle authority"
);
}
#[tokio::test]
async fn test_mob_builder_persists_supervisor_runtime_metadata_on_create() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let definition = sample_definition();
let mob_id = definition.id.clone();
let storage = MobStorage::custom_with_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
Arc::new(InMemoryMobRunStore::new()),
Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata.clone(),
Arc::new(InMemoryMobIdentityStore::new()),
Arc::new(InMemoryMobIdentityStatusStore::new()),
);
let _handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let record = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor authority")
.expect("create should persist supervisor authority");
assert_eq!(record.epoch, 0, "fresh mobs should start at epoch 0");
assert_eq!(
record.public_peer_id,
record.keypair().public_key().to_peer_id().to_string(),
"persisted public peer id should match the reconstructed keypair"
);
assert_eq!(
record.protocol_version,
meerkat_contracts::wire::supervisor_bridge::supervisor_bridge_default_protocol_version(),
"fresh supervisor metadata should persist the canonical default bridge protocol version"
);
}
#[tokio::test]
async fn test_mob_resume_accepts_typed_supervisor_protocol_metadata() {
let definition = sample_definition();
let mob_id = definition.id.clone();
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition),
},
})
.await
.expect("append mob created");
let runtime_metadata: Arc<dyn MobRuntimeMetadataStore> =
Arc::new(InMemoryMobRuntimeMetadataStore::new());
let persisted = SupervisorAuthorityRecord::generate(
super::bridge_protocol::supervisor_bridge_default_protocol_version(),
);
persist_supervisor_authority_for_test(runtime_metadata.as_ref(), &mob_id, &persisted).await;
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let _handle = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("typed supervisor protocol metadata should be accepted on resume");
let loaded = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority")
.expect("authority should remain present");
assert_eq!(
loaded.protocol_version,
super::bridge_protocol::supervisor_bridge_default_protocol_version(),
"resume should preserve the canonical bridge protocol version"
);
assert_eq!(
loaded.public_peer_id, persisted.public_peer_id,
"resume must preserve supervisor identity"
);
assert_eq!(
loaded.epoch, persisted.epoch,
"resume must preserve authority epoch"
);
}
#[test]
fn test_supervisor_authority_record_rejects_unsupported_protocol_metadata_at_decode() {
let record = SupervisorAuthorityRecord::generate(
super::bridge_protocol::supervisor_bridge_default_protocol_version(),
);
let mut raw = serde_json::to_value(record).expect("serialize supervisor authority");
raw["protocol_version"] = serde_json::json!(999);
let error = serde_json::from_value::<SupervisorAuthorityRecord>(raw)
.expect_err("unsupported persisted supervisor protocol metadata must fail at decode");
assert!(
error
.to_string()
.contains("unsupported supervisor bridge protocol version"),
"decode error should report protocol incompatibility, got: {error}"
);
}
#[tokio::test]
async fn test_mob_builder_allows_spec_warnings() {
let definition = MobDefinition::from_toml(
r#"
[mob]
id = "warn-mob"
[profiles.lead]
model = "claude-opus-4-8"
[flows.demo]
[flows.demo.steps.start]
role = "lead"
message = "start"
[flows.demo.steps.join]
role = "lead"
message = "join"
depends_on = ["start"]
depends_on_mode = "any"
"#,
)
.expect("parse warning definition");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.create()
.await
.expect("warning diagnostics should not block create");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
}
#[tokio::test]
async fn test_mob_builder_blocks_spec_errors() {
let definition = MobDefinition::from_toml(
r#"
[mob]
id = "error-mob"
[profiles.lead]
model = "claude-opus-4-8"
[flows.demo]
[flows.demo.steps.start]
role = "ghost"
message = "x"
"#,
)
.expect("parse error definition");
let service = Arc::new(MockSessionService::new());
let result = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.create()
.await;
let diagnostics = match result {
Ok(_) => panic!("spec errors should block create"),
Err(MobError::DefinitionError(diagnostics)) => diagnostics,
Err(other) => panic!("expected DefinitionError, got {other}"),
};
assert!(
diagnostics
.iter()
.any(|diag| diag.code == crate::DiagnosticCode::FlowUnknownRole),
"expected flow unknown role diagnostic"
);
}
#[tokio::test]
async fn test_mob_resume_allows_spec_warnings() {
let definition = MobDefinition::from_toml(
r#"
[mob]
id = "warn-resume-mob"
[profiles.lead]
model = "claude-opus-4-8"
[flows.demo]
[flows.demo.steps.start]
role = "lead"
message = "start"
[flows.demo.steps.join]
role = "lead"
message = "join"
depends_on = ["start"]
depends_on_mode = "any"
"#,
)
.expect("parse warning definition");
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
events
.append(NewMobEvent {
mob_id: definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition.clone()),
},
})
.await
.expect("append mob created");
let runtime_metadata: Arc<dyn MobRuntimeMetadataStore> =
Arc::new(InMemoryMobRuntimeMetadataStore::new());
persist_supervisor_authority_for_test(
runtime_metadata.as_ref(),
&definition.id,
&default_supervisor_authority_record(),
)
.await;
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("warning diagnostics should not block resume");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
}
#[tokio::test]
async fn test_mob_resume_blocks_spec_errors() {
let definition = MobDefinition::from_toml(
r#"
[mob]
id = "error-resume-mob"
[profiles.lead]
model = "claude-opus-4-8"
[flows.demo]
[flows.demo.steps.start]
role = "ghost"
message = "x"
"#,
)
.expect("parse error definition");
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
events
.append(NewMobEvent {
mob_id: definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition.clone()),
},
})
.await
.expect("append mob created");
let runtime_metadata: Arc<dyn MobRuntimeMetadataStore> =
Arc::new(InMemoryMobRuntimeMetadataStore::new());
persist_supervisor_authority_for_test(
runtime_metadata.as_ref(),
&definition.id,
&default_supervisor_authority_record(),
)
.await;
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let result = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await;
let diagnostics = match result {
Ok(_) => panic!("spec errors should block resume"),
Err(MobError::DefinitionError(diagnostics)) => diagnostics,
Err(other) => panic!("expected DefinitionError, got {other}"),
};
assert!(
diagnostics
.iter()
.any(|diag| diag.code == crate::DiagnosticCode::FlowUnknownRole),
"expected flow unknown role diagnostic"
);
}
#[tokio::test]
async fn test_mob_builder_accepts_persistent_session_service() {
let handle = create_test_mob_with_persistent_service(sample_definition()).await;
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn with persistent session service");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"spawn should succeed when mob is built with PersistentSessionService"
);
}
#[tokio::test]
async fn test_mob_builder_rejects_non_persistent_session_service() {
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
let result = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.create()
.await;
assert!(
matches!(result, Err(MobError::Internal(message)) if message.contains("persistent-session contract")),
"create should reject non-persistent session services for REQ-MOB-030"
);
}
#[tokio::test]
async fn test_mob_builder_allows_ephemeral_session_service_when_opted_in() {
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create should allow ephemeral sessions when explicitly enabled");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
}
#[tokio::test]
async fn test_ephemeral_session_service_exposes_live_session_to_mob_read_seam() {
// Regression for the Inherit containment escape: the MobSessionService
// read seam must see the LIVE session on the ephemeral backend. The trait
// default's `Ok(None)` silently coalesced "session invisible to this read
// seam" into "session has no such fact", letting a policy-gated ephemeral
// parent mint unrestricted children through Inherit resolution.
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
let created = SessionService::create_session(
service.as_ref(),
CreateSessionRequest {
injected_context: Vec::new(),
model: "mock-model".to_string(),
prompt: ContentInput::Text("hello".to_string()),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
build: None,
labels: None,
},
)
.await
.expect("create ephemeral session");
let loaded = crate::runtime::MobSessionService::load_persisted_session(
service.as_ref(),
&created.session_id,
)
.await
.expect("live session must be readable through the mob read seam");
assert_eq!(
loaded.map(|session| session.id().clone()),
Some(created.session_id.clone()),
"ephemeral MobSessionService::load_persisted_session must export the live session"
);
let missing = crate::runtime::MobSessionService::load_persisted_session(
service.as_ref(),
&SessionId::new(),
)
.await
.expect("unknown session id is the documented None shape");
assert!(
missing.is_none(),
"absent session must map to None, not an error"
);
}
#[tokio::test]
async fn test_mob_handle_is_clone() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let handle2 = handle.clone();
assert_eq!(handle2.mob_id().as_str(), "test-mob");
}
#[tokio::test]
async fn test_mob_shutdown() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle.shutdown().await.expect("shutdown");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
}
#[tokio::test]
async fn test_mob_shutdown_interrupts_active_autonomous_member() {
let (handle, service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("w-active-autonomous-shutdown");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::AutonomousHost);
handle
.spawn_spec(spec)
.await
.expect("spawn active autonomous member");
let baseline_interrupts = service.interrupt_call_count();
handle
.shutdown()
.await
.expect("shutdown must classify the active autonomous member as interruptible");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
assert!(
service.interrupt_call_count() > baseline_interrupts,
"shutdown must realize the MobMachine Interrupt action for an active autonomous member"
);
}
#[tokio::test]
async fn test_lifecycle_state_machine_enforcement() {
let (handle, _service) = create_test_mob(sample_definition()).await;
assert_eq!(handle.status().await.unwrap(), MobState::Running);
handle.stop().await.expect("stop");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
handle.resume().await.expect("resume");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
handle.complete().await.expect("complete");
assert_eq!(handle.status().await.unwrap(), MobState::Completed);
let err = handle
.resume()
.await
.expect_err("resume must fail from completed");
assert!(
matches!(err, MobError::InvalidTransition { .. }),
"completed -> running must be rejected"
);
handle.destroy().await.expect("destroy");
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
let err = handle
.destroy()
.await
.expect_err("destroy from Destroyed must fail once actor exits");
assert!(
matches!(
err,
crate::runtime::handle::MobDestroyError::Mob(MobError::ActorCommandChannelClosed)
),
"destroy should be terminal once completed: {err:?}"
);
}
#[tokio::test]
async fn test_destroy_is_terminal_for_commands() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let report = handle.destroy().await.expect("destroy");
assert!(
report.force_destroyed_members.is_empty(),
"local destroy should not require force-destroy bookkeeping yet"
);
assert!(
report.orphaned_remote_members.is_empty(),
"local destroy should not orphan remote members in the in-memory test path"
);
assert!(
!report.remote_cleanup_deadline_exceeded,
"local destroy should not exceed the remote cleanup deadline in the in-memory path"
);
assert!(
report.metadata_scrubbed,
"destroy should report metadata scrub"
);
assert!(
report.events_cleared,
"destroy should report cleared events"
);
assert!(
report.namespace_cleaned,
"destroy should report namespace cleanup"
);
assert!(
report.errors.is_empty(),
"successful destroy should not report cleanup errors: {:?}",
report.errors
);
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
let err = handle
.stop()
.await
.expect_err("stop after destroy must fail");
assert!(
matches!(err, MobError::ActorCommandChannelClosed),
"destroy should terminate the actor loop: {err:?}"
);
}
#[tokio::test]
async fn test_destroy_bulk_scrubs_member_event_cursors() {
let definition = with_unique_mob_id(sample_definition(), "destroy-cursor-scrub");
let mob_id = definition.id.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let runtime_metadata = Arc::clone(&storage.runtime_metadata);
runtime_metadata
.put_member_event_cursor(
&mob_id,
&MobMemberEventCursorRecord {
agent_identity: "retired-worker".to_string(),
host_id: "retired-host".to_string(),
host_binding_generation: Some(3),
member_session_id: SessionId::new().to_string(),
generation: 7,
fence_token: Some(11),
next_seq: 19,
},
)
.await
.expect("seed stale pump cursor");
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let report = handle.destroy().await.expect("destroy");
assert!(report.metadata_scrubbed);
assert!(
runtime_metadata
.list_member_event_cursors(&mob_id)
.await
.expect("list cursors after destroy")
.is_empty(),
"the storage-finalizing fence must scrub cursors left by best-effort pump teardown"
);
}
#[tokio::test]
async fn test_destroy_preflight_rejects_unfinished_host_revoke_before_terminal_marker() {
let events = Arc::new(InMemoryMobEventStore::new());
let definition = with_unique_mob_id(sample_definition(), "destroy-revoke-anchor-preflight");
let (handle, _service) =
create_test_mob_with_events(definition, events.clone() as Arc<dyn MobEventStore>).await;
events
.append(crate::event::NewMobEvent {
mob_id: handle.mob_id().clone(),
timestamp: None,
kind: crate::event::MobEventKind::RemoteHostRevokeStarted {
operation_id: meerkat_core::time_compat::new_uuid_v7().to_string(),
host_id: "host-pending-revoke".to_string(),
epoch: 7,
binding_generation: 2,
},
})
.await
.expect("inject unfinished revoke anchor");
let error = handle
.destroy()
.await
.expect_err("fresh destroy must not cross an unfinished host revoke");
let crate::runtime::handle::MobDestroyError::Mob(error) = error else {
panic!("a proved-nonmutating destroy preflight must surface as a mob error");
};
assert!(
error.to_string().contains("cannot begin destroy")
&& error.to_string().contains("revoke operation"),
"destroy must explain the remote-authority preflight fence: {error}"
);
assert_eq!(
handle
.status()
.await
.expect("status after rejected destroy"),
MobState::Running,
"preflight rejection must not publish the terminal phase"
);
assert!(
events
.replay_all()
.await
.expect("replay events after rejected destroy")
.iter()
.all(|event| !matches!(event.kind, crate::event::MobEventKind::MobDestroying)),
"the pure preflight must run before the durable destroy marker"
);
}
#[tokio::test]
async fn test_destroy_detaches_mob_owned_session_ingress_before_runtime_destroy() {
let (handle, service) = create_test_mob_with_runtime_adapter(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let adapter = service.enable_runtime_adapter();
let session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-ingress-destroy"),
None,
)
.await
.expect("spawn session-backed member")
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let owner = adapter.peer_ingress_owner(&session_id).await;
assert!(
matches!(owner, meerkat_runtime::PeerIngressOwner::MobOwned { .. }),
"spawn should claim mob-owned session ingress before destroy, got {owner:?}"
);
handle
.destroy()
.await
.expect("destroy should close detach ack before final destroy");
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
let owner = adapter.peer_ingress_owner(&session_id).await;
assert!(
matches!(owner, meerkat_runtime::PeerIngressOwner::Unattached),
"destroy must leave no mob-owned ingress after the generated detach acknowledgement, got {owner:?}"
);
}
#[tokio::test]
async fn test_destroy_retry_after_member_archive_commit_before_roster_prune() {
let definition = with_unique_mob_id(sample_definition(), "destroy-archive-commit-retry");
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, service) = create_test_mob_with_events(definition, events.clone()).await;
service.set_keep_alive_turns_complete_immediately(true);
let worker = AgentIdentity::from("destroy-archive-commit-worker");
handle
.spawn(ProfileName::from("worker"), worker.clone(), None)
.await
.expect("spawn worker before injected destroy cancellation");
super::actor::fail_after_destroy_member_archive_for_test(&worker);
let error = handle
.destroy()
.await
.expect_err("post-archive cancellation must leave destroy incomplete");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.errors.iter().any(|error| error
.contains("fault-injected cancellation after destroy member archive commit")),
"incomplete destroy must surface the injected archive-commit window: {report:?}"
);
assert!(
handle
.roster
.read()
.await
.list_all()
.any(|entry| entry.agent_identity == worker),
"post-commit cancellation must retain the raw roster retry anchor without re-projecting a history-only member"
);
let interrupted = events
.replay_all()
.await
.expect("replay interrupted destroy");
assert_eq!(
interrupted
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"archive commit must publish exactly one durable retirement before cancellation"
);
handle
.destroy()
.await
.expect("duplicate destroy must accept archived admission and finish cleanup");
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
}
#[tokio::test]
async fn test_destroy_final_member_event_append_failure_survives_cold_restart() {
let definition = with_unique_mob_id(sample_definition(), "destroy-final-event-cold-retry");
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let worker = AgentIdentity::from("destroy-final-event-worker");
let mut spec = SpawnMemberSpec::new("worker", worker.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle
.spawn_spec(spec)
.await
.expect("spawn turn-driven worker before destroy");
let session_id = handle
.resolve_bridge_session_id(&worker)
.await
.expect("session-backed worker");
events.fail_appends_for("MemberRetired").await;
let error = handle
.destroy()
.await
.expect_err("final member-retired append failure must leave destroy incomplete");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report
.errors
.iter()
.any(|error| error.contains("MemberRetired") || error.contains("event append")),
"partial destroy must surface final retirement publication failure: {report:?}"
);
assert_eq!(
service.active_session_count().await,
0,
"archive must complete before the final retirement event append"
);
assert!(
!adapter.contains_session(&session_id).await,
"successful archive must unregister the runtime before final publication"
);
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == worker),
"failed final append must retain the roster retry anchor"
);
let interrupted = events
.replay_all()
.await
.expect("replay interrupted destroy");
assert!(
interrupted
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying))
);
assert!(
interrupted
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
drop(handle);
events.allow_appends_for("MemberRetired").await;
events.fail_clear();
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume partial destroy from durable marker");
let retry_error = resumed
.destroy()
.await
.expect_err("final event clear remains fault-injected after member cleanup");
let retry_report = match retry_error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy retry, got {other:?}"),
};
assert!(
retry_report
.errors
.iter()
.all(|error| !error.contains("routed effect dispatch failed")),
"archive-complete cold retry must not reroute retirement into the absent runtime: {retry_report:?}"
);
assert!(
resumed.list_members_including_retiring().await.is_empty(),
"archive authority must allow cold retry to remove the roster anchor"
);
let completed = events
.replay_all()
.await
.expect("replay completed member cleanup");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"cold retry must publish exactly one terminal member event"
);
events.allow_clear();
resumed
.destroy()
.await
.expect("final destroy retry must succeed after event clear recovers");
}
#[tokio::test]
async fn test_destroy_remote_orphan_retains_events_and_metadata_for_retry() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"destroy-remote-orphan-retains-authority",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external worker");
drop(external);
let err = handle
.destroy()
.await
.expect_err("unreachable external member should make destroy incomplete");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report
.orphaned_remote_members
.contains(&AgentIdentity::from("w-ext")),
"incomplete destroy should report the unreachable external member as orphaned: {report:?}"
);
assert!(
!report.metadata_scrubbed && !report.events_cleared,
"remote orphan must fail before metadata/events are removed: {report:?}"
);
let replayed = events.replay_all().await.expect("replay retained events");
assert!(
replayed
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)),
"remote orphan must persist destroy admission so resume cannot reopen the mob"
);
assert!(
replayed.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::MemberSpawned(spawned)
if spawned.agent_identity == "w-ext"
)
}),
"remote orphan must leave the member-spawn event available for retry/recovery"
);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor metadata after incomplete destroy")
.is_some(),
"remote orphan must retain supervisor authority for a later cleanup attempt"
);
}
#[tokio::test]
async fn test_destroy_remote_retire_failure_uses_force_destroy_before_publication() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"destroy-remote-force-fallback",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata),
)
.with_session_service(service)
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("force-remote");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external worker");
external.fail_next_retire();
events.fail_clear();
let error = handle
.destroy()
.await
.expect_err("event clear fault keeps force-destroy evidence inspectable");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.force_destroyed_members.contains(&identity),
"failed graceful remote retire must be replaced by a proven force destroy: {report:?}"
);
assert!(report.orphaned_remote_members.is_empty(), "{report:?}");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(handle.list_members_including_retiring().await.is_empty());
let retained = events
.replay_all()
.await
.expect("replay force-destroy completion");
assert_eq!(
retained
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1,
"remote completion publishes exactly once after force destroy and revoke"
);
events.allow_clear();
handle
.destroy()
.await
.expect("destroy completes after the event store recovers");
}
#[tokio::test]
async fn test_destroy_remote_revoke_failure_retains_retry_anchor() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"destroy-remote-revoke-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata),
)
.with_session_service(service)
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("revoke-remote");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external worker");
external.fail_next_revoke();
let error = handle
.destroy()
.await
.expect_err("supervisor revoke failure must make destroy incomplete");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.orphaned_remote_members.contains(&identity),
"{report:?}"
);
assert!(external.authorized_supervisor_peer_id().await.is_some());
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == identity),
"revoke failure must retain the exact remote roster anchor"
);
assert!(
events
.replay_all()
.await
.expect("replay revoke failure")
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. })),
"remote terminal publication must wait for supervisor revoke"
);
events.fail_clear();
let retry_error = handle
.destroy()
.await
.expect_err("event clear fault keeps retry completion inspectable");
let retry_report = match retry_error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy retry, got {other:?}"),
};
assert!(
retry_report.orphaned_remote_members.is_empty(),
"{retry_report:?}"
);
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(handle.list_members_including_retiring().await.is_empty());
assert_eq!(
events
.replay_all()
.await
.expect("replay completed revoke retry")
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1
);
events.allow_clear();
handle.destroy().await.expect("final destroy retry");
}
#[tokio::test]
async fn test_destroy_remote_archive_before_revoke_survives_cold_restart() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"destroy-remote-archive-cold-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("archive-before-revoke");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external worker");
super::actor::fail_after_destroy_remote_archive_for_test(&identity);
let error = handle
.destroy()
.await
.expect_err("fault after remote archive must retain cleanup authority");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report
.errors
.iter()
.any(|error| error.contains("fault-injected cancellation after remote archive")),
"incomplete report must identify the archive-before-revoke window: {report:?}"
);
assert!(external.authorized_supervisor_peer_id().await.is_some());
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == identity)
);
assert!(
events
.replay_all()
.await
.expect("replay interrupted remote archive")
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
drop(handle);
events.fail_clear();
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume remote archive-before-revoke authority");
let retry_error = resumed
.destroy()
.await
.expect_err("event clear fault keeps cold retry evidence inspectable");
let retry_report = match retry_error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete cold retry, got {other:?}"),
};
assert!(
retry_report.orphaned_remote_members.is_empty(),
"{retry_report:?}"
);
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(resumed.list_members_including_retiring().await.is_empty());
assert_eq!(
events
.replay_all()
.await
.expect("replay cold remote cleanup")
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1,
"cold retry must publish one terminal event after revoke"
);
events.allow_clear();
resumed.destroy().await.expect("final cold destroy retry");
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_sqlite_destroy_remote_archive_before_revoke_skips_tokenless_rebind() {
let _serial = lock_real_comms_tests();
let directory = tempfile::tempdir().expect("temporary sqlite directory");
let database_path = directory.path().join("remote-retirement-cold.sqlite");
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"sqlite-destroy-remote-archive-cold-retry",
);
let mob_id = definition.id.clone();
let storage = MobStorage::persistent(&database_path).expect("open sqlite mob storage");
let events = storage.events.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create sqlite-backed mob");
let identity = AgentIdentity::from("sqlite-archive-before-revoke");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn sqlite-backed external worker");
super::actor::fail_after_destroy_remote_archive_for_test(&identity);
handle
.destroy()
.await
.expect_err("fault after checkpoint must leave sqlite destroy incomplete");
assert!(external.authorized_supervisor_peer_id().await.is_some());
drop(handle);
let replayed = events
.replay_all()
.await
.expect("replay serialized sqlite events");
let serialized_member_ref = replayed.iter().find_map(|event| match &event.kind {
MobEventKind::MemberSpawned(spawned) if spawned.agent_identity == identity => {
spawned.bridge_member_ref()
}
_ => None,
});
assert!(
matches!(
serialized_member_ref,
Some(MemberRef::BackendPeer {
bootstrap_token: None,
..
})
),
"SQLite replay must exercise the intentionally redacted bootstrap-token shape"
);
assert!(
replayed.iter().any(|event| matches!(
&event.kind,
MobEventKind::RemoteMemberRuntimeRetired { agent_identity, .. }
if agent_identity == &identity
)),
"remote runtime checkpoint must be durable before the injected crash"
);
drop(events);
let resumed = MobBuilder::for_resume(
MobStorage::persistent(&database_path).expect("reopen sqlite mob storage"),
)
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume tokenless serialized retirement checkpoint");
resumed
.destroy()
.await
.expect("checkpoint retry must skip reauthorize/rebind and revoke directly");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert_eq!(
resumed.status().await.expect("destroyed status"),
MobState::Destroyed
);
}
#[tokio::test]
async fn test_destroy_remote_final_append_after_revoke_survives_cold_restart() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"destroy-remote-final-append-cold-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("append-after-revoke");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external worker");
events.fail_appends_for("MemberRetired").await;
let error = handle
.destroy()
.await
.expect_err("terminal append failure after revoke must retain retry authority");
let report = match error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.orphaned_remote_members.contains(&identity),
"{report:?}"
);
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == identity),
"failed terminal append must retain the remote roster anchor"
);
assert!(
events
.replay_all()
.await
.expect("replay failed append")
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
assert!(
events
.replay_all()
.await
.expect("replay destroy supervisor checkpoint")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::RemoteMemberSupervisorRevoked { agent_identity, .. }
if agent_identity == &identity
))
);
let intents_before_cold_retry = external.received_intents().await.len();
external.task.abort();
drop(handle);
events.allow_appends_for("MemberRetired").await;
events.fail_clear();
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume revoked remote with pending terminal publication");
let retry_error = resumed
.destroy()
.await
.expect_err("event clear fault keeps append retry evidence inspectable");
let retry_report = match retry_error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete append retry, got {other:?}"),
};
assert!(
retry_report.orphaned_remote_members.is_empty(),
"durable supervisor revocation must let terminal publication finish locally: {retry_report:?}"
);
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"destroy terminal-append retry must not contact an offline retired runtime"
);
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(resumed.list_members_including_retiring().await.is_empty());
assert_eq!(
events
.replay_all()
.await
.expect("replay completed append retry")
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1
);
events.allow_clear();
resumed
.destroy()
.await
.expect("final destroy after append retry");
}
#[tokio::test]
async fn test_destroy_event_clear_failure_restores_runtime_metadata_for_retry() {
let definition = with_unique_mob_id(sample_definition(), "destroy-clear-restores-metadata");
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("destroy-retry-worker"),
None,
)
.await
.expect("spawn worker before partial destroy");
let bridge_session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("destroy-retry-worker"))
.await
.expect("worker bridge session");
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load initial supervisor metadata")
.is_some(),
"test should start with supervisor metadata"
);
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("overlay-retry-anchor"),
generation: crate::ids::Generation::INITIAL,
normalized_member_ref: None,
bootstrap_token: Some("restore-me".to_string().into()),
status: ExternalBindingOverlayStatus::Failed {
reason: "test overlay must be restored".to_string(),
},
updated_at: Utc::now(),
},
)
.await
.expect("insert overlay before partial destroy");
let pending_request = crate::store::MobMemberOperatorRequestRecord::pending(
crate::store::MobMemberOperatorRequestKey::new(
"overlay-retry-anchor",
1,
1,
"host-a",
1,
"member-session-a",
"req-restore-after-clear-failure",
),
crate::store::member_operator_op_digest(
&crate::runtime::bridge_protocol::MemberOperatorOp::ListMembers,
)
.expect("request operation digest"),
);
assert!(matches!(
runtime_metadata
.begin_member_operator_request(&mob_id, &pending_request)
.await
.expect("insert durable request before partial destroy"),
crate::store::MobMemberOperatorRequestBegin::Started
));
let terminal_request = pending_request
.terminal(crate::runtime::bridge_protocol::MemberOperatorReply {
request_id: pending_request.request_id.clone(),
outcome: crate::runtime::bridge_protocol::MemberOperatorOutcome::Rejected {
cause: crate::runtime::bridge_protocol::BridgeRejectionCause::Unavailable,
reason: "durable test terminal".to_string(),
},
})
.expect("terminal request record");
assert!(
runtime_metadata
.compare_and_put_member_operator_request(&mob_id, &pending_request, &terminal_request,)
.await
.expect("persist terminal request before partial destroy")
);
events.fail_clear();
let err = handle
.destroy()
.await
.expect_err("event clear failure should make destroy incomplete");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
!report.metadata_scrubbed && !report.events_cleared,
"metadata scrub must be rolled back when final event clear fails: {report:?}"
);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load restored supervisor metadata")
.is_some(),
"incomplete destroy must restore supervisor metadata for retry"
);
assert!(
runtime_metadata
.list_external_binding_overlays(&mob_id)
.await
.expect("list restored overlays")
.iter()
.any(|overlay| overlay.agent_identity == "overlay-retry-anchor"),
"incomplete destroy must restore external binding overlays for retry"
);
assert_eq!(
runtime_metadata
.load_member_operator_request(&mob_id, &terminal_request.key())
.await
.expect("load restored member operator request"),
Some(terminal_request),
"incomplete destroy must restore immutable terminal upcall replies for retry"
);
let replayed_after_partial = events
.replay_all()
.await
.expect("events remain after failed clear");
assert!(
replayed_after_partial
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)),
"partial destroy must persist destroy admission across restart"
);
assert!(
replayed_after_partial.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::MemberRetired {
agent_identity,
..
} if agent_identity == &AgentIdentity::from("destroy-retry-worker")
)
}),
"successful member cleanup during partial destroy must persist a retire marker"
);
assert!(
!replayed_after_partial.is_empty(),
"failed event clear should leave event authority available"
);
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("check worker session after partial destroy"),
"member cleanup should have archived the bridge session before later partial failure"
);
let supervisor_before_rotate = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor before rejected rotate")
.expect("supervisor metadata retained before rejected rotate");
let rotate_error = handle
.rotate_supervisor()
.await
.expect_err("partial destroy must close supervisor rotation authority");
assert!(
matches!(
rotate_error,
MobError::InvalidTransition {
from: MobState::Destroyed,
to: MobState::Destroyed,
}
),
"post-destroy supervisor rotation should be rejected fail-closed: {rotate_error:?}"
);
assert_eq!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor after rejected rotate"),
Some(supervisor_before_rotate),
"rejected rotate must not mutate retained supervisor authority"
);
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume retained partial destroy storage");
assert_eq!(
resumed.status().await.unwrap(),
MobState::Destroyed,
"resume after partial destroy must keep destroy authority terminal"
);
assert!(
resumed.list_members_including_retiring().await.is_empty(),
"resume after partial destroy must not resurrect already-cleaned members"
);
assert_eq!(
service.active_session_count().await,
0,
"resume must not recreate the archived bridge session for a durably retired member"
);
events.allow_clear();
let retry_report = resumed
.destroy()
.await
.expect("resumed retry should complete after event store recovers");
assert!(retry_report.metadata_scrubbed);
assert!(retry_report.events_cleared);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load metadata after retry")
.is_none(),
"complete retry should scrub supervisor metadata"
);
assert!(
runtime_metadata
.list_external_binding_overlays(&mob_id)
.await
.expect("list overlays after retry")
.is_empty(),
"complete retry should scrub external binding overlays"
);
assert!(
events
.replay_all()
.await
.expect("events after complete retry")
.is_empty(),
"complete retry should clear events"
);
}
#[tokio::test]
async fn test_destroy_overlay_delete_failure_does_not_keep_roster_retry_authority() {
let definition = with_unique_mob_id(sample_definition(), "destroy-overlay-delete-order");
let mob_id = definition.id.clone();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let worker = AgentIdentity::from("destroy-overlay-delete-worker");
let bridge_session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(worker.as_str()),
None,
)
.await
.expect("spawn worker before partial destroy")
.bridge_session_id()
.cloned()
.expect("session-backed worker");
runtime_metadata.fail_next_delete_overlay();
let err = handle
.destroy()
.await
.expect_err("per-member overlay delete failure should make destroy incomplete");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report
.errors
.iter()
.any(|error| error.contains("fault-injected runtime overlay delete failure")),
"report should surface the overlay delete failure: {report:?}"
);
assert!(
handle.list_members_including_retiring().await.is_empty(),
"successful generated member retirement must remove the roster retry anchor before later overlay delete failure"
);
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("check worker session after overlay delete failure"),
"member archive must complete before the later overlay delete failure"
);
assert!(
events
.replay_all()
.await
.expect("replay retained events")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::MemberRetired {
agent_identity,
..
} if agent_identity == &worker
)),
"generated member-retired authority must be durable before the retry anchor is removed"
);
let retry_report = handle
.destroy()
.await
.expect("retry should complete without re-running member cleanup from roster");
assert!(retry_report.metadata_scrubbed);
assert!(retry_report.events_cleared);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load metadata after retry")
.is_none(),
"complete retry should scrub runtime metadata"
);
assert!(
events
.replay_all()
.await
.expect("events after retry")
.is_empty(),
"complete retry should clear durable destroy events"
);
}
#[tokio::test]
async fn test_resume_destroy_storage_finalizing_without_metadata_fails_closed() {
let definition = with_unique_mob_id(sample_definition(), "destroy-finalizing-no-metadata");
let mob_id = definition.id.clone();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition),
},
})
.await
.expect("append created event");
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MobDestroying,
})
.await
.expect("append destroy marker");
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MobDestroyStorageFinalizing,
})
.await
.expect("append storage-finalizing marker");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let error = match MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
{
Ok(_) => {
panic!("storage-finalizing destroy without supervisor metadata must fail closed")
}
Err(error) => error,
};
let message = error.to_string();
assert!(
message.contains("destroy storage finalization is in progress")
&& message.contains("supervisor runtime metadata is absent"),
"resume should report missing generated supervisor authority, got: {message}"
);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor after resume")
.is_none(),
"resume must not mint replacement persisted supervisor authority after finalizing scrub"
);
assert!(
events
.replay_all()
.await
.expect("events after failed closed resume")
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroyStorageFinalizing)),
"failed closed resume must leave durable destroy-finalizing authority intact"
);
}
#[tokio::test]
async fn test_resume_after_destroying_marker_replays_retire_before_storage_cleanup() {
let definition = with_unique_mob_id(sample_definition(), "destroying-replay-retire");
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let worker = AgentIdentity::from("destroying-retry-worker");
let bridge_session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(worker.as_str()),
None,
)
.await
.expect("spawn worker before simulated destroy crash")
.bridge_session_id()
.cloned()
.expect("session-backed worker");
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MobDestroying,
})
.await
.expect("append durable destroy marker");
drop(handle);
// The first automatic cleanup attempts reach the final storage clear and
// fail transiently. Recovery must keep retrying in this same process;
// the test never sends an operator destroy command.
events.fail_clear();
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume after durable destroy marker");
assert_eq!(
resumed.status().await.unwrap(),
MobState::Destroyed,
"durable destroy marker must close public authority after resume"
);
tokio::time::timeout(Duration::from_secs(5), async {
loop {
let archived = !service
.has_live_session(&bridge_session_id)
.await
.expect("check bridge session during automatic destroy retry");
let retired_carrier = events
.replay_all()
.await
.expect("replay retained destroy events")
.iter()
.any(|event| {
matches!(
event.kind,
MobEventKind::MemberRetired { ref agent_identity, .. }
if agent_identity == &worker
)
});
if archived && retired_carrier {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("automatic destroy reaches retained final-clear retry anchor");
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("check bridge session after resumed destroy retry"),
"retry after MobDestroying must still archive the member bridge session before storage clear"
);
let retained_events = events.replay_all().await.expect("replay retained events");
assert!(
retained_events.iter().any(|event| matches!(
event.kind,
MobEventKind::MemberRetired { ref agent_identity, .. } if agent_identity == &worker
)),
"retry must persist the member retire event before attempting final storage clear"
);
events.allow_clear();
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if events
.replay_all()
.await
.expect("events after complete retry")
.is_empty()
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("same-process bounded-backoff destroy retry clears storage");
}
#[tokio::test]
async fn test_resume_retries_retirement_started_anchor_without_operator_command() {
let definition = with_unique_mob_id(sample_definition(), "retirement-started-auto-retry");
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let worker = AgentIdentity::from("retirement-started-worker");
handle
.spawn(ProfileName::from("worker"), worker.clone(), None)
.await
.expect("spawn worker");
let survivor = AgentIdentity::from("retirement-started-survivor");
let survivor_bridge_session_id = handle
.spawn(ProfileName::from("worker"), survivor.clone(), None)
.await
.expect("spawn survivor")
.bridge_session_id()
.cloned()
.expect("survivor bridge session id");
handle
.wire(worker.clone(), survivor.clone())
.await
.expect("wire retiring worker to survivor");
let entry = handle
.list_members_including_retiring()
.await
.into_iter()
.find(|entry| entry.agent_identity == worker)
.expect("spawned roster entry");
let agent_runtime_id = entry.agent_runtime_id.clone().expect("spawned runtime id");
let generation = agent_runtime_id.generation;
let bridge_session_id = entry
.current_bridge_session_id
.clone()
.expect("spawned bridge session id");
assert!(
matches!(
adapter.peer_ingress_owner(&bridge_session_id).await,
meerkat_runtime::PeerIngressOwner::MobOwned { .. }
),
"fixture must begin with mob-owned peer ingress"
);
let retiring_peer_endpoint = events
.replay_all()
.await
.expect("replay spawned endpoint")
.into_iter()
.find_map(|event| match event.kind {
MobEventKind::MemberSpawned(spawned) if spawned.agent_identity == worker => {
spawned.member_peer_endpoint().cloned()
}
_ => None,
})
.expect("retiring worker has a durable peer endpoint");
events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: MobEventKind::MemberRetirementStarted {
agent_identity: entry.agent_identity.clone(),
agent_runtime_id,
generation,
role: entry.role.clone(),
releasing: None,
session_id: Some(bridge_session_id.clone()),
retiring_peer_endpoint: Some(retiring_peer_endpoint),
preserve_machine_topology: true,
},
})
.await
.expect("persist retirement-started crash anchor");
service.set_archive_failure(&bridge_session_id).await;
handle
.crash_stop_preserving_durable_work_for_test()
.await
.expect("quiesce old actor with crash semantics");
let closed_error = tokio::time::timeout(Duration::from_secs(1), handle.stop())
.await
.expect("old actor command channel closes after crash stop")
.expect_err("crash-stopped actor must reject later commands");
assert!(matches!(closed_error, MobError::ActorCommandChannelClosed));
// Generated owner tokens are process-local volatile state. A real crash
// destroys them with the runtimes; this retained-runtime mock must clear
// them explicitly before the recovered MobMachine installs its new owner.
service
.clear_mob_machine_trust_owner(&bridge_session_id)
.await;
service
.clear_mob_machine_trust_owner(&survivor_bridge_session_id)
.await;
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume retirement-started anchor");
// The first recovered attempts fail archive, but Retire consequences must
// still be re-realized: ingress detaches and the non-routable roster
// anchor remains for automatic same-process retry.
tokio::time::timeout(Duration::from_secs(5), async {
loop {
let retained = resumed
.list_members_including_retiring()
.await
.into_iter()
.find(|entry| entry.agent_identity == worker);
if retained.is_some()
&& matches!(
adapter.peer_ingress_owner(&bridge_session_id).await,
meerkat_runtime::PeerIngressOwner::Unattached
)
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("recovered Retire re-realizes ingress detach before failed archive");
service.clear_archive_failure(&bridge_session_id).await;
let convergence = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if !resumed
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == worker)
{
break;
}
tokio::task::yield_now().await;
}
})
.await;
if convergence.is_err() {
eprintln!(
"retirement retry diagnostic: members={:?} dsl={:?} events={:?} live_session={:?}",
resumed.list_members_including_retiring().await,
resumed.debug_dsl_t2_snapshot().await,
events.replay_all().await,
service.has_live_session(&bridge_session_id).await,
);
}
convergence.expect("background retire convergence completes");
let surviving_entry = resumed
.list_members()
.await
.into_iter()
.find(|entry| entry.agent_identity == survivor)
.expect("survivor remains after recovered respawn retirement");
assert!(
!surviving_entry.wired_to.contains(&worker),
"a recovered retirement start without a durable successor must abandon respawn topology"
);
let convergence_events = events
.replay_all()
.await
.expect("replay retirement convergence");
assert_eq!(
convergence_events
.iter()
.filter(|event| {
event.mob_id == mob_id
&& matches!(
&event.kind,
MobEventKind::MemberRetirementStarted { agent_identity, .. }
if agent_identity == &worker
)
})
.count(),
1,
"cold replay must reuse the one durable retirement-start carrier"
);
assert_eq!(
convergence_events
.iter()
.filter(|event| {
event.mob_id == mob_id
&& matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &worker
)
})
.count(),
1,
"background convergence must append exactly one final MemberRetired carrier"
);
}
#[tokio::test]
async fn test_shutdown_joins_recovered_retirement_retry_worker_before_reply() {
let definition = with_unique_mob_id(sample_definition(), "retirement-retry-shutdown-join");
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let worker = AgentIdentity::from("retirement-shutdown-worker");
handle
.spawn(ProfileName::from("worker"), worker.clone(), None)
.await
.expect("spawn worker");
let entry = handle
.list_members_including_retiring()
.await
.into_iter()
.find(|entry| entry.agent_identity == worker)
.expect("spawned roster entry");
let agent_runtime_id = entry.agent_runtime_id.clone().expect("spawned runtime id");
let generation = agent_runtime_id.generation;
let bridge_session_id = entry
.current_bridge_session_id
.clone()
.expect("spawned bridge session id");
events
.append(NewMobEvent {
mob_id,
timestamp: None,
kind: MobEventKind::MemberRetirementStarted {
agent_identity: entry.agent_identity,
agent_runtime_id,
generation,
role: entry.role,
releasing: Some(bridge_session_id.clone()),
session_id: Some(bridge_session_id.clone()),
retiring_peer_endpoint: None,
preserve_machine_topology: false,
},
})
.await
.expect("persist retirement-started crash anchor");
service.set_archive_failure(&bridge_session_id).await;
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume retirement-started anchor");
let resumed_mob_id = resumed.mob_id().clone();
// Drive the recovered worker to its capped two-second retry delay. A
// detached worker would still retain its MobHandle well after Shutdown
// replied; actor-owned shutdown must abort and join it first.
tokio::time::timeout(Duration::from_secs(6), async {
while service.archive_call_count(&bridge_session_id).await < 8 {
tokio::task::yield_now().await;
}
})
.await
.expect("recovered retirement worker reaches capped retry delay");
assert!(
super::builder::latest_actor_owned_startup_worker_count_for_test(&resumed_mob_id) >= 2,
"retirement retry and remote-intent reconciler must both be live before shutdown"
);
resumed.shutdown().await.expect("shutdown resumed mob");
let attempts_after_shutdown = service.archive_call_count(&bridge_session_id).await;
assert_eq!(
super::builder::latest_actor_owned_startup_worker_count_for_test(&resumed_mob_id),
0,
"shutdown must abort and join every actor-owned startup worker before replying"
);
tokio::time::sleep(Duration::from_millis(250)).await;
assert_eq!(
service.archive_call_count(&bridge_session_id).await,
attempts_after_shutdown,
"joined recovery worker must perform no archive retries after shutdown"
);
}
#[tokio::test]
async fn test_retire_final_replay_prunes_wiring_and_preserves_generation_history() {
let definition = with_unique_mob_id(sample_definition(), "retire-final-replay-parity");
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let retired = AgentIdentity::from("retire-final-a");
let survivor = AgentIdentity::from("retire-final-b");
handle
.spawn(ProfileName::from("worker"), retired.clone(), None)
.await
.expect("spawn retiring member");
let original_generation = handle
.get_member(&retired)
.await
.expect("query retiring member")
.expect("retiring member exists")
.generation;
let survivor_session = handle
.spawn(ProfileName::from("worker"), survivor.clone(), None)
.await
.expect("spawn surviving member")
.bridge_session_id()
.cloned()
.expect("survivor bridge session");
handle
.wire(retired.clone(), survivor.clone())
.await
.expect("wire members");
let retired_dsl = crate::machines::mob_machine::AgentIdentity::from_domain(&retired);
handle.retire(retired.clone()).await.expect("retire A");
handle
.retire(retired.clone())
.await
.expect("same-process second retire is a machine-authorized historical no-op");
assert!(
!service
.trusted_peer_names(&survivor_session)
.await
.contains(&test_comms_name("worker", retired.as_str())),
"live retirement removes A from B's trust before final publication"
);
let retired_state = handle
.query_machine_state()
.await
.expect("query retired machine state");
assert!(
!retired_state.identity_to_runtime.contains_key(&retired_dsl),
"exact final removes the current binding"
);
assert_eq!(
retired_state
.identity_runtime_generations
.get(&retired_dsl)
.map(|generation| generation.0),
Some(original_generation.get()),
"generation history survives exact final"
);
assert!(
retired_state.member_kickoff_started.contains(&retired_dsl)
|| retired_state.member_kickoff_failed.contains(&retired_dsl)
|| retired_state
.member_kickoff_cancelled
.contains(&retired_dsl),
"retirement must leave the old incarnation's settled kickoff fact for successor cleanup"
);
assert!(
handle
.list_members_observation_snapshot()
.await
.iter()
.all(|entry| entry.agent_identity != retired),
"observation lists must not synthesize a member from history-only state"
);
let live_terminal = handle
.wait_one(&retired)
.await
.expect("wait_one resolves exact live retirement history");
assert_eq!(
live_terminal.status,
crate::runtime::handle::MobMemberStatus::Completed
);
assert!(live_terminal.is_final);
assert!(
handle
.list_members()
.await
.iter()
.all(|entry| entry.agent_identity != retired),
"live wait terminality must not require synthetic list membership"
);
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("cold resume after wired retirement");
assert!(
resumed
.get_member(&retired)
.await
.expect("query retired member")
.is_none(),
"retired A must stay absent after replay"
);
let cold_terminal = resumed
.wait_one(&retired)
.await
.expect("wait_one resolves exact cold-replayed retirement history");
assert_eq!(
cold_terminal.status,
crate::runtime::handle::MobMemberStatus::Completed
);
assert!(cold_terminal.is_final);
assert!(
resumed
.list_members()
.await
.iter()
.all(|entry| entry.agent_identity != retired),
"cold wait terminality must not require synthetic list membership"
);
assert!(
resumed
.get_member(&survivor)
.await
.expect("query survivor")
.is_some(),
"wired survivor B must remain live"
);
let snapshot = resumed
.debug_dsl_t2_snapshot()
.await
.expect("replayed machine snapshot");
assert!(
snapshot
.wiring_edges
.iter()
.all(|edge| edge.a.0 != retired.as_str() && edge.b.0 != retired.as_str()),
"MemberRetired replay must prune every edge incident to A"
);
assert!(
!service
.trusted_peer_names(&survivor_session)
.await
.contains(&test_comms_name("worker", retired.as_str())),
"cold resume must not repair stale A-B trust"
);
resumed
.retire(retired.clone())
.await
.expect("cold second retire is a machine-authorized historical no-op");
resumed
.spawn_with_options(
ProfileName::from("worker"),
retired.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("cold respawn same identity");
let replacement = resumed
.get_member(&retired)
.await
.expect("query replacement")
.expect("replacement exists");
assert_eq!(
replacement.generation,
original_generation
.next()
.expect("test generation has a successor"),
"cold replay must preserve retired generation history"
);
drop(resumed);
let resumed_replacement = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("cold resume with generation-two replacement live");
let replayed_replacement = resumed_replacement
.get_member(&retired)
.await
.expect("query replayed replacement")
.expect("generation-two replacement remains live after cold replay");
assert_eq!(
replayed_replacement.generation,
original_generation
.next()
.expect("test generation has a successor"),
"cold replay must install the strict successor as current"
);
let replacement_state = resumed_replacement
.query_machine_state()
.await
.expect("query replacement machine state");
assert!(
!replacement_state
.member_kickoff_pending
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_starting
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_callback_pending
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_started
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_failed
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_cancelled
.contains(&retired_dsl)
&& !replacement_state
.member_kickoff_error
.contains_key(&retired_dsl),
"generation-two replay must clear every kickoff fact keyed by generation one"
);
resumed_replacement
.retire(retired.clone())
.await
.expect("retire replacement");
let finals = events
.replay_all()
.await
.expect("replay both retirement finals")
.into_iter()
.filter_map(|event| match event.kind {
MobEventKind::MemberRetired {
agent_identity,
generation,
..
} if agent_identity == retired => Some(generation),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
finals,
vec![
original_generation,
original_generation
.next()
.expect("test generation has a successor"),
]
);
}
#[tokio::test]
async fn test_destroy_marker_append_failure_precedes_cleanup_side_effects() {
let definition = with_unique_mob_id(sample_definition(), "destroy-marker-first");
let events = Arc::new(FaultInjectedMobEventStore::new());
let storage = MobStorage::with_events(events.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let bridge_session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("marker-first-worker"),
None,
)
.await
.expect("spawn worker before marker failure")
.bridge_session_id()
.cloned()
.expect("session-backed worker");
events.fail_appends_for("MobDestroying").await;
let err = handle
.destroy()
.await
.expect_err("destroy marker append failure should fail closed");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report
.errors
.iter()
.any(|error| error.contains("destroy marker append failed")),
"report should identify marker append failure: {report:?}"
);
let replayed = events.replay_all().await.expect("replay events");
assert!(
replayed
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. })),
"destroy must not retire members before the durable destroy marker is written"
);
assert!(
service
.has_live_session(&bridge_session_id)
.await
.expect("check worker session after marker failure"),
"destroy must not archive member sessions before the durable destroy marker is written"
);
}
#[tokio::test]
async fn test_destroy_storage_finalizing_append_failure_precedes_storage_scrub() {
let definition = with_unique_mob_id(sample_definition(), "destroy-finalizing-marker-first");
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(definition, events.clone()).await;
events.fail_appends_for("MobDestroyStorageFinalizing").await;
let err = handle
.destroy()
.await
.expect_err("destroy storage-finalizing append failure should fail closed");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.namespace_cleaned,
"namespace cleanup precedes storage-finalizing marker"
);
assert!(
!report.metadata_scrubbed && !report.events_cleared,
"failed storage-finalizing marker must stop before metadata scrub/event clear: {report:?}"
);
assert!(
report
.errors
.iter()
.any(|error| error.contains("record destroy storage finalizing event failed")),
"report should identify storage-finalizing marker append failure: {report:?}"
);
let replayed = events.replay_all().await.expect("replay events");
assert!(
replayed
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MobDestroyStorageFinalizing)),
"failed storage-finalizing append must not persist the marker"
);
}
#[tokio::test]
async fn test_destroy_retire_admission_failure_after_marker_retains_retry_anchor_and_closes_authority()
{
let definition = with_unique_mob_id(sample_definition(), "destroy-retire-admission-fail");
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, service) = create_test_mob_with_events(definition, events.clone()).await;
service.set_keep_alive_turns_complete_immediately(true);
let adapter = service.enable_runtime_adapter();
let worker = AgentIdentity::from("destroy-stop-loop-worker");
let bridge_session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(worker.as_str()),
None,
)
.await
.expect("spawn autonomous worker before partial destroy")
.bridge_session_id()
.cloned()
.expect("session-backed worker");
unregister_runtime_session_until_complete(adapter.as_ref(), &bridge_session_id)
.await
.expect("bridge session should unregister cleanly");
let err = handle
.destroy()
.await
.expect_err("missing runtime adapter registration should make destroy incomplete");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.errors.iter().any(|error| {
error.contains("destroy retire admission failed")
&& error.contains("routed_session_not_durability_ready")
&& error.contains(
"cannot accept routed input until its persistent runtime is cold reloaded",
)
&& error.contains("destroyed")
}),
"destroy should surface the failed retire admission: {report:?}"
);
assert!(
!report.metadata_scrubbed && !report.events_cleared,
"local cleanup failure must retain metadata/events for retry: {report:?}"
);
assert_eq!(
handle.status().await.unwrap(),
MobState::Destroyed,
"durable destroy admission should close live authority after partial cleanup"
);
let stop_err = handle
.stop()
.await
.expect_err("destroy-admitted mob must reject non-destroy mutation");
assert!(
matches!(stop_err, MobError::InvalidTransition { .. }),
"partial destroy should fail closed for live mutation commands: {stop_err:?}"
);
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == worker),
"failed StopHostLoop must retain the member roster anchor for retry"
);
// The campaign-0.7.2 D1 unregister drain cleanly stops the runtime loop,
// whose pre-existing executor-stop teardown discards the LIVE session copy
// (durable state is preserved — discard is not archival). The retry-safety
// contract is that the partial-destroy failure must not ARCHIVE (terminally
// lose) the bridge session; the durable copy stays recoverable.
assert!(
!service.test_is_session_archived(&bridge_session_id).await,
"failed StopHostLoop must not archive the bridge session"
);
assert!(
events
.replay_all()
.await
.expect("replay retained events")
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)),
"partial destroy must retain the durable destroy marker"
);
let retry_err = handle
.destroy()
.await
.expect_err("retry should report the retained retire-admission failure");
let retry_report = match retry_err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy retry, got {other:?}"),
};
// The admitted destroy owns cleanup retry, but it does not own
// registration-authorized cold materialization of the consumer runtime.
// Redispatch must therefore preserve the same typed refusal and roster
// anchor until a cold host rebuilds the runtime from durable truth.
assert!(
retry_report.errors.iter().any(|error| {
error.contains("destroy retire admission failed")
&& error.contains("routed_session_not_durability_ready")
&& error.contains(
"cannot accept routed input until its persistent runtime is cold reloaded",
)
}),
"same-process retry must preserve the cold-reload requirement and partial destroy state: {retry_report:?}"
);
}
#[tokio::test]
async fn test_destroy_autonomous_archive_failure_retry_reaches_archive_after_runtime_stopped() {
let definition = with_unique_mob_id(sample_definition(), "destroy-autonomous-archive-retry");
let (handle, service) = create_test_mob(definition).await;
service.set_keep_alive_turns_complete_immediately(true);
let worker = AgentIdentity::from("destroy-autonomous-archive-worker");
let bridge_session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(worker.as_str()),
None,
)
.await
.expect("spawn autonomous worker before partial destroy")
.bridge_session_id()
.cloned()
.expect("autonomous worker bridge session");
service
.set_archive_precommit_failure(&bridge_session_id)
.await;
let err = handle
.destroy()
.await
.expect_err("archive failure should make destroy incomplete");
let report = match err {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy, got {other:?}"),
};
assert!(
report.errors.iter().any(|error| {
error.contains("ArchiveSession") && error.contains("mock archive precommit failure")
}),
"destroy should surface the failed archive step: {report:?}"
);
assert!(
!report.metadata_scrubbed && !report.events_cleared,
"archive failure must retain metadata/events for retry: {report:?}"
);
assert_eq!(
handle.status().await.unwrap(),
MobState::Destroyed,
"partial destroy should close live authority"
);
assert!(
handle
.list_members_including_retiring()
.await
.iter()
.any(|entry| entry.agent_identity == worker),
"archive failure must retain the member roster anchor for retry"
);
assert!(
service
.has_live_session(&bridge_session_id)
.await
.expect("check worker session after archive failure"),
"failed archive must leave the bridge session available for retry"
);
service
.clear_archive_precommit_failure(&bridge_session_id)
.await;
let retry_report = handle
.destroy()
.await
.expect("retry should reach ArchiveSession after runtime was already stopped");
assert!(retry_report.metadata_scrubbed);
assert!(retry_report.events_cleared);
assert!(retry_report.namespace_cleaned);
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("check worker session after successful retry"),
"successful retry must archive the retained bridge session"
);
}
#[tokio::test]
async fn test_resume_prefers_destroying_marker_over_completed_marker() {
let definition = with_unique_mob_id(sample_definition(), "destroying-over-completed");
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.complete().await.expect("complete empty mob");
events.fail_clear();
let err = handle
.destroy()
.await
.expect_err("event clear failure should leave partial destroy");
assert!(
matches!(
err,
crate::runtime::handle::MobDestroyError::Incomplete { .. }
),
"expected incomplete destroy after failed event clear"
);
let replayed = events.replay_all().await.expect("replay retained events");
assert!(
replayed
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobCompleted)),
"test should retain a completed marker before destroy"
);
assert!(
replayed
.iter()
.any(|event| matches!(event.kind, MobEventKind::MobDestroying)),
"partial destroy should retain the destroy marker"
);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("resume retained completed-then-destroying storage");
assert_eq!(
resumed.status().await.unwrap(),
MobState::Destroyed,
"MobDestroying must dominate earlier MobCompleted during resume"
);
assert_eq!(resumed.definition().id, mob_id);
let _ = resumed.shutdown().await;
}
#[tokio::test]
async fn test_stopped_retire_detaches_mob_owned_session_ingress() {
let (handle, service) = create_test_mob_with_runtime_adapter(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let adapter = service.enable_runtime_adapter();
let session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-stopped-ingress-retire"),
None,
)
.await
.expect("spawn session-backed member")
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
handle.stop().await.expect("stop mob");
handle
.retire(AgentIdentity::from("w-stopped-ingress-retire"))
.await
.expect("stopped retire should close detach ack");
assert!(
matches!(
adapter.peer_ingress_owner(&session_id).await,
meerkat_runtime::PeerIngressOwner::Unattached
),
"retire from Stopped should detach mob-owned peer ingress"
);
}
#[tokio::test]
async fn test_stopped_wired_member_retire_converges_topology_and_trust() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("z-stopped-retiring"),
None,
)
.await
.expect("spawn retiring member");
let surviving_session = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("a-stopped-survivor"),
None,
)
.await
.expect("spawn surviving member")
.bridge_session_id()
.cloned()
.expect("session-backed survivor");
let retiring = AgentIdentity::from("z-stopped-retiring");
let survivor = AgentIdentity::from("a-stopped-survivor");
handle
.wire(retiring.clone(), survivor.clone())
.await
.expect("wire members before stop");
handle.stop().await.expect("stop mob");
handle
.retire(retiring.clone())
.await
.expect("stopped retirement must admit scoped topology cleanup");
assert_eq!(
handle.status().await.expect("stopped mob status"),
MobState::Stopped,
"per-phase retirement cleanup must not drift the mob back to Running"
);
let surviving_entry = handle
.get_member(&survivor)
.await
.expect("read surviving roster")
.expect("surviving member");
assert!(!surviving_entry.wired_to.contains(&retiring));
assert!(
!service
.trusted_peer_names(&surviving_session)
.await
.iter()
.any(|name| name == &test_comms_name("worker", retiring.as_str())),
"stopped retirement must revoke reciprocal trust before terminal publication"
);
assert!(
handle
.events()
.replay_all()
.await
.expect("replay stopped retirement")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &retiring
))
);
}
#[tokio::test]
async fn test_detach_failure_defers_runtime_retire_until_correlated_retry() {
let (handle, service) = create_test_mob_with_runtime_adapter(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let adapter = service.enable_runtime_adapter();
let identity = AgentIdentity::from("w-detach-failure");
let session_id = handle
.spawn(ProfileName::from("worker"), identity.clone(), None)
.await
.expect("spawn session-backed member")
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
assert!(matches!(
adapter.peer_ingress_owner(&session_id).await,
meerkat_runtime::PeerIngressOwner::MobOwned { .. }
));
super::actor::fail_session_ingress_detach_for_test(&session_id);
let error = handle
.retire(identity.clone())
.await
.expect_err("fault-injected detach must retain retirement for retry");
assert!(
error
.to_string()
.contains("fault-injected session-ingress detach failure"),
"unexpected detach failure: {error}"
);
// Let the actor reach its automatic boundary flush. The queued runtime
// retire must remain deferred while the machine still owns the detach
// obligation.
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(matches!(
adapter.peer_ingress_owner(&session_id).await,
meerkat_runtime::PeerIngressOwner::MobOwned { .. }
));
let runtime_state = <meerkat_runtime::MeerkatMachine as meerkat_runtime::SessionServiceRuntimeExt>::runtime_state(
adapter.as_ref(),
&session_id,
)
.await
.expect("read runtime state after failed detach");
assert!(
!matches!(
runtime_state,
meerkat_runtime::RuntimeState::Retired | meerkat_runtime::RuntimeState::Destroyed
),
"runtime retire escaped before detach acknowledgement: {runtime_state:?}"
);
handle
.retire(identity)
.await
.expect("retry must close detach, dispatch one correlated retire, and archive");
assert!(matches!(
adapter.peer_ingress_owner(&session_id).await,
meerkat_runtime::PeerIngressOwner::Unattached
));
}
#[tokio::test]
async fn test_destroy_scrubs_runtime_metadata() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = sample_definition();
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor metadata")
.is_some(),
"create should persist supervisor metadata before destroy"
);
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("worker-overlay"),
generation: crate::ids::Generation::INITIAL,
normalized_member_ref: None,
bootstrap_token: None,
status: ExternalBindingOverlayStatus::Failed {
reason: "test overlay".to_string(),
},
updated_at: Utc::now(),
},
)
.await
.expect("insert overlay");
let request = crate::store::MobMemberOperatorRequestRecord::pending(
crate::store::MobMemberOperatorRequestKey::new(
"worker-overlay",
1,
1,
"host-a",
1,
"member-session-a",
"req-destroy-scrub",
),
crate::store::member_operator_op_digest(
&crate::runtime::bridge_protocol::MemberOperatorOp::ListMembers,
)
.expect("request operation digest"),
);
assert!(matches!(
runtime_metadata
.begin_member_operator_request(&mob_id, &request)
.await
.expect("insert member operator request"),
crate::store::MobMemberOperatorRequestBegin::Started
));
handle.destroy().await.expect("destroy");
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor metadata after destroy")
.is_none(),
"destroy should scrub supervisor metadata"
);
assert!(
runtime_metadata
.list_external_binding_overlays(&mob_id)
.await
.expect("list overlays after destroy")
.is_empty(),
"destroy should scrub binding overlays"
);
assert!(
runtime_metadata
.list_member_operator_requests(&mob_id)
.await
.expect("list member operator requests after destroy")
.is_empty(),
"destroy should scrub durable member operator requests"
);
}
#[tokio::test]
async fn test_rotate_supervisor_updates_runtime_metadata() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(sample_definition(), "rotate-supervisor-metadata");
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original supervisor")
.expect("original supervisor metadata");
let report = handle.rotate_supervisor().await.expect("rotate supervisor");
let rotated = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load rotated supervisor")
.expect("rotated supervisor metadata");
assert_eq!(report.previous_epoch, original.epoch);
assert_eq!(report.current_epoch, original.epoch + 1);
assert_eq!(rotated.epoch, report.current_epoch);
assert_eq!(rotated.public_peer_id, report.public_peer_id);
assert_ne!(
rotated.public_peer_id, original.public_peer_id,
"rotation should replace the supervisor keypair"
);
}
#[tokio::test]
async fn test_rotate_supervisor_allows_final_epoch_then_refuses_wrap_without_write() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-final-representable-epoch",
);
let mob_id = definition.id.clone();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut recovered_authority = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load initial authority")
.expect("initial authority");
handle
.crash_stop_preserving_durable_work_for_test()
.await
.expect("crash-stop original actor");
drop(handle);
recovered_authority.epoch = u64::MAX - 1;
runtime_metadata
.inner
.seed_legacy_supervisor_authority(&mob_id, recovered_authority.clone())
.await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("recover near-exhausted supervisor authority");
let final_rotation = resumed
.rotate_supervisor()
.await
.expect("u64::MAX remains a valid final supervisor epoch");
assert_eq!(final_rotation.previous_epoch, u64::MAX - 1);
assert_eq!(final_rotation.current_epoch, u64::MAX);
let final_authority = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load final supervisor authority")
.expect("final supervisor authority");
assert_eq!(final_authority.epoch, u64::MAX);
assert!(final_authority.pending_rotation.is_none());
// Arm a store failure and prove exhaustion returns before consuming it:
// no pending anchor/final CAS is attempted at the wrap boundary.
runtime_metadata.fail_next_put_supervisor();
let exhausted = resumed
.rotate_supervisor()
.await
.expect_err("rotation after the final epoch must fail closed");
assert!(matches!(
exhausted,
MobError::SupervisorEpochExhausted {
current_epoch: u64::MAX
}
));
assert_eq!(
runtime_metadata
.fail_put_supervisor_countdown
.load(Ordering::Relaxed),
1,
"epoch exhaustion must not attempt a supervisor metadata write",
);
assert_eq!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("reload authority after exhausted rotation"),
Some(final_authority),
"epoch exhaustion must leave the durable authority byte-identical",
);
}
#[tokio::test]
async fn test_rotate_supervisor_initial_pending_completion_loss_converges_exactly() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-initial-pending-completion-loss",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
),
)
.with_session_service(service)
.create()
.await
.expect("create mob");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
// With no remote members, compare #1 records the initial pending
// operation and compare #2 activates it. Lose only compare #1's return.
runtime_metadata.lose_nth_compare_supervisor_completion(1);
let report = handle
.rotate_supervisor()
.await
.expect("exact reread should absorb the committed pending anchor");
let durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load converged authority")
.expect("converged authority");
assert_eq!(report.previous_epoch, original.epoch);
assert_eq!(report.current_epoch, original.epoch + 1);
assert_eq!(durable.epoch, report.current_epoch);
assert_eq!(durable.public_peer_id, report.public_peer_id);
assert!(durable.pending_rotation.is_none());
}
#[tokio::test]
async fn test_rotate_supervisor_final_activation_completion_loss_converges_exactly() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-final-activation-completion-loss",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
),
)
.with_session_service(service)
.create()
.await
.expect("create mob");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
runtime_metadata.lose_nth_compare_supervisor_completion(2);
let report = handle
.rotate_supervisor()
.await
.expect("exact reread should absorb the committed final authority");
let durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load converged authority")
.expect("converged authority");
assert_eq!(report.previous_epoch, original.epoch);
assert_eq!(report.current_epoch, original.epoch + 1);
assert_eq!(durable.epoch, report.current_epoch);
assert_eq!(durable.public_peer_id, report.public_peer_id);
assert!(durable.pending_rotation.is_none());
let machine = handle
.query_machine_state()
.await
.expect("query converged machine authority");
assert_eq!(
machine
.supervisor_authority_peer_id
.as_ref()
.map(|peer| peer.0.as_str()),
Some(durable.public_peer_id.as_str()),
);
assert_eq!(machine.supervisor_authority_epoch, Some(durable.epoch));
assert!(machine.supervisor_pending_authority_peer_id.is_none());
}
#[tokio::test]
async fn test_rotate_supervisor_unreadable_final_completion_loss_fail_stops_and_cold_recovers() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-final-completion-loss-unreadable",
);
let mob_id = definition.id.clone();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
runtime_metadata.lose_nth_compare_supervisor_completion_with_unreadable_reread(2);
let error = handle
.rotate_supervisor()
.await
.expect_err("unreadable reconciliation must fail-stop the live actor");
assert!(
error
.to_string()
.contains("exact reread remained unreadable")
&& error
.to_string()
.contains("fail-stopping for cold recovery"),
"unexpected reconciliation error: {error}",
);
let durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after one-shot reread fault")
.expect("postcommit authority must be durable");
assert_eq!(durable.epoch, original.epoch + 1);
assert!(durable.pending_rotation.is_none());
let closed = tokio::time::timeout(Duration::from_secs(2), handle.rotate_supervisor())
.await
.expect("fail-stopped actor closes its command channel")
.expect_err("fail-stopped actor must reject same-process retry");
assert!(matches!(closed, MobError::ActorCommandChannelClosed));
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("cold recovery should rebuild from the exact committed authority");
let machine = resumed
.query_machine_state()
.await
.expect("query cold-recovered machine authority");
assert_eq!(
machine
.supervisor_authority_peer_id
.as_ref()
.map(|peer| peer.0.as_str()),
Some(durable.public_peer_id.as_str()),
);
assert_eq!(machine.supervisor_authority_epoch, Some(durable.epoch));
assert!(machine.supervisor_pending_authority_peer_id.is_none());
}
#[tokio::test]
async fn test_rotate_supervisor_timeout_keeps_durable_operation_and_retry_reuses_id() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-operation-retry",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
external.suppress_rotation_observations(true);
let rotation_handle = handle.clone();
let rotation_task = tokio::spawn(async move { rotation_handle.rotate_supervisor().await });
let remote_completion_deadline =
tokio::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
if !external.rotation_operation_ids().await.is_empty()
&& external.authorized_supervisor_peer_id().await.as_deref()
!= Some(original.public_peer_id.as_str())
{
break;
}
if rotation_task.is_finished() {
let early_result = rotation_task.await.expect("rotation task should not panic");
panic!(
"caller stopped observing before the remote completed: {early_result:?}; received operations: {:?}",
external.rotation_operation_ids().await
);
}
assert!(
tokio::time::Instant::now() < remote_completion_deadline,
"remote did not complete the submitted operation before the test deadline"
);
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
}
assert!(
!rotation_task.is_finished(),
"member must complete while the caller is still waiting on observation"
);
let error = rotation_task
.await
.expect("rotation task should not panic")
.expect_err("completed remote operation with unavailable observation must remain pending");
assert!(matches!(
error,
MobError::SupervisorRotationIncomplete {
pending_authority_recorded: true,
rollback_succeeded: false,
..
}
));
let pending = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load pending authority")
.expect("pending authority");
assert_eq!(
pending.epoch, original.epoch,
"timeout cannot advance locally"
);
let operation_id = pending
.pending_rotation
.as_ref()
.and_then(|pending| pending.operation_id)
.expect("operation id persisted before submit");
let attempted_public_peer_id = pending
.pending_rotation
.as_ref()
.expect("pending operation remains durable")
.public_peer_id
.clone();
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str()),
"member completion must survive the caller's observation timeout"
);
assert!(
!external
.trusted_peer_routes()
.await
.iter()
.any(|(_, peer_id, _)| peer_id == &original.public_peer_id),
"the member must keep the old supervisor fenced after timeout"
);
assert_eq!(
external.rotation_operation_ids().await,
vec![operation_id],
"the member must have completed the durably recorded operation"
);
external.suppress_rotation_observations(false);
let late_external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-late")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-late"),
None,
late_external.binding(),
)
.await
.expect("spawn member added after the incomplete rotation");
let report = handle.rotate_supervisor().await.expect(
"retry should observe the existing receipt even when old-signed resubmit is fenced",
);
let rotated = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load rotated authority")
.expect("rotated authority");
assert_eq!(rotated.epoch, report.current_epoch);
assert!(rotated.pending_rotation.is_none());
assert_ne!(rotated.public_peer_id, original.public_peer_id);
assert_eq!(
external.rotation_operation_ids().await,
vec![operation_id],
"fenced old authority must not create a second operation; retry succeeds by observing the persisted id"
);
assert_eq!(
late_external.rotation_operation_ids().await,
vec![operation_id],
"retry must monotonically add the late member target to the same operation"
);
assert_eq!(
late_external
.authorized_supervisor_peer_id()
.await
.as_deref(),
Some(rotated.public_peer_id.as_str())
);
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_rotate_supervisor_observes_rejection_through_retained_authority() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-cross-authority-rejection",
);
let mob_id = definition.id.clone();
let supervisor_port = unused_loopback_port();
let supervisor_address = format!("tcp://127.0.0.1:{supervisor_port}");
definition
.backend
.external
.as_mut()
.expect("external backend")
.supervisor_bridge = Some(crate::definition::SupervisorBridgeEndpointConfig {
bind_address: Some(format!("127.0.0.1:{supervisor_port}")),
advertised_address: Some(supervisor_address),
});
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external =
spawn_live_external_tcp_peer(&test_comms_name_for(&mob_id, "worker", "w-reject")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-reject"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
external.reject_next_rotation();
external.drop_next_attempted_rotation_observation_response();
let error = handle
.rotate_supervisor()
.await
.expect_err("terminal member rejection must fail the rotation");
let MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
pending_authority_recorded,
reason,
..
} = error
else {
panic!("expected typed incomplete rotation, got {error:?}");
};
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert!(pending_authority_recorded);
assert!(
!external.attempted_rotation_observation_response_drop_is_armed(),
"the attempted-authority observation response must be dropped exactly once before retained-authority fallback"
);
assert!(
reason.contains("was rejected by peer")
&& reason.contains("injected terminal rotation rejection"),
"the retained-current observation path must surface the durable rejection, got: {reason}"
);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(original.public_peer_id.as_str()),
"a rejected target must not replace the retained supervisor"
);
let durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load pending authority")
.expect("pending authority");
assert_eq!(durable.epoch, original.epoch);
assert!(
durable.pending_rotation.is_some(),
"the exact rejected operation remains durably diagnosable"
);
assert_eq!(
external.rotation_operation_ids().await.len(),
1,
"the rejection belongs to the one write-ahead operation"
);
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_legacy_pending_rotation_prunes_inactive_acceptance_and_survives_restart() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-legacy-operation-migration",
);
let mob_id = definition.id.clone();
let supervisor_port = unused_loopback_port();
let supervisor_address = format!("tcp://127.0.0.1:{supervisor_port}");
definition
.backend
.external
.as_mut()
.expect("external backend")
.supervisor_bridge = Some(crate::definition::SupervisorBridgeEndpointConfig {
bind_address: Some(format!("127.0.0.1:{supervisor_port}")),
advertised_address: Some(supervisor_address),
});
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external =
spawn_live_external_tcp_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority");
let mut next = crate::store::SupervisorAuthorityRecord::generate(
super::bridge_protocol::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
);
next.epoch = original.epoch + 1;
let previous_spec = external
.authorized_supervisor_spec()
.await
.expect("bound legacy supervisor spec");
let target = super::bridge_protocol::BridgePeerSpec {
name: previous_spec.name,
peer_id: next.public_peer_id.clone(),
address: previous_spec.address,
pubkey: next.public_signing_key(),
};
let mut legacy_target = target.clone();
let legacy_stale_address = "inproc://legacy-stale-supervisor-route".to_string();
legacy_target.address = legacy_stale_address.clone();
external
.install_legacy_rotated_supervisor(legacy_target, next.epoch)
.await;
let external_peer_id = match external.binding() {
crate::RuntimeBinding::External { peer_id, .. } => peer_id,
other => panic!("expected external binding, got {other:?}"),
};
let mut legacy = original.clone();
legacy.pending_rotation = Some(crate::store::SupervisorPendingRotationRecord {
operation_id: None,
secret_key: next.secret_key,
public_peer_id: next.public_peer_id.clone(),
epoch: next.epoch,
protocol_version: next.protocol_version,
accepted_peer_ids: vec![
external_peer_id.clone(),
"peer-retired-before-v4-migration".to_string(),
],
member_targets: BTreeMap::new(),
});
runtime_metadata
.seed_legacy_supervisor_authority(&mob_id, legacy)
.await;
handle
.shutdown()
.await
.expect("shutdown before legacy migration recovery");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume legacy pending rotation");
external.suppress_rotation_observations(true);
let error = resumed
.rotate_supervisor()
.await
.expect_err("suppressed observation must leave migrated operation pending");
assert!(
matches!(
&error,
MobError::SupervisorRotationIncomplete {
pending_authority_recorded: true,
rollback_succeeded: false,
..
}
),
"suppressed legacy observation must remain durably pending, got: {error:?}"
);
let migrated = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load migrated authority")
.expect("migrated authority")
.pending_rotation
.expect("migrated pending operation");
let operation_id = migrated
.operation_id
.expect("stable operation id assigned by CAS");
assert_eq!(migrated.accepted_peer_ids, vec![external_peer_id.clone()]);
let migrated_target = migrated
.member_targets
.get(&external_peer_id)
.expect("migrated active member target");
assert_eq!(migrated_target.name, target.name);
assert_eq!(migrated_target.peer_id, target.peer_id);
assert_eq!(migrated_target.pubkey, target.pubkey);
assert!(
migrated_target.address.starts_with("tcp://"),
"migration must persist the resumed controller's authenticated TCP callback: {}",
migrated_target.address
);
assert_ne!(
migrated_target.address, legacy_stale_address,
"migration must not retain the injected legacy route"
);
assert!(
external.rotation_operation_ids().await.is_empty(),
"unavailable observation must not guess legacy status or submit before a confirmed route refresh"
);
resumed
.shutdown()
.await
.expect("shutdown after partial legacy migration");
external.suppress_rotation_observations(false);
let retried = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("resume migrated operation for exact retry");
// Arm this after resume so the lost reply deterministically lands on the
// first NotFound-triggered route refresh. The peer commits that refresh,
// but the same call must retry it exactly within the reserved budget.
external.drop_next_authorize_response();
let report = retried
.rotate_supervisor()
.await
.expect("restart retry must adopt and observe the same operation");
assert!(
!external.authorize_response_drop_is_armed(),
"the first legacy route-refresh response must be dropped exactly once"
);
assert_eq!(report.public_peer_id, next.public_peer_id);
let authorized = external
.authorized_supervisor_spec()
.await
.expect("authorized supervisor after restart recovery");
let supervisor_addresses = external.supervisor_addresses().await;
assert_eq!(
supervisor_addresses.last(),
Some(&authorized.address),
"NotFound-triggered next-authority route refresh must repair legacy address drift before adoption"
);
assert_ne!(authorized.address, legacy_stale_address);
assert_eq!(
external.rotation_operation_ids().await,
vec![operation_id],
"legacy adoption retry must preserve the exact operation id"
);
assert!(
runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load committed authority")
.expect("committed authority")
.pending_rotation
.is_none()
);
}
#[tokio::test]
async fn test_rotate_supervisor_reauthorizes_live_remote_members_and_rejects_stale_epoch() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-live-remote",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
let report = handle.rotate_supervisor().await.expect("rotate supervisor");
handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("after-rotate".to_string()))
.await
.expect("rotated supervisor should still control live peer-only member");
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"rotated supervisor should still be able to deliver one logical input"
);
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token: _,
pubkey,
} = external.binding()
else {
panic!("live external peer must expose external binding");
};
let peer_name = address
.strip_prefix("inproc://")
.map(|value| value.split('?').next().unwrap_or(value).to_string())
.unwrap_or_else(|| format!("mob_member/backend_peer/{peer_id}"));
let peer = meerkat_core::comms::TrustedPeerDescriptor::unsigned_with_pubkey(
peer_name,
peer_id.clone(),
pubkey,
address.clone(),
)
.expect("peer spec");
let old_bridge = crate::runtime::MobSupervisorBridge::new(&mob_id, original.clone(), None)
.await
.expect("build old supervisor bridge");
old_bridge
.trust_recipient(&peer)
.await
.expect("old supervisor bridge should explicitly trust peer for stale-epoch probe");
let stale_command = super::bridge_protocol::BridgeCommand::DeliverMemberInput(
super::bridge_protocol::BridgeDeliveryPayload {
objective_id: None,
system_prompt: None,
injected_context: Vec::new(),
transient_turn_context: None,
supervisor: old_bridge
.supervisor_spec()
.await
.expect("old supervisor spec")
.into(),
epoch: report.previous_epoch,
protocol_version: original.protocol_version,
input_id: "stale-epoch-input".to_string(),
transcript_interaction_id: None,
content: ContentInput::from("stale-epoch".to_string()),
handling_mode: HandlingMode::Queue,
expected_member: None,
turn: None,
outcome_tracking: None,
},
);
let stale_reply = old_bridge
.send_bridge_command(&peer, &stale_command, std::time::Duration::from_secs(1))
.await
.expect("bridge should return a rejected reply for stale supervisor epochs");
let stale_reply: super::bridge_protocol::BridgeReply =
serde_json::from_value(stale_reply).expect("decode stale bridge reply");
match stale_reply {
super::bridge_protocol::BridgeReply::Rejected { cause, reason } => {
assert_eq!(
cause,
super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
"stale supervisor should be rejected with typed cause, got reason: {reason}"
);
assert!(
reason.contains("stale supervisor"),
"stale supervisor should be rejected explicitly, got: {reason}"
);
}
other => panic!("expected stale supervisor rejection, got {other:?}"),
}
}
/// Build the supervisor-side peer descriptor for a live external binding,
/// mirroring `MultiBackendProvisioner::peer_only_spec_from_parts`.
fn external_peer_descriptor_for_binding(
binding: &crate::RuntimeBinding,
) -> meerkat_core::comms::TrustedPeerDescriptor {
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token: _,
pubkey,
} = binding
else {
panic!("live external peer must expose external binding");
};
let peer_name = address
.strip_prefix("inproc://")
.map(|value| value.split('?').next().unwrap_or(value).to_string())
.unwrap_or_else(|| format!("mob_member/backend_peer/{peer_id}"));
meerkat_core::comms::TrustedPeerDescriptor::unsigned_with_pubkey(
peer_name,
peer_id.clone(),
*pubkey,
address.clone(),
)
.expect("peer spec")
}
/// Peer ids currently trusted by the mob's live supervisor bridge runtime —
/// the behavioral trust surface (comms admission), not source text.
async fn supervisor_bridge_trusted_peer_ids(handle: &MobHandle) -> Vec<String> {
handle
.supervisor_bridge
.runtime()
.await
.peers()
.await
.into_iter()
.map(|entry| entry.peer_id.to_string())
.collect()
}
/// Snapshot of the MobMachine `pending_recipient_trust` obligation set.
fn pending_recipient_trust_snapshot(handle: &MobHandle) -> Vec<String> {
handle
.machine_state_watch_rx
.borrow()
.pending_recipient_trust
.iter()
.map(|peer_id| peer_id.0.clone())
.collect()
}
/// Dogma row R044 (actor path): a terminal `AuthorizeSupervisor` rejection
/// must leave NO trust that the authorization attempt itself installed, and
/// the machine-owned `pending_recipient_trust` obligation must be closed.
#[tokio::test]
async fn test_authorize_rejection_rolls_back_newly_installed_recipient_trust() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"authorize-reject-rollback",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"bind during spawn must have installed recipient trust"
);
// Remove the bind-installed trust through the canonical removal seam so
// the next authorization attempt newly installs trust itself.
handle
.supervisor_bridge
.untrust_recipient(&peer)
.await
.expect("untrust recipient ahead of rejected authorize attempt");
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"precondition: peer must be untrusted before the rejected attempt"
);
external.fail_next_authorize();
let turn = handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("after-rejected-authorize".to_string()))
.await;
assert!(
turn.is_err(),
"turn must fail when supervisor authorization is rejected"
);
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"rejected authorization must roll back the trust it installed"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be closed after failure terminality"
);
}
/// Dogma row R044 (prior-trust scope): a rejected re-authorization of a peer
/// that was ALREADY trusted before the attempt must not rip out the
/// previously confirmed trust — rollback is scoped to trust installed by the
/// failing attempt only.
#[tokio::test]
async fn test_authorize_rejection_preserves_preexisting_recipient_trust() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"authorize-reject-preserve",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"bind during spawn must have installed recipient trust"
);
external.fail_next_authorize();
let turn = handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("after-rejected-authorize".to_string()))
.await;
assert!(
turn.is_err(),
"turn must fail when supervisor authorization is rejected"
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"trust confirmed by the earlier bind must survive a later rejected re-authorization"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be closed after failure terminality"
);
}
/// Dogma row R044 (provisioner path, terminal rejection): a rejected
/// `BindMember` during external provisioning must leave NO recipient trust
/// installed and must close the machine-owned obligation window.
#[tokio::test]
async fn test_bind_rejection_rolls_back_provisioner_recipient_trust() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"bind-reject-rollback",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
external.fail_next_bind();
let spawn = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await;
assert!(spawn.is_err(), "spawn must fail when the bind is rejected");
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"rejected bind must roll back the recipient trust it installed"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be closed after the failed provision"
);
}
/// Dogma row R044 (provisioner path, send failure): a bind send that cannot
/// reach the peer must leave NO recipient trust installed and must close the
/// machine-owned obligation window.
#[tokio::test]
async fn test_bind_send_failure_rolls_back_provisioner_recipient_trust() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"bind-send-failure-rollback",
);
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
// A well-formed external binding whose peer is not registered anywhere:
// the inproc transport fails the send immediately after trust install.
let ghost_keypair = meerkat_comms::Keypair::generate();
let ghost_binding = crate::RuntimeBinding::External {
peer_id: ghost_keypair.public_key().to_peer_id().to_string(),
address: format!(
"inproc://ghost-peer-{}",
meerkat_core::time_compat::new_uuid_v7()
),
bootstrap_token: Some("ghost-bootstrap-token".to_string().into()),
pubkey: *ghost_keypair.public_key().as_bytes(),
};
let ghost_peer_id = ghost_keypair.public_key().to_peer_id().to_string();
let spawn = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ghost"),
None,
ghost_binding,
)
.await;
assert!(
spawn.is_err(),
"spawn must fail when the bind cannot reach the peer"
);
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&ghost_peer_id),
"failed bind send must roll back the recipient trust it installed"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be closed after the failed provision"
);
}
/// A BindMember reply that becomes undecodable after the remote mutation is
/// not rollback proof. The exact PeerId reservation and recipient trust stay
/// quarantined, the machine-owned obligation remains open, and the actor
/// fail-stops before a second BindMember can escape.
#[tokio::test]
async fn test_post_commit_bind_decode_failure_quarantines_and_fail_stops() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"bind-decode-failure-rollback",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
external.garble_next_bind_reply();
let error = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect_err("undecodable post-commit bind reply is uncertain");
assert!(
matches!(&error, MobError::ExternalMemberCleanupUncertain { .. }),
"post-send ambiguity must remain typed, got: {error}"
);
assert_eq!(
external.bind_count(),
1,
"the harness must mutate before garbling the terminal reply"
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"post-commit ambiguity must retain trust for cleanup reconciliation"
);
assert_eq!(
pending_recipient_trust_snapshot(&handle),
vec![peer_id],
"post-commit ambiguity must keep the machine cleanup obligation open"
);
let retry = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext-retry"),
None,
external.binding(),
)
.await;
assert!(retry.is_err(), "fail-stopped actor must reject retry");
assert_eq!(
external.bind_count(),
1,
"quarantined PeerId must not reach a second BindMember"
);
}
/// Dogma row R044 (obligation lifecycle, success): a successful external
/// spawn plus an authorized turn leaves the peer trusted and the
/// `pending_recipient_trust` obligation set empty — the window is recorded
/// and resolved, never leaked.
#[tokio::test]
async fn test_pending_recipient_trust_obligation_resolves_on_success() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"pending-trust-success",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be resolved after a successful provision"
);
handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("after-authorized-spawn".to_string()))
.await
.expect("authorized supervisor should control the live peer-only member");
assert_eq!(external.delivered_input_ids().await.len(), 1);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"confirmed authorization must leave the peer trusted"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"pending_recipient_trust obligation must be empty after success terminality"
);
}
#[tokio::test]
async fn test_rotate_supervisor_bind_fallback_binds_next_authority() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-bind-fallback",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
external.forget_supervisor().await;
let report = handle.rotate_supervisor().await.expect("rotate supervisor");
let rotated = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load rotated authority")
.expect("rotated authority record");
assert_eq!(rotated.public_peer_id, report.public_peer_id);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(rotated.public_peer_id.as_str()),
"bind fallback during rotation must install the next supervisor, not the old one"
);
}
#[tokio::test]
async fn test_rotate_supervisor_does_not_attempt_old_bind_fallback() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-bind-fallback-failure",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
external.forget_supervisor().await;
external.fail_next_bind();
let report = handle
.rotate_supervisor()
.await
.expect("rotation should install only the exact next authority");
assert!(
external.fail_next_bind.load(Ordering::Relaxed),
"rotation must not consume an injected old-authority bind failure"
);
let current = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after rotation")
.expect("authority after rotation");
assert_eq!(
current.epoch,
original.epoch + 1,
"rotation should advance without reconstructing the old authority"
);
assert_eq!(current.public_peer_id, report.public_peer_id);
assert!(
current.pending_rotation.is_none(),
"successful rotation should clear its durable operation anchor"
);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(current.public_peer_id.as_str()),
"the member must be fenced onto the exact committed next authority"
);
}
#[tokio::test]
async fn test_rotate_supervisor_does_not_persist_old_rebind_metadata() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-bind-fallback-persist-failure",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
external.forget_supervisor().await;
runtime_metadata.fail_next_upsert_overlay();
let report = handle
.rotate_supervisor()
.await
.expect("rotation should not persist an old-authority rebound");
assert!(
runtime_metadata.fail_upsert_overlay.load(Ordering::Relaxed),
"rotation must not consume the injected old-rebind overlay failure"
);
let current = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after rotation")
.expect("authority after rotation");
assert_eq!(current.epoch, original.epoch + 1);
assert_eq!(current.public_peer_id, report.public_peer_id);
assert!(
current.pending_rotation.is_none(),
"successful rotation should clear its durable operation anchor"
);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(current.public_peer_id.as_str()),
"member must stay on the committed next authority, never an old rebound"
);
}
#[tokio::test]
async fn test_rotate_supervisor_blocks_old_rebind_and_reuses_hidden_completion() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-hidden-completion-retry",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
external_b.suppress_rotation_observations(true);
let error = handle
.rotate_supervisor()
.await
.expect_err("hidden second completion should leave the exact operation pending");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
attempted_public_peer_id,
..
} => attempted_public_peer_id,
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let pending = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after incomplete rotation")
.expect("authority after incomplete rotation")
.pending_rotation
.expect("incomplete rotation should persist the operation anchor");
let crate::RuntimeBinding::External {
peer_id: external_a_peer_id,
..
} = external_a.binding()
else {
panic!("live external peer should use an external binding");
};
let crate::RuntimeBinding::External {
peer_id: external_b_peer_id,
..
} = external_b.binding()
else {
panic!("live external peer should use an external binding");
};
assert_eq!(
pending.accepted_peer_ids,
vec![external_a_peer_id.clone()],
"only the observable completion may be checkpointed"
);
let operation_id = pending.operation_id.expect("durable operation id");
assert_eq!(
external_a.rotation_operation_ids().await,
vec![operation_id]
);
assert_eq!(
external_b.rotation_operation_ids().await,
vec![operation_id],
"the unobserved peer must already own the same terminal receipt"
);
let external_a_target = pending
.member_targets
.get(&external_a_peer_id)
.cloned()
.expect("durable exact target for accepted peer");
external_a.forget_supervisor().await;
let bind_count_before = external_a.bind_count();
let rebind_error = handle
.member(&AgentIdentity::from("w-a"))
.await
.expect("member handle")
.internal_turn(ContentInput::from(
"current-authority-rebind-must-be-refused".to_string(),
))
.await
.expect_err("pending operation must block old-authority fallback");
assert!(
matches!(
&rebind_error,
MobError::SupervisorRotationIncomplete { reason, .. }
if reason.contains("refusing to reauthorize old authority")
),
"old-authority fallback should fail with the pending operation intact: {rebind_error}"
);
assert_eq!(external_a.bind_count(), bind_count_before);
assert_eq!(external_a.authorized_supervisor_peer_id().await, None);
let after_rebind = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after current rebind")
.expect("authority after current rebind");
assert_eq!(after_rebind.public_peer_id, original.public_peer_id);
assert_eq!(
after_rebind.pending_rotation.as_ref(),
Some(&pending),
"refused old-authority fallback must preserve the exact id, receipts, and targets"
);
// Restore only the test harness's lost volatile authorization. Its durable
// operation receipt was never removed. The actor must then observe both the
// checkpointed and hidden completions and finalize this same operation.
external_a
.install_legacy_rotated_supervisor(external_a_target, pending.epoch)
.await;
external_b.suppress_rotation_observations(false);
let retry_report = handle
.rotate_supervisor()
.await
.expect("retry should reuse both terminal receipts and finalize the same operation");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
assert_eq!(retry_report.public_peer_id, attempted_public_peer_id);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after retry")
.expect("authority after retry");
assert!(retried.pending_rotation.is_none());
assert_eq!(retried.public_peer_id, attempted_public_peer_id);
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"checkpointed receipt should remain on the attempted authority"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"hidden receipt should remain on the attempted authority"
);
assert!(
external_b
.rotation_operation_ids()
.await
.iter()
.all(|observed| *observed == operation_id),
"every hidden-completion retry submission must reuse the exact operation id"
);
assert!(
pending.member_targets.contains_key(&external_b_peer_id),
"the hidden completion must retain its exact durable target"
);
}
#[tokio::test]
async fn test_rotate_supervisor_refused_old_rebind_preserves_operation_across_restart() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-refused-old-rebind-restart",
);
let mob_id = definition.id.clone();
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
external_b.fail_next_authorize();
let error = handle
.rotate_supervisor()
.await
.expect_err("first rotation should persist one accepted peer");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
attempted_public_peer_id,
..
} => attempted_public_peer_id,
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let crate::RuntimeBinding::External {
peer_id: external_a_peer_id,
..
} = external_a.binding()
else {
panic!("live external peer should use an external binding");
};
let pending = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load pending authority")
.expect("pending authority")
.pending_rotation
.expect("pending authority should be durable");
assert_eq!(pending.accepted_peer_ids, vec![external_a_peer_id.clone()]);
let operation_id = pending.operation_id.expect("durable operation id");
let external_a_target = pending
.member_targets
.get(&external_a_peer_id)
.cloned()
.expect("durable accepted-peer target");
external_a.forget_supervisor().await;
let rebind_error = handle
.member(&AgentIdentity::from("w-a"))
.await
.expect("member handle")
.internal_turn(ContentInput::from(
"current-rebind-is-refused-before-restart".to_string(),
))
.await
.expect_err("pending operation must refuse old-authority fallback");
assert!(
matches!(
&rebind_error,
MobError::SupervisorRotationIncomplete { reason, .. }
if reason.contains("refusing to reauthorize old authority")
),
"old-authority fallback should be refused: {rebind_error}"
);
assert_eq!(external_a.authorized_supervisor_peer_id().await, None);
let durable_after_refusal = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load durable pending after refusal")
.expect("durable pending after refusal")
.pending_rotation
.expect("durable pending must survive refusal");
assert_eq!(
durable_after_refusal, pending,
"refusal must preserve the exact operation id, accepted receipts, and targets"
);
external_a
.install_legacy_rotated_supervisor(external_a_target, pending.epoch)
.await;
handle
.shutdown()
.await
.expect("shutdown original actor before restart-style retry");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("resume mob with the exact refused-rebind operation anchor");
let retry_report = resumed
.rotate_supervisor()
.await
.expect("restart retry should observe and commit the exact pending operation");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
assert_eq!(
retry_report.public_peer_id, attempted_public_peer_id,
"restart retry must commit the same attempted authority"
);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after retry")
.expect("authority after retry");
assert!(retried.pending_rotation.is_none());
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"accepted peer should remain fenced onto the attempted authority"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should rotate the peer that rejected the first attempt"
);
assert!(
external_a
.rotation_operation_ids()
.await
.iter()
.all(|observed| *observed == operation_id)
);
}
#[tokio::test]
async fn test_rotate_supervisor_final_commit_conflict_reports_lost_retry_anchor() {
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-final-commit-conflict",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
// First compare records the durable operation; the second is the final
// authority CAS after local bridge activation.
runtime_metadata.conflict_nth_compare_supervisor(2);
let error = handle
.rotate_supervisor()
.await
.expect_err("stale final commit should fail closed");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 0);
assert!(!rollback_succeeded);
assert!(
!pending_authority_recorded,
"final CAS conflict must report the durable retry anchor as lost: {reason}"
);
assert!(
reason.contains("supervisor authority changed before final commit"),
"CAS conflict should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_conflict = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after conflict")
.expect("authority after conflict");
assert_eq!(after_conflict.epoch, original.epoch + 1);
assert_ne!(
after_conflict.public_peer_id, attempted_public_peer_id,
"stale actor must not overwrite the newer committed authority"
);
assert!(after_conflict.pending_rotation.is_none());
let retry_error = match handle.rotate_supervisor().await {
Ok(_) => panic!("same actor must not rotate from an external durable authority change"),
Err(error) => error,
};
assert!(
matches!(retry_error, MobError::ActorCommandChannelClosed),
"divergent durable authority must terminate the stale actor before retry, got: {retry_error}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_pending_commit_conflict_fails_closed_without_local_authority() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-pending-cas-conflict-reconcile",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
runtime_metadata.conflict_next_compare_supervisor();
let error = handle
.rotate_supervisor()
.await
.expect_err("initial operation-anchor CAS conflict should fail closed");
assert!(
matches!(
&error,
MobError::Internal(reason)
if reason.contains("exact reread found divergent supervisor authority")
&& reason.contains("fail-stopping for cold recovery")
),
"operation-anchor CAS divergence must be a durable fail-stop, got: {error:?}",
);
let after_conflict = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after conflict")
.expect("authority after conflict");
assert_eq!(after_conflict.epoch, original.epoch + 1);
assert_ne!(after_conflict.public_peer_id, original.public_peer_id);
assert!(after_conflict.pending_rotation.is_none());
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(original.public_peer_id.as_str()),
"anchor persistence must succeed before the operation reaches a member"
);
let retry_error = match handle.rotate_supervisor().await {
Ok(_) => panic!("same actor must not rotate from an external durable authority change"),
Err(error) => error,
};
assert!(
matches!(retry_error, MobError::ActorCommandChannelClosed),
"divergent durable authority must terminate the stale actor before retry, got: {retry_error}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_receipt_checkpoint_conflict_does_not_reauthorize_old_authority() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-pending-cas-conflict-reconcile-fails",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
// The operation anchor is compare #1. Force compare #2, the first
// completed-member receipt checkpoint, to lose its CAS after the member
// has durably accepted the exact next authority.
runtime_metadata.conflict_nth_compare_supervisor(2);
let error = handle
.rotate_supervisor()
.await
.expect_err("receipt-checkpoint CAS loss should fail closed");
assert!(
matches!(
&error,
MobError::Internal(reason)
if reason.contains("exact reread found divergent supervisor authority")
&& reason.contains("fail-stopping for cold recovery")
),
"receipt-checkpoint CAS divergence must be a durable fail-stop, got: {error:?}",
);
let attempted_public_peer_id = external
.authorized_supervisor_peer_id()
.await
.expect("member accepted the attempted authority before checkpoint divergence");
let after_conflict = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after conflict")
.expect("authority after conflict");
assert_eq!(after_conflict.epoch, original.epoch + 1);
assert_ne!(after_conflict.public_peer_id, attempted_public_peer_id);
assert!(after_conflict.pending_rotation.is_none());
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str()),
"CAS loss must not reauthorize the member onto the old authority"
);
let retry_error = match handle.rotate_supervisor().await {
Ok(_) => panic!("same actor must not rotate from an external durable authority change"),
Err(error) => error,
};
assert!(
matches!(retry_error, MobError::ActorCommandChannelClosed),
"divergent durable authority must terminate the stale actor before retry, got: {retry_error}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_pending_verification_conflict_fails_closed_without_local_authority()
{
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-pending-verify-cas-conflict",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let crate::RuntimeBinding::External {
peer_id: external_peer_id,
..
} = external.binding()
else {
panic!("live external peer should use an external binding");
};
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
// Operation anchor + accepted-receipt checkpoint precede the final
// activation write.
runtime_metadata.fail_nth_put_supervisor(3);
let first_error = handle
.rotate_supervisor()
.await
.expect_err("final commit failure should leave a durable pending rotation");
let attempted_public_peer_id = match first_error {
MobError::SupervisorRotationIncomplete {
attempted_public_peer_id,
pending_authority_recorded,
..
} => {
assert!(pending_authority_recorded);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let durable_pending = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load durable pending authority")
.expect("durable pending authority")
.pending_rotation
.expect("first failure should persist pending rotation");
assert_eq!(durable_pending.accepted_peer_ids, vec![external_peer_id]);
external.fail_next_authorize();
runtime_metadata.conflict_next_compare_supervisor();
let error = handle
.rotate_supervisor()
.await
.expect_err("pending verification failure with CAS conflict should be typed incomplete");
match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 1);
assert!(!rollback_succeeded);
assert!(
!pending_authority_recorded,
"divergent final-CAS authority must clear the attempted pending witness: {reason}"
);
assert!(
reason.contains("local authority activation failed")
&& reason.contains("supervisor authority changed before final commit"),
"retry final-CAS conflict should remain visible, got: {reason}"
);
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
}
let after_conflict = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after verification conflict")
.expect("authority after verification conflict");
assert_eq!(after_conflict.epoch, original.epoch + 1);
assert_ne!(after_conflict.public_peer_id, attempted_public_peer_id);
assert!(after_conflict.pending_rotation.is_none());
assert_ne!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(after_conflict.public_peer_id.as_str()),
"pending verification CAS loss must not reconcile to a durable authority absent local machine authorization"
);
let retry_error = match handle.rotate_supervisor().await {
Ok(_) => panic!("same actor must not rotate from an external durable authority change"),
Err(error) => error,
};
assert!(
matches!(retry_error, MobError::ActorCommandChannelClosed),
"divergent durable authority must terminate the stale actor before retry, got: {retry_error}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_final_commit_conflict_fails_closed_without_local_authority() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-final-cas-conflict-reconcile",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
// Operation anchor + accepted-receipt checkpoint precede final CAS.
runtime_metadata.conflict_nth_compare_supervisor(3);
let error = handle
.rotate_supervisor()
.await
.expect_err("final CAS conflict should fail closed");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 1);
assert!(!rollback_succeeded);
assert!(!pending_authority_recorded);
assert!(
reason.contains("local authority activation failed")
&& reason.contains("supervisor authority changed before final commit"),
"final CAS conflict should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_conflict = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after conflict")
.expect("authority after conflict");
assert_eq!(after_conflict.epoch, original.epoch + 1);
assert_ne!(after_conflict.public_peer_id, attempted_public_peer_id);
assert!(after_conflict.pending_rotation.is_none());
assert_ne!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(after_conflict.public_peer_id.as_str()),
"same actor must not pretend it reconciled to an externally changed durable authority"
);
let retry_error = match handle.rotate_supervisor().await {
Ok(_) => panic!("same actor must not rotate from an external durable authority change"),
Err(error) => error,
};
assert!(
matches!(retry_error, MobError::ActorCommandChannelClosed),
"divergent durable authority must terminate the stale actor before retry, got: {retry_error}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_final_commit_failure_preserves_attempted_authority_for_retry() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-final-commit-failure",
);
let mob_id = definition.id.clone();
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
let crate::RuntimeBinding::External {
peer_id: external_a_peer_id,
..
} = external_a.binding()
else {
panic!("live external peer should use an external binding");
};
let crate::RuntimeBinding::External {
peer_id: external_b_peer_id,
..
} = external_b.binding()
else {
panic!("live external peer should use an external binding");
};
// Initial operation anchor + one accepted checkpoint per remote precede
// the final authority CAS.
runtime_metadata.fail_nth_put_supervisor(4);
let error = handle
.rotate_supervisor()
.await
.expect_err("final commit write failure should surface typed incomplete rotation");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 2);
assert!(!rollback_succeeded);
assert!(
pending_authority_recorded,
"accepted attempted authority must remain durable after final commit failure"
);
assert!(
reason.contains("local authority activation failed")
&& reason.contains("fault-injected runtime metadata supervisor write failure"),
"commit failure should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let pending = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load pending authority after final commit failure")
.expect("pending authority after final commit failure");
assert_eq!(pending.epoch, original.epoch);
assert_eq!(pending.public_peer_id, original.public_peer_id);
let pending_rotation = pending
.pending_rotation
.expect("final commit failure should retain pending attempted authority");
assert_eq!(pending_rotation.public_peer_id, attempted_public_peer_id);
assert_eq!(
pending_rotation
.accepted_peer_ids
.iter()
.cloned()
.collect::<BTreeSet<_>>(),
[external_a_peer_id, external_b_peer_id]
.into_iter()
.collect::<BTreeSet<_>>(),
"durable pending metadata should identify every remote that accepted the attempted authority"
);
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str())
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str())
);
handle
.shutdown()
.await
.expect("shutdown original actor before restart-style retry");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("resume mob with pending final-commit attempt");
let retry_report = resumed
.rotate_supervisor()
.await
.expect("restart-style retry should commit the already accepted attempted authority");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
assert_eq!(retry_report.public_peer_id, attempted_public_peer_id);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load retried authority")
.expect("retried authority");
assert!(retried.pending_rotation.is_none());
assert_eq!(retried.public_peer_id, attempted_public_peer_id);
}
#[tokio::test]
async fn test_rotate_supervisor_private_trust_failure_keeps_durable_operation_for_retry() {
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-private-trust-activation-failure",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let agent_identity = AgentIdentity::from("w-activation-fail");
let receipt = handle
.spawn(ProfileName::from("worker"), agent_identity.clone(), None)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let comms_name = test_comms_name_for(&mob_id, "worker", agent_identity.as_str());
let member_comms = {
let sessions = service.sessions.read().await;
sessions
.get(&session_id)
.cloned()
.expect("member comms runtime")
};
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
service
.set_comms_behavior(
&comms_name,
MockCommsBehavior {
fail_remove_trust_once: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.rotate_supervisor()
.await
.expect_err("private trust activation failure should be typed incomplete");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 0);
assert!(!rollback_succeeded);
assert!(pending_authority_recorded);
assert!(
reason.contains("local authority activation failed")
&& reason.contains("previous supervisor private trust removal failed"),
"local activation failure should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_failure = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after failed activation")
.expect("authority after failed activation");
assert_eq!(after_failure.epoch, original.epoch);
assert_eq!(after_failure.public_peer_id, original.public_peer_id);
assert_eq!(
after_failure
.pending_rotation
.as_ref()
.map(|pending| pending.public_peer_id.as_str()),
Some(attempted_public_peer_id.as_str()),
"local activation failure must retain the exact durable operation target"
);
let trusted = member_comms.trusted_peers.read().await;
assert!(
trusted.contains_key(&original.public_peer_id),
"failed removal may leave the original trust in place, but recovery must not reinstall it"
);
assert!(
!trusted.contains_key(&attempted_public_peer_id),
"failed activation must not claim attempted supervisor trust was installed"
);
drop(trusted);
let retry_report = handle
.rotate_supervisor()
.await
.expect("retry should finish the same durable local activation");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
assert_eq!(retry_report.public_peer_id, attempted_public_peer_id);
}
#[tokio::test]
async fn test_rotate_supervisor_partial_publish_leaves_pending_without_old_reauthorization() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-private-trust-partial-publish",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-private"),
None,
)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let member_comms = {
let sessions = service.sessions.read().await;
sessions
.get(&session_id)
.cloned()
.expect("member comms runtime")
};
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
member_comms.fail_next_supervisor_private_add_after_insert_replacing(&original.public_peer_id);
let error = handle
.rotate_supervisor()
.await
.expect_err("partial private trust publication should be typed incomplete");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 0);
assert!(!rollback_succeeded);
assert!(pending_authority_recorded);
assert!(
reason.contains("supervisor private trust publication failed"),
"partial publish failure should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_failure = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after failed activation")
.expect("authority after failed activation");
assert_eq!(after_failure.epoch, original.epoch);
assert_eq!(after_failure.public_peer_id, original.public_peer_id);
assert_eq!(
after_failure
.pending_rotation
.as_ref()
.map(|pending| pending.public_peer_id.as_str()),
Some(attempted_public_peer_id.as_str())
);
let trusted = member_comms.trusted_peers.read().await;
assert!(
!trusted.contains_key(&original.public_peer_id),
"partial publish failure must not reconstruct revoked old-supervisor trust"
);
assert!(
!trusted.contains_key(&attempted_public_peer_id),
"partial publish cleanup may remove attempted trust while the durable operation stays pending"
);
}
#[tokio::test]
async fn test_rotate_supervisor_ack_rejection_leaves_pending_without_old_reauthorization() {
let definition = with_unique_mob_id(
sample_definition(),
"rotate-supervisor-private-trust-ack-rejection",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-private"),
None,
)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let comms_name = test_comms_name_for(&mob_id, "worker", "w-private");
let member_comms = {
let sessions = service.sessions.read().await;
sessions
.get(&session_id)
.cloned()
.expect("member comms runtime")
};
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
service
.set_comms_behavior(
&comms_name,
MockCommsBehavior {
reject_private_publish_ack_after_insert: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.rotate_supervisor()
.await
.expect_err("rejected private trust ack should be typed incomplete");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
rollback_succeeded,
pending_authority_recorded,
reason,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 0);
assert!(!rollback_succeeded);
assert!(pending_authority_recorded);
assert!(
reason.contains("supervisor private trust publication ack rejected"),
"ack rejection should be visible, got: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_failure = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after failed activation")
.expect("authority after failed activation");
assert_eq!(after_failure.epoch, original.epoch);
assert_eq!(after_failure.public_peer_id, original.public_peer_id);
assert_eq!(
after_failure
.pending_rotation
.as_ref()
.map(|pending| pending.public_peer_id.as_str()),
Some(attempted_public_peer_id.as_str())
);
let trusted = member_comms.trusted_peers.read().await;
assert!(
!trusted.contains_key(&original.public_peer_id),
"ack rejection must not reconstruct revoked old-supervisor trust"
);
assert!(
!trusted.contains_key(&attempted_public_peer_id),
"ack rejection cleanup may remove attempted trust while durable state remains pending"
);
drop(trusted);
if let meerkat_runtime::meerkat_machine::SupervisorBinding::Bound { peer_id, .. } =
adapter.supervisor_binding(&session_id).await
{
assert_ne!(
peer_id, original.public_peer_id,
"runtime binding recovery must not reauthorize the old supervisor"
);
}
}
#[tokio::test]
async fn test_rotate_supervisor_retains_exact_operation_when_checkpoint_write_fails() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-stale-durable-pending",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
let external_c = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-c")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-c"),
None,
external_c.binding(),
)
.await
.expect("spawn third live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
let crate::RuntimeBinding::External {
peer_id: external_a_peer_id,
..
} = external_a.binding()
else {
panic!("live external peer should use an external binding");
};
let crate::RuntimeBinding::External {
peer_id: external_b_peer_id,
..
} = external_b.binding()
else {
panic!("live external peer should use an external binding");
};
// Operation anchor + A's accepted checkpoint persist. B then accepts the
// exact operation, but its accepted-peer checkpoint is the third write and
// fails before mutation. This leaves durable metadata intentionally one
// receipt behind live peer state without relying on observation timing.
runtime_metadata.fail_nth_put_supervisor(3);
let checkpoint_error = handle
.rotate_supervisor()
.await
.expect_err("accepted-peer checkpoint failure must stop local activation");
assert!(
checkpoint_error
.to_string()
.contains("fault-injected runtime metadata supervisor write failure"),
"proven no-write must preserve the original typed store error: {checkpoint_error:?}",
);
let stale_durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load stale durable pending")
.expect("stale durable pending")
.pending_rotation
.expect("durable pending should still exist");
let attempted_public_peer_id = stale_durable.public_peer_id.clone();
assert_eq!(
stale_durable.accepted_peer_ids,
vec![external_a_peer_id],
"durable pending metadata should still have the stale accepted set after injected write failure"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str()),
"failed checkpoint persistence must not reauthorize an accepted peer onto the old authority"
);
let retry_report = handle
.rotate_supervisor()
.await
.expect("same-process retry should derive from durable supervisor metadata");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
assert_eq!(
retry_report.public_peer_id, attempted_public_peer_id,
"retry may reuse the attempted authority because it remained in durable pending metadata"
);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load retried authority")
.expect("retried authority");
assert!(retried.pending_rotation.is_none());
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should keep the first accepted peer on the committed authority"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should observe and retain the peer that completed before checkpoint persistence failed"
);
assert_eq!(
external_c.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should rotate the remaining peer before commit"
);
}
#[tokio::test]
async fn test_rotate_supervisor_does_not_submit_before_operation_anchor_is_durable() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rotate-supervisor-pending-write-failure",
);
let mob_id = definition.id.clone();
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage = MobStorage::with_events_and_runtime_metadata(
Arc::new(InMemoryMobEventStore::new()),
runtime_metadata.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first live external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second live external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
runtime_metadata.fail_next_put_supervisor();
let error = handle
.rotate_supervisor()
.await
.expect_err("pending rotation write failure should abort rotation");
assert!(
error
.to_string()
.contains("fault-injected runtime metadata supervisor write failure"),
"exact reread proving no write must return the original typed store error: {error:?}",
);
let attempted_epoch = original.epoch + 1;
let current = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after pending write failure")
.expect("authority after pending write failure");
assert_eq!(
current.epoch, original.epoch,
"persisted current authority must remain pre-rotation after pending write failure"
);
assert_eq!(
current.public_peer_id, original.public_peer_id,
"persisted current authority must remain the original peer"
);
assert!(
current.pending_rotation.is_none(),
"fault-injected write should leave no persisted pending metadata"
);
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(original.public_peer_id.as_str()),
"the operation must not reach the first remote before its anchor is durable"
);
handle
.member(&AgentIdentity::from("w-b"))
.await
.expect("member handle")
.internal_turn(ContentInput::from(
"recover-after-pending-write-failure".to_string(),
))
.await
.expect("current authority should remain usable after pre-submission anchor failure");
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(original.public_peer_id.as_str()),
"current-authority peer should remain controllable after pending write failure"
);
let retry_report = handle
.rotate_supervisor()
.await
.expect("retry should start from durable authority after failed pending persistence");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, attempted_epoch);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load retried authority")
.expect("retried authority");
assert!(
retried.pending_rotation.is_none(),
"successful retry should persist the newly committed authority"
);
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should rotate the recovered peer to the committed authority"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should rotate the peer that rejected the first attempt"
);
}
#[tokio::test]
async fn test_restarted_peer_only_member_rebinds_when_supervisor_state_is_lost() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-rebind-after-restart",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_real_comms(definition).await;
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let default_protocol_version =
meerkat_contracts::wire::supervisor_bridge::supervisor_bridge_default_protocol_version();
assert_eq!(external.bind_count(), 1, "initial spawn should bind once");
assert_eq!(
external.bind_protocol_versions().await,
vec![default_protocol_version],
"initial bridge bind should use the canonical default protocol version"
);
external.forget_supervisor().await;
handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("after-restart".to_string()))
.await
.expect("lost supervisor state should trigger a rebind fallback");
assert_eq!(
external.bind_count(),
2,
"restart recovery should re-establish supervisor authority via BindMember"
);
assert_eq!(
external.bind_protocol_versions().await,
vec![default_protocol_version, default_protocol_version],
"restart rebind should keep reporting the canonical bridge protocol version"
);
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"rebound peer should still receive the requested input"
);
}
#[tokio::test]
async fn test_resume_rebind_ambiguity_retains_outer_trust_for_exact_retry() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-rebind-ambiguous-bind",
);
let external_config = definition
.backend
.external
.clone()
.expect("external backend config");
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob_with_real_comms(definition).await;
let external = spawn_live_external_peer(&test_comms_name_for(
&mob_id,
"worker",
"w-rebind-ambiguous",
))
.await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-rebind-ambiguous"),
None,
external.binding(),
)
.await
.expect("spawn live external worker");
let member_ref = handle
.get_member(&AgentIdentity::from("w-rebind-ambiguous"))
.await
.expect("member lookup")
.expect("external member")
.member_ref;
let peer = external_peer_descriptor_for_binding(&external.binding());
let crate::RuntimeBinding::External {
bootstrap_token: Some(bootstrap_token),
..
} = external.binding()
else {
panic!("live external binding carries a bootstrap token");
};
let rebind_authority = super::provisioner::PeerOnlyRebindAuthority {
peer: peer.clone(),
bootstrap_token,
};
let provisioner = super::provisioner::MultiBackendProvisioner::new(
service,
handle.runtime_adapter.clone(),
None,
Some(external_config),
Arc::clone(&handle.supervisor_bridge),
);
// Reproduce the resume fallback ownership split: the outer authorization
// newly installs trust, the rejected AuthorizeSupervisor drives an inline
// BindMember, and that inner bind observes the edge as AlreadyTrusted.
external.forget_supervisor().await;
handle
.supervisor_bridge
.untrust_recipient(&peer)
.await
.expect("remove old trust before resume-style reconciliation");
external.garble_next_bind_reply();
let error = provisioner
.reconcile_peer_only_trust(&member_ref, None, Some(&rebind_authority))
.await
.expect_err("post-send ambiguous fallback bind must remain quarantined");
assert!(
matches!(&error, MobError::ExternalMemberCleanupUncertain { .. }),
"fallback ambiguity must stay typed, got: {error}"
);
assert_eq!(
external.bind_count(),
2,
"the ambiguous fallback bind must have committed remotely"
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer.peer_id.to_string()),
"the outer NewlyInstalled trust must survive inner sent-but-unterminated BindMember"
);
provisioner
.reconcile_peer_only_trust(&member_ref, None, Some(&rebind_authority))
.await
.expect("exact retry observes the remote bind and converges");
assert_eq!(
external.bind_count(),
2,
"exact retry must not apply a second remote bind mutation"
);
}
#[tokio::test]
async fn test_external_member_spawn_rejects_bind_peer_id_pubkey_mismatch() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-bind-peer-id-mismatch",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_real_comms(definition).await;
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let mismatched_peer_id = PeerId::from_ed25519_pubkey(&[77u8; 32]).to_string();
external
.override_next_bind_peer_id(mismatched_peer_id)
.await;
let err = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect_err("bind response peer id must derive from the stored pubkey");
assert!(
err.to_string().contains("pubkey-derived id"),
"unexpected error: {err}"
);
let events = handle.events().replay_all().await.expect("replay events");
assert!(
!events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::MemberSpawned(spawned)
if spawned.agent_identity.as_str() == "w-ext"
)
}),
"rejected bind response must not append a member spawn event"
);
}
/// Post-Bind compensation must never certify local rollback while the remote
/// member may still be bound. A rejected compensating RetireMember leaves the
/// exact operation/PeerId reservation and recipient-trust obligation open,
/// fail-stops the actor, and prevents any second BindMember side effect.
#[tokio::test]
async fn test_external_post_bind_retire_failure_quarantines_peer_and_fail_stops() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-post-bind-retire-failure",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_real_comms(definition).await;
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-quarantined")).await;
let peer = external_peer_descriptor_for_binding(&external.binding());
let peer_id = peer.peer_id.to_string();
// Force a local post-Bind validation failure, then reject the compensating
// remote retirement. Bind succeeded and therefore crossed the side-effect
// boundary before cleanup became uncertain.
external
.override_next_bind_peer_id(PeerId::from_ed25519_pubkey(&[77u8; 32]).to_string())
.await;
external.fail_next_retire();
let error = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-quarantined"),
None,
external.binding(),
)
.await
.expect_err("unconfirmed compensating retire must fail closed");
assert!(
matches!(&error, MobError::ExternalMemberCleanupUncertain { .. }),
"cleanup uncertainty must stay typed, got: {error}"
);
assert_eq!(external.bind_protocol_versions().await.len(), 1);
assert_eq!(external.retire_count(), 1);
assert_eq!(
pending_recipient_trust_snapshot(&handle),
vec![peer_id.clone()],
"actor must not emit RollbackPendingRecipientTrust while remote retirement is unconfirmed"
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&peer_id),
"trust must remain installed for a cleanup retry while the exact binding quarantines the PeerId"
);
// The typed uncertainty fail-stops the actor. No later generic spawn may
// reach BindMember; the exact-reservation unit pins the complementary
// generic/exact collision behavior beneath this actor boundary.
let retry = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-quarantined-retry"),
None,
external.binding(),
)
.await;
assert!(retry.is_err(), "fail-stopped actor must reject retry");
assert_eq!(
external.bind_protocol_versions().await.len(),
1,
"quarantined PeerId must not receive a second BindMember"
);
}
#[tokio::test]
async fn test_query_string_bootstrap_token_fallback_is_rejected() {
// Dogma regression: bind with only a query-string `?mob_supervisor_bootstrap_token=...`
// (no typed `bootstrap_token` field) must be rejected. The typed field is the
// sole authoritative carrier; the query-string fallback was deleted in Wave 3 E.
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-query-bootstrap-rejected",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_real_comms(definition).await;
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token,
pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let bootstrap_token = bootstrap_token.expect("live external binding bootstrap token");
let binding = crate::RuntimeBinding::External {
peer_id,
address: format!(
"{}?{}={}",
address,
super::bridge_protocol::SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM,
bootstrap_token.as_str()
),
bootstrap_token: None,
pubkey,
};
let err = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
binding,
)
.await
.expect_err("query-string-only bootstrap token must be rejected");
let rendered = err.to_string();
assert!(
rendered.contains("typed bootstrap_token field"),
"error must name the missing typed field, got: {rendered}"
);
}
#[tokio::test]
async fn test_concurrent_terminal_lifecycle_commands_observe_live_state_drift() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let stop = {
let handle = handle.clone();
tokio::spawn(async move { ("stop", handle.stop().await) })
};
let complete = {
let handle = handle.clone();
tokio::spawn(async move { ("complete", handle.complete().await) })
};
let (stop, complete) = tokio::join!(stop, complete);
let results = [stop.expect("stop join"), complete.expect("complete join")];
let successes = results.iter().filter(|(_, result)| result.is_ok()).count();
assert_eq!(
successes, 1,
"only the first admitted terminal lifecycle command may run shell mechanics: {results:?}"
);
let failed = results
.iter()
.find_map(|(_, result)| result.as_ref().err())
.expect("one command should observe live-state drift");
assert!(
matches!(failed, MobError::InvalidTransition { .. }),
"losing lifecycle command must fail at live admission, got {failed:?}"
);
let state = handle.status().await.expect("status after lifecycle race");
assert!(
matches!(state, MobState::Stopped | MobState::Completed),
"terminal lifecycle race should leave a terminal lifecycle phase, got {state:?}"
);
}
#[tokio::test]
async fn test_stop_persists_all_state_and_rejects_mutations() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
let events_before = handle.events().replay_all().await.expect("replay");
handle.stop().await.expect("stop");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
assert_eq!(
service.active_session_count().await,
1,
"stop should not archive active sessions"
);
let events_after = handle.events().replay_all().await.expect("replay");
assert!(
events_after.len() >= events_before.len(),
"stop should not delete persisted events"
);
let before_cursors = events_before
.iter()
.map(|event| event.cursor)
.collect::<Vec<_>>();
let after_prefix_cursors = events_after[..events_before.len()]
.iter()
.map(|event| event.cursor)
.collect::<Vec<_>>();
assert_eq!(
after_prefix_cursors, before_cursors,
"stop should preserve the existing event log prefix even if a concurrent lifecycle projection appends later"
);
let err = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect_err("spawn must be rejected while stopped");
assert!(matches!(err, MobError::InvalidTransition { .. }));
assert_eq!(
service.active_session_count().await,
1,
"MobMachine spawn admission must reject before provisioning a stopped mob"
);
}
#[tokio::test]
async fn test_stopped_runtime_commands_are_rejected_by_machine_admission() {
let (handle, _service) =
create_test_mob(sample_definition_with_single_step_flow(60_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
handle.stop().await.expect("stop");
let spawn = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await;
assert!(
matches!(
spawn,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"Spawn must surface the MobMachine stopped-phase rejection: {spawn:?}"
);
let submit = handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("queued while stopped".to_string(), WorkOrigin::Internal),
)
.await;
assert!(
matches!(
submit,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"SubmitWork must surface the MobMachine stopped-phase rejection: {submit:?}"
);
let cancel_all_work = handle
.cancel_all_work(entry.agent_runtime_id.clone(), entry.fence_token)
.await;
assert!(
matches!(
cancel_all_work,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"CancelAllWork must surface the MobMachine stopped-phase rejection: {cancel_all_work:?}"
);
let run_flow = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await;
assert!(
matches!(
run_flow,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"RunFlow must surface the MobMachine stopped-phase rejection: {run_flow:?}"
);
let respawn = handle.respawn(AgentIdentity::from("w-1"), None).await;
assert!(
matches!(
respawn,
Err(crate::runtime::handle::MobRespawnError::Mob(
MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
}
))
),
"Respawn must surface the MobMachine stopped-phase rejection: {respawn:?}"
);
}
#[tokio::test]
async fn test_stopped_unknown_profile_spawn_is_rejected_by_machine_admission() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle.stop().await.expect("stop");
let result = handle
.spawn(
ProfileName::from("missing-profile"),
AgentIdentity::from("w-missing-profile"),
None,
)
.await;
assert!(
matches!(
result,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"unknown profile must not shadow MobMachine stopped-phase admission: {result:?}"
);
assert_eq!(
service.active_session_count().await,
0,
"stopped rejected spawn must not provision a session"
);
}
#[tokio::test]
async fn test_stopped_unknown_flow_run_flow_is_rejected_by_machine_admission() {
let (handle, _service) =
create_test_mob(sample_definition_with_single_step_flow(60_000, 8)).await;
handle.stop().await.expect("stop");
let result = handle
.run_flow(FlowId::from("missing-flow"), serde_json::json!({}))
.await;
assert!(
matches!(
result,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"unknown flow must not shadow MobMachine stopped-phase admission: {result:?}"
);
}
#[tokio::test]
async fn test_unknown_flow_does_not_invoke_target_provisioner() {
let (handle, _service) =
create_test_mob(sample_definition_with_single_step_flow(60_000, 8)).await;
let provision_calls = Arc::new(AtomicUsize::new(0));
let observed_calls = Arc::clone(&provision_calls);
handle.install_flow_target_provisioner(Arc::new(move || {
observed_calls.fetch_add(1, Ordering::Relaxed);
Box::pin(async { Ok(()) })
}));
let result = handle
.run_flow(FlowId::from("missing-flow"), serde_json::json!({}))
.await;
assert!(matches!(result, Err(MobError::FlowNotFound(_))));
assert_eq!(
provision_calls.load(Ordering::Relaxed),
0,
"unknown flow ids must fail before side-effecting target provisioning"
);
}
fn mob_command_arm_source<'a>(source: &'a str, command: &str) -> &'a str {
let marker = format!(" MobCommand::{command}");
let start = source
.find(&marker)
.unwrap_or_else(|| panic!("MobCommand::{command} arm exists"));
let rest = &source[start + marker.len()..];
let end = rest
.find("\n MobCommand::")
.map(|offset| start + marker.len() + offset)
.unwrap_or(source.len());
&source[start..end]
}
#[test]
fn test_lifecycle_command_admission_arms_do_not_shadow_mob_machine_guards() {
let source = include_str!("actor.rs");
for (command, input) in [
("Stop", "Stop"),
("ResumeLifecycle", "Resume"),
("Complete", "Complete"),
("Reset", "Reset"),
] {
let arm = mob_command_arm_source(source, command);
for disallowed in [
"require_state(",
"require_live_lifecycle_phase(",
"require_live_reset_admission(",
"require_live_stop_admission(",
] {
assert!(
!arm.contains(disallowed),
"MobCommand::{command} admission must be delegated to MobMachineInput::{input}, not shell `{disallowed}`"
);
}
if command == "Complete" {
assert!(
arm.contains("self.handle_complete()")
&& source.contains(
"prepare_dsl_input_transition(mob_dsl::MobMachineInput::Complete",
),
"MobCommand::Complete must delegate final admission to MobMachineInput::Complete after member retirement mechanics"
);
} else {
assert!(
arm.contains(&format!("MobMachineInput::{input}")),
"MobCommand::{command} must submit MobMachineInput::{input} for command admission before shell mechanics"
);
}
}
}
#[test]
fn test_mob_command_admission_arms_do_not_shadow_mob_machine_guards() {
let source = include_str!("actor.rs");
for command in [
"Spawn",
"Retire",
"RetireAll",
"Respawn",
"RunFlow",
"CancelFlow",
"SubmitWork",
"CancelAllWork",
"ForceCancel",
] {
let arm = mob_command_arm_source(source, command);
for disallowed in [
"require_state(",
"require_live_lifecycle_phase(",
"require_live_reset_admission(",
"require_live_stop_admission(",
] {
assert!(
!arm.contains(disallowed),
"MobCommand::{command} admission must be delegated to MobMachine guards, not shell `{disallowed}`"
);
}
}
}
#[test]
fn test_generated_dsl_paths_do_not_shadow_destroy_admission() {
let source = include_str!("actor.rs");
assert!(
!source.contains("ensure_destroy_mutation_allowed"),
"generated MobMachine apply/prepare paths must let MobMachine guards decide destroy-admitted legality"
);
assert!(
!source.contains("destroy_admitted_error"),
"actor-local destroyed admission errors must not preempt generated MobMachine transitions"
);
}
#[test]
fn test_pending_spawn_insertion_requires_generated_start_authority() {
let source = include_str!("actor.rs");
let insert_pending_spawn = source
.find("fn insert_pending_spawn")
.expect("insert_pending_spawn");
let insert_body = &source[insert_pending_spawn..];
let pending_start_count = source.matches(".start(&pending)").count();
assert!(
source.contains("struct AuthorizedMobSpawnStart")
&& source.contains("struct AuthorizedMobSpawnStarted")
&& source.contains("struct AuthorizedMobSpawnCompleted")
&& source.contains(
"generated_can_start: generated_mob_command_capabilities::CommandPlanKind"
)
&& source
.contains("generated_owner: generated_mob_command_capabilities::CommandPlanKind")
&& source
.contains("generated_started: generated_mob_command_capabilities::CommandPlanKind")
&& source
.contains("generated_effect: generated_mob_command_capabilities::CommandPlanKind")
&& !source.contains("mint_from_generated_command_plan(")
&& source.contains(
"generated_mob_command_capabilities::CommandPlanKind::AuthorizedMobSpawnStart"
)
&& source
.contains("generated_mob_command_capabilities::CommandPlanKind::CanStartSpawn")
&& source.contains("generated_mob_command_capabilities::CommandPlanKind::SpawnStarted")
&& source.contains("generated_mob_command_capabilities::CommandPlanKind::SpawnEffect")
&& source.contains("generated_mob_command_capabilities::CommandPlanKind::FailSpawn")
&& source.contains("MobMachineSignal::StageSpawn")
&& source.contains("PendingSpawnOperationOwnerAuthorized")
&& source.contains("MobMachineSignal::CompleteSpawn")
&& source.contains("EmitMemberLifecycleNotice")
&& insert_body.contains("started: AuthorizedMobSpawnStarted")
&& pending_start_count >= 2
&& source.contains(".start(&respawn_pending)"),
"mob pending-spawn insertion and completion must consume generated StageSpawn/CompleteSpawn authorities"
);
}
#[tokio::test]
async fn test_stopped_missing_member_wire_is_rejected_by_machine_admission() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle.stop().await.expect("stop mob");
let result = handle
.wire(
AgentIdentity::from("missing-a"),
AgentIdentity::from("missing-b"),
)
.await;
assert!(
matches!(
result,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"WireMembers must surface the MobMachine stopped-phase admission before shell member lookup: {result:?}"
);
}
#[tokio::test]
async fn test_cancel_all_work_after_respawn_preserves_stale_fence_error() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("w-1");
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from(member_id.as_str()),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original member");
let original = handle
.get_member(&member_id)
.await
.unwrap()
.expect("original member exists");
let old_runtime_id = original.agent_runtime_id.clone();
let old_fence_token = original.fence_token;
let receipt = handle
.respawn(member_id.clone(), None)
.await
.expect("respawn member");
let replacement = handle
.get_member(&member_id)
.await
.unwrap()
.expect("replacement member exists");
assert_eq!(replacement.agent_runtime_id, receipt.agent_runtime_id);
assert_ne!(
old_runtime_id, replacement.agent_runtime_id,
"respawn should replace the runtime generation"
);
let result = handle
.cancel_all_work(old_runtime_id.clone(), old_fence_token)
.await;
match result {
Err(MobError::StaleFenceToken {
runtime_id,
expected,
actual,
}) => {
assert_eq!(runtime_id, old_runtime_id);
assert_eq!(expected, replacement.fence_token);
assert_eq!(actual, old_fence_token);
}
other => panic!(
"cancel_all_work with a pre-respawn runtime id/fence must preserve StaleFenceToken, got {other:?}"
),
}
}
struct StaticWorkerSpawnPolicy;
#[async_trait]
impl super::spawn_policy::SpawnPolicy for StaticWorkerSpawnPolicy {
async fn resolve(&self, _target: &AgentIdentity) -> Option<super::spawn_policy::SpawnSpec> {
Some(super::spawn_policy::SpawnSpec {
profile: ProfileName::from("worker"),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
})
}
}
#[tokio::test]
async fn test_stopped_submit_work_auto_spawn_rejects_before_policy_provisioning() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.set_spawn_policy(Some(Arc::new(StaticWorkerSpawnPolicy)))
.await
.expect("set spawn policy");
handle.stop().await.expect("stop");
let target = AgentIdentity::from("auto-stopped");
let result = handle
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new("queued while stopped".to_string(), WorkOrigin::External),
)
.await;
assert!(
matches!(
result,
Err(MobError::InvalidTransition {
from: MobState::Stopped,
to: MobState::Running,
})
),
"stopped auto-spawn SubmitWork must surface MobMachine phase admission: {result:?}"
);
assert!(
handle.get_member(&target).await.unwrap().is_none(),
"stopped SubmitWork must not auto-spawn the absent target"
);
assert_eq!(
service.recorded_create_requests().await.len(),
0,
"stopped SubmitWork must reject before policy provisioning side effects"
);
assert_eq!(
service.active_session_count().await,
0,
"stopped SubmitWork must not leak a provisioned session"
);
}
#[tokio::test]
async fn test_running_submit_work_auto_spawn_non_addressable_rejects_before_policy_provisioning() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.set_spawn_policy(Some(Arc::new(StaticWorkerSpawnPolicy)))
.await
.expect("set spawn policy");
let target = AgentIdentity::from("auto-running");
let result = handle
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new(
"queued for non-addressable policy profile".to_string(),
WorkOrigin::External,
),
)
.await;
assert!(
matches!(result, Err(MobError::NotExternallyAddressable(_))),
"running external auto-spawn SubmitWork must surface MobMachine addressability admission: {result:?}"
);
assert!(
handle.get_member(&target).await.unwrap().is_none(),
"non-addressable auto-spawn SubmitWork must not insert the rejected target"
);
assert_eq!(
service.recorded_create_requests().await.len(),
0,
"non-addressable auto-spawn SubmitWork must reject before policy provisioning side effects"
);
assert_eq!(
service.active_session_count().await,
0,
"non-addressable auto-spawn SubmitWork must not leak a provisioned session"
);
}
#[tokio::test]
async fn test_register_tool_bundle_is_wired_into_spawn() {
let definition = sample_definition_with_tool_bundle("bundle-a");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.register_tool_bundle("bundle-a", Arc::new(EchoBundleDispatcher))
.create()
.await
.expect("create mob");
let session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn")
.bridge_session_id()
.expect("session-backed")
.clone();
let flags = service.recorded_external_tools_flags().await;
assert_eq!(
flags,
vec![true],
"spawn should pass registered rust bundle as external_tools"
);
let tool_names = service.external_tool_names(&session_id).await;
assert!(
tool_names.contains(&"bundle_echo".to_string()),
"registered rust bundle tool should be exposed"
);
let result = service
.dispatch_external_tool(
&session_id,
"bundle_echo",
serde_json::json!({"value": "hello"}),
)
.await
.expect("bundle tool dispatch");
let payload: serde_json::Value =
serde_json::from_str(&result.text_content()).expect("bundle payload");
assert_eq!(payload["echo"], "hello");
}
#[tokio::test]
async fn test_spawn_fails_when_tool_bundle_not_registered() {
let definition = sample_definition_with_tool_bundle("missing-bundle");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(matches!(result, Err(MobError::Internal(_))));
assert!(handle.list_members().await.is_empty());
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberSpawned(..))),
"failed spawn should not emit any spawn event"
);
}
#[tokio::test]
async fn test_mob_management_tools_visible_with_default_authority_but_management_restricted() {
// profile.tools.mob = true is the policy declaration. The canonical
// resolver synthesizes a generated create-only authority on spawn, so the
// operator dispatcher mounts and the tools are visible in the catalog.
// Without `managed_mob_scope`, management dispatch on the current mob is
// denied with AccessDenied — capability scope is the gate, not visibility.
let (handle, service) = create_test_mob(sample_definition_with_mob_tools()).await;
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let tool_names = service.external_tool_names(&sid_1).await;
for required in [
"spawn_member",
"spawn_many_members",
"retire_member",
"wire_members",
"unwire_members",
"list_members",
] {
assert!(
tool_names.contains(&required.to_string()),
"operator tool '{required}' must be visible when profile.tools.mob = true"
);
}
let list_err = service
.dispatch_external_tool_outcome(&sid_1, "list_members", serde_json::json!({}))
.await
.expect_err("list_members should be denied without managed_mob_scope");
assert!(
matches!(list_err, ToolError::AccessDenied { .. }),
"expected AccessDenied (scope gate), got: {list_err:?}"
);
}
#[tokio::test]
async fn test_default_mob_authority_can_spawn_definition_profiles_only() {
let (handle, service) = create_test_mob(sample_definition_with_mob_tools()).await;
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let spawn_result = service
.dispatch_external_tool_outcome(
&sid_1,
"spawn_member",
serde_json::json!({"profile": "worker", "member_id": "w-2"}),
)
.await
.expect("default mob authority should spawn profiles declared by this mob definition");
let spawn_payload: serde_json::Value =
serde_json::from_str(&spawn_result.result.text_content()).expect("spawn payload");
assert_eq!(spawn_payload["agent_identity"], "w-2");
assert!(
spawn_payload["member_ref"]
.as_str()
.is_some_and(|value| !value.is_empty()),
"definition-profile spawn should return a canonical member ref"
);
let unknown_profile_err = service
.dispatch_external_tool_outcome(
&sid_1,
"spawn_member",
serde_json::json!({"profile": "not-in-definition", "member_id": "blocked"}),
)
.await
.expect_err("default mob authority must not spawn profiles outside the mob definition");
assert!(
matches!(unknown_profile_err, ToolError::AccessDenied { .. }),
"expected AccessDenied for profile outside spawn scope, got: {unknown_profile_err:?}"
);
assert!(
handle
.get_member(&AgentIdentity::from("blocked"))
.await
.unwrap()
.is_none(),
"scope-denied profile spawn must not mutate roster state"
);
}
#[tokio::test]
async fn profile_tools_mcp_allowlist_scopes_default_external_tools() {
use meerkat_core::types::{ToolProvenance, ToolSourceId, ToolSourceKind};
fn mcp_tool(name: &str, source: &str) -> Arc<ToolDef> {
Arc::new(ToolDef {
name: name.into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new(source),
}),
})
}
struct StaticDefsDispatcher {
defs: Arc<[Arc<ToolDef>]>,
}
#[async_trait]
impl AgentToolDispatcher for StaticDefsDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.defs)
}
async fn dispatch(&self, call: ToolCallView<'_>) -> Result<ToolDispatchOutcome, ToolError> {
if self.defs.iter().any(|d| d.name.as_str() == call.name) {
Ok(ToolResult::new(call.id.to_string(), "{}".to_string(), false).into())
} else {
Err(ToolError::not_found(call.name))
}
}
}
// Build a definition where the worker profile lists only "linear".
let mut definition = sample_definition();
let worker = definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|b| b.as_inline_mut())
.expect("worker profile");
worker.tools.mcp = vec!["linear".to_string()];
let (handle, _service) = create_test_mob(definition).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.and_then(|b| b.as_inline().cloned())
.expect("worker profile");
let host_mcp_dispatcher: Arc<dyn AgentToolDispatcher> = Arc::new(StaticDefsDispatcher {
defs: Arc::from(vec![
mcp_tool("search", "linear"),
mcp_tool("create_issue", "linear"),
mcp_tool("ping", "github"),
mcp_tool("trigger", "zapier"),
]),
});
let composed = super::tools::compose_external_tools_for_profile(
&profile,
&BTreeMap::new(),
handle.clone(),
Some(host_mcp_dispatcher),
None,
None,
)
.expect("compose dispatcher")
.expect("dispatcher should mount when default_external_tools is provided");
let visible: std::collections::BTreeSet<String> = composed
.tools()
.iter()
.map(|t| t.name.to_string())
.collect();
assert!(
visible.contains("search") && visible.contains("create_issue"),
"linear-source MCP tools must remain visible"
);
assert!(
!visible.contains("ping") && !visible.contains("trigger"),
"non-allowlisted MCP source tools must be hidden: visible={visible:?}"
);
}
#[tokio::test]
async fn profile_tools_mcp_empty_passes_all_default_external_tools() {
use meerkat_core::types::{ToolProvenance, ToolSourceId, ToolSourceKind};
fn mcp_tool(name: &str, source: &str) -> Arc<ToolDef> {
Arc::new(ToolDef {
name: name.into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new(source),
}),
})
}
struct StaticDefsDispatcher {
defs: Arc<[Arc<ToolDef>]>,
}
#[async_trait]
impl AgentToolDispatcher for StaticDefsDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.defs)
}
async fn dispatch(
&self,
_call: ToolCallView<'_>,
) -> Result<ToolDispatchOutcome, ToolError> {
Ok(ToolResult::new("id".to_string(), "{}".to_string(), false).into())
}
}
// Worker profile keeps tools.mcp = vec![] (default).
let definition = sample_definition();
let (handle, _service) = create_test_mob(definition).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.and_then(|b| b.as_inline().cloned())
.expect("worker profile");
assert!(profile.tools.mcp.is_empty());
let host_mcp_dispatcher: Arc<dyn AgentToolDispatcher> = Arc::new(StaticDefsDispatcher {
defs: Arc::from(vec![
mcp_tool("search", "linear"),
mcp_tool("ping", "github"),
]),
});
let composed = super::tools::compose_external_tools_for_profile(
&profile,
&BTreeMap::new(),
handle.clone(),
Some(host_mcp_dispatcher),
None,
None,
)
.expect("compose dispatcher")
.expect("dispatcher should mount when default_external_tools is provided");
let visible: std::collections::BTreeSet<String> = composed
.tools()
.iter()
.map(|t| t.name.to_string())
.collect();
assert!(
visible.contains("search") && visible.contains("ping"),
"empty allowlist must let all host MCP tools through; visible={visible:?}"
);
}
#[tokio::test]
async fn profile_tools_mob_true_grants_operator_dispatcher_without_persisted_authority() {
// Regression: profile.tools.mob = true is the policy declaration. The
// canonical resolver must synthesize a generated create-only authority
// when no persisted authority is supplied (production spawn default), so
// the operator dispatcher mounts on the spawned member.
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
assert!(
profile.tools.mob,
"fixture must declare profile.tools.mob = true"
);
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
None,
)
.expect("compose dispatcher")
.expect(
"profile.tools.mob = true must mount the operator dispatcher even without \
persisted authority — the canonical resolver synthesizes a create-only shape",
);
let tool_names: std::collections::BTreeSet<String> = dispatcher
.tools()
.iter()
.map(|tool| tool.name.to_string())
.collect();
for required in [
"spawn_member",
"retire_member",
"wire_members",
"unwire_members",
"list_members",
] {
assert!(
tool_names.contains(required),
"operator tool '{required}' must be visible when profile.tools.mob = true"
);
}
}
#[tokio::test]
async fn profile_tools_mob_false_keeps_operator_dispatcher_off() {
// Negative half of the regression: when tools.mob is not declared, the
// resolver yields no authority and the dispatcher is not mounted, even
// with default (None) persisted authority.
let (handle, _service) = create_test_mob(sample_definition_without_mob_flags()).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline()
.unwrap();
assert!(!profile.tools.mob);
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
None,
)
.expect("compose dispatcher");
if let Some(dispatcher) = dispatcher {
let tool_names: std::collections::BTreeSet<String> = dispatcher
.tools()
.iter()
.map(|tool| tool.name.to_string())
.collect();
for forbidden in [
"spawn_member",
"retire_member",
"wire_members",
"unwire_members",
"list_members",
] {
assert!(
!tool_names.contains(forbidden),
"operator tool '{forbidden}' must stay hidden when profile.tools.mob is false"
);
}
}
}
#[tokio::test]
async fn profile_tools_mob_true_rejects_deserialized_operator_authority_projection() {
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
assert!(profile.tools.mob);
let authority_projection: meerkat_core::service::MobToolAuthorityContext =
serde_json::from_value(
serde_json::to_value(generated_mob_operator_authority_with_scope(
handle.definition().id.as_str(),
))
.unwrap(),
)
.unwrap();
assert!(!authority_projection.is_generated_authority_context());
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(authority_projection),
)
.expect("compose dispatcher");
if let Some(dispatcher) = dispatcher {
assert!(
dispatcher.tools().is_empty(),
"deserialized mob authority projection must not surface operator tools"
);
}
}
#[tokio::test]
async fn test_visible_mob_operator_reads_still_require_exact_scope() {
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(generated_mob_operator_authority_with_scope("different-mob")),
)
.expect("compose dispatcher")
.expect("operator dispatcher should be visible when authority is injected");
let tools = dispatcher.tools();
let tool_names = tools
.iter()
.map(|tool| tool.name.as_str())
.collect::<Vec<_>>();
assert!(
tool_names.contains(&"list_members"),
"tool visibility should depend on injected authority presence, not scope success"
);
let args = RawValue::from_string("{}".to_string()).expect("raw args");
let error = dispatcher
.dispatch(ToolCallView {
id: "call-1",
name: "list_members",
args: &args,
})
.await
.expect_err("out-of-scope operator read should be denied");
assert!(matches!(error, ToolError::AccessDenied { .. }));
}
#[tokio::test]
async fn test_visible_mob_operator_tools_deny_before_arg_validation_when_scope_missing() {
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(generated_mob_operator_authority_with_scope("different-mob")),
)
.expect("compose dispatcher")
.expect("operator dispatcher should be visible when authority is injected");
let args = RawValue::from_string("{}".to_string()).expect("raw args");
let error = dispatcher
.dispatch(ToolCallView {
id: "call-2",
name: "spawn_member",
args: &args,
})
.await
.expect_err("out-of-scope operator call should be denied before args are parsed");
assert!(matches!(error, ToolError::AccessDenied { .. }));
}
#[tokio::test]
async fn test_coarse_spawn_tool_admission_is_machine_routed() {
// The coarse spawn-tool gate (spawn_member / spawn_many_members) is now
// decided by MobMachine (ResolveSpawnToolAdmission -> SpawnToolAdmissionResolved)
// from the operator's pure can-spawn-any-profile-in-current-mob observation.
// This test pins the verdict-driven behavior on both sides, including the
// unique-coverage empty-specs spawn_many_members case (where the per-member
// loop fires zero per-member admissions, so this coarse gate is the only
// thing that can deny).
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let mob_id = handle.definition().id.to_string();
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
// --- No spawnable profile in the current mob (manage scope is on a
// DIFFERENT mob): the machine must DENY every spawn-tool call, including
// empty-specs spawn_many_members. ---
let denied_dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(generated_mob_operator_authority_with_scope("different-mob")),
)
.expect("compose dispatcher")
.expect("operator dispatcher should be visible when authority is injected");
let empty_specs = RawValue::from_string(serde_json::json!({ "specs": [] }).to_string())
.expect("empty specs args");
let empty_specs_denied = denied_dispatcher
.dispatch(ToolCallView {
id: "deny-empty-spawn-many",
name: "spawn_many_members",
args: &empty_specs,
})
.await
.expect_err(
"empty-specs spawn_many_members must be denied for an operator with no spawnable \
profile — the coarse machine-routed gate is the only thing that can deny it",
);
assert!(matches!(empty_specs_denied, ToolError::AccessDenied { .. }));
let spawn_member_args = RawValue::from_string(
serde_json::json!({ "profile": "worker", "member_id": "w-deny" }).to_string(),
)
.expect("spawn args");
let spawn_member_denied = denied_dispatcher
.dispatch(ToolCallView {
id: "deny-spawn-member",
name: "spawn_member",
args: &spawn_member_args,
})
.await
.expect_err("spawn_member must be denied for an operator with no spawnable profile");
assert!(matches!(
spawn_member_denied,
ToolError::AccessDenied { .. }
));
// --- A spawnable profile IS granted in the current mob (no manage scope):
// the machine must ALLOW the coarse gate, so the call proceeds past it. An
// empty-specs spawn_many_members then admits and yields an empty batch. ---
let allowed_dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(generated_mob_operator_authority_with_spawn_profile(
mob_id.as_str(),
"worker",
)),
)
.expect("compose dispatcher")
.expect("operator dispatcher should be visible when authority is injected");
let empty_specs_allowed = allowed_dispatcher
.dispatch(ToolCallView {
id: "allow-empty-spawn-many",
name: "spawn_many_members",
args: &empty_specs,
})
.await
.expect(
"empty-specs spawn_many_members must pass the coarse gate when the operator can \
spawn a profile, then admit and produce an empty batch",
);
let payload: serde_json::Value =
serde_json::from_str(&empty_specs_allowed.result.text_content()).expect("payload");
assert_eq!(
payload["results"].as_array().map(|entries| entries.len()),
Some(0),
"empty-specs spawn_many_members must yield zero result entries once admitted"
);
}
#[tokio::test]
async fn test_visible_mob_operator_tools_emit_identity_native_member_payloads() {
let (handle, _service) = create_test_mob(sample_definition_with_mob_tools()).await;
let mob_id = handle.definition().id.to_string();
let profile = handle
.definition()
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.unwrap();
let dispatcher = super::tools::compose_external_tools_for_profile(
profile,
&BTreeMap::new(),
handle.clone(),
None,
None,
Some(generated_mob_operator_authority_with_scope(mob_id.as_str())),
)
.expect("compose dispatcher")
.expect("operator dispatcher should be visible when authority is injected");
let spawn_args = RawValue::from_string(
serde_json::json!({
"profile": "worker",
"member_id": "w-bridge"
})
.to_string(),
)
.expect("spawn args");
let spawn_result = dispatcher
.dispatch(ToolCallView {
id: "call-bridge-spawn",
name: "spawn_member",
args: &spawn_args,
})
.await
.expect("spawn_member should dispatch");
let spawn_payload: serde_json::Value =
serde_json::from_str(&spawn_result.result.text_content()).expect("spawn payload");
assert!(
spawn_payload["agent_identity"].is_string(),
"spawn result should surface agent_identity"
);
assert!(
spawn_payload["member_ref"]
.as_str()
.is_some_and(|value| !value.is_empty()),
"spawn result should surface canonical member_ref"
);
assert!(spawn_payload.get("agent_runtime_id").is_none());
assert!(spawn_payload.get("fence_token").is_none());
let list_args = RawValue::from_string("{}".to_string()).expect("list args");
let list_result = dispatcher
.dispatch(ToolCallView {
id: "call-bridge-list",
name: "list_members",
args: &list_args,
})
.await
.expect("list_members should dispatch");
let list_payload: serde_json::Value =
serde_json::from_str(&list_result.result.text_content()).expect("list payload");
let member = list_payload["members"]
.as_array()
.expect("members array")
.iter()
.find(|entry| entry["agent_identity"] == "w-bridge")
.expect("spawned member should appear in list");
assert_eq!(member["agent_identity"], "w-bridge");
assert!(
member["member_ref"]
.as_str()
.is_some_and(|value| !value.is_empty()),
"list output should surface canonical member_ref"
);
assert!(member.get("agent_runtime_id").is_none());
assert!(member.get("fence_token").is_none());
assert_eq!(member["role"], "worker");
assert!(member.get("session_id").is_none());
assert!(member.get("bridge_session_id").is_none());
assert!(member.get("current_session_id").is_none());
assert!(member.get("current_bridge_session_id").is_none());
}
#[tokio::test]
async fn test_spawn_helper_contract_aligns_with_retired_terminal_state() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let helper_id = AgentIdentity::from("helper-spawn");
let result = tokio::time::timeout(
std::time::Duration::from_secs(1),
handle.spawn_helper(
helper_id.clone(),
"summarize this",
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
..HelperOptions::default()
},
),
)
.await
.expect("spawn_helper must not wait on helper terminality")
.expect("spawn_helper succeeds");
assert_eq!(result.agent_identity, helper_id);
assert!(
handle
.get_member(&result.agent_identity)
.await
.unwrap()
.is_none(),
"spawn_helper must remove the helper from the active roster once it returns"
);
}
#[tokio::test]
async fn test_fork_helper_contract_aligns_with_retired_terminal_state() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let source_id = AgentIdentity::from("fork-source");
let source_member_identity = AgentIdentity::from("fork-source");
let source_ref = handle
.spawn_with_options(
ProfileName::from("worker"),
source_member_identity,
Some("source context".into()),
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn source member");
let helper_id = AgentIdentity::from("fork-helper");
let result = tokio::time::timeout(
std::time::Duration::from_secs(1),
handle.fork_helper(
&source_id,
helper_id.clone(),
"continue from source",
crate::launch::ForkContext::FullHistory,
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
..HelperOptions::default()
},
),
)
.await
.expect("fork_helper must not wait on helper terminality")
.expect("fork_helper succeeds");
assert_eq!(result.agent_identity, helper_id);
assert!(
handle
.get_member(&result.agent_identity)
.await
.unwrap()
.is_none(),
"fork_helper must remove the helper from the active roster once it returns"
);
assert_eq!(
source_ref.bridge_session_id().cloned(),
handle
.get_member(&AgentIdentity::from(source_id.as_str()))
.await
.unwrap()
.and_then(|entry| entry.member_ref.bridge_session_id().cloned()),
"fork_helper must not perturb the source member's canonical session binding"
);
}
/// Ask 6 spawn-site threading: an explicit `tool_access_policy` on the spawn
/// spec must ride `SpawnMemberSpec` -> actor -> `BuildAgentConfigParams` ->
/// `AgentBuildConfig` -> `SessionBuildOptions` and land in the member
/// session's persisted metadata tooling section (the field children inherit
/// from). Regression for the spec field previously being destructured and
/// ignored at the actor spawn sites.
#[tokio::test]
async fn test_spawn_spec_tool_access_policy_reaches_session_metadata() {
let (handle, service) = create_test_mob(sample_definition()).await;
let policy = meerkat_core::ops::ToolAccessPolicy::AllowList(
["send_message", "peers"].into_iter().collect(),
);
let spec = SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("gated-worker"),
)
.with_tool_access_policy(policy.clone());
handle.spawn_spec(spec).await.expect("spawn gated member");
let session_id = service
.session_id_for_comms_name("test-mob/worker/gated-worker")
.await
.expect("gated member session must exist");
let metadata = service
.persisted_session_clone(&session_id)
.await
.expect("persisted session")
.session_metadata()
.expect("session metadata");
assert_eq!(
metadata.tooling.tool_access_policy,
Some(policy),
"spec policy must reach the member's persisted metadata tooling section"
);
}
/// Ask 6: `HelperOptions.tool_access_policy` on `fork_helper` is honored —
/// the forked helper's session persists the policy in its metadata tooling
/// section.
#[tokio::test]
async fn test_fork_helper_tool_access_policy_reaches_session_metadata() {
let (handle, service) = create_test_mob(sample_definition()).await;
let source_id = AgentIdentity::from("fork-gated-source");
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("fork-gated-source"),
Some("source context".into()),
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn source member");
let policy = meerkat_core::ops::ToolAccessPolicy::DenyList(["bash"].into_iter().collect());
let helper_id = AgentIdentity::from("fork-gated-helper");
tokio::time::timeout(
std::time::Duration::from_secs(1),
handle.fork_helper(
&source_id,
helper_id,
"continue gated",
crate::launch::ForkContext::FullHistory,
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
tool_access_policy: Some(policy.clone()),
..HelperOptions::default()
},
),
)
.await
.expect("fork_helper must not hang")
.expect("fork_helper succeeds");
// The one-shot helper is retired (and its comms-name mapping dropped) by
// the time `fork_helper` returns, so identify its session by the persisted
// policy itself: exactly one persisted session — the helper's — must carry
// the tooling policy, and the source member's session must not.
let persisted = service.persisted_sessions.read().await;
let gated_policies: Vec<_> = persisted
.values()
.filter_map(|session| {
session
.session_metadata()
.expect("session metadata")
.tooling
.tool_access_policy
})
.collect();
assert_eq!(
gated_policies,
vec![policy],
"fork_helper policy must reach exactly the helper's persisted metadata tooling section"
);
}
#[tokio::test]
async fn test_spawn_helper_defaults_to_turn_driven_even_when_profile_is_autonomous() {
let (handle, service) = create_test_mob(sample_definition()).await;
let _ = tokio::time::timeout(
std::time::Duration::from_secs(1),
handle.spawn_helper(
AgentIdentity::from("helper-default"),
"summarize this",
HelperOptions {
role_name: Some(ProfileName::from("worker")),
..HelperOptions::default()
},
),
)
.await
.expect("spawn_helper default runtime mode must not inherit autonomous host behavior")
.expect("spawn_helper succeeds");
assert_eq!(
service.keep_alive_start_turn_call_count(),
0,
"spawn_helper must default to TurnDriven semantics instead of inheriting AutonomousHost"
);
}
#[tokio::test]
async fn test_respawn_contract_aligns_receipt_with_canonical_member_state() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("respawn-target");
let original_ref = handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original member");
let old_session_id = original_ref
.bridge_session_id()
.cloned()
.expect("original session id");
let original_snapshot = handle
.member_status(&AgentIdentity::from(member_id.as_str()))
.await
.expect("original member snapshot");
let (original_runtime_id, original_fence_token) = original_snapshot
.runtime_identity_fields()
.expect("original member should have runtime identity");
let receipt = handle
.respawn(
AgentIdentity::from(member_id.as_str()),
Some("resume task".into()),
)
.await
.expect("respawn succeeds");
assert_eq!(receipt.identity, AgentIdentity::from(member_id.as_str()));
assert_eq!(receipt.previous_fence_token, original_fence_token);
assert_eq!(
receipt.agent_runtime_id.identity,
AgentIdentity::from(member_id.as_str())
);
assert_eq!(
receipt.agent_runtime_id.generation,
original_runtime_id
.generation
.next()
.expect("test generation has a successor")
);
let snapshot = handle
.member_status(&receipt.identity)
.await
.expect("member snapshot");
let new_bridge_session_id = snapshot
.current_bridge_session_id
.clone()
.expect("new session id");
assert_ne!(new_bridge_session_id, old_session_id);
assert!(
service.read(&old_session_id).await.is_err(),
"respawn must archive the retired session before returning"
);
let (snapshot_runtime_id, snapshot_fence_token) = snapshot
.runtime_identity_fields()
.expect("respawned member should have runtime identity");
assert_eq!(snapshot_runtime_id, &receipt.agent_runtime_id);
assert_eq!(snapshot_fence_token, receipt.fence_token);
assert_eq!(
snapshot.current_session_id,
Some(new_bridge_session_id.clone())
);
assert_eq!(
service
.read(&new_bridge_session_id)
.await
.expect("new session readable")
.state
.session_id,
new_bridge_session_id,
"respawn receipt must align with the canonical replacement session"
);
}
#[tokio::test]
async fn test_respawn_abandonment_append_failure_closes_reply_and_actor_channels() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
let member_id = AgentIdentity::from("respawn-abandonment-fail-stop");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original member");
// Fail the replacement before membership publication, then fail the
// exact abandonment marker that must precede any definitive respawn
// failure. The actor must expose crash semantics, not launder either
// failure into an ordinary SpawnAfterRetire result.
events.fail_appends_for("MemberSpawned").await;
events.fail_appends_for("RespawnTopologyAbandoned").await;
let respawn_error = tokio::time::timeout(
Duration::from_secs(2),
handle.respawn(AgentIdentity::from(member_id.as_str()), None),
)
.await
.expect("fail-stop must close the in-flight respawn reply channel")
.expect_err("replacement and abandonment append failures must not report success");
assert!(
matches!(
respawn_error,
crate::runtime::handle::MobRespawnError::Mob(MobError::ActorReplyChannelClosed)
),
"durable abandonment failure must surface only reply-channel closure, got {respawn_error:?}"
);
// The accepted command's reply closes before the actor finishes its
// quiescence barrier, so a racing lifecycle command may itself be
// accepted and then lose its reply. Neither outcome may be an ordinary
// lifecycle result; eventually the command channel must be closed.
tokio::time::timeout(Duration::from_secs(2), async {
loop {
match handle.stop().await {
Err(MobError::ActorCommandChannelClosed) => break,
Err(MobError::ActorReplyChannelClosed) => tokio::task::yield_now().await,
other => panic!(
"fail-stopped actor leaked an ordinary lifecycle reply instead of channel closure: {other:?}"
),
}
}
})
.await
.expect("fail-stopped actor must close its command channel after quiescence");
let recorded = events.replay_all().await.expect("replay faulted respawn");
assert!(
recorded
.iter()
.all(|event| !matches!(event.kind, MobEventKind::RespawnTopologyAbandoned { .. })),
"confirmed-no-write abandonment failure must not fabricate a durable marker"
);
}
#[tokio::test]
async fn test_respawn_success_restores_existing_peer_wiring() {
let (handle, service) = create_test_mob(sample_definition()).await;
let left = AgentIdentity::from("respawn-left");
let right = AgentIdentity::from("respawn-right");
let original_left = handle
.spawn_with_options(
ProfileName::from("worker"),
left.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn left");
handle
.spawn_with_options(
ProfileName::from("worker"),
right.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn right");
handle
.wire(AgentIdentity::from(left.as_str()), right.clone())
.await
.expect("wire peers before respawn");
let original_left_snapshot = handle
.member_status(&AgentIdentity::from(left.as_str()))
.await
.expect("left snapshot before respawn");
let (original_left_runtime_id, original_left_fence_token) = original_left_snapshot
.runtime_identity_fields()
.expect("left member should have runtime identity before respawn");
let old_session_id = original_left
.bridge_session_id()
.cloned()
.expect("left session id");
let receipt = handle
.respawn(
AgentIdentity::from(left.as_str()),
Some("rebuild wiring".into()),
)
.await
.expect("respawn succeeds");
assert_eq!(receipt.previous_fence_token, original_left_fence_token);
assert_eq!(
receipt.agent_runtime_id.generation,
original_left_runtime_id
.generation
.next()
.expect("test generation has a successor")
);
assert!(
service.read(&old_session_id).await.is_err(),
"respawn must archive the retired session before returning"
);
let left_entry = handle
.get_member(&AgentIdentity::from(left.as_str()))
.await
.unwrap()
.expect("left remains active");
let right_entry = handle
.get_member(&AgentIdentity::from(right.as_str()))
.await
.unwrap()
.expect("right remains active");
assert!(
left_entry
.wired_to
.contains(&AgentIdentity::from(right.as_str())),
"respawn replacement must restore canonical wiring on the replacement member"
);
assert!(
right_entry
.wired_to
.contains(&AgentIdentity::from(left.as_str())),
"respawn replacement must preserve the peer's canonical wiring projection"
);
assert_eq!(left_entry.agent_runtime_id, receipt.agent_runtime_id);
assert_eq!(left_entry.fence_token, receipt.fence_token);
assert_eq!(
left_entry.agent_runtime_id, receipt.agent_runtime_id,
"respawn receipt must align with the replacement member's canonical session binding"
);
}
#[tokio::test]
async fn test_respawn_repairs_local_machine_edge_without_roster_projection() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let left = AgentIdentity::from("respawn-machine-left");
let right = AgentIdentity::from("respawn-machine-right");
handle
.spawn_with_options(
ProfileName::from("worker"),
left.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn left");
handle
.spawn_with_options(
ProfileName::from("worker"),
right.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn right");
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from(left.as_str()),
crate::machines::mob_machine::AgentIdentity::from(right.as_str()),
);
handle
.project_machine_input(crate::machines::mob_machine::MobMachineInput::WireMembers {
edge: edge.clone(),
})
.await
.expect("seed machine-owned edge without roster projection");
let left_before = handle
.get_member(&AgentIdentity::from(left.as_str()))
.await
.unwrap()
.expect("left before respawn");
assert!(
left_before
.wired_to
.contains(&AgentIdentity::from(right.as_str())),
"public member projection must read machine-owned topology"
);
handle
.respawn(
AgentIdentity::from(left.as_str()),
Some("restore machine edge".into()),
)
.await
.expect("respawn succeeds");
let left_after = handle
.get_member(&AgentIdentity::from(left.as_str()))
.await
.unwrap()
.expect("left after respawn");
let right_after = handle
.get_member(&AgentIdentity::from(right.as_str()))
.await
.unwrap()
.expect("right after respawn");
assert!(
left_after
.wired_to
.contains(&AgentIdentity::from(right.as_str())),
"respawn trust repair must preserve machine-owned left projection"
);
assert!(
right_after
.wired_to
.contains(&AgentIdentity::from(left.as_str())),
"respawn trust repair must preserve machine-owned right projection"
);
}
#[tokio::test]
async fn test_respawn_reports_machine_local_edge_to_retired_peer() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let left = AgentIdentity::from("respawn-stale-left");
let right = AgentIdentity::from("respawn-stale-right");
handle
.spawn_with_options(
ProfileName::from("worker"),
left.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn left");
handle
.spawn_with_options(
ProfileName::from("worker"),
right.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn right");
handle
.retire(AgentIdentity::from(right.as_str()))
.await
.expect("retire old peer");
assert!(
handle
.get_member(&AgentIdentity::from(right.as_str()))
.await
.unwrap()
.is_none(),
"test setup should remove the old peer from the live roster"
);
let stale_edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from(left.as_str()),
crate::machines::mob_machine::AgentIdentity::from(right.as_str()),
);
handle
.project_machine_input(crate::machines::mob_machine::MobMachineInput::WireMembers {
edge: stale_edge,
})
.await
.expect("seed stale machine-owned edge after terminal retire cleanup");
let error = handle
.respawn(
AgentIdentity::from(left.as_str()),
Some("report stale peer".into()),
)
.await
.expect_err("respawn should report machine edge to retired peer");
let failed_peer_ids = match error {
crate::runtime::handle::MobRespawnError::TopologyRestoreFailed {
failed_peer_ids, ..
} => failed_peer_ids,
other => panic!("expected TopologyRestoreFailed, got {other:?}"),
};
assert_eq!(
failed_peer_ids,
vec![crate::ids::RespawnTopologyPeerId::from(right.as_str())],
"machine-owned local peer edge must reach generated respawn topology result authority"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|event| matches!(event.kind, MobEventKind::MembersWired { .. })),
"failed respawn trust repair must not synthesize a MembersWired projection event"
);
}
/// Task #34 regression: respawn of a role-wired member must not be destroyed
/// by a peer whose live comms runtime is gone.
///
/// Live shape: the peer's runtime turn fails terminally (e.g. provider 400 /
/// rate-limit exhaustion), the session layer fail-closed discards the peer's
/// live session — and with it the comms runtime — while the MobMachine still
/// owns the peer as an Active member. Respawning the OTHER member then
/// re-wires the machine-owned edge. Before the fix, the spawn-contract
/// role-wiring fan-out inside `finalize_spawn_from_pending` ran the same edge
/// with hard-fail semantics, rolled back the replacement, and surfaced
/// `spawn failed after retire for member …: wiring error: wire requires
/// comms runtime for '<peer>'`. The machine-owned restore plan owns those
/// edges: the unrestorable peer must resolve through the generated
/// `ResolveRespawnTopologyRestore` authority into a typed
/// `TopologyRestoreFailed` result that preserves the replacement member.
#[tokio::test]
async fn test_respawn_role_wired_member_survives_peer_without_comms_runtime() {
let (handle, service) = create_test_mob(sample_definition_with_role_wiring()).await;
let peer_ref = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2 (role-wired to w-1)");
let original = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 in roster");
assert!(
original.wired_to.contains(&AgentIdentity::from("w-1")),
"role wiring should connect w-2 to w-1 at spawn"
);
let original_generation = original.agent_runtime_id.generation;
// Model the live failure: w-1's live session was fail-closed discarded,
// so its comms runtime is no longer resolvable while the MobMachine
// still owns w-1 as Active.
let peer_session = peer_ref
.bridge_session_id()
.cloned()
.expect("w-1 bridge session id");
service.set_missing_comms_runtime(&peer_session).await;
let error = handle
.respawn(AgentIdentity::from("w-2"), Some("come back online".into()))
.await
.expect_err("respawn topology restore cannot reach w-1's comms runtime");
match error {
crate::runtime::handle::MobRespawnError::TopologyRestoreFailed {
receipt,
failed_peer_ids,
} => {
assert_eq!(receipt.identity, AgentIdentity::from("w-2"));
assert_eq!(
failed_peer_ids,
vec![crate::ids::RespawnTopologyPeerId::from("w-1")],
"the degraded peer must surface through the typed topology-restore result"
);
}
other => {
panic!("expected typed TopologyRestoreFailed preserving the replacement, got {other:?}")
}
}
// The replacement member must survive with a fresh generation; the
// machine-owned wiring edge is preserved for repair once w-1 itself is
// respawned.
let replacement = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("replacement member must remain in the roster");
assert_eq!(
replacement.agent_runtime_id.generation,
original_generation
.next()
.expect("test generation has a successor"),
"replacement must carry the next runtime generation"
);
assert!(
replacement.wired_to.contains(&AgentIdentity::from("w-1")),
"machine-owned wiring edge must be preserved for later repair"
);
}
#[tokio::test]
async fn test_respawn_archive_failure_removes_stale_anchor_and_respawns() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("respawn-ambiguous");
let member_ref = handle
.spawn(ProfileName::from("worker"), member_id.clone(), None)
.await
.expect("spawn member");
let old_session_id = member_ref
.bridge_session_id()
.cloned()
.expect("session-backed member");
let original_snapshot = handle
.member_status(&AgentIdentity::from(member_id.as_str()))
.await
.expect("snapshot before respawn");
service.set_archive_failure(&old_session_id).await;
let error = handle
.respawn(AgentIdentity::from(member_id.as_str()), None)
.await
.expect_err("session-bound respawn should fail and retain the anchor when archive fails");
assert!(
matches!(
error,
crate::runtime::handle::MobRespawnError::Mob(MobError::SessionError(
SessionError::Store(ref source)
)) if source.to_string().contains("mock archive failure")
),
"session-bound respawn should retain the cleanup anchor and preserve the typed session-store archive failure: {error:?}"
);
let retained = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("archive failure should retain retiring member for retry");
let (original_runtime_id, original_fence_token) = original_snapshot
.runtime_identity_fields()
.expect("original member should have runtime identity before archive failure");
assert_eq!(&retained.agent_runtime_id, original_runtime_id);
assert_eq!(retained.fence_token, original_fence_token);
let including_retiring = handle.list_members_including_retiring().await;
assert_eq!(
including_retiring.len(),
1,
"failed cleanup must leave exactly the retained retiring member, not a replacement"
);
assert_eq!(
including_retiring[0].status,
crate::runtime::handle::MobMemberStatus::Retiring,
"retained member must carry the machine-owned Retiring status"
);
service.clear_archive_failure(&old_session_id).await;
let receipt = handle
.respawn(
AgentIdentity::from(member_id.as_str()),
Some("retry replacement".into()),
)
.await
.expect("respawn retry should clean up the old session and spawn replacement");
assert_eq!(receipt.identity, AgentIdentity::from(member_id.as_str()));
let replacement = handle
.member_status(&AgentIdentity::from(member_id.as_str()))
.await
.expect("replacement status");
let (replacement_runtime_id, _) = replacement
.runtime_identity_fields()
.expect("replacement member should have runtime identity");
assert_ne!(
replacement_runtime_id, original_runtime_id,
"successful retry must rotate the runtime binding"
);
}
#[tokio::test]
async fn test_list_members_returns_after_respawn_without_hanging() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("respawn-list-members");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original member");
let receipt = handle
.respawn(
AgentIdentity::from(member_id.as_str()),
Some("refresh listing".into()),
)
.await
.expect("respawn succeeds");
let members = tokio::time::timeout(std::time::Duration::from_secs(2), handle.list_members())
.await
.expect("list_members should not hang after respawn");
let listed = members
.into_iter()
.find(|entry| entry.agent_identity == member_id)
.expect("respawned member remains listed");
assert_eq!(listed.agent_runtime_id, Some(receipt.agent_runtime_id));
assert_eq!(listed.fence_token, Some(receipt.fence_token));
assert_eq!(
listed.status,
crate::runtime::handle::MobMemberStatus::Active
);
}
#[tokio::test]
async fn test_wait_one_fails_closed_for_missing_member_without_machine_material() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let error = handle
.wait_one(&AgentIdentity::from("missing-helper"))
.await
.expect_err("wait_one should fail closed for missing member");
assert!(
matches!(error, crate::MobError::Internal(message) if message.contains("MobMachine runtime material is absent")),
"missing-member wait must not invent runtime material"
);
}
#[tokio::test]
async fn test_wait_one_observes_retiring_member_as_non_terminal_until_archive() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("wait-retiring");
service.set_flow_turn_never_terminal(true);
let session_id = handle
.spawn(ProfileName::from("worker"), member_id.clone(), None)
.await
.expect("spawn retiring member")
.bridge_session_id()
.expect("session-backed member")
.clone();
service.set_flow_turn_never_terminal(false);
service.set_archive_delay_ms(250);
let retire = {
let handle = handle.clone();
let member_id = member_id.clone();
tokio::spawn(async move { handle.retire(AgentIdentity::from(member_id.as_str())).await })
};
tokio::time::sleep(Duration::from_millis(25)).await;
let in_flight = handle
.member_status(&AgentIdentity::from(member_id.as_str()))
.await
.expect("retiring member snapshot");
assert_eq!(
in_flight.status,
crate::runtime::handle::MobMemberStatus::Retiring,
"helper classification should continue to expose retiring canonical truth while disposal is still in flight"
);
assert!(
!in_flight.is_final,
"retiring member must not classify as terminal before archive/removal completes"
);
let terminal = handle
.wait_one(&AgentIdentity::from(member_id.as_str()))
.await
.expect("wait_one should observe terminal retirement");
assert!(
matches!(
terminal.status,
crate::runtime::handle::MobMemberStatus::Unknown
| crate::runtime::handle::MobMemberStatus::Completed
),
"terminal helper snapshot must come from canonical post-retire truth"
);
assert!(terminal.is_final);
assert!(
service.read(&session_id).await.is_err(),
"wait_one must not return terminal until the backing session is archived or removed from canonical truth"
);
retire
.await
.expect("retire join")
.expect("retire completes");
}
#[tokio::test]
async fn test_wait_all_fails_closed_for_missing_members_without_machine_material() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let ids = vec![
AgentIdentity::from("missing-a"),
AgentIdentity::from("missing-b"),
];
let error = handle
.wait_all(&ids)
.await
.expect_err("wait_all should fail closed for missing members");
assert!(
matches!(error, crate::MobError::Internal(message) if message.contains("MobMachine runtime material is absent")),
"missing-member wait must not invent runtime material"
);
}
#[tokio::test]
async fn test_wait_for_kickoff_complete_returns_current_autonomous_snapshots() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
service.set_start_turn_delay_ms(150);
let lead = AgentIdentity::from("lead-ready");
let worker = AgentIdentity::from("worker-ready");
handle
.spawn(ProfileName::from("lead"), lead.clone(), None)
.await
.expect("spawn lead");
handle
.spawn(ProfileName::from("worker"), worker.clone(), None)
.await
.expect("spawn worker");
let snapshots = handle
.wait_for_kickoff_complete(Some(Duration::from_secs(2)))
.await
.expect("kickoff barrier succeeds");
assert_eq!(
snapshots.len(),
2,
"all current autonomous members should be returned"
);
assert_eq!(snapshots[0].0, AgentIdentity::from(lead.as_str()));
assert_eq!(snapshots[1].0, AgentIdentity::from(worker.as_str()));
assert!(
snapshots
.iter()
.all(|(_, snapshot)| snapshot.status == crate::runtime::handle::MobMemberStatus::Active),
"kickoff completion should return current active snapshots rather than terminalizing members"
);
}
#[tokio::test]
async fn test_wait_for_members_kickoff_complete_only_waits_requested_members() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
service.set_start_turn_delay_ms(120);
let autonomous = AgentIdentity::from("lead-only");
let turn_driven = AgentIdentity::from("worker-td");
handle
.spawn(ProfileName::from("lead"), autonomous.clone(), None)
.await
.expect("spawn autonomous lead");
handle
.spawn_with_options(
ProfileName::from("worker"),
turn_driven.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let snapshots = handle
.wait_for_members_kickoff_complete(
&[
AgentIdentity::from(autonomous.as_str()),
AgentIdentity::from(turn_driven.as_str()),
],
Some(Duration::from_secs(2)),
)
.await
.expect("member-scoped barrier succeeds");
assert_eq!(snapshots.len(), 2);
assert_eq!(snapshots[0].0, AgentIdentity::from(autonomous.as_str()));
assert_eq!(snapshots[1].0, AgentIdentity::from(turn_driven.as_str()));
assert_eq!(
snapshots[1].1.status,
crate::runtime::handle::MobMemberStatus::Active,
"non-autonomous members should be treated as immediately satisfied"
);
}
#[tokio::test]
async fn test_wait_for_kickoff_complete_returns_after_initial_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
// Runtime-backed autonomous kickoff still resolves asynchronously, but
// enabling immediate completion should let the barrier clear quickly.
service.set_keep_alive_turns_complete_immediately(true);
let member = AgentIdentity::from("lead-hung");
handle
.spawn(ProfileName::from("lead"), member.clone(), None)
.await
.expect("spawn lead");
let snapshots = handle
.wait_for_kickoff_complete(Some(Duration::from_secs(2)))
.await
.expect("barrier should return after initial turn completes");
assert!(
!snapshots.is_empty(),
"barrier should return snapshots for spawned members"
);
let kickoff = snapshots[0]
.1
.kickoff
.as_ref()
.expect("autonomous member should expose kickoff state");
assert_eq!(
kickoff.phase,
crate::roster::MobMemberKickoffPhase::Started,
"barrier must return only after kickoff reaches a terminal started phase"
);
}
#[tokio::test]
async fn test_wait_for_kickoff_complete_times_out_while_runtime_backed_kickoff_pending() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(250);
let member = AgentIdentity::from("lead-pending");
handle
.spawn(ProfileName::from("lead"), member.clone(), None)
.await
.expect("spawn lead");
let error = handle
.wait_for_kickoff_complete(Some(Duration::from_millis(50)))
.await
.expect_err("barrier must not return before kickoff resolves");
match error {
MobError::KickoffWaitTimedOut { pending_member_ids } => {
assert_eq!(
pending_member_ids,
vec![member],
"pending autonomous kickoff should surface the still-starting member"
);
}
other => panic!("expected kickoff timeout, got {other:?}"),
}
}
#[tokio::test]
async fn test_wait_for_members_ready_treats_turn_driven_members_as_ready() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member = AgentIdentity::from("worker-ready-td");
handle
.spawn_with_options(
ProfileName::from("worker"),
member.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let snapshots = handle
.wait_for_members_ready(
&[AgentIdentity::from(member.as_str())],
Some(Duration::from_secs(2)),
)
.await
.expect("turn-driven members should be ready immediately");
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].0, AgentIdentity::from(member.as_str()));
}
#[tokio::test]
async fn test_wait_for_members_ready_returns_for_autonomous_members_after_startup_readiness() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(250);
let member = AgentIdentity::from("lead-ready-autonomous");
handle
.spawn(ProfileName::from("lead"), member.clone(), None)
.await
.expect("spawn autonomous lead");
let snapshots = handle
.wait_for_members_ready(
&[AgentIdentity::from(member.as_str())],
Some(Duration::from_secs(2)),
)
.await
.expect("autonomous member should become startup-ready without waiting for kickoff");
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].0, AgentIdentity::from(member.as_str()));
assert_eq!(
snapshots[0].1.status,
crate::runtime::handle::MobMemberStatus::Active,
"startup-ready barrier should return the live autonomous snapshot"
);
}
#[tokio::test]
async fn test_wait_for_members_ready_marks_missing_bridge_session_broken() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(250);
let member = AgentIdentity::from("worker-ready-missing-session");
let receipt = handle
.spawn(ProfileName::from("worker"), member.clone(), None)
.await
.expect("spawn autonomous worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
service
.archive(&bridge_session_id)
.await
.expect("test should remove the live bridge session");
let snapshots = handle
.wait_for_members_ready(
&[AgentIdentity::from(member.as_str())],
Some(Duration::from_secs(2)),
)
.await
.expect("missing bridge session should resolve ready wait as broken");
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].0, AgentIdentity::from(member.as_str()));
assert_eq!(
snapshots[0].1.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert_eq!(snapshots[0].1.current_session_id, Some(bridge_session_id));
assert!(
snapshots[0]
.1
.error
.as_deref()
.is_some_and(|message| message.contains("missing bridge session")),
"broken member should surface missing bridge-session reason: {:?}",
snapshots[0].1
);
}
#[tokio::test]
async fn test_wait_for_members_kickoff_complete_excludes_later_spawns() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let barrier = tokio::spawn({
let handle = handle.clone();
async move {
handle
.wait_for_members_kickoff_complete(&[], Some(Duration::from_secs(2)))
.await
}
});
tokio::time::sleep(Duration::from_millis(10)).await;
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("late-lead"),
None,
)
.await
.expect("spawn late lead");
let snapshots = barrier
.await
.expect("barrier join")
.expect("empty target barrier succeeds");
assert!(
snapshots.is_empty(),
"members spawned after the barrier call must not be included"
);
}
#[tokio::test]
async fn test_wait_for_kickoff_complete_returns_broken_snapshot_without_hanging() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let broken = AgentIdentity::from("lead-broken");
handle
.spawn(ProfileName::from("lead"), broken.clone(), None)
.await
.expect("spawn lead");
handle.stop().await.expect("stop mob");
let old_sid = handle
.get_member(&AgentIdentity::from(broken.as_str()))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume should succeed with Broken projection");
resumed
.resume()
.await
.expect("explicit resume should materialize the stopped mob");
let snapshots = resumed
.wait_for_kickoff_complete(Some(Duration::from_secs(2)))
.await
.expect("barrier should not hang on Broken members");
let broken_snapshot = snapshots
.into_iter()
.find(|(id, _)| *id == broken)
.expect("broken member snapshot");
assert_eq!(
broken_snapshot.1.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert!(broken_snapshot.1.error.is_some());
}
#[tokio::test]
async fn test_mob_flow_tools_visible_but_scope_restricted_with_default_authority() {
let mut definition =
with_cancel_grace_timeout(sample_definition_with_single_step_flow(60_000, 8), 25);
let worker = definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile exists")
.as_inline_mut()
.unwrap();
worker.tools.mob = true;
worker.tools.comms = true;
let (handle, service) = create_test_mob(definition).await;
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let tool_names = service.external_tool_names(&sid_1).await;
for required in [
"mob_list_flows",
"mob_run_flow",
"mob_flow_status",
"mob_cancel_flow",
] {
assert!(
tool_names.contains(&required.to_string()),
"flow operator tool '{required}' must be visible when profile.tools.mob = true"
);
}
let listed_err = service
.dispatch_external_tool(&sid_1, "mob_list_flows", serde_json::json!({}))
.await
.expect_err("mob_list_flows should be denied without managed_mob_scope");
assert!(matches!(listed_err, ToolError::AccessDenied { .. }));
let started_err = service
.dispatch_external_tool(
&sid_1,
"mob_run_flow",
serde_json::json!({
"flow_id": "demo",
"params": {"ticket": "REQ-017"}
}),
)
.await
.expect_err("mob_run_flow should be denied without managed_mob_scope");
assert!(matches!(started_err, ToolError::AccessDenied { .. }));
}
#[tokio::test]
async fn test_mob_flow_status_denies_before_arg_validation_when_scope_missing() {
// With default create-only authority (no managed_mob_scope), the tool is
// visible but dispatch is denied at the scope gate — *before* args are
// parsed. Bad-arg payloads must not leak as ArgValidation when the caller
// lacks scope; the scope check must short-circuit.
let mut definition = sample_definition_with_single_step_flow(500, 8);
let worker = definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile exists")
.as_inline_mut()
.unwrap();
worker.tools.mob = true;
let (handle, service) = create_test_mob(definition).await;
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let error = service
.dispatch_external_tool(
&sid_1,
"mob_flow_status",
serde_json::json!({"run_id":"not-a-uuid"}),
)
.await
.expect_err("mob_flow_status should be denied without managed_mob_scope");
assert!(
matches!(error, ToolError::AccessDenied { .. }),
"scope check must short-circuit before arg validation; got: {error:?}"
);
}
#[tokio::test]
async fn test_flow_step_tool_overlay_is_step_scoped() {
let mut definition = sample_definition_with_single_step_flow(2_000, 8);
let flow = definition
.flows
.get_mut(&flow_id("demo"))
.expect("demo flow exists");
let step = flow
.steps
.get_mut(&step_id("start"))
.expect("start step exists");
step.allowed_tools = Some(vec!["alpha".to_string(), "beta".to_string()]);
step.blocked_tools = Some(vec!["beta".to_string()]);
let (handle, service) = create_test_mob(definition).await;
let sid = handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-overlay"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("start flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(5)).await;
assert!(
terminal.status == MobRunStatus::Completed,
"flow failed; failure ledger: {:?}",
terminal.failure_ledger
);
service
.start_turn(
&sid,
StartTurnRequest {
injected_context: Vec::new(),
prompt: "non-flow turn".to_string().into(),
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::default(),
},
)
.await
.expect("plain start_turn after flow");
let overlays = service
.recorded_flow_turn_overlays()
.await
.into_iter()
.filter(|(session_id, _)| session_id == &sid)
.map(|(_, overlay)| overlay)
.collect::<Vec<_>>();
assert_eq!(overlays.len(), 2, "expected flow turn + plain turn");
assert_eq!(
overlays[0],
Some(TurnToolOverlay {
allowed_tools: Some(vec!["alpha".into(), "beta".into()]),
blocked_tools: Some(vec!["beta".into()]),
..TurnToolOverlay::default()
}),
"flow step turn should pass flow overlay"
);
assert_eq!(
overlays[1], None,
"overlay must not persist into unrelated turns"
);
}
#[tokio::test]
async fn test_flow_step_tool_overlay_changes_runtime_visible_tools_and_restores_baseline() {
let mut definition = sample_definition_with_single_step_flow(2_000, 8);
let flow = definition
.flows
.get_mut(&flow_id("demo"))
.expect("demo flow exists");
let step = flow
.steps
.get_mut(&step_id("start"))
.expect("start step exists");
step.allowed_tools = Some(vec!["alpha".to_string(), "beta".to_string()]);
step.blocked_tools = Some(vec!["beta".to_string()]);
let (handle, provider_visible_tools) =
create_test_mob_with_overlay_probe_service(definition).await;
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-runtime-overlay"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run demo flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert!(
terminal.status == MobRunStatus::Completed,
"flow failed; failure ledger: {:?}",
terminal.failure_ledger
);
handle
.member(&AgentIdentity::from("w-runtime-overlay"))
.await
.expect("member handle")
.internal_turn("baseline after flow".to_string())
.await
.expect("run baseline turn");
let seen = provider_visible_tools
.lock()
.expect("provider_visible_tools lock poisoned")
.clone();
assert_eq!(
seen.len(),
2,
"expected one provider call for flow and one for baseline"
);
assert_eq!(
seen[0],
vec!["alpha".to_string()],
"flow step overlay should restrict provider-visible tools"
);
assert_eq!(
seen[1],
vec!["alpha".to_string(), "beta".to_string()],
"next non-flow turn should restore baseline visible tools"
);
}
#[tokio::test]
async fn test_member_turn_options_dispatch_context_reaches_session_dispatcher() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(ProfileBinding::as_inline_mut)
.expect("worker profile")
.external_addressable = true;
let identity = AgentIdentity::from("w-member-turn-dispatch-context");
let (handle, _provider_visible_tools, provider_turn_overlays) =
create_test_mob_with_overlay_probe_service_and_workgraph(definition, None).await;
handle
.spawn_with_options(
ProfileName::from("worker"),
identity.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let dispatch_context = BTreeMap::from([(
"host.routing_hint".to_string(),
serde_json::json!({ "shard": "blue" }),
)]);
let turn = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("preserve trusted dispatch metadata".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::new().with_turn_tool_overlay(TurnToolOverlay {
allowed_tools: Some(vec!["alpha".into()]),
dispatch_context: dispatch_context.clone(),
..Default::default()
}),
None,
)
.await
.expect("member turn with dispatch context should be admitted");
turn.wait()
.await
.expect("member turn with dispatch context should execute");
let applied_overlay = provider_turn_overlays
.lock()
.expect("provider_turn_overlays lock poisoned")
.last()
.cloned()
.flatten()
.expect("session dispatcher should receive the turn overlay");
assert_eq!(applied_overlay.dispatch_context, dispatch_context);
assert_eq!(
applied_overlay.allowed_tools,
Some(vec![meerkat_core::ToolName::from("alpha")])
);
handle.shutdown().await.expect("shutdown test mob");
}
#[tokio::test]
async fn test_workgraph_owner_attention_survives_respawn_and_scopes_member_turn() {
let definition = sample_definition();
let identity = AgentIdentity::from("w-attention");
let workgraph_service = meerkat::WorkGraphService::with_scope(
Arc::new(meerkat::MemoryWorkGraphStore::new()),
"mob-attention-realm",
meerkat::WorkNamespace::new("goals").expect("namespace"),
);
let goal = workgraph_service
.create_goal(meerkat::GoalCreateRequest {
realm_id: None,
namespace: None,
title: "Keep reviewer focused after respawn".to_string(),
description: None,
target: crate::lower_agent_identity_attention_target(&definition.id, &identity)
.expect("lower mob identity to owner target"),
mode: meerkat::WorkAttentionMode::Coordinate,
completion_policy: meerkat::WorkCompletionPolicy::SelfAttest,
delegated_authority: meerkat::AttentionDelegatedAuthority::AddEvidence,
projection_policy: meerkat::AttentionProjectionPolicy::default(),
})
.await
.expect("create owner-bound goal");
let (handle, provider_visible_tools, provider_turn_overlays) =
create_test_mob_with_overlay_probe_service_and_workgraph(
definition,
Some(workgraph_service.clone()),
)
.await;
let original_session = handle
.spawn_with_options(
ProfileName::from("worker"),
identity.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.respawn(identity.clone(), None)
.await
.expect("respawn worker");
let member = handle.member(&identity).await.expect("member handle");
let respawned_session = member
.current_bridge_session_id()
.await
.expect("bridge lookup")
.expect("respawned bridge session");
assert_ne!(
respawned_session, original_session,
"respawn must rotate the mortal session id"
);
member
.internal_turn("continue under owner-bound goal attention")
.await
.expect("deliver respawned member turn");
let seen_tools = provider_visible_tools
.lock()
.expect("provider_visible_tools lock poisoned")
.clone();
let scoped_tools = seen_tools.last().expect("provider call recorded");
assert!(
scoped_tools
.iter()
.all(|name| name.starts_with("workgraph_")),
"attention turn should replace baseline tools with WorkGraph-scoped tools: {scoped_tools:?}"
);
for expected in [
"workgraph_get",
"workgraph_create",
"workgraph_update",
"workgraph_link",
"workgraph_add_evidence",
"workgraph_attention_reassign",
] {
assert!(
scoped_tools.iter().any(|name| name == expected),
"attention-scoped turn should expose {expected}: {scoped_tools:?}"
);
}
let overlay = provider_turn_overlays
.lock()
.expect("provider_turn_overlays lock poisoned")
.last()
.cloned()
.flatten()
.expect("attention turn overlay");
let projection = meerkat::workgraph_attention_projection_from_overlay(Some(&overlay))
.expect("decode attention projection from overlay")
.expect("attention projection present");
assert_eq!(projection.binding_id, goal.attention.binding_id);
assert_eq!(projection.work_ref.item_id, goal.item.id);
workgraph_service
.pause_attention(meerkat::AttentionPauseRequest {
binding_id: goal.attention.binding_id.clone(),
realm_id: None,
namespace: None,
expected_revision: goal.attention.machine_state.revision,
until: None,
})
.await
.expect("revise binding mid-flight");
let stale_surface =
meerkat::WorkGraphToolSurface::with_attention_projection(workgraph_service, projection);
let raw = RawValue::from_string(serde_json::json!({ "id": goal.item.id }).to_string())
.expect("raw workgraph_get args");
let stale_error = stale_surface
.dispatch(ToolCallView {
id: "stale-get",
name: "workgraph_get",
args: &raw,
})
.await
.expect_err("stale projection must fail closed");
assert!(
format!("{stale_error:?}").contains("stale or inactive WorkGraph attention projection"),
"unexpected stale projection error: {stale_error:?}"
);
}
#[tokio::test]
async fn test_spawn_member_tool_dispatches_backend_selection() {
let mut definition = sample_definition_with_mob_tools();
definition.backend.external = Some(crate::definition::ExternalBackendConfig {
address_base: "https://backend.example.invalid/mesh".to_string(),
supervisor_bridge: None,
});
let (handle, service) = create_test_mob(definition).await;
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let err = service
.dispatch_external_tool(
&sid_1,
"spawn_member",
serde_json::json!({
"profile": "worker",
"member_id": "w-ext",
"backend": "external"
}),
)
.await
.expect_err("spawn external via tool should be denied without managed_mob_scope");
assert!(matches!(err, ToolError::AccessDenied { .. }));
assert!(
handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.is_none(),
"scope-denied operator spawn tool must not provision a backend peer"
);
}
#[tokio::test]
async fn test_spawn_member_profile_scope_allows_auto_wire_parent() {
let (handle, service) = create_test_mob(sample_definition_with_mob_tools()).await;
let sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
service
.dispatch_external_tool_outcome(
&sid,
"spawn_member",
serde_json::json!({
"profile": "worker",
"member_id": "w-auto-wire",
"auto_wire_parent": true
}),
)
.await
.expect("profile-scoped spawn should allow auto_wire_parent");
assert!(
handle
.get_member(&AgentIdentity::from("w-auto-wire"))
.await
.unwrap()
.is_some(),
"profile-scoped auto-wired spawn should provision the requested member"
);
}
#[tokio::test]
async fn test_tool_flag_enforcement_blocks_mob_tools() {
let (handle, service) = create_test_mob(sample_definition_without_mob_flags()).await;
let sid = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let names = service.external_tool_names(&sid).await;
assert!(
!names.iter().any(|name| name == "spawn_member"),
"tools.mob=false should hide spawn_member"
);
let spawn_err = service
.dispatch_external_tool(
&sid,
"spawn_member",
serde_json::json!({"profile": "worker", "member_id": "w-2"}),
)
.await
.expect_err("spawn tool must be unavailable");
assert!(matches!(spawn_err, ToolError::NotFound { .. }));
}
#[tokio::test]
async fn test_for_resume_rebuilds_definition_and_roster() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let mut definition = sample_definition();
for profile in definition
.profiles
.values_mut()
.filter_map(|profile| profile.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
// `for_resume` models a new MobMachine authority after process loss. Do
// not point it at the still-live runtimes owned by `handle`: generated
// trust ownership is intentionally non-transferable between concurrent
// authorities. Restart the service from durable session state so the new
// runtimes begin unowned, as they do after a real cold restart.
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
let restarted_adapter = restarted_service.enable_runtime_adapter();
drop(handle);
drop(service);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(restarted_service)
.resume()
.await
.expect("resume");
assert_eq!(resumed.mob_id().as_str(), "test-mob");
let entry_1 = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
let entry_2 = resumed
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.unwrap();
assert!(entry_1.wired_to.contains(&AgentIdentity::from("w-2")));
assert!(entry_2.wired_to.contains(&AgentIdentity::from("w-1")));
assert_eq!(
resumed.status().await.expect("cold-resumed mob status"),
MobState::Running,
"a recovered Running mob must keep the actor-owned exact attachment sidecars"
);
let snapshot = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("actor-routed status after cold Running recovery");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active,
"the actor must observe the exact sidecar committed during reconciliation"
);
resumed
.member(&AgentIdentity::from("w-1"))
.await
.expect("cold-resumed member handle")
.internal_turn(ContentInput::from(
"post-cold-resume exact sidecar check".to_string(),
))
.await
.expect("the actor provisioner must own the attachment sidecar committed by recovery");
}
#[tokio::test]
async fn test_cold_running_resume_reestablishes_autonomous_startup_ready_without_kickoff() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runs = storage.runs.clone();
let specs = storage.specs.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = storage.identity.clone();
let identity_member = storage.identity_member.clone();
let identity_status = storage.identity_status.clone();
let realm_profiles = storage.realm_profiles.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let member = AgentIdentity::from("cold-ready-autonomous");
handle
.spawn(ProfileName::from("worker"), member.clone(), None)
.await
.expect("spawn autonomous member");
handle
.wait_for_members_ready(std::slice::from_ref(&member), Some(Duration::from_secs(2)))
.await
.expect("fresh member reaches startup readiness");
wait_for_keep_alive_start_turn_count(service.as_ref(), 1).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"fresh spawn must execute exactly one kickoff turn"
);
// Rebuild from durable state using a process-restart-faithful service.
// Its live runtime mechanics and counters start empty, so readiness after
// `for_resume` can only come from checks performed by the recovered actor.
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
let _ = restarted_service.enable_runtime_adapter();
drop(handle);
drop(service);
let resumed = MobBuilder::for_resume(MobStorage {
events,
runs,
specs,
runtime_metadata,
identity,
identity_member,
identity_status,
realm_profiles,
})
.with_session_service(restarted_service.clone())
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("cold resume running mob");
let ready = resumed
.wait_for_members_ready(
std::slice::from_ref(&member),
Some(Duration::from_millis(250)),
)
.await
.expect("recovered live autonomous member must become startup-ready");
assert_eq!(ready.len(), 1);
assert_eq!(ready[0].0, member);
assert_eq!(ready[0].1.status, MobMemberStatus::Active);
assert_eq!(
restarted_service.keep_alive_start_turn_call_count(),
0,
"cold readiness recovery must not synthesize a second kickoff turn"
);
}
#[derive(Clone, Copy)]
enum ColdLocalReadinessFault {
RuntimeAdapter,
TurnDrivenRuntimeAdapter,
CommsRuntime,
}
async fn wait_for_keep_alive_start_turn_count(service: &MockSessionService, expected: u64) {
tokio::time::timeout(Duration::from_secs(1), async {
while service.keep_alive_start_turn_call_count() < expected {
tokio::task::yield_now().await;
}
})
.await
.expect("keep-alive kickoff observation timed out");
}
fn assert_missing_outbound_comms_capability(
error: MobError,
expected_member: &AgentIdentity,
expected_context: &'static str,
) {
match error {
MobError::MissingMemberCapability {
member_id,
capability,
context,
} => {
assert_eq!(&member_id, expected_member);
assert_eq!(
capability,
crate::error::MobMemberCapability::OutboundCommsRuntime
);
assert_eq!(context, expected_context);
}
other => panic!("expected typed missing outbound comms capability, got {other:?}"),
}
}
async fn assert_cold_running_local_resume_fails_closed(
fault: ColdLocalReadinessFault,
expected_context: &'static str,
) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runs = storage.runs.clone();
let specs = storage.specs.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = storage.identity.clone();
let identity_member = storage.identity_member.clone();
let identity_status = storage.identity_status.clone();
let realm_profiles = storage.realm_profiles.clone();
let mut definition = sample_definition();
if matches!(fault, ColdLocalReadinessFault::TurnDrivenRuntimeAdapter) {
for profile in definition
.profiles
.values_mut()
.filter_map(|profile| profile.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
}
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let member = AgentIdentity::from(match fault {
ColdLocalReadinessFault::RuntimeAdapter => "cold-missing-runtime-adapter",
ColdLocalReadinessFault::TurnDrivenRuntimeAdapter => {
"cold-turn-driven-missing-runtime-adapter"
}
ColdLocalReadinessFault::CommsRuntime => "cold-missing-comms-runtime",
});
handle
.spawn(ProfileName::from("worker"), member.clone(), None)
.await
.expect("spawn local member");
handle
.wait_for_members_ready(std::slice::from_ref(&member), Some(Duration::from_secs(2)))
.await
.expect("fresh member reaches startup readiness");
if !matches!(fault, ColdLocalReadinessFault::TurnDrivenRuntimeAdapter) {
wait_for_keep_alive_start_turn_count(service.as_ref(), 1).await;
}
let member_entry = handle
.get_member(&member)
.await
.expect("fresh member lookup")
.expect("fresh member roster entry");
assert!(
member_entry.member_ref.bridge_session_id().is_some(),
"local member must be session-backed before cold restart"
);
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
match fault {
ColdLocalReadinessFault::RuntimeAdapter
| ColdLocalReadinessFault::TurnDrivenRuntimeAdapter => {}
ColdLocalReadinessFault::CommsRuntime => {
let _ = restarted_service.enable_runtime_adapter();
// Cold builder reconciliation observes the live runtime four
// times while rebuilding endpoint/trust projections. The fifth
// observation is the recovered actor's readiness check.
restarted_service.set_comms_runtime_missing_after_successes(4);
}
}
drop(handle);
drop(service);
let resume_result = MobBuilder::for_resume(MobStorage {
events: events.clone(),
runs,
specs,
runtime_metadata,
identity,
identity_member,
identity_status,
realm_profiles,
})
.with_session_service(restarted_service.clone())
.notify_orchestrator_on_resume(false)
.resume()
.await;
if matches!(
fault,
ColdLocalReadinessFault::RuntimeAdapter | ColdLocalReadinessFault::TurnDrivenRuntimeAdapter
) {
let error = match resume_result {
Ok(_) => panic!("missing composition adapter must reject cold resume synchronously"),
Err(error) => error,
};
assert_missing_outbound_comms_capability(error, &member, expected_context);
assert_eq!(
restarted_service.keep_alive_start_turn_call_count(),
0,
"a synchronous composition failure must not start a replacement kickoff turn"
);
return;
}
// Per-session comms is discovered only after durable reconstruction. The
// recovered actor owns that lifecycle failure: it commits Stopped and
// retains the handle, whose explicit resume is the canonical typed retry.
let resumed = resume_result
.expect("per-session readiness failure must return its durably fail-stopped handle");
let resumed_status = resumed.status().await.expect("failed cold-resume status");
let comms_observations = restarted_service
.comms_runtime_observations
.load(Ordering::Relaxed);
let remaining_successes = restarted_service
.comms_runtime_successes_before_missing
.load(Ordering::Relaxed);
assert_eq!(
resumed_status,
MobState::Stopped,
"local cold resume must fail closed when comms-drain readiness is unavailable; observations={comms_observations}, remaining_successes={remaining_successes}"
);
let durable_events = events
.replay_all()
.await
.expect("replay startup fail-stop events");
assert_eq!(
durable_events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MobStopped))
.count(),
1,
"polling a pending exact interrupt must commit one structural Stop transition"
);
assert_eq!(
durable_events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MemberKickoffUpdated {
member: event_member,
kickoff,
} if event_member == &member
&& kickoff.phase == crate::MobMemberKickoffPhase::Cancelled
)
})
.count(),
1,
"polling the retained interrupt receiver must not duplicate its durable kickoff cancellation"
);
let error = resumed
.resume()
.await
.expect_err("explicit retry must preserve the typed missing capability failure");
assert_missing_outbound_comms_capability(error, &member, expected_context);
assert_eq!(
restarted_service.keep_alive_start_turn_call_count(),
0,
"a readiness failure must not synthesize a replacement kickoff turn"
);
}
#[tokio::test]
async fn test_cold_running_resume_fails_closed_without_runtime_adapter() {
assert_cold_running_local_resume_fails_closed(
ColdLocalReadinessFault::RuntimeAdapter,
"local member comms-drain runtime adapter",
)
.await;
}
#[tokio::test]
async fn test_cold_running_turn_driven_resume_fails_closed_without_runtime_adapter() {
assert_cold_running_local_resume_fails_closed(
ColdLocalReadinessFault::TurnDrivenRuntimeAdapter,
"local member comms-drain runtime adapter",
)
.await;
}
#[tokio::test]
async fn test_cold_running_resume_fails_closed_without_comms_runtime() {
assert_cold_running_local_resume_fails_closed(
ColdLocalReadinessFault::CommsRuntime,
"local member comms-drain startup",
)
.await;
}
#[tokio::test]
async fn test_resume_preserves_owner_bridge_authority_across_reset_epoch() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let owner_session_id = SessionId::new();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.with_owner_bridge_session_create_authority(owner_session_id.clone(), true, false)
.create()
.await
.expect("create owner-bound mob");
handle.reset().await.expect("reset");
assert_eq!(
handle
.owner_bridge_session_lifecycle_authority()
.map(|authority| authority.bridge_session_id),
Some(owner_session_id.clone()),
"live reset must preserve generated owner authority"
);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("resume after reset");
let owner_authority = resumed
.owner_bridge_session_lifecycle_authority()
.expect("owner authority should recover from full owner history");
assert_eq!(owner_authority.bridge_session_id, owner_session_id);
assert!(owner_authority.destroy_on_owner_archive);
assert!(!owner_authority.implicit_delegation_mob);
}
#[tokio::test]
async fn test_resume_ignores_legacy_owner_projection_without_generated_authority() {
let owner_session_id = SessionId::new();
let mut legacy_definition =
serde_json::to_value(sample_definition()).expect("serialize definition");
legacy_definition["owner_bridge_session_id"] = serde_json::json!(owner_session_id.to_string());
legacy_definition["session_cleanup_policy"] = serde_json::json!("destroy_on_owner_archive");
legacy_definition["is_implicit"] = serde_json::json!(true);
let legacy_definition: MobDefinition =
serde_json::from_value(legacy_definition).expect("legacy definition");
let storage = MobStorage::in_memory();
storage
.events
.append(NewMobEvent {
mob_id: MobId::from("test-mob"),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(legacy_definition),
},
})
.await
.expect("append legacy created event");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let resumed = MobBuilder::for_resume(storage)
.with_session_service(service)
.resume()
.await
.expect("resume legacy owner projection");
assert_eq!(
resumed.owner_bridge_session_lifecycle_authority(),
None,
"legacy definition-only owner projection must not recover behavior authority"
);
}
#[tokio::test]
async fn test_owner_bridge_create_authority_rejects_implicit_without_cleanup() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let result = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.with_owner_bridge_session_create_authority(SessionId::new(), false, true)
.create()
.await;
assert!(
matches!(result, Err(MobError::Internal(message)) if message.contains("BindOwnerBridgeSession")),
"generated MobMachine authority must reject implicit owner bindings without cleanup"
);
}
#[tokio::test]
async fn test_resume_fails_when_orchestrator_resume_notification_fails() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut def = sample_definition();
for profile in def.profiles.values_mut().filter_map(|b| b.as_inline_mut()) {
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(def, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn orchestrator");
handle.stop().await.expect("stop");
service.set_fail_start_turn(true);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not notify or materialize the orchestrator"
);
let error = resumed
.resume()
.await
.expect_err("explicit resume must surface orchestrator start_turn failures");
assert!(
matches!(
&error,
MobError::Internal(message) if message.contains("mock start_turn failure")
),
"resume must surface orchestrator start_turn failures, got {error:?}"
);
}
#[tokio::test]
async fn test_resume_reconciles_orphaned_sessions() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let _handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let orphan = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "orphan".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/orphan".to_string()),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create orphan");
assert_eq!(service.active_session_count().await, 1);
let _resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
let sessions = service
.list(SessionQuery::default())
.await
.expect("list sessions");
assert!(
!sessions.iter().any(|s| s.session_id == orphan.session_id),
"resume should archive orphaned sessions"
);
}
#[tokio::test]
async fn test_resume_restores_missing_sessions_with_same_session_and_history() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let mut persisted = service
.live_session_clone(&old_sid)
.await
.expect("live session");
persisted.append_system_message("Persisted worker prompt".to_string());
persisted.push(meerkat_core::Message::User(
meerkat_core::types::UserMessage::text("Remember session continuity.".to_string()),
));
service.replace_live_session(persisted).await;
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal: the real service returns no durable snapshot for
// archived sessions).
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly restore missing member sessions"
);
resumed
.resume()
.await
.expect("explicit resume should restore the missing member session");
let restored_sid = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("restored entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
assert_eq!(
restored_sid, old_sid,
"persistent resume should restore the original session id"
);
let history = service
.read_history(
&restored_sid,
meerkat_core::service::SessionHistoryQuery::default(),
)
.await
.expect("history for restored session");
assert!(
history.messages.iter().any(|message| message
.as_indexable_text()
.contains("Remember session continuity.")),
"restored session should preserve persisted history"
);
assert_eq!(service.active_session_count().await, 1);
}
#[tokio::test]
async fn test_resume_restores_missing_sessions_with_tool_wiring() {
let mut definition = sample_definition_with_mob_tools();
let worker = definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap();
worker.tools.rust_bundles = vec!["bundle-a".to_string()];
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.register_tool_bundle("bundle-a", Arc::new(EchoBundleDispatcher))
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal for durable snapshots).
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.register_tool_bundle("bundle-a", Arc::new(EchoBundleDispatcher))
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly restore member tool wiring"
);
resumed
.resume()
.await
.expect("explicit resume should restore member tool wiring");
let restored_sid = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("restored entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
assert_eq!(
restored_sid, old_sid,
"persistent resume should restore the original session id"
);
let names = service.external_tool_names(&restored_sid).await;
assert!(
names.contains(&"spawn_member".to_string()),
"restored sessions must keep operator tools that the profile declares"
);
assert!(
names.contains(&"bundle_echo".to_string()),
"rust bundle tools must be wired on restored sessions"
);
}
#[tokio::test]
async fn test_resume_marks_missing_persisted_session_as_broken() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly materialize a stopped mob"
);
resumed
.resume()
.await
.expect("explicit resume should classify the missing session");
let snapshot = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("broken member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert!(snapshot.is_final, "broken members should be terminal");
assert_eq!(snapshot.current_session_id, Some(old_sid.clone()));
assert!(
snapshot
.error
.as_deref()
.is_some_and(|message| message.contains("missing bridge session snapshot")),
"broken member should surface restore failure reason"
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query MobMachine state after restore failure");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
assert_eq!(
machine_state.member_lifecycle_for_identity(&machine_identity),
crate::machines::mob_machine::MobMemberLifecycleMaterial {
status: crate::machines::mob_machine::MobMemberLifecycleStatus::Broken,
terminal_class: crate::machines::mob_machine::MobMemberTerminalClass::TerminalFailure,
error: Some(format!("missing bridge session snapshot for '{old_sid}'")),
},
"restore-failure member status must be owned by MobMachine, not reconstructed by projection code"
);
let members = resumed.list_members().await;
let broken = members
.into_iter()
.find(|entry| entry.agent_identity == "w-1")
.expect("broken member should remain visible in list_members");
assert_eq!(
broken.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert!(broken.is_final);
assert_eq!(broken.current_session_id, Some(old_sid));
assert!(
broken
.error
.as_deref()
.is_some_and(|message| message.contains("missing bridge session snapshot")),
"projected member listing should carry the restore failure"
);
let including_retiring = resumed.list_members_including_retiring().await;
let broken_including_retiring = including_retiring
.into_iter()
.find(|entry| entry.agent_identity == "w-1")
.expect("broken member should remain visible in list_members_including_retiring");
assert_eq!(
broken_including_retiring.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert!(broken_including_retiring.is_final);
}
#[tokio::test]
async fn test_resume_repoints_snapshotless_member_head_to_latest_persisted_session() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = AgentIdentity::from("mk--rt_creview_csingleton_c0");
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("worker"), identity.clone(), None)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&identity)
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard old live session");
service.delete_persisted_session(&old_sid).await;
let replacement = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "replacement".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/rt:review:singleton:0".to_string()),
mob_member_binding: None,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create replacement");
service
.session_comms_names
.write()
.await
.remove(&replacement.session_id);
let mut recovered_head = service
.persisted_session_clone(&replacement.session_id)
.await
.expect("replacement committed body");
let mut recovered_metadata = recovered_head
.session_metadata()
.expect("replacement metadata");
recovered_metadata.model = "recovered-head-model".to_string();
recovered_head
.set_session_metadata(recovered_metadata)
.expect("recovered-head metadata should serialize");
service.stage_prepared_resume_session(recovered_head).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly self-heal member sessions"
);
resumed
.resume()
.await
.expect("explicit resume should self-heal snapshotless head");
assert!(
service.resume_prepare_calls.load(Ordering::Relaxed) > 0,
"replacement materialization must cross explicit resume preparation"
);
assert_eq!(
service
.create_requests
.read()
.await
.last()
.expect("replacement materialization request")
.model,
"recovered-head-model",
"replacement config must be compiled from metadata on recovered H, not stale A"
);
let snapshot = resumed
.member_status(&identity)
.await
.expect("member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert_eq!(
snapshot.current_session_id,
Some(replacement.session_id.clone())
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query MobMachine state after self-heal");
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from_domain(&identity);
let dsl_session_id =
crate::machines::mob_machine::SessionId::from_domain(&replacement.session_id);
assert_eq!(
machine_state.member_session_bindings.get(&dsl_identity),
Some(&dsl_session_id),
"snapshotless continuity heads should be repointed through MobMachine recovery"
);
let replacement_peer_id = service
.comms_runtime(&replacement.session_id)
.await
.expect("replacement comms runtime")
.peer_id()
.expect("replacement peer id");
assert_eq!(
machine_state
.member_peer_endpoints
.get(&dsl_identity)
.map(|endpoint| endpoint.peer_id.0.clone()),
Some(replacement_peer_id.as_str()),
"snapshotless recovery must replace the old spawn endpoint with the exact recovered session endpoint"
);
let resumed_again = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("repeat resume should replay durable self-heal");
let repeated_snapshot = resumed_again
.member_status(&identity)
.await
.expect("repeat member status");
assert_eq!(
repeated_snapshot.current_session_id,
Some(replacement.session_id.clone()),
"durable self-heal should survive the next restart without fresh-spawn churn"
);
let stored_events = events.replay_all().await.expect("replay events");
assert_eq!(
stored_events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberSessionBindingRecovered(..)))
.count(),
2,
"foreign attachment takeover must persist binding first, then the materialized replacement endpoint"
);
let recovered_endpoint = stored_events.iter().find_map(|event| match &event.kind {
MobEventKind::MemberSessionBindingRecovered(recovered)
if recovered.agent_identity == identity =>
{
recovered.member_peer_endpoint()
}
_ => None,
});
assert_eq!(
recovered_endpoint.map(|endpoint| endpoint.peer_id),
Some(replacement_peer_id),
"the recovered binding event must durably carry the replacement endpoint"
);
assert_eq!(
stored_events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberSpawned(..)))
.count(),
1,
"repeat resume should not fresh-spawn a replacement member"
);
}
#[tokio::test]
async fn test_snapshotless_recovery_never_publishes_foreign_attachment_endpoint() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = AgentIdentity::from("mk--rt_creview_csingleton_c0");
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("worker"), identity.clone(), None)
.await
.expect("spawn member");
let old_sid = handle
.get_member(&identity)
.await
.expect("roster query")
.expect("spawned member")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let old_runtime = service
.comms_runtime(&old_sid)
.await
.expect("old comms runtime");
let old_peer_id = old_runtime.peer_id().expect("old peer id");
let old_public_key_bytes = old_runtime
.public_key_bytes()
.expect("old public key bytes");
handle.stop().await.expect("stop before recovery");
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard old live session");
service.delete_persisted_session(&old_sid).await;
let recovered_comms_name = "test-mob/worker/rt:review:singleton:0";
let replacement = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "replacement".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some(recovered_comms_name.to_string()),
mob_member_binding: None,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create replacement session head");
let mut drifted_runtime = MockCommsRuntime::new(
recovered_comms_name,
MockCommsBehavior::default(),
replacement.session_id.clone(),
Some(service.enable_runtime_adapter()),
);
drifted_runtime.default_peer_id = old_peer_id;
drifted_runtime.default_public_key_bytes = old_public_key_bytes;
drifted_runtime.default_public_key =
meerkat_comms::PubKey::new(old_public_key_bytes).to_pubkey_string();
let foreign_attachment_address = format!("inproc://{recovered_comms_name}/descriptor-drift");
drifted_runtime
.default_address
.clone_from(&foreign_attachment_address);
assert_eq!(
drifted_runtime.peer_id(),
Some(old_peer_id),
"fixture must preserve the cryptographic peer identity"
);
service
.sessions
.write()
.await
.insert(replacement.session_id.clone(), Arc::new(drifted_runtime));
service
.session_comms_names
.write()
.await
.remove(&replacement.session_id);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly take over the foreign attachment"
);
resumed
.resume()
.await
.expect("explicit resume must retire foreign A and materialize exact replacement B");
let machine_after_resume = resumed
.query_machine_state()
.await
.expect("query machine after exact replacement materialization");
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from_domain(&identity);
assert_eq!(
machine_after_resume
.member_session_bindings
.get(&dsl_identity),
Some(&crate::machines::mob_machine::SessionId::from_domain(
&replacement.session_id
)),
"snapshotless recovery must publish the replacement session binding"
);
let rebuilt_runtime = service
.comms_runtime(&replacement.session_id)
.await
.expect("replacement B comms runtime");
let rebuilt_address = rebuilt_runtime
.advertised_address()
.expect("replacement B advertised address");
let machine_endpoint_after_resume = machine_after_resume
.member_peer_endpoints
.get(&dsl_identity)
.expect("replacement B endpoint is machine-owned");
assert_eq!(
machine_endpoint_after_resume.address.0, rebuilt_address,
"machine endpoint must come from materialized replacement B"
);
assert_ne!(
machine_endpoint_after_resume.address.0, foreign_attachment_address,
"foreign attachment A must never publish durable endpoint state over B"
);
let roster_after_resume = resumed
.get_member(&identity)
.await
.expect("query roster after exact replacement")
.expect("member remains projected after exact replacement");
assert_eq!(
roster_after_resume.bridge_session_id(),
Some(&replacement.session_id),
"roster must publish the same exact replacement binding as MobMachine"
);
let recovery_events = events
.replay_all()
.await
.expect("replay exact replacement events")
.into_iter()
.filter_map(|event| match event.kind {
MobEventKind::MemberSessionBindingRecovered(recovered)
if recovered.agent_identity == identity =>
{
Some(recovered)
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
recovery_events.len(),
2,
"foreign takeover must write the binding before materialization, then B's endpoint"
);
assert!(
recovery_events[0].member_peer_endpoint().is_none(),
"the pre-materialization binding carrier must not contain foreign A's endpoint"
);
assert_eq!(
recovery_events[1]
.member_peer_endpoint()
.map(|endpoint| endpoint.address.to_string()),
Some(rebuilt_address.clone()),
"the endpoint upgrade must carry materialized replacement B"
);
}
#[test]
fn test_mob_machine_rejects_peer_id_reuse_across_generation_owners() {
use crate::machines::mob_machine as dsl;
fn recover_member(
authority: &mut dsl::MobMachineAuthority,
identity: &AgentIdentity,
session_id: &SessionId,
) -> AgentRuntimeId {
let runtime_id = AgentRuntimeId::initial(identity.clone());
authority
.apply_signal(dsl::MobMachineSignal::RecoverRosterMember {
agent_identity: dsl::AgentIdentity::from_domain(identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&runtime_id),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
profile_name: "worker".to_string(),
runtime_mode: dsl::SpawnPolicyRuntimeMode::TurnDriven,
external_addressable: false,
})
.expect("recover roster member");
authority
.apply_signal(dsl::MobMachineSignal::RecoverMemberSessionBinding {
agent_identity: dsl::AgentIdentity::from_domain(identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&runtime_id),
bridge_session_id: dsl::SessionId::from_domain(session_id),
replacing: None,
})
.expect("recover member session binding");
runtime_id
}
fn endpoint(seed: u8, name: &str) -> TrustedPeerDescriptor {
let mut pubkey = [0; 32];
pubkey[0] = seed;
pubkey[31] = seed.wrapping_mul(7).max(1);
let peer_id = PeerId::from_ed25519_pubkey(&pubkey);
TrustedPeerDescriptor::unsigned_with_pubkey(
name,
peer_id.to_string(),
pubkey,
format!("inproc://{name}"),
)
.expect("test endpoint")
}
let mut authority = dsl::MobMachineAuthority::new();
let first_identity = AgentIdentity::from("first");
let first_session = SessionId::new();
let first_runtime = recover_member(&mut authority, &first_identity, &first_session);
let first_old = endpoint(11, "first-old");
let first_current = endpoint(12, "first-current");
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl::AgentIdentity::from_domain(&first_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&first_runtime),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
peer_endpoint: dsl::MemberPeerEndpoint::from(&first_old),
},
)
.expect("register first endpoint");
authority
.apply_signal(dsl::MobMachineSignal::RecoverMemberPeerEndpoint {
agent_identity: dsl::AgentIdentity::from_domain(&first_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&first_runtime),
bridge_session_id: dsl::SessionId::from_domain(&first_session),
peer_endpoint: dsl::MemberPeerEndpoint::from(&first_current),
})
.expect("rotate first endpoint");
let second_identity = AgentIdentity::from("second");
let second_session = SessionId::new();
let second_runtime = recover_member(&mut authority, &second_identity, &second_session);
let second_current = endpoint(13, "second-current");
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl::AgentIdentity::from_domain(&second_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&second_runtime),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
peer_endpoint: dsl::MemberPeerEndpoint::from(&second_current),
},
)
.expect("register second endpoint");
authority
.apply_signal(dsl::MobMachineSignal::RecoverMemberPeerEndpoint {
agent_identity: dsl::AgentIdentity::from_domain(&second_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&second_runtime),
bridge_session_id: dsl::SessionId::from_domain(&second_session),
peer_endpoint: dsl::MemberPeerEndpoint::from(&first_old),
})
.expect_err("another identity must not claim a retained historical PeerId");
assert_eq!(
authority
.state()
.member_peer_endpoints
.get(&dsl::AgentIdentity::from_domain(&second_identity)),
Some(&dsl::MemberPeerEndpoint::from(&second_current)),
"rejected collision must leave the current endpoint unchanged"
);
let third_identity = AgentIdentity::from("third");
let third_session = SessionId::new();
let third_runtime = recover_member(&mut authority, &third_identity, &third_session);
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl::AgentIdentity::from_domain(&third_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&third_runtime),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
peer_endpoint: dsl::MemberPeerEndpoint::from(&first_current),
},
)
.expect_err("another identity must not claim a current PeerId");
assert!(
!authority
.state()
.member_peer_endpoints
.contains_key(&dsl::AgentIdentity::from_domain(&third_identity))
);
let edge = dsl::WiringEdge::new(
dsl::AgentIdentity::from_domain(&first_identity),
dsl::AgentIdentity::from_domain(&second_identity),
);
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::WireMembers { edge: edge.clone() },
)
.expect("wire current member endpoints");
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::BeginPlacedCompletionLifecycleQuiesce {
intent: dsl::PlacedCompletionLifecycleIntentKind::Stop,
},
)
.expect("begin stopped-machine completion quiesce");
dsl::MobMachineMutator::apply(&mut authority, dsl::MobMachineInput::Stop)
.expect("stop machine");
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::Retire {
mob_id: dsl::MobId::from_domain(&MobId::from("test-mob")),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&first_runtime),
agent_identity: dsl::AgentIdentity::from_domain(&first_identity),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
releasing: None,
session_id: Some(dsl::SessionId::from_domain(&first_session)),
},
)
.expect("admit stopped retirement");
authority
.apply_signal(dsl::MobMachineSignal::ObserveRuntimeRetired {
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&first_runtime),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
})
.expect("observe stopped retiring runtime gone");
dsl::MobMachineMutator::apply(
&mut authority,
dsl::MobMachineInput::AuthorizeMemberEndpointMigrationTrustCleanup {
edge,
agent_identity: dsl::AgentIdentity::from_domain(&first_identity),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&first_runtime),
retained_peer_endpoint: dsl::MemberPeerEndpoint::from(&first_old),
},
)
.expect("stopped Retiring tuple must retain historical cleanup authority");
assert_eq!(authority.state().lifecycle_phase, dsl::MobPhase::Stopped);
let mut legacy_authority = dsl::MobMachineAuthority::new();
let legacy_first = AgentIdentity::from("legacy-first");
let legacy_second = AgentIdentity::from("legacy-second");
let legacy_first_session = SessionId::new();
let legacy_second_session = SessionId::new();
let legacy_first_runtime =
recover_member(&mut legacy_authority, &legacy_first, &legacy_first_session);
let legacy_second_runtime = recover_member(
&mut legacy_authority,
&legacy_second,
&legacy_second_session,
);
let legacy_second_endpoint = endpoint(21, "legacy-second");
dsl::MobMachineMutator::apply(
&mut legacy_authority,
dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl::AgentIdentity::from_domain(&legacy_second),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&legacy_second_runtime),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
peer_endpoint: dsl::MemberPeerEndpoint::from(&legacy_second_endpoint),
},
)
.expect("register one legacy endpoint");
legacy_authority
.apply_signal(dsl::MobMachineSignal::RecoverRosterWiring {
edge: dsl::WiringEdge::new(
dsl::AgentIdentity::from_domain(&legacy_first),
dsl::AgentIdentity::from_domain(&legacy_second),
),
})
.expect("recover legacy topology before endpoint material");
dsl::MobMachineMutator::apply(
&mut legacy_authority,
dsl::MobMachineInput::RegisterMemberPeer {
agent_identity: dsl::AgentIdentity::from_domain(&legacy_first),
agent_runtime_id: dsl::AgentRuntimeId::from_domain(&legacy_first_runtime),
generation: dsl::Generation::from_domain(crate::ids::Generation::INITIAL),
fence_token: dsl::FenceToken::from_domain(FenceToken::new(1)),
peer_endpoint: dsl::MemberPeerEndpoint::from(&endpoint(22, "legacy-first-live")),
},
)
.expect_err("legacy topology without a durable baseline endpoint must fail closed");
}
#[tokio::test]
async fn test_resume_stamps_legacy_recovered_binding_endpoint_exactly_once() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = AgentIdentity::from("legacy-recovered-worker");
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("worker"), identity.clone(), None)
.await
.expect("spawn member");
let entry = handle
.get_member(&identity)
.await
.expect("roster query")
.expect("spawned member");
let bridge_session_id = entry
.bridge_session_id()
.cloned()
.expect("session-backed member");
let expected_peer_id = service
.comms_runtime(&bridge_session_id)
.await
.expect("member comms runtime")
.peer_id()
.expect("member peer id");
handle.stop().await.expect("stop before legacy replay");
// Journals written before endpoint persistence can contain a recovered
// binding for the current session without the exact endpoint. Even when
// the live observation equals the spawn endpoint, resume must consume that
// legacy exception with one durable enriched stamp so a later observation
// cannot silently authorize a migration.
events
.append(NewMobEvent {
mob_id: MobId::from("test-mob"),
timestamp: None,
kind: MobEventKind::MemberSessionBindingRecovered(
crate::event::MemberSessionBindingRecoveredEvent::new(
identity.clone(),
entry.agent_runtime_id.clone(),
bridge_session_id.clone(),
),
),
})
.await
.expect("append legacy endpoint-less recovered binding");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly enrich the legacy recovered binding"
);
resumed
.resume()
.await
.expect("explicit resume should enrich the legacy recovered binding");
let first_state = resumed
.query_machine_state()
.await
.expect("query first resumed machine state");
drop(resumed);
let resumed_again = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("repeat resume should replay without another endpoint stamp");
let second_state = resumed_again
.query_machine_state()
.await
.expect("query second resumed machine state");
assert_eq!(
second_state.topology_epoch, first_state.topology_epoch,
"exact endpoint replay must not advance the topology epoch"
);
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from_domain(&identity);
assert!(
!second_state
.member_prior_peer_endpoints
.contains_key(&dsl_identity),
"an equal legacy observation is an upgrade stamp, not an endpoint migration"
);
let stored_events = events.replay_all().await.expect("replay upgraded events");
let recovered = stored_events
.iter()
.filter_map(|event| match &event.kind {
MobEventKind::MemberSessionBindingRecovered(binding)
if binding.agent_identity == identity =>
{
Some(binding)
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
recovered.len(),
2,
"the legacy record must be followed by exactly one enriched record"
);
let enriched = recovered
.iter()
.filter_map(|binding| binding.member_peer_endpoint())
.collect::<Vec<_>>();
assert_eq!(
enriched.len(),
1,
"repeat resume must not append a second endpoint-bearing recovery stamp"
);
assert_eq!(
enriched[0].peer_id, expected_peer_id,
"the one-shot upgrade must persist the exact live endpoint"
);
}
#[tokio::test]
async fn test_retire_after_snapshotless_member_head_recovery_removes_old_and_new_peer_trust() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let retiring = AgentIdentity::from("mk--rt_creview_csingleton_c0");
let survivor = AgentIdentity::from("w-survivor");
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let old_sid = handle
.spawn(ProfileName::from("worker"), retiring.clone(), None)
.await
.expect("spawn retiring member")
.bridge_session_id()
.cloned()
.expect("session-backed retiring member");
let survivor_sid = handle
.spawn(ProfileName::from("worker"), survivor.clone(), None)
.await
.expect("spawn surviving member")
.bridge_session_id()
.cloned()
.expect("session-backed surviving member");
handle
.wire(retiring.clone(), survivor.clone())
.await
.expect("wire members before snapshotless recovery");
let recovered_comms_name = "test-mob/worker/rt:review:singleton:0";
let old_runtime = service
.comms_runtime(&old_sid)
.await
.expect("old retiring comms runtime");
let old_peer_id = old_runtime.peer_id().expect("old retiring peer id");
let old_peer_endpoint = super::provisioner::SessionBackend::trusted_peer_spec_from_runtime(
recovered_comms_name,
old_runtime.as_ref(),
)
.expect("old peer endpoint material")
.expect("old peer endpoint");
let initial_survivor_runtime = {
let sessions = service.sessions.read().await;
sessions
.get(&survivor_sid)
.cloned()
.expect("surviving comms runtime")
};
assert!(
initial_survivor_runtime
.trusted_peers
.read()
.await
.contains_key(&old_peer_id.to_string()),
"setup must install the original retiring peer's trust"
);
handle
.stop()
.await
.expect("stop before snapshotless recovery");
service.clear_mob_machine_trust_owner(&survivor_sid).await;
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard old live session");
service.delete_persisted_session(&old_sid).await;
let replacement = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "replacement".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some(recovered_comms_name.to_string()),
mob_member_binding: None,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create replacement session head");
// Explicit resume must not adopt the surface-created actor above: it has
// no exact mob attachment sidecar. Ask the mock's next authoritative
// materialization to rotate only the signing identity while retaining the
// canonical public name/address, so the recovered head still exercises a
// genuine endpoint migration.
service
.set_comms_identity_seed(
&replacement.session_id,
"test-mob/worker/rt:review:singleton:0#replacement-key",
)
.await;
service
.session_comms_names
.write()
.await
.remove(&replacement.session_id);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly recover the snapshotless session head"
);
resumed
.resume()
.await
.expect("explicit resume should recover the snapshotless session head");
let resumed_survivor_runtime = {
let sessions = service.sessions.read().await;
sessions
.get(&survivor_sid)
.cloned()
.expect("resumed surviving comms runtime")
};
assert_eq!(
resumed
.member_status(&retiring)
.await
.expect("recovered member status")
.current_session_id,
Some(replacement.session_id.clone())
);
let replacement_runtime = service
.comms_runtime(&replacement.session_id)
.await
.expect("exact resumed replacement comms runtime");
let replacement_peer_id = replacement_runtime.peer_id().expect("replacement peer id");
let replacement_peer_endpoint =
super::provisioner::SessionBackend::trusted_peer_spec_from_runtime(
recovered_comms_name,
replacement_runtime.as_ref(),
)
.expect("replacement peer endpoint material")
.expect("replacement peer endpoint");
assert_ne!(
replacement_peer_id, old_peer_id,
"fixture must exercise peer-key rotation across exact resume materialization"
);
assert!(
resumed_survivor_runtime
.trusted_peers
.read()
.await
.contains_key(&replacement_peer_id.to_string()),
"recovery must wire the survivor to the replacement peer identity"
);
let recovered_entry = resumed
.get_member(&retiring)
.await
.expect("recovered roster query")
.expect("recovered roster entry");
drop(resumed);
// Simulate a crash immediately after the durable retirement-start append.
// Replay must retain the historical endpoint cleanup anchor even though
// the runtime may already be gone and the member is already Retiring.
events
.append(NewMobEvent {
mob_id: MobId::from("test-mob"),
timestamp: None,
kind: MobEventKind::MemberRetirementStarted {
agent_identity: retiring.clone(),
agent_runtime_id: recovered_entry.agent_runtime_id.clone(),
generation: recovered_entry.generation,
role: recovered_entry.role.clone(),
releasing: Some(replacement.session_id.clone()),
session_id: Some(replacement.session_id.clone()),
retiring_peer_endpoint: Some(replacement_peer_endpoint),
preserve_machine_topology: false,
},
})
.await
.expect("append crash-point retirement start");
service.clear_mob_machine_trust_owner(&survivor_sid).await;
service
.clear_mob_machine_trust_owner(&replacement.session_id)
.await;
let restarted = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("restart should replay recovered endpoint authority");
let restarted_survivor_runtime = {
let sessions = service.sessions.read().await;
sessions
.get(&survivor_sid)
.cloned()
.expect("restarted surviving comms runtime")
};
// Model a durable trust projection restoring the old generation row after
// retirement admission. The actor retry, not ordinary resume repair, must
// sweep it before terminal MemberRetired clears endpoint history.
restarted_survivor_runtime
.trusted_peers
.write()
.await
.insert(old_peer_id.to_string(), old_peer_endpoint);
restarted
.retire(retiring)
.await
.expect("retry retirement from durable start marker");
let trusted = restarted_survivor_runtime.trusted_peers.read().await;
let residual_peer_ids = [old_peer_id, replacement_peer_id]
.into_iter()
.map(|peer_id| peer_id.to_string())
.filter(|peer_id| trusted.contains_key(peer_id))
.collect::<Vec<_>>();
assert!(
residual_peer_ids.is_empty(),
"terminal retirement must remove both historical and current peer trust; old={old_peer_id}, current={replacement_peer_id}, residual={residual_peer_ids:?}"
);
}
#[tokio::test]
async fn test_resume_skips_wiring_for_broken_peer_and_keeps_partial_resume() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
handle.stop().await.expect("stop");
service.clear_mob_machine_trust_owner(&sid_1).await;
service.clear_mob_machine_trust_owner(&sid_2).await;
service
.archive(&sid_2)
.await
.expect("archive broken session");
service.delete_persisted_session(&sid_2).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly recover member sessions"
);
resumed
.resume()
.await
.expect("explicit resume should keep recovery partial");
let left = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("healthy member status");
assert_eq!(left.status, crate::runtime::handle::MobMemberStatus::Active);
assert_eq!(left.current_session_id, Some(sid_1.clone()));
let right = resumed
.member_status(&AgentIdentity::from("w-2"))
.await
.expect("broken member status");
assert_eq!(
right.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert_eq!(right.current_session_id, Some(sid_2.clone()));
let trusted_by_left = service.trusted_peer_names(&sid_1).await;
assert!(
!trusted_by_left.contains(&test_comms_name("worker", "w-2")),
"healthy members must prune trust to broken peers during partial resume"
);
}
#[tokio::test]
async fn test_resume_restores_missing_live_session_even_when_list_reports_inactive_persisted_summary()
{
let inner = Arc::new(MockSessionService::new());
let _ = inner.enable_runtime_adapter();
let service = Arc::new(PersistedListingSessionService::new(inner.clone()));
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
handle.stop().await.expect("stop");
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal for durable snapshots).
MobSessionService::discard_live_session(inner.as_ref(), &sid)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not materialize an inactive persisted session"
);
resumed
.resume()
.await
.expect("explicit resume should restore the persisted snapshot");
let snapshot = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status after resume");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert_eq!(snapshot.current_session_id, Some(sid.clone()));
assert!(
inner.comms_runtime(&sid).await.is_some(),
"resume should recreate the live session bridge for persisted-but-inactive sessions"
);
}
#[tokio::test]
async fn test_resume_skips_broken_orchestrator_notification_and_keeps_partial_resume() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
)
.await
.expect("spawn orchestrator");
handle.stop().await.expect("stop");
let orchestrator_sid = handle
.get_member(&AgentIdentity::from("lead-1"))
.await
.unwrap()
.expect("orchestrator entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&orchestrator_sid)
.await
.expect("archive orchestrator");
service.delete_persisted_session(&orchestrator_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly recover the orchestrator session"
);
resumed
.resume()
.await
.expect("explicit resume should keep recovery partial");
let orchestrator = resumed
.member_status(&AgentIdentity::from("lead-1"))
.await
.expect("orchestrator status");
assert_eq!(
orchestrator.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert_eq!(orchestrator.current_session_id, Some(orchestrator_sid));
}
#[tokio::test]
async fn test_broken_member_turn_returns_restore_failed_error() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly classify the missing member session"
);
match resumed.resume().await {
Ok(()) => {}
Err(MobError::InvalidTransition {
from: MobState::Running,
to: MobState::Running,
}) => {}
Err(error) => {
panic!(
"broken members should still surface restore failures after lifecycle resume: {error}"
)
}
}
let error = match resumed.member(&AgentIdentity::from("w-1")).await {
Err(error) => error,
Ok(member) => member
.internal_turn(ContentInput::from("repair me".to_string()))
.await
.expect_err("Broken members must reject turn delivery"),
};
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
..
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(old_sid));
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
}
#[tokio::test]
async fn test_wire_broken_member_returns_restore_failed_error() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn broken candidate");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("partial resume should still succeed");
match resumed.resume().await {
Ok(()) => {}
Err(MobError::InvalidTransition {
from: MobState::Running,
to: MobState::Running,
}) => {}
Err(error) => {
panic!("broken members should still allow mob lifecycle resume: {error}")
}
}
resumed
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn healthy worker");
let error = resumed
.wire(AgentIdentity::from("w-2"), AgentIdentity::from("w-1"))
.await
.expect_err("wiring to Broken member must be rejected");
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
..
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(old_sid));
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
}
#[tokio::test]
async fn test_retire_broken_member_succeeds_and_removes_it() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly classify the missing member session"
);
resumed
.resume()
.await
.expect("explicit resume should classify the missing member as Broken");
resumed
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire should work on broken member");
assert!(
resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"retire should remove the broken member from the roster"
);
}
#[tokio::test]
async fn test_respawn_broken_member_clears_restore_diagnostic() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
service
.archive(&old_sid)
.await
.expect("archive live session");
service.delete_persisted_session(&old_sid).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("partial resume should still succeed");
match resumed.resume().await {
Ok(()) => {}
Err(MobError::InvalidTransition {
from: MobState::Running,
to: MobState::Running,
}) => {}
Err(error) => {
panic!("broken members should still be repairable after lifecycle resume: {error}")
}
}
let receipt = resumed
.respawn(AgentIdentity::from("w-1"), None)
.await
.expect("respawn should repair broken member");
let snapshot = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status after repair");
let new_sid = snapshot
.current_bridge_session_id
.clone()
.expect("respawned member should have a new session");
assert_ne!(
new_sid, old_sid,
"respawn should rotate the session identity"
);
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
let (snapshot_runtime_id, snapshot_fence_token) = snapshot
.runtime_identity_fields()
.expect("repaired member should have runtime identity");
assert_eq!(snapshot_runtime_id, &receipt.agent_runtime_id);
assert_eq!(snapshot_fence_token, receipt.fence_token);
assert_eq!(snapshot.current_session_id, Some(new_sid.clone()));
assert!(
snapshot.error.is_none(),
"repair should clear the old broken diagnostic"
);
let members = resumed.list_members().await;
let repaired = members
.into_iter()
.find(|entry| entry.agent_identity == "w-1")
.expect("repaired member remains listed");
assert_eq!(
repaired.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert!(repaired.error.is_none());
assert_eq!(repaired.current_session_id, Some(new_sid));
}
#[tokio::test]
async fn test_respawn_broken_wired_member_replays_peer_endpoint_after_cold_service_restart() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire members before partial resume");
handle.stop().await.expect("stop");
service.delete_persisted_session(&sid_w2).await;
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
let restarted_adapter = restarted_service.enable_runtime_adapter();
assert_eq!(restarted_service.active_session_count().await, 0);
assert!(restarted_service.live_session_data.read().await.is_empty());
assert!(
restarted_service
.trusted_peer_names(&sid_w1)
.await
.is_empty(),
"cold restart must begin without stale generated trust evidence"
);
drop(handle);
drop(service);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(restarted_service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"cold reconstruction must not implicitly recover member sessions"
);
resumed
.resume()
.await
.expect("explicit resume should keep recovery partial");
let broken = resumed
.member_status(&AgentIdentity::from("w-2"))
.await
.expect("broken member status");
assert_eq!(
broken.status,
crate::runtime::handle::MobMemberStatus::Broken
);
let recovered_machine = resumed
.debug_dsl_t2_snapshot()
.await
.expect("read recovered MobMachine endpoint authority");
assert!(
recovered_machine
.member_peer_endpoints
.contains_key(&crate::machines::mob_machine::AgentIdentity::from("w-2")),
"resume must retain the broken member's generated peer endpoint before pruning stale trust"
);
let receipt = resumed
.respawn(AgentIdentity::from("w-2"), None)
.await
.expect("respawn should repair broken wired member");
let repaired = resumed
.member_status(&AgentIdentity::from("w-2"))
.await
.expect("repaired member status");
let new_sid = repaired
.current_bridge_session_id
.clone()
.expect("respawned worker should have new session");
assert_ne!(new_sid, sid_w2);
assert_eq!(
repaired.status,
crate::runtime::handle::MobMemberStatus::Active
);
let (runtime_id, fence_token) = repaired
.runtime_identity_fields()
.expect("repaired member should have runtime identity");
assert_eq!(runtime_id, &receipt.agent_runtime_id);
assert_eq!(fence_token, receipt.fence_token);
assert!(repaired.error.is_none());
assert!(
!restarted_adapter.contains_session(&sid_w2).await,
"respawn must unregister the exact retired operation-owner registration"
);
let trusted_by_w1 = restarted_service.trusted_peer_names(&sid_w1).await;
assert!(
trusted_by_w1.contains(&test_comms_name("worker", "w-2")),
"respawn should restore trust from healthy peer to repaired member"
);
let trusted_by_w2 = restarted_service.trusted_peer_names(&new_sid).await;
assert!(
trusted_by_w2.contains(&test_comms_name("worker", "w-1")),
"respawn should restore trust from repaired member to healthy peer"
);
}
#[tokio::test]
async fn test_resume_restores_persisted_behavior_metadata() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let mut persisted = service
.live_session_clone(&old_sid)
.await
.expect("live session");
persisted.append_system_message("Persisted worker prompt".to_string());
let mut metadata = persisted
.session_metadata()
.expect("seeded session metadata should exist");
metadata.model = "gpt-5.4-pro".to_string();
metadata.provider = Provider::OpenAI;
metadata.provider_params = Some(
meerkat_core::lifecycle::run_primitive::ProviderParamsOverride {
provider_tag: Some(meerkat_core::lifecycle::run_primitive::ProviderTag::OpenAi(
meerkat_core::lifecycle::run_primitive::OpenAiProviderTag {
reasoning_effort: Some(
meerkat_core::lifecycle::run_primitive::ReasoningEffort::High,
),
..Default::default()
},
)),
..Default::default()
},
);
metadata.max_tokens = 8192;
metadata.tooling.builtins = ToolCategoryOverride::Disable;
metadata.tooling.shell = ToolCategoryOverride::Enable;
metadata.tooling.mob = ToolCategoryOverride::Enable;
metadata.tooling.memory = ToolCategoryOverride::Enable;
metadata.tooling.active_skills = Some(vec![meerkat_core::skills::SkillKey::builtin(
meerkat_core::skills::SkillName::parse("mob-communication").expect("valid skill name"),
)]);
metadata.comms_name = Some(test_comms_name("worker", "w-1"));
let mut peer_meta = metadata.peer_meta.clone().expect("peer metadata");
peer_meta.labels.insert("resume".into(), "yes".into());
metadata.peer_meta = Some(peer_meta.clone());
persisted
.set_session_metadata(metadata.clone())
.expect("metadata update");
service.replace_live_session(persisted).await;
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal: the real service returns no durable snapshot for
// archived sessions).
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly restore session behavior"
);
resumed
.resume()
.await
.expect("explicit resume should restore session behavior");
let restored_sid = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("restored entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let restored = service
.live_session_clone(&restored_sid)
.await
.expect("restored live session");
let restored_metadata = restored
.session_metadata()
.expect("restored session metadata");
assert_eq!(restored_sid, old_sid);
assert_eq!(restored_metadata.model, metadata.model);
assert_eq!(restored_metadata.provider, metadata.provider);
assert_eq!(restored_metadata.provider_params, metadata.provider_params);
assert_eq!(restored_metadata.max_tokens, metadata.max_tokens);
assert_eq!(restored_metadata.tooling, metadata.tooling);
assert!(
restored_metadata.keep_alive,
"autonomous host member should be restored with keep_alive=true"
);
assert_eq!(restored_metadata.comms_name, metadata.comms_name);
assert_eq!(restored_metadata.peer_meta, metadata.peer_meta);
assert!(
matches!(
restored.messages().first(),
Some(Message::System(system)) if system.content.contains("Persisted worker prompt")
),
"restored session should keep the persisted system prompt"
);
}
#[tokio::test]
async fn test_resume_marks_comms_name_mismatch_as_broken() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.stop().await.expect("stop");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let mut persisted = service
.live_session_clone(&old_sid)
.await
.expect("live session");
let mut metadata = persisted.session_metadata().expect("metadata");
metadata.comms_name = Some("test-mob/worker/other-name".to_string());
persisted
.set_session_metadata(metadata)
.expect("set metadata");
service.replace_live_session(persisted).await;
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal for durable snapshots).
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly classify the missing live session"
);
resumed
.resume()
.await
.expect("explicit resume should succeed with Broken projection");
let snapshot = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("broken member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Broken
);
assert!(
snapshot
.error
.as_deref()
.is_some_and(|message| message.contains("persisted comms_name")),
"comms_name mismatch should surface as a restore failure"
);
}
#[tokio::test]
async fn test_attach_existing_session_rejects_comms_name_mismatch() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let initial = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "seed".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Set(
"Persisted resume prompt".to_string(),
),
max_tokens: Some(4096),
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-resume".to_string()),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("seed session");
let session_id = initial.session_id.clone();
let mut persisted = service
.live_session_clone(&session_id)
.await
.expect("live session");
let mut metadata = persisted.session_metadata().expect("metadata");
metadata.comms_name = Some("test-mob/worker/not-w-resume".to_string());
persisted
.set_session_metadata(metadata)
.expect("set metadata");
service.replace_live_session(persisted).await;
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal: the real service returns no durable snapshot for
// archived sessions).
MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard live session");
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let error = handle
.attach_existing_session_as_member(
ProfileName::from("worker"),
AgentIdentity::from("w-resume"),
session_id.clone(),
)
.await
.expect_err("comms_name mismatch should be rejected for explicit resume");
assert!(
error.to_string().contains("persisted comms_name"),
"explicit member resume should fail with the comms_name mismatch"
);
}
#[tokio::test]
async fn test_explicit_durable_resume_preparation_does_not_block_actor_commands() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let initial = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "seed".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Set(
"Persisted resume prompt".to_string(),
),
max_tokens: Some(4096),
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-resume-nonblocking".to_string()),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("seed session");
let session_id = initial.session_id.clone();
MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard live session");
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
service.set_load_persisted_session_delay_ms(300);
let attach_handle = handle.clone();
let attach_session_id = session_id.clone();
let attach = tokio::spawn(async move {
attach_handle
.attach_existing_session_as_member(
ProfileName::from("worker"),
AgentIdentity::from("w-resume-nonblocking"),
attach_session_id,
)
.await
});
tokio::time::timeout(
Duration::from_secs(1),
service.wait_for_load_persisted_session(),
)
.await
.expect("cold durable load should begin");
let state = tokio::time::timeout(Duration::from_millis(100), handle.status())
.await
.expect("durable resume preparation must not block actor commands")
.expect("query mob status");
assert_eq!(state, MobState::Running);
let duplicate = tokio::time::timeout(
Duration::from_millis(100),
handle.attach_existing_session_as_member(
ProfileName::from("worker"),
AgentIdentity::from("w-resume-nonblocking"),
session_id.clone(),
),
)
.await
.expect("duplicate admission must not wait for durable preparation")
.expect_err("pending durable resume must reserve its identity");
assert!(matches!(duplicate, MobError::MemberAlreadyExists(_)));
let member_ref = tokio::time::timeout(Duration::from_secs(2), attach)
.await
.expect("attach should finish")
.expect("attach task should not panic")
.expect("attach should restore the durable session");
assert_eq!(member_ref.bridge_session_id(), Some(&session_id));
}
#[tokio::test]
async fn test_explicit_durable_resume_preparation_runs_concurrently() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut sessions = Vec::new();
for index in 0..4 {
let identity = format!("w-resume-parallel-{index}");
let initial = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "seed".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Set(
"Persisted resume prompt".to_string(),
),
max_tokens: Some(4096),
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some(format!("test-mob/worker/{identity}")),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("seed session");
MobSessionService::discard_live_session(service.as_ref(), &initial.session_id)
.await
.expect("discard live session");
sessions.push((identity, initial.session_id));
}
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
service.set_load_persisted_session_delay_ms(300);
let mut attaches = Vec::new();
for (identity, session_id) in sessions {
let attach_handle = handle.clone();
attaches.push(tokio::spawn(async move {
attach_handle
.attach_existing_session_as_member(
ProfileName::from("worker"),
AgentIdentity::from(identity),
session_id,
)
.await
}));
}
for attach in attaches {
tokio::time::timeout(Duration::from_secs(2), attach)
.await
.expect("parallel attach should finish")
.expect("attach task should not panic")
.expect("attach should restore the durable session");
}
assert!(
service.max_concurrent_load_persisted_session_calls() > 1,
"cold durable loads should overlap instead of serializing in the actor"
);
}
#[tokio::test]
async fn test_build_resumed_agent_config_rejects_mismatched_session_identity() {
let definition = sample_definition();
let mob_id = MobId::from("test-mob");
let profile_name = ProfileName::from("worker");
let member_identity = AgentIdentity::from("w-1");
let profile = definition
.profiles
.get(&profile_name)
.expect("worker profile")
.as_inline()
.unwrap();
let mut resumed = Session::new();
resumed
.set_session_metadata(SessionMetadata {
schema_version: meerkat_core::session_metadata_schema_version(),
model: "claude-sonnet-4-5".to_string(),
max_tokens: 4096,
structured_output_retries: 2,
provider: Provider::Anthropic,
self_hosted_server_id: None,
provider_params: None,
tooling: SessionTooling {
builtins: ToolCategoryOverride::Enable,
shell: ToolCategoryOverride::Disable,
comms: ToolCategoryOverride::Enable,
mob: ToolCategoryOverride::Disable,
memory: ToolCategoryOverride::Disable,
schedule: ToolCategoryOverride::Inherit,
workgraph: ToolCategoryOverride::Inherit,
image_generation: ToolCategoryOverride::Inherit,
web_search: ToolCategoryOverride::Inherit,
tool_access_policy: None,
active_skills: Some(vec![meerkat_core::skills::SkillKey::builtin(
meerkat_core::skills::SkillName::parse("mob-communication")
.expect("valid skill name"),
)]),
},
keep_alive: false,
comms_name: Some(test_comms_name("worker", "w-1")),
peer_meta: Some(
meerkat_core::PeerMeta::default()
.with_label("mob_id", "test-mob")
.with_label("role", "worker")
.with_label("member_id", "w-1"),
),
realm_id: Some(meerkat_core::RealmId::parse("mob.test-mob").unwrap()),
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: Some(meerkat_core::MobMemberBinding {
mob_id: "test-mob".to_string(),
role: "worker".to_string(),
member: "w-1".to_string(),
}),
})
.expect("resume metadata");
let wrong_session_id = SessionId::new();
let error =
crate::build::build_resumed_agent_config(crate::build::BuildResumedAgentConfigParams {
base: crate::build::BuildAgentConfigParams {
mob_id: &mob_id,
profile_name: &profile_name,
agent_identity: &member_identity,
profile,
definition: &definition,
external_tools: None,
context: None,
labels: None,
additional_instructions: None,
shell_env: None,
mob_tool_authority_context: None,
tool_access_policy: None,
inherited_tool_filter: None,
system_prompt_override: None,
},
expected_session_id: &wrong_session_id,
resumed_session: resumed,
})
.await
.expect_err("resume helper must validate the target session identity");
assert!(
error.to_string().contains("resume session id mismatch"),
"mismatched injected resume_session should fail immediately"
);
}
#[tokio::test]
async fn test_attach_existing_session_restores_persisted_inactive_session() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let initial = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "seed".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Set(
"Persisted resume prompt".to_string(),
),
max_tokens: Some(4096),
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-resume".to_string()),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("seed session");
let session_id = initial.session_id.clone();
let mut persisted = service
.live_session_clone(&session_id)
.await
.expect("live session");
persisted.push(meerkat_core::Message::User(
meerkat_core::types::UserMessage::text("Persist this resume target.".to_string()),
));
service.replace_live_session(persisted).await;
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal: the real service returns no durable snapshot for
// archived sessions).
MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard live session");
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let resume_prompt = "new build configuration, not durable resume input";
let mut resume_spec = SpawnMemberSpec::new("worker", "w-resume")
.with_resume_bridge_session_id(session_id.clone());
resume_spec.system_prompt_override = Some(SpawnSystemPromptOverride::Replace(
resume_prompt.to_string(),
));
resume_spec.additional_instructions = Some(vec![
"process-local resume configuration must not be replayed".to_string(),
]);
let spawned = handle
.spawn_spec(resume_spec)
.await
.expect("attach should restore persisted session");
assert_eq!(
handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.as_ref(),
Some(&session_id),
"explicit member resume should preserve the requested session id"
);
let history = service
.read_history(
&session_id,
meerkat_core::service::SessionHistoryQuery::default(),
)
.await
.expect("restored history");
assert!(
history.messages.iter().any(|message| message
.as_indexable_text()
.contains("Persist this resume target.")),
"explicit member resume should restore persisted history"
);
assert_eq!(
history
.messages
.iter()
.filter(|message| matches!(message, Message::System(system) if system.content == "Persisted resume prompt"))
.count(),
1,
"the persisted System row must remain the sole resume prompt authority",
);
assert!(
history.messages.iter().all(
|message| !matches!(message, Message::System(system) if system.content == resume_prompt)
),
"resume build configuration must not append a new System row",
);
}
#[tokio::test]
async fn test_resume_recreates_missing_external_bridge_preserving_backend_identity() {
let _serial = lock_real_comms_tests();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-external-peer-only",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external");
assert_eq!(
external.bind_count(),
1,
"peer-only external spawn should bind the mob supervisor during provisioning"
);
handle.stop().await.expect("stop");
let old_entry = handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("external roster entry");
let (old_peer_id, old_address, old_sid) = match old_entry.member_ref {
MemberRef::BackendPeer {
ref peer_id,
ref address,
ref session_id,
..
} => (peer_id.clone(), address.clone(), session_id.clone()),
ref other => panic!("expected external backend member ref, got {other:?}"),
};
if let Some(old_sid) = &old_sid {
service
.archive(old_sid)
.await
.expect("archive legacy external bridge session");
}
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
let resumed_entry = resumed
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("resumed external entry");
match resumed_entry.member_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(peer_id, old_peer_id, "resume must preserve backend peer_id");
assert_eq!(address, old_address, "resume must preserve backend address");
assert!(
session_id.is_none(),
"resume should preserve peer-only external members without recreating a bridge session"
);
}
other => panic!("expected backend peer after resume, got {other:?}"),
}
}
#[tokio::test]
async fn test_resume_treats_normalized_external_binding_overlay_as_projection_only() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = sample_definition_with_external_backend();
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
let old_peer_id = "ed25519:test-key:w-ext".to_string();
let old_address = "tcp://test.invalid/w-ext".to_string();
let old_sid = SessionId::new();
let generation = crate::ids::Generation::INITIAL;
events
.append(crate::event::NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: crate::event::MobEventKind::MemberSpawned(
crate::event::MemberSpawnedEvent::new(
AgentIdentity::from("w-ext"),
generation,
FenceToken::new(0),
AgentRuntimeId::initial(AgentIdentity::from("w-ext")),
ProfileName::from("worker"),
)
.with_bridge_member_ref(Some(MemberRef::BackendPeer {
peer_id: old_peer_id.clone(),
address: old_address.clone(),
bootstrap_token: None,
session_id: Some(old_sid.clone()),
pubkey: [7u8; 32],
})),
),
})
.await
.expect("append legacy external member event");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("w-ext"),
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: old_peer_id.clone(),
address: old_address.clone(),
bootstrap_token: None,
session_id: None,
pubkey: [7u8; 32],
}),
bootstrap_token: None,
status: ExternalBindingOverlayStatus::Normalized,
updated_at: Utc::now(),
},
)
.await
.expect("persist overlay");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume with normalized overlay");
let resumed_entry = resumed
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("resumed external entry");
match resumed_entry.member_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(
peer_id, old_peer_id,
"event replay must preserve peer identity"
);
assert_eq!(
address, old_address,
"event replay must preserve peer address"
);
assert!(
session_id.is_some(),
"normalized overlay must not remove the replay-owned bridge binding"
);
}
other => panic!("expected backend peer after resume, got {other:?}"),
}
let snapshot = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("external status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert!(
!snapshot.is_final,
"external member should stay non-terminal under MobMachine lifecycle truth"
);
assert!(
snapshot.current_bridge_session_id().is_some(),
"status must report replay/reconcile-owned bridge binding"
);
let external = snapshot
.external_member
.as_ref()
.expect("external member should expose observation snapshot");
assert_eq!(
external.owner.agent_identity,
AgentIdentity::from("w-ext"),
"external observation is keyed by stable AgentIdentity"
);
assert_eq!(external.owner.mob_id, mob_id);
assert_eq!(
external.binding_mode,
crate::runtime::ExternalMemberBindingMode::BridgeSessionBacked
);
assert!(external.bridge_session_present);
assert_eq!(
external.rebind,
crate::runtime::ExternalMemberRebindStatus::NotRequired
);
let serialized_external =
serde_json::to_value(external).expect("serialize external observation");
assert!(
serialized_external.get("peer_id").is_none(),
"external observation must not expose raw peer ids as public authority"
);
assert!(
serialized_external.get("address").is_none(),
"external observation must not expose transport addresses as public authority"
);
assert!(
serialized_external.get("bootstrap_token").is_none(),
"external observation must not expose bootstrap proofs"
);
assert!(
serialized_external.get("agent_runtime_id").is_none(),
"external observation must not expose raw runtime ids as public authority"
);
assert!(
serialized_external.get("fence_token").is_none(),
"external observation must not expose fence tokens as public authority"
);
}
#[tokio::test]
async fn test_resume_treats_failed_external_binding_overlay_as_projection_only() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = sample_definition_with_external_backend();
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
let old_sid = SessionId::new();
let generation = crate::ids::Generation::INITIAL;
events
.append(crate::event::NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: crate::event::MobEventKind::MemberSpawned(
crate::event::MemberSpawnedEvent::new(
AgentIdentity::from("w-ext"),
generation,
FenceToken::new(0),
AgentRuntimeId::initial(AgentIdentity::from("w-ext")),
ProfileName::from("worker"),
)
.with_bridge_member_ref(Some(MemberRef::BackendPeer {
peer_id: "ed25519:test-key:w-ext".to_string(),
address: "tcp://test.invalid/w-ext".to_string(),
bootstrap_token: None,
session_id: Some(old_sid.clone()),
pubkey: [7u8; 32],
})),
),
})
.await
.expect("append legacy external member event");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("w-ext"),
generation,
normalized_member_ref: None,
bootstrap_token: None,
status: ExternalBindingOverlayStatus::Failed {
reason: "legacy external bridge normalization failed".to_string(),
},
updated_at: Utc::now(),
},
)
.await
.expect("persist failed overlay");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume should ignore stale failed overlay authority");
let resumed_entry = resumed
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("resumed external entry");
match resumed_entry.member_ref {
MemberRef::BackendPeer { session_id, .. } => {
assert!(
session_id.is_some(),
"failed overlay must not strip the replay-owned bridge binding"
);
}
other => panic!("expected backend peer after resume, got {other:?}"),
}
let snapshot = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("external status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert!(
snapshot.error.is_none(),
"failed overlay reason must not surface as member status truth"
);
let external = snapshot
.external_member
.as_ref()
.expect("external member should still expose observation snapshot");
assert_eq!(
external.reachability,
crate::runtime::ExternalMemberReachability::Unknown
);
assert_eq!(
external.rebind,
crate::runtime::ExternalMemberRebindStatus::NotRequired
);
assert_eq!(
external.forwarding.approvals.owner.agent_identity,
AgentIdentity::from("w-ext"),
"approval forwarding hook should be stable-identity owned"
);
assert_eq!(
external.forwarding.artifacts.owner.agent_identity,
AgentIdentity::from("w-ext"),
"artifact forwarding hook should be stable-identity owned"
);
}
async fn resume_with_stale_external_binding_overlay(
label: &str,
status: ExternalBindingOverlayStatus,
overlay_member_ref: Option<MemberRef>,
) -> MobHandle {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(sample_definition_with_external_backend(), label);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
let old_sid = SessionId::new();
let generation = crate::ids::Generation::INITIAL;
events
.append(crate::event::NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: crate::event::MobEventKind::MemberSpawned(
crate::event::MemberSpawnedEvent::new(
AgentIdentity::from("w-ext"),
generation,
FenceToken::new(0),
AgentRuntimeId::initial(AgentIdentity::from("w-ext")),
ProfileName::from("worker"),
)
.with_bridge_member_ref(Some(MemberRef::BackendPeer {
peer_id: "ed25519:test-key:w-ext".to_string(),
address: "tcp://test.invalid/w-ext".to_string(),
bootstrap_token: None,
session_id: Some(old_sid),
pubkey: [7u8; 32],
})),
),
})
.await
.expect("append external member event");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("w-ext"),
generation,
normalized_member_ref: overlay_member_ref,
bootstrap_token: None,
status,
updated_at: Utc::now(),
},
)
.await
.expect("persist stale overlay");
MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("resume must be driven by event/MobMachine authority")
}
#[tokio::test]
async fn test_resume_ignores_stale_normalized_external_binding_overlay_for_member_material() {
let resumed = resume_with_stale_external_binding_overlay(
"stale-normalized-overlay",
ExternalBindingOverlayStatus::Normalized,
Some(MemberRef::BackendPeer {
peer_id: "ed25519:test-key:w-ext".to_string(),
address: "tcp://test.invalid/w-ext".to_string(),
bootstrap_token: None,
session_id: None,
pubkey: [7u8; 32],
}),
)
.await;
let entry = resumed
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("replayed member must remain visible");
assert!(
entry.member_ref.bridge_session_id().is_some(),
"stale overlay-only normalization must not hide the replayed bridge binding"
);
let snapshot = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active,
"stale overlay-only facts must not override MobMachine lifecycle truth"
);
assert!(
snapshot.current_bridge_session_id().is_some(),
"status must report the MobMachine/replay-owned bridge binding, not overlay-only absence"
);
}
#[tokio::test]
async fn test_resume_ignores_stale_failed_external_binding_overlay_for_status_truth() {
let resumed = resume_with_stale_external_binding_overlay(
"stale-failed-overlay",
ExternalBindingOverlayStatus::Failed {
reason: "stale overlay failure".to_string(),
},
None,
)
.await;
let snapshot = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active,
"stale overlay-only failure must not mark a replayed live member broken"
);
assert!(
snapshot.error.is_none(),
"stale overlay-only failure must not surface as member status truth"
);
resumed
.member(&AgentIdentity::from("w-ext"))
.await
.expect("stale overlay failure must not block member access");
}
#[tokio::test]
async fn test_resume_does_not_require_external_binding_overlay_load_for_authority() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
runtime_metadata
.fail_list_overlays
.store(true, Ordering::Relaxed);
MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("overlay projection load failure must not block MobMachine-owned resume");
}
#[tokio::test]
async fn test_reconcile_spawns_member_despite_stale_overlay_only_record() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = with_unique_mob_id(sample_definition(), "overlay-only-reconcile");
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("ghost"),
generation: crate::ids::Generation::INITIAL,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id: "ed25519:test-key:ghost".to_string(),
address: "tcp://test.invalid/ghost".to_string(),
bootstrap_token: None,
session_id: None,
pubkey: [7u8; 32],
}),
bootstrap_token: None,
status: ExternalBindingOverlayStatus::Normalized,
updated_at: Utc::now(),
},
)
.await
.expect("persist overlay-only record");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("resume");
assert!(
resumed
.get_member(&AgentIdentity::from("ghost"))
.await
.unwrap()
.is_none(),
"overlay-only records must not materialize members on restart"
);
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must remain inert before explicit resume"
);
resumed
.resume()
.await
.expect("explicit resume should make the mob available for reconciliation");
let report = resumed
.reconcile(
vec![SpawnMemberSpec::new("worker", "ghost")],
crate::runtime::reconcile::ReconcileOptions::default(),
)
.await
.expect("reconcile");
assert_eq!(
report.spawned.len(),
1,
"reconcile must spawn desired members that exist only as stale overlays"
);
assert!(
report.retained.is_empty(),
"overlay-only records must not be retained as existing roster members"
);
}
#[tokio::test]
#[ignore = "Integration-shaped: spawns a live external peer and performs a real \
supervisor-bridge round-trip over loopback comms. The async trust/peer-registration \
propagation lag is unbounded under the high-parallelism RBE unit lane (--jobs=64 on \
shared executors), so the bridge request times out nondeterministically even at a 60s \
budget; the test passes deterministically in ~0.03s locally and on the e2e/local lane. \
Run with --run-ignored. (The requester bridge timeout was already raised 30s->60s and \
the responder registration deadline 10s->60s; the lag still exceeds both under RBE.)"]
async fn test_peer_only_members_reject_agent_event_subscriptions_explicitly() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = sample_definition_with_external_backend();
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.stop().await.expect("stop");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token,
pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let generation = crate::ids::Generation::INITIAL;
events
.append(crate::event::NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: crate::event::MobEventKind::MemberSpawned(
crate::event::MemberSpawnedEvent::new(
AgentIdentity::from("w-ext"),
generation,
FenceToken::new(0),
AgentRuntimeId::initial(AgentIdentity::from("w-ext")),
ProfileName::from("worker"),
)
.with_bridge_member_ref(Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: address.clone(),
bootstrap_token: bootstrap_token.clone(),
session_id: None,
pubkey,
})),
),
})
.await
.expect("append peer-only external member event");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("w-ext"),
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id,
address,
bootstrap_token: bootstrap_token.clone(),
session_id: None,
pubkey,
}),
bootstrap_token,
status: ExternalBindingOverlayStatus::Normalized,
updated_at: Utc::now(),
},
)
.await
.expect("persist overlay");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume peer-only external member");
let single_error = match resumed
.subscribe_agent_events(&AgentIdentity::from("w-ext"))
.await
{
Ok(_) => panic!("peer-only members must reject direct event subscriptions"),
Err(error) => error,
};
assert!(matches!(
single_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
..
}
));
let all_error = match resumed.subscribe_all_agent_events().await {
Ok(_) => panic!("peer-only members must reject aggregate event subscriptions"),
Err(error) => error,
};
assert!(matches!(
all_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
..
}
));
}
#[tokio::test]
async fn test_peer_only_members_accept_direct_turn_delivery_without_bridge_session() {
let _serial = lock_real_comms_tests();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-direct-turn",
);
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let owner_bridge_session_id = SessionId::new();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_owner_bridge_session_create_authority(owner_bridge_session_id.clone(), false, false)
.create()
.await
.expect("create mob");
assert_eq!(
handle
.owner_bridge_session_lifecycle_authority()
.map(|authority| authority.bridge_session_id),
Some(owner_bridge_session_id.clone()),
"peer-only restore setup must use generated owner bridge authority"
);
assert!(
events
.replay_all()
.await
.expect("replay owner bridge event")
.into_iter()
.any(|event| matches!(
event.kind,
crate::event::MobEventKind::MobOwnerBridgeSessionBound {
ref bridge_session_id,
destroy_on_owner_archive: false,
implicit_delegation_mob: false,
} if bridge_session_id == &owner_bridge_session_id
)),
"owner bridge authority must be persisted as generated MobMachine event"
);
handle.stop().await.expect("stop");
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token,
pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let generation = crate::ids::Generation::INITIAL;
events
.append(crate::event::NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: crate::event::MobEventKind::MemberSpawned(
crate::event::MemberSpawnedEvent::new(
AgentIdentity::from("w-ext"),
generation,
FenceToken::new(0),
AgentRuntimeId::initial(AgentIdentity::from("w-ext")),
ProfileName::from("worker"),
)
.with_bridge_member_ref(Some(MemberRef::BackendPeer {
peer_id: peer_id.clone(),
address: address.clone(),
bootstrap_token: bootstrap_token.clone(),
session_id: None,
pubkey,
})),
),
})
.await
.expect("append peer-only external member event");
runtime_metadata
.upsert_external_binding_overlay(
&mob_id,
&ExternalBindingOverlayRecord {
agent_identity: AgentIdentity::from("w-ext"),
generation,
normalized_member_ref: Some(MemberRef::BackendPeer {
peer_id,
address,
bootstrap_token: None,
session_id: None,
pubkey,
}),
bootstrap_token: bootstrap_token.clone(),
status: ExternalBindingOverlayStatus::Normalized,
updated_at: Utc::now(),
},
)
.await
.expect("persist overlay");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped peer-only mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly start peer-only delivery"
);
resumed
.resume()
.await
.expect("explicit resume should start peer-only delivery");
assert_eq!(
resumed.status().await.expect("resumed mob status"),
MobState::Running
);
assert_eq!(
resumed
.owner_bridge_session_lifecycle_authority()
.map(|authority| authority.bridge_session_id),
Some(owner_bridge_session_id),
"resumed peer-only mob should retain generated owner bridge authority"
);
resumed
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("ping".to_string()))
.await
.expect("peer-only members should accept direct turn delivery without a bridge session");
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"peer-only direct turn should use request/ack delivery with one logical input admission"
);
let peer_member = resumed
.member(&AgentIdentity::from("w-ext"))
.await
.expect("peer-only member handle");
let metadata_error = peer_member
.start_turn(
ContentInput::from("unsupported peer-only override".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"remote-hot-swap",
)),
..Default::default()
},
None,
)
.await
.expect_err("peer-only delivery must not silently drop runtime metadata");
assert!(matches!(
metadata_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
ref reason,
} if reason.contains("tracked turn event streams are not supported")
&& reason.contains("peer-only")
));
let (event_tx, _event_rx) = tokio::sync::mpsc::channel(1);
let event_error = peer_member
.start_turn(
ContentInput::from("unsupported peer-only event stream".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
Some(event_tx),
)
.await
.expect_err("peer-only delivery must reject tracked event streams before admission");
assert!(matches!(
event_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
..
}
));
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"unsupported peer-only carriers must be rejected before remote delivery"
);
let snapshot = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("peer-only member status after direct turn");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active
);
assert_eq!(snapshot.current_bridge_session_id(), None);
let machine_state = resumed
.query_machine_state()
.await
.expect("machine state after peer-only direct turn");
assert!(
!machine_state.member_session_bindings.contains_key(
&crate::machines::mob_machine::AgentIdentity("w-ext".to_string())
),
"peer-only members must remain absent from generated session bindings"
);
let external_snapshot = snapshot
.external_member
.as_ref()
.expect("live peer-only external member should expose observation snapshot");
assert_eq!(
external_snapshot.rebind,
crate::runtime::ExternalMemberRebindStatus::Available,
"live peer-only member with bootstrap proof should be rebind-capable"
);
assert!(
external
.trusted_peer_names()
.await
.iter()
.any(|name| name.contains("__mob_supervisor__")),
"peer-only direct turn should authorize the mob supervisor on the live external peer"
);
let respawn_receipt = resumed
.respawn(AgentIdentity::from("w-ext"), None)
.await
.expect("peer-only members should respawn through runtime control");
assert_eq!(respawn_receipt.identity, AgentIdentity::from("w-ext"));
assert_ne!(
respawn_receipt.previous_fence_token, respawn_receipt.fence_token,
"respawn must advance the member fence token"
);
let post_respawn = resumed
.member_status(&AgentIdentity::from("w-ext"))
.await
.expect("peer-only member status after respawn");
assert_eq!(post_respawn.current_bridge_session_id(), None);
let post_respawn_machine_state = resumed
.query_machine_state()
.await
.expect("machine state after peer-only respawn");
assert!(
!post_respawn_machine_state
.member_session_bindings
.contains_key(&crate::machines::mob_machine::AgentIdentity(
"w-ext".to_string()
)),
"peer-only respawn must not recreate a generated session binding"
);
let post_external = post_respawn
.external_member
.as_ref()
.expect("external observation should survive respawn");
assert_eq!(
post_external.owner.agent_identity,
AgentIdentity::from("w-ext"),
"respawn must preserve stable AgentIdentity in external observation"
);
assert_eq!(
post_external.rebind,
crate::runtime::ExternalMemberRebindStatus::Available
);
resumed
.retire(AgentIdentity::from("w-ext"))
.await
.expect("retire the replacement incarnation");
let replayed = events
.replay_all()
.await
.expect("replay peer-only respawn retirement history");
let retired_generations = replayed
.iter()
.filter_map(|event| match &event.kind {
MobEventKind::MemberRetired {
agent_identity,
generation,
..
} if agent_identity == &AgentIdentity::from("w-ext") => Some(generation.get()),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
retired_generations.len(),
2,
"each peer-only incarnation needs its own generation-scoped final carrier"
);
assert_ne!(
retired_generations[0], retired_generations[1],
"respawn retirement must not deduplicate against the prior generation"
);
drop(resumed);
tokio::task::yield_now().await;
let cold = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("cold resume after two peer-only retirement generations");
assert!(
cold.get_member(&AgentIdentity::from("w-ext"))
.await
.expect("query cold-resumed peer-only member")
.is_none(),
"cold replay must not resurrect the terminal replacement incarnation"
);
}
#[tokio::test]
async fn test_resume_reconciles_mixed_topology_without_losing_external_member_refs() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-mixed-topology",
);
let mut definition = definition;
for profile in definition
.profiles
.values_mut()
.filter_map(|binding| binding.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let mob_id = definition.id.clone();
let service = Arc::new(RealCommsSessionService::new());
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let old_sub_sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-sub"),
None,
)
.await
.expect("spawn session-backed member")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external");
handle
.wire(AgentIdentity::from("w-sub"), AgentIdentity::from("w-ext"))
.await
.expect("wire mixed topology");
let old_ext_entry = handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("external entry before resume");
let (old_ext_peer_id, old_ext_addr, old_ext_sid) = match old_ext_entry.member_ref {
MemberRef::BackendPeer {
ref peer_id,
ref address,
ref session_id,
..
} => (peer_id.clone(), address.clone(), session_id.clone()),
ref other => panic!("expected external backend member ref, got {other:?}"),
};
let old_sub_comms = service
.real_comms(&old_sub_sid)
.await
.expect("session-backed member comms");
remove_mob_member_peer_trust_for_test(
&handle,
old_sub_comms.as_ref(),
AgentIdentity::from("w-sub"),
&old_ext_peer_id,
"remove external trust before resume",
)
.await;
handle.stop().await.expect("stop");
service
.restart_comms_for_session(
&old_sub_sid,
&test_comms_name_for(&mob_id, "worker", "w-sub"),
)
.await;
if let Some(old_ext_sid) = &old_ext_sid {
service
.archive(old_ext_sid)
.await
.expect("archive legacy external bridge session");
}
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mixed-topology mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly reconcile live mixed-topology state"
);
resumed
.resume()
.await
.expect("explicit resume should reconcile mixed topology");
let resumed_sub = resumed
.get_member(&AgentIdentity::from("w-sub"))
.await
.unwrap()
.expect("resumed session-backed member entry");
let resumed_sub_sid = match resumed_sub.member_ref {
MemberRef::Session { ref session_id } => session_id.clone(),
ref other => panic!("expected session member ref, got {other:?}"),
};
assert_eq!(
resumed_sub_sid, old_sub_sid,
"resume should restore missing session-backed member session with the same session_id"
);
let resumed_ext = resumed
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("resumed external entry");
match resumed_ext.member_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(peer_id, old_ext_peer_id, "external peer_id must be stable");
assert_eq!(address, old_ext_addr, "external address must be stable");
assert!(
session_id.is_none(),
"mixed-topology resume should preserve peer-only externals without recreating a bridge session"
);
}
other => panic!("expected backend peer member ref, got {other:?}"),
}
let trusted_sub = service
.real_comms(&resumed_sub_sid)
.await
.expect("resumed session-backed member comms")
.peers()
.await
.into_iter()
.map(|entry| entry.address)
.collect::<Vec<_>>();
assert!(
trusted_sub
.iter()
.any(|addr| addr.to_string() == old_ext_addr),
"mixed resume should re-establish trust using the external member's real transport address"
);
}
#[tokio::test]
async fn test_resume_replaces_foreign_live_owner_before_trust_repair() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-live-owner-trust-repair-fails",
);
let mut definition = definition;
for profile in definition
.profiles
.values_mut()
.filter_map(|binding| binding.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let mob_id = definition.id.clone();
let service = Arc::new(RealCommsSessionService::new());
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sub_sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-sub"),
None,
)
.await
.expect("spawn session-backed member")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external");
handle
.wire(AgentIdentity::from("w-sub"), AgentIdentity::from("w-ext"))
.await
.expect("wire mixed topology");
let ext_entry = handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("external entry before resume");
let ext_peer_id = match ext_entry.member_ref {
MemberRef::BackendPeer { ref peer_id, .. } => peer_id.clone(),
ref other => panic!("expected external backend member ref, got {other:?}"),
};
let sub_comms = service
.real_comms(&sub_sid)
.await
.expect("session-backed member comms");
remove_mob_member_peer_trust_for_test(
&handle,
sub_comms.as_ref(),
AgentIdentity::from("w-sub"),
&ext_peer_id,
"remove external trust before same-process resume",
)
.await;
handle.stop().await.expect("stop");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly repair trust"
);
resumed
.resume()
.await
.expect("explicit resume should replace the foreign attachment before trust repair");
assert_eq!(
resumed.status().await.expect("resumed mob status"),
MobState::Running
);
let repaired_comms = service
.real_comms(&sub_sid)
.await
.expect("explicit resume should rebuild the local member runtime");
assert!(
repaired_comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == ext_peer_id),
"the replacement attachment should receive machine-owned external trust"
);
}
#[tokio::test]
async fn test_resume_rebuilds_all_foreign_attachments_before_trust_repair() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-trust-repair-preflights",
);
let mut definition = definition;
for profile in definition
.profiles
.values_mut()
.filter_map(|binding| binding.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let mob_id = definition.id.clone();
let service = Arc::new(RealCommsSessionService::new());
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let ok_sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-a-ok"),
None,
)
.await
.expect("spawn first session-backed member")
.bridge_session_id()
.expect("session-backed")
.clone();
let blocked_sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-z-blocked"),
None,
)
.await
.expect("spawn second session-backed member")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external");
handle
.wire(AgentIdentity::from("w-a-ok"), AgentIdentity::from("w-ext"))
.await
.expect("wire first member to external");
handle
.wire(
AgentIdentity::from("w-z-blocked"),
AgentIdentity::from("w-ext"),
)
.await
.expect("wire second member to external");
let ext_entry = handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.expect("external entry before resume");
let ext_peer_id = match ext_entry.member_ref {
MemberRef::BackendPeer { ref peer_id, .. } => peer_id.clone(),
ref other => panic!("expected external backend member ref, got {other:?}"),
};
let blocked_comms = service
.real_comms(&blocked_sid)
.await
.expect("blocked member comms");
remove_mob_member_peer_trust_for_test(
&handle,
blocked_comms.as_ref(),
AgentIdentity::from("w-z-blocked"),
&ext_peer_id,
"remove blocked external trust before same-process resume",
)
.await;
handle.stop().await.expect("stop");
let foreign_ok = service
.real_comms(&ok_sid)
.await
.expect("foreign first-member comms before reconstruction");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not apply trust repair projections"
);
resumed
.resume()
.await
.expect("explicit resume should rebuild every foreign attachment before trust repair");
let rebuilt_ok = service
.real_comms(&ok_sid)
.await
.expect("rebuilt first-member comms");
let rebuilt_blocked = service
.real_comms(&blocked_sid)
.await
.expect("rebuilt second-member comms");
assert!(
!Arc::ptr_eq(&foreign_ok, &rebuilt_ok),
"the reconstructed provisioner must not adopt the first foreign attachment"
);
for (label, comms) in [("first", rebuilt_ok), ("second", rebuilt_blocked)] {
assert!(
comms
.peers()
.await
.iter()
.any(|peer| peer.peer_id.to_string() == ext_peer_id),
"the {label} replacement attachment should receive machine-owned external trust"
);
}
}
#[tokio::test]
async fn test_resume_reconciles_peer_only_trust_edges() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-peer-only-trust-edges",
);
let mut definition = definition;
for profile in definition
.profiles
.values_mut()
.filter_map(|binding| binding.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let mob_id = definition.id.clone();
let service = Arc::new(RealCommsSessionService::new());
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn external a");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn external b");
handle
.wire(AgentIdentity::from("w-a"), AgentIdentity::from("w-b"))
.await
.expect("wire peer-only pair");
let crate::RuntimeBinding::External {
peer_id: a_peer_id, ..
} = external_a.binding()
else {
panic!("expected external a binding");
};
let crate::RuntimeBinding::External {
peer_id: b_peer_id, ..
} = external_b.binding()
else {
panic!("expected external b binding");
};
handle.stop().await.expect("stop mob");
external_a.remove_trusted_peer(&b_peer_id).await;
external_b.remove_trusted_peer(&a_peer_id).await;
external_a.forget_supervisor().await;
external_b.forget_supervisor().await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("stopped peer-only mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly reconcile peer-only trust"
);
resumed
.resume()
.await
.expect("explicit resume should reconcile peer-only trust topology");
assert_eq!(
external_a.bind_count(),
2,
"resume should route worker a supervisor rebind through MobMachine authority"
);
assert_eq!(
external_b.bind_count(),
2,
"resume should route worker b supervisor rebind through MobMachine authority"
);
let name_a = test_comms_name_for(&mob_id, "worker", "w-a");
let name_b = test_comms_name_for(&mob_id, "worker", "w-b");
assert!(
external_a.trusted_peer_names().await.contains(&name_b),
"resume should restore worker b trust on peer-only worker a"
);
assert!(
external_b.trusted_peer_names().await.contains(&name_a),
"resume should restore worker a trust on peer-only worker b"
);
let trusted_a = external_a.trusted_peer_routes().await;
assert!(
trusted_a
.iter()
.any(|(name, peer_id, _)| name == &name_b && peer_id == &b_peer_id),
"resume should restore worker b trust using MobMachine's current peer id"
);
let trusted_b = external_b.trusted_peer_routes().await;
assert!(
trusted_b
.iter()
.any(|(name, peer_id, _)| name == &name_a && peer_id == &a_peer_id),
"resume should restore worker a trust using MobMachine's current peer id"
);
resumed.stop().await.expect("stop resumed mob");
}
#[tokio::test]
async fn test_resume_reestablishes_missing_trust() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
handle.stop().await.expect("stop");
service.force_remove_trust(&sid_1, &sid_2).await;
service.force_remove_trust(&sid_2, &sid_1).await;
service.clear_mob_machine_trust_owner(&sid_1).await;
service.clear_mob_machine_trust_owner(&sid_2).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly restore missing peer trust"
);
resumed
.resume()
.await
.expect("explicit resume should reestablish missing trust");
let resumed_sid_1 = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("w-1")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let resumed_sid_2 = resumed
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let trusted_1 = service.trusted_peer_names(&resumed_sid_1).await;
let trusted_2 = service.trusted_peer_names(&resumed_sid_2).await;
assert!(
trusted_1.contains(&test_comms_name("worker", "w-2")),
"resume should restore trust from w-1 to w-2"
);
assert!(
trusted_2.contains(&test_comms_name("worker", "w-1")),
"resume should restore trust from w-2 to w-1"
);
}
#[tokio::test]
async fn test_resume_leaves_stale_trust_without_machine_revoke_authority() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let _sid_2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle.stop().await.expect("stop");
let stale = test_trusted_peer_descriptor(
"remote-mob/worker/stale-peer",
"inproc://remote-mob/worker/stale-peer",
);
service
.force_add_trust_from_spec(&sid_1, stale.clone())
.await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
let resumed_sid_1 = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("w-1")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let trusted = service.trusted_peer_names(&resumed_sid_1).await;
assert!(
trusted.iter().any(|n| n == stale.name.as_str()),
"resume must not prune stale trust without generated revoke authority"
);
}
#[tokio::test]
async fn test_resume_restores_external_wiring_from_event_log() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/agent-b",
"inproc://remote-mob/worker/agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
handle.stop().await.expect("stop");
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
let _ = restarted_service.enable_runtime_adapter();
drop(handle);
drop(service);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(restarted_service.clone())
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly restore external wiring"
);
resumed
.resume()
.await
.expect("explicit resume should restore external wiring");
let resumed_sid = resumed
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("resumed member")
.bridge_session_id()
.cloned()
.expect("session-backed member");
let trusted = restarted_service.trusted_peer_names(&resumed_sid).await;
assert!(
trusted.iter().any(|n| n == external.name.as_str()),
"resume should restore external trusted peers from persisted wiring events"
);
let entry = resumed
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("resumed member");
assert_eq!(
entry
.external_peer_specs
.get(&AgentIdentity::from(external.name.as_str()))
.cloned(),
Some(external)
);
}
#[tokio::test]
async fn test_complete_archives_and_emits_mob_completed() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle.complete().await.expect("complete");
assert_eq!(handle.status().await.unwrap(), MobState::Completed);
assert!(
handle.list_members().await.is_empty(),
"complete should retire all active meerkats"
);
assert_eq!(
service.active_session_count().await,
0,
"complete should archive all sessions"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MobCompleted)),
"complete should emit MobCompleted"
);
}
#[tokio::test]
async fn test_destroy_deletes_storage() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle.destroy().await.expect("destroy");
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
assert!(
handle.list_members().await.is_empty(),
"destroy should clear roster"
);
assert_eq!(
service.active_session_count().await,
0,
"destroy should archive active sessions"
);
assert!(
handle
.events()
.replay_all()
.await
.expect("replay")
.is_empty(),
"destroy should delete persisted events"
);
}
#[tokio::test]
async fn test_for_resume_requires_mob_created_event() {
let service = Arc::new(MockSessionService::new());
let result = MobBuilder::for_resume(MobStorage::in_memory())
.with_session_service(service)
.resume()
.await;
assert!(matches!(result, Err(MobError::Internal(_))));
}
#[tokio::test]
async fn test_for_resume_rejects_non_persistent_session_service_by_default() {
let storage = MobStorage::in_memory();
storage
.events
.append(NewMobEvent {
mob_id: MobId::from("test-mob"),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(sample_definition()),
},
})
.await
.expect("append created event");
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
let result = MobBuilder::for_resume(storage)
.with_session_service(service)
.resume()
.await;
assert!(
matches!(result, Err(MobError::Internal(message)) if message.contains("persistent-session contract")),
"resume should reject non-persistent session services by default"
);
}
#[tokio::test]
async fn test_for_resume_allows_ephemeral_session_service_when_opted_in() {
let storage = MobStorage::in_memory();
persist_supervisor_authority_for_test(
storage.runtime_metadata.as_ref(),
&MobId::from("test-mob"),
&default_supervisor_authority_record(),
)
.await;
storage
.events
.append(NewMobEvent {
mob_id: MobId::from("test-mob"),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(sample_definition()),
},
})
.await
.expect("append created event");
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
let resumed = MobBuilder::for_resume(storage)
.with_session_service(service)
.allow_ephemeral_sessions(true)
.resume()
.await
.expect("resume should allow ephemeral sessions when explicitly enabled");
assert_eq!(resumed.status().await.unwrap(), MobState::Running);
}
// -----------------------------------------------------------------------
// P1-T04: spawn() creates a real session
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_spawn_creates_session() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn should succeed")
.bridge_session_id()
.expect("session-backed")
.clone();
// Verify roster updated
let meerkats = handle.list_members().await;
assert_eq!(meerkats.len(), 1);
assert_eq!(meerkats[0].agent_identity.as_str(), "w-1");
assert_eq!(meerkats[0].role.as_str(), "worker");
assert_eq!(
meerkats[0].current_bridge_session_id.as_ref(),
Some(&session_id)
);
}
#[tokio::test]
async fn test_spawn_create_session_request_sets_peer_meta_labels() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
let create_requests = service.recorded_create_requests().await;
assert_eq!(
create_requests.len(),
1,
"exactly one create_session expected"
);
let req = &create_requests[0];
assert_eq!(
req.initial_turn,
meerkat_core::service::InitialTurnPolicy::Defer,
"spawn must defer initial turn; mob actor starts autonomous loop explicitly"
);
assert_eq!(req.comms_name.as_deref(), Some("test-mob/worker/w-1"));
assert_eq!(
req.peer_meta_labels.get("mob_id").map(String::as_str),
Some("test-mob")
);
assert_eq!(
req.peer_meta_labels.get("role").map(String::as_str),
Some("worker")
);
assert_eq!(
req.peer_meta_labels.get("meerkat_id").map(String::as_str),
Some("w-1")
);
assert!(
req.budget_limits.is_none(),
"ordinary spawns must not synthesize per-member budget caps"
);
}
#[tokio::test]
async fn test_spawn_spec_budget_limits_reach_session_build_options() {
let (handle, service) = create_test_mob(sample_definition()).await;
let limits = meerkat_core::BudgetLimits::default()
.with_max_tokens(17)
.with_max_tool_calls(2)
.with_max_duration(meerkat_core::time_compat::Duration::from_secs(3));
handle
.spawn_spec(SpawnMemberSpec::new("worker", "w-budget").with_budget_limits(limits.clone()))
.await
.expect("spawn");
let create_requests = service.recorded_create_requests().await;
assert_eq!(
create_requests.len(),
1,
"exactly one create_session expected"
);
let recorded_limits = create_requests[0]
.budget_limits
.as_ref()
.expect("spawn budget limits should be carried into SessionBuildOptions");
assert_eq!(recorded_limits.max_tokens, limits.max_tokens);
assert_eq!(recorded_limits.max_tool_calls, limits.max_tool_calls);
assert_eq!(recorded_limits.max_duration, limits.max_duration);
}
#[tokio::test]
async fn test_spawn_emits_meerkat_spawned_event() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let session_id = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn")
.bridge_session_id()
.expect("session-backed")
.clone();
let events = handle.events().replay_all().await.expect("replay");
// Should have MobCreated + MemberSpawned
assert!(
events.len() >= 2,
"expected at least 2 events, got {}",
events.len()
);
let spawned = events
.iter()
.find(|e| matches!(e.kind, MobEventKind::MemberSpawned(..)));
assert!(spawned.is_some(), "should have MemberSpawned event");
if let MobEventKind::MemberSpawned(member_spawned) = &spawned.unwrap().kind {
assert_eq!(member_spawned.agent_identity.as_str(), "w-1");
assert_eq!(member_spawned.role.as_str(), "worker");
assert_eq!(member_spawned.generation, crate::ids::Generation::INITIAL);
assert!(
member_spawned.fence_token.get() > 0,
"fence token should be non-zero"
);
assert_eq!(
member_spawned
.bridge_member_ref()
.and_then(|member_ref| member_ref.bridge_session_id()),
Some(&session_id),
);
}
}
#[tokio::test]
async fn test_spawn_duplicate_member_identity_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("first spawn");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(matches!(result, Err(MobError::MemberAlreadyExists(_))));
}
#[tokio::test]
async fn test_spawn_supports_session_and_external_backends() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"spawn-session-and-external",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_real_comms(definition).await;
let session_ref = handle
.spawn_with_backend(
ProfileName::from("worker"),
AgentIdentity::from("w-sub"),
None,
Some(MobBackendKind::Session),
)
.await
.expect("spawn session backend");
assert!(
matches!(session_ref, MemberRef::Session { .. }),
"session backend should emit session member ref"
);
let external = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
let crate::RuntimeBinding::External {
peer_id: expected_peer_id,
address: expected_address,
bootstrap_token: _expected_bootstrap_token,
pubkey: _expected_pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let external_ref = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external.binding(),
)
.await
.expect("spawn external backend");
match external_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(
peer_id, expected_peer_id,
"external backend should use the committed peer_id returned by bind_member"
);
assert_eq!(
address,
canonical_external_address(&expected_address),
"external backend should use the committed transport address returned by bind_member"
);
assert!(
session_id.is_none(),
"external backend should now persist peer-only member refs"
);
}
other => panic!("expected backend peer member ref, got {other:?}"),
}
assert_eq!(
external.bind_count(),
1,
"external spawn should issue exactly one bind request"
);
assert!(
external
.trusted_peer_names()
.await
.iter()
.any(|name| name.contains("__mob_supervisor__")),
"external spawn should authorize the mob supervisor during bind"
);
}
#[tokio::test]
async fn test_peer_only_external_spawn_without_owner_binding_fails_closed() {
let (handle, _service) = create_test_mob(sample_definition_with_external_backend()).await;
let mut spec = SpawnMemberSpec::new("worker", "w-unbound-owner");
spec.binding = Some(test_external_binding("w-unbound-owner"));
let err = handle
.spawn_spec_internal(spec)
.await
.expect_err("peer-only external spawn without generated owner binding must fail closed");
assert!(
err.to_string()
.contains("external peer-only member operation requires generated owner binding"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_external_backend_rejects_invalid_peer_name_components() {
let (handle, _service) = create_test_mob(sample_definition_with_external_backend()).await;
let result = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("../w-ext"),
None,
test_external_binding("../w-ext"),
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"invalid peer name components must fail external member registration"
);
}
#[tokio::test]
async fn test_external_backend_wiring_uses_sendable_transport_addresses() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-wiring-addresses",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
let member_a = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn external w-a");
let member_b = handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn external w-b");
assert_eq!(member_a.bridge_session_id(), None);
assert_eq!(member_b.bridge_session_id(), None);
handle
.wire(AgentIdentity::from("w-a"), AgentIdentity::from("w-b"))
.await
.expect("wire external peers");
let roster = handle.roster().await;
let entry_a = roster
.get_by_identity(&AgentIdentity::from("w-a"))
.expect("entry a");
let entry_b = roster
.get_by_identity(&AgentIdentity::from("w-b"))
.expect("entry b");
assert!(
entry_a.wired_to.contains(&AgentIdentity::from("w-b")),
"peer-only external wiring should remain symmetric in the roster"
);
assert!(
entry_b.wired_to.contains(&AgentIdentity::from("w-a")),
"peer-only external wiring should remain symmetric in the roster"
);
let trusted_a = external_a.trusted_peer_names().await;
let trusted_b = external_b.trusted_peer_names().await;
assert!(
trusted_a
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", "w-b")),
"w-a should trust the peer-only remote target after wire"
);
assert!(
trusted_b
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", "w-a")),
"w-b should trust the peer-only remote source after wire"
);
let intents_a = external_a.received_intents().await;
let intents_b = external_b.received_intents().await;
assert!(
intents_a
.iter()
.any(|intent| intent == super::bridge_protocol::SUPERVISOR_BRIDGE_INTENT),
"w-a should receive an explicit wire_member bridge command"
);
assert!(
intents_b
.iter()
.any(|intent| intent == super::bridge_protocol::SUPERVISOR_BRIDGE_INTENT),
"w-b should receive an explicit wire_member bridge command"
);
}
#[tokio::test]
async fn test_external_backend_unwiring_revokes_remote_trust() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-unwiring-trust",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external_a = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "u-a")).await;
let external_b = spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "u-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("u-a"),
None,
external_a.binding(),
)
.await
.expect("spawn external u-a");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("u-b"),
None,
external_b.binding(),
)
.await
.expect("spawn external u-b");
handle
.wire(AgentIdentity::from("u-a"), AgentIdentity::from("u-b"))
.await
.expect("wire external peers");
handle
.unwire(AgentIdentity::from("u-a"), AgentIdentity::from("u-b"))
.await
.expect("unwire external peers");
let trusted_a = external_a.trusted_peer_names().await;
let trusted_b = external_b.trusted_peer_names().await;
assert!(
!trusted_a
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", "u-b")),
"u-a should drop remote peer trust after unwire"
);
assert!(
!trusted_b
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", "u-a")),
"u-b should drop remote peer trust after unwire"
);
let intents_a = external_a.received_intents().await;
let intents_b = external_b.received_intents().await;
assert!(
intents_a
.iter()
.any(|intent| intent == super::bridge_protocol::SUPERVISOR_BRIDGE_INTENT),
"u-a should receive an explicit unwire_member bridge command"
);
assert!(
intents_b
.iter()
.any(|intent| intent == super::bridge_protocol::SUPERVISOR_BRIDGE_INTENT),
"u-b should receive an explicit unwire_member bridge command"
);
}
#[tokio::test]
async fn test_peer_only_retire_unwire_failure_retains_retry_anchor() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-retire-unwire-retry",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external_a =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "z-retry-u")).await;
let external_b =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "a-retry-u")).await;
// Deliberately retire the reverse-lexicographic endpoint so cleanup must
// use the canonical edge's a/b fields rather than assuming subject order.
let identity_a = AgentIdentity::from("z-retry-u");
let identity_b = AgentIdentity::from("a-retry-u");
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity_a.clone(),
None,
external_a.binding(),
)
.await
.expect("spawn external z-retry-u");
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity_b.clone(),
None,
external_b.binding(),
)
.await
.expect("spawn external a-retry-u");
handle
.wire(identity_a.clone(), identity_b.clone())
.await
.expect("wire external peers");
external_b.fail_next_unwire();
let error = handle
.retire(identity_a.clone())
.await
.expect_err("remote trust refusal must interrupt terminal retirement");
assert!(
matches!(&error, MobError::RetirementTopologyIncomplete(_)),
"remote unwire failure must retain typed retry authority: {error}"
);
let retained_a = handle
.get_member(&identity_a)
.await
.expect("read retiring member")
.expect("failed unwire must retain retiring member");
let retained_b = handle
.get_member(&identity_b)
.await
.expect("read surviving member")
.expect("surviving member");
assert!(retained_a.wired_to.contains(&identity_b));
assert!(retained_b.wired_to.contains(&identity_a));
// The one-way lifecycle notice may already prune the recipient's physical
// trust projection before the explicit unwire refusal arrives. Durable
// retry authority is the retained machine edge and roster anchor above,
// not rollback of a partially completed remote mutation.
assert!(
handle
.events()
.replay_all()
.await
.expect("replay interrupted retirement")
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. })),
"terminal retirement must not publish before remote trust cleanup"
);
handle
.retire(identity_a.clone())
.await
.expect("same-handle retry must converge remote trust and retirement");
assert!(
handle
.get_member(&identity_a)
.await
.expect("read completed retirement")
.is_none()
);
assert!(
!external_b
.trusted_peer_names()
.await
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", identity_a.as_str())),
"successful retry must revoke the retiring peer's trust"
);
assert!(
!external_a
.trusted_peer_names()
.await
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", identity_b.as_str())),
"terminal retirement must also remove the surviving peer from the retiring remote runtime"
);
assert_eq!(
external_a.authorized_supervisor_peer_id().await,
None,
"terminal ordinary retirement must revoke supervisor authority from the retired remote runtime"
);
}
#[tokio::test]
async fn test_peer_only_retire_unwire_failure_survives_cold_restart() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-retire-unwire-cold-retry",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let owner_bridge_session_id = SessionId::new();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_owner_bridge_session_create_authority(owner_bridge_session_id.clone(), false, false)
.create()
.await
.expect("create mob");
let retiring = AgentIdentity::from("z-cold-retiring");
let survivor = AgentIdentity::from("a-cold-survivor");
let external_retiring =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", retiring.as_str())).await;
let external_survivor =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", survivor.as_str())).await;
let mut retiring_spec = SpawnMemberSpec::new("worker", retiring.clone());
retiring_spec.binding = Some(external_retiring.binding());
handle
.spawn_spec_receipt_with_generated_owner_context(
retiring_spec,
owner_bridge_session_id.clone(),
)
.await
.expect("spawn retiring external member");
let mut survivor_spec = SpawnMemberSpec::new("worker", survivor.clone());
survivor_spec.binding = Some(external_survivor.binding());
handle
.spawn_spec_receipt_with_generated_owner_context(
survivor_spec,
owner_bridge_session_id.clone(),
)
.await
.expect("spawn surviving external member");
handle
.wire(retiring.clone(), survivor.clone())
.await
.expect("wire peer-only members");
external_survivor.fail_next_unwire();
handle
.retire(retiring.clone())
.await
.expect_err("first remote unwire must fail closed");
let retained = handle
.get_member(&retiring)
.await
.expect("read retained cold-retry anchor")
.expect("failed remote unwire must retain the retiring member");
assert!(
retained.wired_to.contains(&survivor),
"failed remote unwire must retain the machine edge for cold retry"
);
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume durable retiring endpoint and wiring authority");
resumed
.retire(retiring.clone())
.await
.expect("cold retry must remove remote trust before terminal retirement");
assert!(
!external_survivor
.trusted_peer_names()
.await
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", retiring.as_str())),
"cold retry must not strand peer-only reciprocal trust"
);
assert!(
!external_retiring
.trusted_peer_names()
.await
.iter()
.any(|name| name == &test_comms_name_for(&mob_id, "worker", survivor.as_str())),
"cold retry must converge the retiring remote runtime's reciprocal trust half"
);
assert_eq!(
external_retiring.authorized_supervisor_peer_id().await,
None,
"cold retry must revoke supervisor authority before terminal publication"
);
assert!(
resumed
.get_member(&retiring)
.await
.expect("read completed cold retirement")
.is_none()
);
assert_eq!(
events
.replay_all()
.await
.expect("replay cold peer-only retirement")
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &retiring
))
.count(),
1
);
}
#[tokio::test]
async fn test_peer_only_retire_revoke_failure_retains_overlay_and_retry_anchor() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-retire-revoke-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service)
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("ordinary-revoke-retry");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external member");
external.fail_next_revoke();
let error = handle
.retire(identity.clone())
.await
.expect_err("revoke failure must interrupt ordinary retirement");
assert!(
matches!(&error, MobError::RetirementTopologyIncomplete(_)),
"ordinary revoke failure needs typed retry authority: {error}"
);
assert!(external.authorized_supervisor_peer_id().await.is_some());
assert!(
handle
.get_member(&identity)
.await
.expect("read retiring member")
.is_some()
);
assert!(
runtime_metadata
.list_external_binding_overlays(&mob_id)
.await
.expect("list retained overlay")
.iter()
.any(|overlay| overlay.agent_identity == identity),
"revoke failure must retain the binding overlay used by cleanup retry"
);
assert!(
events
.replay_all()
.await
.expect("replay revoke failure")
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
let supervisor_before_rejected_rotation = external
.authorized_supervisor_peer_id()
.await
.expect("retiring peer retains its supervisor before revoke retry");
let intents_before_rejected_rotation = external.received_intents().await.len();
let rotation_error = handle
.rotate_supervisor()
.await
.expect_err("supervisor rotation must be closed while any member is retiring");
assert!(
matches!(
&rotation_error,
MobError::MobMachineRejected {
context: "handle_rotate_supervisor_admission",
..
}
),
"rotation during retirement needs a typed machine rejection: {rotation_error}"
);
assert_eq!(
external.received_intents().await.len(),
intents_before_rejected_rotation,
"rejected rotation must not send authority-change effects to the retiring runtime"
);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(supervisor_before_rejected_rotation.as_str()),
"rejected rotation must leave the retiring runtime's authority unchanged"
);
handle
.retire(identity.clone())
.await
.expect("same-actor retry must finish revoke and terminal retirement");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(
handle
.get_member(&identity)
.await
.expect("read completed retirement")
.is_none()
);
assert!(
runtime_metadata
.list_external_binding_overlays(&mob_id)
.await
.expect("list completed overlays")
.is_empty()
);
}
#[tokio::test]
async fn test_peer_only_retire_supervisor_checkpoint_cold_retry_never_recontacts_remote() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-supervisor-checkpoint-cold-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("supervisor-checkpoint-cold-retry");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external member");
super::actor::fail_after_remote_supervisor_revoked_for_test(&identity);
handle
.retire(identity.clone())
.await
.expect_err("fault after supervisor checkpoint must retain retirement anchor");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(
events
.replay_all()
.await
.expect("replay supervisor checkpoint")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::RemoteMemberSupervisorRevoked { agent_identity, .. }
if agent_identity == &identity
))
);
let intents_before_cold_retry = external.received_intents().await.len();
external.task.abort();
handle
.shutdown()
.await
.expect("shutdown must skip a peer-only terminal-publication retry anchor");
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"shutdown must not recontact an offline peer after retirement and supervisor revocation"
);
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume supervisor-revoked retirement checkpoint");
resumed
.retire(identity.clone())
.await
.expect("checkpoint retry must finish locally while remote is offline");
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"a durable supervisor-revoked checkpoint must suppress every remote cleanup command"
);
assert!(
resumed
.get_member(&identity)
.await
.expect("read completed checkpoint retry")
.is_none()
);
}
#[tokio::test]
async fn test_peer_only_retire_local_trust_remove_repair_is_offline_retryable() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-local-trust-remove-repair",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("local-trust-remove-repair");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external member");
let external_peer_id = external.runtime.public_key().to_peer_id().to_string();
super::supervisor_bridge::fail_next_bridge_trust_remove_reconcile_for_test(&external_peer_id);
handle
.retire(identity.clone())
.await
.expect_err("live trust-remove failure must retain retirement authority");
assert!(
events
.replay_all()
.await
.expect("replay supervisor checkpoint after remove fault")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::RemoteMemberSupervisorRevoked { agent_identity, .. }
if agent_identity == &identity
))
);
let intents_before_cold_retry = external.received_intents().await.len();
external.task.abort();
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume committed-DSL/live-trust-remove gap");
resumed
.retire(identity.clone())
.await
.expect("RepairRemoveDirectPeerEndpoint must finish with remote offline");
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"local trust repair must not recontact the retired runtime"
);
assert!(
resumed
.get_member(&identity)
.await
.expect("read completed repair retry")
.is_none()
);
}
#[tokio::test]
async fn test_peer_only_retire_final_append_after_revoke_survives_cold_restart() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-retire-final-append-cold-retry",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("ordinary-append-after-revoke");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
identity.clone(),
None,
external.binding(),
)
.await
.expect("spawn external member");
events.fail_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect_err("terminal append failure must retain ordinary retirement authority");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(
handle
.get_member(&identity)
.await
.expect("read retained member")
.is_some()
);
assert!(
events
.replay_all()
.await
.expect("replay supervisor checkpoint before terminal append")
.iter()
.any(|event| matches!(
&event.kind,
MobEventKind::RemoteMemberSupervisorRevoked { agent_identity, .. }
if agent_identity == &identity
))
);
let intents_before_cold_retry = external.received_intents().await.len();
external.task.abort();
handle
.shutdown()
.await
.expect("shutdown must skip a peer-only final-publication retry anchor");
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"shutdown must not recontact an offline peer after final publication fails"
);
drop(handle);
events.allow_appends_for("MemberRetired").await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume remote runtime retired before terminal append");
resumed
.retire(identity.clone())
.await
.expect("cold retry must finish locally after durable supervisor revocation");
assert_eq!(
external.received_intents().await.len(),
intents_before_cold_retry,
"terminal-append retry must not contact an offline retired runtime"
);
assert_eq!(external.authorized_supervisor_peer_id().await, None);
assert!(
resumed
.get_member(&identity)
.await
.expect("read completed cold retry")
.is_none()
);
assert_eq!(
events
.replay_all()
.await
.expect("replay completed ordinary cold retry")
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1
);
}
#[tokio::test]
async fn test_peer_only_respawn_records_retirement_per_generation() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"peer-only-retirement-generation-key",
);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let owner_bridge_session_id = SessionId::new();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.with_owner_bridge_session_create_authority(owner_bridge_session_id.clone(), false, false)
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("generation-worker");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", identity.as_str())).await;
let mut initial_spec = SpawnMemberSpec::new("worker", identity.clone());
initial_spec.binding = Some(external.binding());
handle
.spawn_spec_receipt_with_generated_owner_context(initial_spec, owner_bridge_session_id)
.await
.expect("spawn generation zero");
let receipt = handle
.respawn(identity.clone(), None)
.await
.expect("respawn through the same external binding");
assert_eq!(receipt.agent_runtime_id.generation.get(), 1);
handle
.retire(identity.clone())
.await
.expect("retire generation one");
assert_eq!(external.authorized_supervisor_peer_id().await, None);
let retired_generations = events
.replay_all()
.await
.expect("replay per-generation retirement events")
.into_iter()
.filter_map(|event| match event.kind {
MobEventKind::MemberRetired {
agent_identity,
generation,
..
} if agent_identity == identity => Some(generation.get()),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
retired_generations,
vec![0, 1],
"durable retirement idempotency must be keyed by stable identity plus generation"
);
}
#[tokio::test]
async fn test_spawn_unknown_profile_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let result = handle
.spawn(
ProfileName::from("nonexistent"),
AgentIdentity::from("x-1"),
None,
)
.await;
assert!(matches!(result, Err(MobError::ProfileNotFound(_))));
}
#[tokio::test]
async fn test_spawn_fails_when_profile_comms_disabled() {
let mut definition = sample_definition();
let worker = definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile exists")
.as_inline_mut()
.unwrap();
worker.tools.comms = false;
let (handle, service) = create_test_mob(definition).await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"tools.comms=false must be rejected by spawn path"
);
assert_eq!(
service.recorded_prompts().await.len(),
0,
"spawn should fail before create_session when comms is disabled"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn test_spawn_append_failure_rolls_back_runtime_state() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MemberSpawned").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"spawn should surface append failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"spawn append failure must leave roster untouched"
);
assert_eq!(
service.active_session_count().await,
0,
"spawn append failure must archive the just-created session"
);
let recorded = handle.events().replay_all().await.expect("replay");
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberSpawned(..))),
"failed spawn append must not persist MemberSpawned"
);
}
// -----------------------------------------------------------------------
// P1-T05: retire() removes a meerkat
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_retire_removes_from_roster() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire");
assert!(handle.list_members().await.is_empty());
}
#[test]
fn test_retire_idempotency_checks_do_not_replay_full_event_log() {
// Live mobs also own the remote-turn reconciler. Every structural event
// wakes that shared worker, whose current-epoch scan legitimately reads
// the event log. Counting store-wide reads around `retire()` therefore
// conflates the worker's scan with the retirement idempotency checks this
// regression protects. Inspect the two exact indexed checks instead.
let source = include_str!("actor.rs");
for (start_marker, end_marker, index_name) in [
(
"async fn retirement_started_event_exists",
"async fn preserved_respawn_topology_event_exists",
"retirement_started_event_index",
),
(
"async fn retire_event_exists",
"async fn append_retire_event_for_entry",
"retired_event_index",
),
] {
let start = source.find(start_marker).expect("indexed check exists");
let end = source[start..]
.find(end_marker)
.expect("indexed check has a stable following function");
let body = &source[start..start + end];
assert!(
body.contains(index_name),
"{start_marker} must use its actor-local durable-event index"
);
assert!(
!body.contains("replay_all"),
"{start_marker} must not replay the full event log per retirement request"
);
}
}
#[tokio::test]
async fn test_retire_emits_event() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire");
let events = handle.events().replay_all().await.expect("replay");
let started = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MemberRetirementStarted { agent_identity, .. }
if agent_identity == &AgentIdentity::from("w-1")
)
})
.collect::<Vec<_>>();
let retired = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &AgentIdentity::from("w-1")
)
})
.collect::<Vec<_>>();
assert_eq!(started.len(), 1, "retire must persist one start carrier");
assert_eq!(retired.len(), 1, "retire must persist one final carrier");
assert!(
started[0].cursor < retired[0].cursor,
"retirement start must precede archive-confirmed terminality"
);
}
#[tokio::test]
async fn test_retire_nonexistent_is_idempotent() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let result = handle.retire(AgentIdentity::from("nope")).await;
assert!(result.is_ok(), "retire should be idempotent");
}
#[tokio::test]
async fn test_retire_removes_wiring_from_peers() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
// Verify wired
let entry = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.unwrap();
assert!(entry.wired_to.contains(&AgentIdentity::from("w-1")));
// Retire w-1
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire");
// w-2 should no longer be wired to w-1
let entry = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.unwrap();
assert!(!entry.wired_to.contains(&AgentIdentity::from("w-1")));
}
#[tokio::test]
async fn test_retire_archive_failure_is_not_silent() {
let (handle, service) = create_test_mob(sample_definition()).await;
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
service.set_archive_failure(&session_id).await;
// Archive failure is critical — retire must surface it (dogma §15).
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
result.is_err(),
"retire must return Err when ArchiveSession fails"
);
let retained = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("archive failure must retain the non-routable retry anchor");
assert_eq!(
handle
.member_status(&retained.agent_identity)
.await
.expect("retained member status")
.status,
crate::runtime::handle::MobMemberStatus::Retiring
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"retirement-started carrier must persist before archive so cleanup remains retryable"
);
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"archive failure must not mint terminal MemberRetired"
);
service.clear_archive_failure(&session_id).await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("same-process retry converges after transient archive failure");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"archive-confirmed retry removes the retained anchor"
);
assert_eq!(
handle
.events()
.replay_all()
.await
.expect("replay converged retirement")
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"transient archive retry publishes exactly one final"
);
}
#[tokio::test]
async fn test_retire_terminal_publication_failure_retains_anchor_and_retry_converges() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MemberRetired").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
let first = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
matches!(first, Err(MobError::StorageError(_))),
"terminal publication failure must preserve its typed storage cause, got {first:?}"
);
assert_eq!(
service.active_session_count().await,
0,
"session archive must remain terminal despite a journal append failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("query retained retirement anchor")
.is_some(),
"same-process retry needs a non-routable retiring roster anchor"
);
let after_failure = events.replay_all().await.expect("replay failed retirement");
assert_eq!(
after_failure
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetirementStarted { .. }))
.count(),
1
);
assert!(
!after_failure
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberRetired { .. })),
"failed terminal publication must not fabricate a final carrier"
);
events.allow_appends_for("MemberRetired").await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retry terminal publication");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("query converged retirement")
.is_none(),
"successful terminal publication removes the cleanup anchor"
);
let converged = events
.replay_all()
.await
.expect("replay converged retirement");
assert_eq!(
converged
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetirementStarted { .. }))
.count(),
1
);
assert_eq!(
converged
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1
);
}
#[tokio::test]
async fn test_retire_trust_removal_failure_is_not_silent() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-2"),
MockCommsBehavior {
fail_remove_trust: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
// Reciprocal trust removal is a critical topology boundary: publishing a
// terminal retirement while the surviving peer still trusts this member
// would strand authority with no retry anchor.
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
matches!(result, Err(MobError::RetirementTopologyIncomplete(_))),
"retire must surface trust-removal failure"
);
// The durable retiring member remains as the exact retry anchor and no
// terminal event is published until trust cleanup converges.
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"retire must retain its roster entry when trust removal fails"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.all(|e| !matches!(e.kind, MobEventKind::MemberRetired { .. })),
"retire event must not persist while trust removal is incomplete"
);
service
.set_comms_behavior(
&test_comms_name("worker", "w-2"),
MockCommsBehavior::default(),
)
.await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("same-handle retry must finish trust cleanup and retirement");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"successful retry must remove the roster entry"
);
}
#[tokio::test]
async fn test_retire_fails_when_peer_retired_notification_fails_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-2"),
MockCommsBehavior {
fail_send_peer_retired: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
result.is_ok(),
"retire should remain best-effort under notification failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"retired meerkat should be removed from roster"
);
let entry_w2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 should stay in roster");
assert!(
!entry_w2.wired_to.contains(&AgentIdentity::from("w-1")),
"retire should remove wiring from remaining peers"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"retire event should persist even when notification cleanup fails"
);
}
#[tokio::test]
async fn test_retirement_started_append_failure_is_retryable_without_side_effects() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MemberRetirementStarted").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/start-append-failure",
"inproc://remote-mob/worker/start-append-failure",
);
handle
.wire(
AgentIdentity::from("w-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"retire should surface append failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"retire append failure must keep roster state retryable"
);
let entry_w2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 should remain in roster");
assert!(
entry_w2.wired_to.contains(&AgentIdentity::from("w-1")),
"retire append failure must not apply trust cleanup before event persistence"
);
assert_eq!(
service.active_session_count().await,
2,
"retire append failure must not archive sessions"
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("w-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("machine state after failed retirement-start append");
assert!(
dsl.external_peer_edges.contains(&external_edge),
"retirement-start append failure must precede external topology cleanup"
);
let recorded = handle.events().replay_all().await.expect("replay");
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"failed retirement-started append must not persist MemberRetirementStarted"
);
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"failed retirement-started append must not persist terminal MemberRetired"
);
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::ExternalPeerUnwired { .. })),
"failed retirement-started append must not persist external unwiring"
);
}
#[tokio::test]
async fn test_retirement_start_append_failure_is_retryable_without_side_effects() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MemberRetirementStarted").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"retire should surface append failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"retire append failure must keep roster state retryable"
);
let entry_w2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 should remain in roster");
assert!(
entry_w2.wired_to.contains(&AgentIdentity::from("w-1")),
"retire append failure must not apply trust cleanup before event persistence"
);
assert_eq!(
service.active_session_count().await,
2,
"retire append failure must not archive sessions"
);
let recorded = handle.events().replay_all().await.expect("replay");
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"failed retirement-start append must not persist its retry anchor"
);
}
// -----------------------------------------------------------------------
// P1-T06: wire() establishes bidirectional trust
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_wire_establishes_bidirectional() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
// Check roster wiring
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
assert!(entry_l.wired_to.contains(&AgentIdentity::from("w-1")));
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(entry_w.wired_to.contains(&AgentIdentity::from("l-1")));
}
#[tokio::test]
async fn test_member_roster_surfaces_peer_id() {
let (handle, service) = create_test_mob(sample_definition()).await;
let spawn = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let lead_sid = spawn.bridge_session_id().expect("session-backed").clone();
let expected_peer_id = service
.comms_runtime(&lead_sid)
.await
.expect("lead comms runtime")
.peer_id()
.expect("lead peer id");
let transport_public_key = service
.comms_runtime(&lead_sid)
.await
.expect("lead comms runtime")
.public_key()
.expect("lead transport public key");
let entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member should exist");
assert_eq!(entry.peer_id, Some(expected_peer_id));
assert_eq!(
entry.transport_public_key.as_deref(),
Some(transport_public_key.as_str())
);
assert_ne!(
expected_peer_id.to_string(),
transport_public_key,
"roster peer_id must be the canonical UUID, not the Ed25519 public key"
);
}
#[tokio::test]
async fn test_member_status_projects_unknown_peer_connectivity() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let left_id = AgentIdentity::from("l-1");
let right_id = AgentIdentity::from("w-1");
let _left_session_id = handle
.spawn(ProfileName::from("lead"), left_id.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(ProfileName::from("worker"), right_id.clone(), None)
.await
.expect("spawn worker");
handle
.wire(AgentIdentity::from(left_id.as_str()), right_id.clone())
.await
.expect("wire local peers");
let snapshot = handle
.member_status(&AgentIdentity::from(left_id.as_str()))
.await
.expect("member status should succeed");
let meerkat_contracts::WirePeerConnectivity::Known {
snapshot: connectivity,
} = snapshot
.peer_connectivity
.expect("live comms runtime should project peer connectivity")
else {
panic!("live comms runtime should resolve a Known peer-connectivity snapshot");
};
assert_eq!(connectivity.reachable_peer_count, 0);
assert_eq!(connectivity.unknown_peer_count, 1);
assert!(connectivity.unreachable_peers.is_empty());
}
#[tokio::test]
async fn test_member_status_keeps_connectivity_unknown_when_public_key_differs() {
let (handle, service) = create_test_mob(sample_definition()).await;
let left_id = AgentIdentity::from("l-1");
let right_id = AgentIdentity::from("w-1");
let left_session_id = handle
.spawn(ProfileName::from("lead"), left_id.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let right_session_id = handle
.spawn(ProfileName::from("worker"), right_id.clone(), None)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from(left_id.as_str()), right_id.clone())
.await
.expect("wire local peers");
let right_runtime = service
.comms_runtime(&right_session_id)
.await
.expect("right comms runtime");
let right_peer_id = right_runtime.peer_id().expect("right canonical peer id");
let right_public_key = right_runtime
.public_key()
.expect("right transport public key");
assert_ne!(
right_peer_id.to_string(),
right_public_key,
"test fixture must exercise distinct canonical peer id and public key carriers"
);
let alias_name = format!("{}/worker/right-alias", handle.mob_id());
let right_pubkey = meerkat_comms::PubKey::from_pubkey_string(&right_public_key)
.expect("right transport public key should decode");
let alias_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
alias_name.clone(),
right_peer_id.to_string(),
*right_pubkey.as_bytes(),
format!("inproc://{alias_name}"),
)
.expect("valid alias peer spec");
service
.force_add_trust_from_spec(&left_session_id, alias_spec)
.await;
drop(alias_name);
let snapshot = handle
.member_status(&AgentIdentity::from(left_id.as_str()))
.await
.expect("member status should succeed");
let meerkat_contracts::WirePeerConnectivity::Known {
snapshot: connectivity,
} = snapshot
.peer_connectivity
.expect("live comms runtime should project peer connectivity")
else {
panic!("live comms runtime should resolve a Known peer-connectivity snapshot");
};
assert_eq!(connectivity.reachable_peer_count, 0);
assert_eq!(connectivity.unknown_peer_count, 1);
assert!(connectivity.unreachable_peers.is_empty());
}
#[tokio::test]
async fn test_member_status_omits_peer_connectivity_without_live_comms_runtime() {
let (handle, service) = create_test_mob(sample_definition()).await;
let left_id = AgentIdentity::from("l-1");
let right_id = AgentIdentity::from("w-1");
let left_session_id = handle
.spawn(ProfileName::from("lead"), left_id.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(ProfileName::from("worker"), right_id.clone(), None)
.await
.expect("spawn worker");
handle
.wire(AgentIdentity::from(left_id.as_str()), right_id.clone())
.await
.expect("wire local peers");
service.set_missing_comms_runtime(&left_session_id).await;
let snapshot = handle
.member_status(&AgentIdentity::from(left_id.as_str()))
.await
.expect("member status should succeed");
assert!(
matches!(
snapshot.peer_connectivity,
Some(meerkat_contracts::WirePeerConnectivity::NotApplicable)
),
"member snapshot should report NotApplicable connectivity when no live comms runtime exists, got {:?}",
snapshot.peer_connectivity
);
}
#[tokio::test]
async fn test_wire_external_adds_trusted_peer_and_tracks_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/agent-b",
"inproc://remote-mob/worker/agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member should exist");
assert!(
entry_l
.wired_to
.contains(&AgentIdentity::from(external.name.as_str()))
);
assert_eq!(
entry_l
.external_peer_specs
.get(&AgentIdentity::from(external.name.as_str()))
.cloned(),
Some(external.clone())
);
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
trusted.iter().any(|n| n == external.name.as_str()),
"trusted peer list should include external peer name"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
assert!(
dsl.wiring_edges
.iter()
.all(|edge| edge.a.0 != external.name.as_str() && edge.b.0 != external.name.as_str()),
"external peer descriptors must not be projected into member wiring_edges"
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
assert!(
dsl.external_peer_edges.contains(&external_edge),
"MobMachine should own the descriptor-bearing external peer edge"
);
}
#[tokio::test]
async fn test_wire_external_rejects_same_name_descriptor_replacement() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/duplicate-agent",
"inproc://remote-mob/worker/duplicate-agent-a",
);
let replacement = test_trusted_peer_descriptor(
external.name.as_str(),
"inproc://remote-mob/worker/duplicate-agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
let result = handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(replacement.clone()),
)
.await;
assert!(
result.is_err(),
"MobMachine should reject a second descriptor for the same local/name peer key"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member should exist");
assert_eq!(
entry_l
.external_peer_specs
.get(&AgentIdentity::from(external.name.as_str()))
.cloned(),
Some(external.clone()),
"projection must still reflect the descriptor admitted by MobMachine"
);
let trusted_addresses = service.trusted_peer_addresses(&sid_l).await;
assert!(
trusted_addresses
.iter()
.any(|address| address == "inproc://remote-mob/worker/duplicate-agent-a"),
"trusted peer should keep the first admitted descriptor"
);
assert!(
!trusted_addresses
.iter()
.any(|address| address == "inproc://remote-mob/worker/duplicate-agent-b"),
"rejected replacement descriptor must not be installed into comms trust"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
let external_key = crate::machines::mob_machine::ExternalPeerKey::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::PeerName::from(external.name.as_str()),
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
assert_eq!(
dsl.external_peer_edges_by_key.get(&external_key).cloned(),
Some(external_edge.clone()),
"MobMachine should own one descriptor edge for the local/name peer key"
);
let duplicate_name_edges = dsl
.external_peer_edges
.iter()
.filter(|edge| {
edge.local == crate::machines::mob_machine::AgentIdentity::from("l-1")
&& edge.endpoint.name.0 == external.name.as_str()
})
.count();
assert_eq!(
duplicate_name_edges, 1,
"MobMachine must not retain multiple descriptor edges for one local/name peer key"
);
}
#[tokio::test]
async fn test_rewire_external_repairs_trust_for_existing_machine_edge() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/agent-b",
"inproc://remote-mob/worker/agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
service
.force_remove_trust_by_peer_id(&sid_l, &external.peer_id.to_string())
.await;
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("rewire external should repair trust");
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
trusted.iter().any(|name| name == external.name.as_str()),
"rewire external should restore external trust"
);
let events = handle.events().replay_all().await.expect("replay");
let external_wired_events = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::ExternalPeerWired { local, spec }
if local == &AgentIdentity::from("l-1") && spec.name == external.name
)
})
.count();
assert_eq!(
external_wired_events, 1,
"external trust repair for an existing machine edge must not duplicate the wire event"
);
}
#[tokio::test]
async fn test_rewire_external_machine_edge_without_roster_projection_repairs_trust_only() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-machine-repair"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/repair-agent",
"inproc://remote-mob/worker/repair-agent",
);
let machine_local = crate::machines::mob_machine::AgentIdentity::from("l-machine-repair");
let machine_key = crate::machines::mob_machine::ExternalPeerKey::new(
machine_local.clone(),
crate::machines::mob_machine::PeerName::from(external.name.as_str()),
);
let machine_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
machine_local,
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::WireExternalPeer {
key: machine_key,
edge: machine_edge,
},
)
.await
.expect("seed machine-owned external edge without roster projection");
handle
.wire(
AgentIdentity::from("l-machine-repair"),
PeerTarget::External(external.clone()),
)
.await
.expect("machine-edge rewire should repair trust only");
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
trusted.iter().any(|name| name == external.name.as_str()),
"machine-edge rewire should install external trust from generated repair authority"
);
let entry = handle
.get_member(&AgentIdentity::from("l-machine-repair"))
.await
.unwrap()
.expect("member should exist");
assert!(
!entry
.external_peer_specs
.contains_key(&AgentIdentity::from(external.name.as_str())),
"trust repair must not synthesize external peer roster projection"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|event| matches!(event.kind, MobEventKind::ExternalPeerWired { .. })),
"trust repair must not append ExternalPeerWired without a generated graph-change effect"
);
}
#[tokio::test]
async fn test_respawn_repairs_external_roster_spec_trust_without_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let old_sid = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/agent-b",
"inproc://remote-mob/worker/agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
let receipt = handle
.respawn(AgentIdentity::from("l-1"), Some("resume".into()))
.await
.expect("respawn");
let entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member should exist");
let new_sid = entry
.member_ref
.bridge_session_id()
.cloned()
.expect("respawn should produce replacement session");
assert_ne!(new_sid, old_sid, "respawn should replace the session");
let trusted = service.trusted_peer_names(&new_sid).await;
assert!(
trusted.iter().any(|n| n == external.name.as_str()),
"respawn should restore external trust from the stored roster spec"
);
assert!(
!entry
.external_peer_specs
.contains_key(&AgentIdentity::from(external.name.as_str())),
"respawn trust repair must not synthesize external roster projection"
);
assert_eq!(entry.agent_runtime_id, receipt.agent_runtime_id);
assert_eq!(entry.fence_token, receipt.fence_token);
}
#[tokio::test]
async fn test_respawn_repairs_external_machine_edge_without_roster_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let old_sid = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-machine"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/machine-agent",
"inproc://remote-mob/worker/machine-agent",
);
let machine_local = crate::machines::mob_machine::AgentIdentity::from("l-machine");
let machine_key = crate::machines::mob_machine::ExternalPeerKey::new(
machine_local.clone(),
crate::machines::mob_machine::PeerName::from(external.name.as_str()),
);
let machine_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
machine_local,
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::WireExternalPeer {
key: machine_key,
edge: machine_edge,
},
)
.await
.expect("seed machine-owned external edge without roster projection");
let entry_before = handle
.get_member(&AgentIdentity::from("l-machine"))
.await
.unwrap()
.expect("member before respawn");
assert!(
!entry_before
.external_peer_specs
.contains_key(&AgentIdentity::from(external.name.as_str())),
"test setup must leave roster projection empty so MobMachine is the only topology source"
);
let receipt = handle
.respawn(AgentIdentity::from("l-machine"), Some("resume".into()))
.await
.expect("respawn");
let entry = handle
.get_member(&AgentIdentity::from("l-machine"))
.await
.unwrap()
.expect("member should exist");
let new_sid = entry
.member_ref
.bridge_session_id()
.cloned()
.expect("respawn should produce replacement session");
assert_ne!(new_sid, old_sid, "respawn should replace the session");
let trusted = service.trusted_peer_names(&new_sid).await;
assert!(
trusted.iter().any(|n| n == external.name.as_str()),
"respawn should restore external trust from MobMachine topology"
);
assert!(
!entry
.external_peer_specs
.contains_key(&AgentIdentity::from(external.name.as_str())),
"respawn trust repair must not synthesize external roster projection"
);
assert_eq!(entry.agent_runtime_id, receipt.agent_runtime_id);
assert_eq!(entry.fence_token, receipt.fence_token);
}
#[tokio::test]
async fn test_unwire_external_removes_trust_and_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external = test_trusted_peer_descriptor(
"remote-mob/worker/agent-b",
"inproc://remote-mob/worker/agent-b",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
handle
.unwire(
AgentIdentity::from("l-1"),
AgentIdentity::from(external.name.as_str()),
)
.await
.expect("unwire external");
let entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member should exist");
assert!(
!entry
.wired_to
.contains(&AgentIdentity::from(external.name.as_str()))
);
assert!(
!entry
.external_peer_specs
.contains_key(&AgentIdentity::from(external.name.as_str()))
);
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
!trusted.iter().any(|n| n == external.name.as_str()),
"trusted peer list should not include the removed external peer"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
assert!(
!dsl.external_peer_edges.contains(&external_edge),
"external unwire should remove the machine-owned descriptor edge"
);
assert!(
dsl.wiring_edges
.iter()
.all(|edge| edge.a.0 != external.name.as_str() && edge.b.0 != external.name.as_str()),
"external unwire should not touch member wiring_edges"
);
}
#[tokio::test]
async fn test_retire_unwires_external_peer_edge_before_releasing_member_peer() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/retire-agent",
"inproc://remote-mob/worker/retire-agent",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
let external_key = crate::machines::mob_machine::ExternalPeerKey::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::PeerName::from(external.name.as_str()),
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
let dsl_before = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot before retire");
assert!(
dsl_before.external_peer_edges.contains(&external_edge),
"test setup must seed MobMachine external edge"
);
handle
.retire(AgentIdentity::from("l-1"))
.await
.expect("retire should clean external peer edge through MobMachine");
let dsl_after = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot after retire");
assert!(
!dsl_after.external_peer_edges.contains(&external_edge),
"retire should remove descriptor-bearing external edge through generated external unwire"
);
assert!(
!dsl_after
.external_peer_edges_by_key
.contains_key(&external_key),
"retire should remove external edge key through generated external unwire"
);
let events = handle.events().replay_all().await.expect("replay events");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::ExternalPeerUnwired { local, peer_name }
if local == &AgentIdentity::from("l-1")
&& peer_name.as_str() == external.name.as_str()
)
}),
"retire should persist the external unwire event before member peer material is released"
);
}
#[tokio::test]
async fn test_stopped_retire_unwires_external_peer_edge_before_terminal_event() {
let (handle, service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("stopped-external-owner");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn external-edge owner")
.bridge_session_id()
.cloned()
.expect("session-backed owner");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/stopped-retire-agent",
"inproc://remote-mob/worker/stopped-retire-agent",
);
handle
.wire(identity.clone(), PeerTarget::External(external.clone()))
.await
.expect("wire external peer before stop");
handle.stop().await.expect("stop mob");
handle
.retire(identity.clone())
.await
.expect("stopped retire must admit external-edge cleanup");
assert!(
!service
.trusted_peer_names(&session_id)
.await
.iter()
.any(|name| name == external.name.as_str()),
"external trust must be removed before stopped retirement completes"
);
let events = handle.events().replay_all().await.expect("replay events");
let unwired_index = events
.iter()
.position(|event| {
matches!(
&event.kind,
MobEventKind::ExternalPeerUnwired { local, peer_name }
if local == &identity && peer_name.as_str() == external.name.as_str()
)
})
.expect("external edge cleanup event");
let retired_index = events
.iter()
.position(|event| {
matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
)
})
.expect("terminal retirement event");
assert!(
unwired_index < retired_index,
"external edge cleanup must be durable before terminal retirement"
);
}
#[tokio::test]
async fn test_retire_external_unwire_append_failure_does_not_persist_member_retired() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/retire-fail-agent",
"inproc://remote-mob/worker/retire-fail-agent",
);
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external");
events.fail_appends_for("ExternalPeerUnwired").await;
let result = handle.retire(AgentIdentity::from("l-1")).await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"retire should surface external unwire append failure; got {result:?}"
);
let recorded = handle.events().replay_all().await.expect("replay events");
assert!(
!recorded
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberRetired { .. })),
"fallible external cleanup must complete before durable MemberRetired is written"
);
assert!(
handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.is_some(),
"failed pre-retire external cleanup must leave member retryable"
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
let dsl_after = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot after failed retire");
assert!(
dsl_after.external_peer_edges.contains(&external_edge),
"external unwire append failure should rollback generated external edge removal"
);
}
#[tokio::test]
async fn test_retire_external_unwire_failure_without_local_comms_survives_cold_restart() {
let definition = with_unique_mob_id(sample_definition(), "external-unwire-cold-retry");
let events = Arc::new(FaultInjectedMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
definition,
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone()),
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("external-cold-owner");
let owner_session = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn external-edge owner")
.bridge_session_id()
.cloned()
.expect("session-backed external-edge owner");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/external-cold-peer",
"inproc://remote-mob/worker/external-cold-peer",
);
handle
.wire(identity.clone(), PeerTarget::External(external.clone()))
.await
.expect("wire external peer");
events.fail_appends_for("ExternalPeerUnwired").await;
handle
.retire(identity.clone())
.await
.expect_err("external unwind append failure must retain retirement authority");
drop(handle);
events.allow_appends_for("ExternalPeerUnwired").await;
service.set_missing_comms_runtime(&owner_session).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume retiring member without rebuilding its released comms binding");
resumed
.retire(identity.clone())
.await
.expect("cold retry must converge machine external topology without local comms");
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from_domain(&identity),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
assert!(
!resumed
.debug_dsl_t2_snapshot()
.await
.expect("query converged external topology")
.external_peer_edges
.contains(&external_edge)
);
let completed = events
.replay_all()
.await
.expect("replay external cold retry");
assert!(completed.iter().any(|event| matches!(
&event.kind,
MobEventKind::ExternalPeerUnwired { local, peer_name }
if local == &identity && peer_name.as_str() == external.name.as_str()
)));
assert_eq!(
completed
.iter()
.filter(|event| matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &identity
))
.count(),
1
);
}
const ED25519_PUBLIC_KEY_7: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
const ED25519_PUBLIC_KEY_ZERO: &str = "ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
fn external_binding_peer_target(public_key: &str) -> PeerTarget {
PeerTarget::ExternalBinding(ExternalPeerBindingSpec::new(
"external-worker",
"inproc://external-worker",
meerkat_contracts::WireTrustedPeerIdentity::Ed25519PublicKey {
public_key: public_key.to_string(),
},
))
}
#[tokio::test]
async fn test_wire_external_binding_resolves_inside_mob_authority() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-binding"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-binding"),
external_binding_peer_target(ED25519_PUBLIC_KEY_7),
)
.await
.expect("wire external binding");
let entry = handle
.get_member(&AgentIdentity::from("l-binding"))
.await
.unwrap()
.expect("member should exist");
let descriptor = entry
.external_peer_specs
.get(&AgentIdentity::from("external-worker"))
.expect("external binding should project descriptor");
let expected_pubkey = [7u8; 32];
assert_eq!(descriptor.pubkey, expected_pubkey);
assert_eq!(
descriptor.peer_id,
PeerId::from_ed25519_pubkey(&expected_pubkey)
);
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
trusted.iter().any(|name| name == "external-worker"),
"mob authority should install trust after resolving external binding"
);
}
#[tokio::test]
async fn test_wire_external_binding_rejects_zero_pubkey_before_trust_install() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-zero"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let err = handle
.wire(
AgentIdentity::from("l-zero"),
external_binding_peer_target(ED25519_PUBLIC_KEY_ZERO),
)
.await
.expect_err("zero pubkey external binding must fail closed");
let message = err.to_string();
assert!(
message.contains("public_key") || message.contains("non-zero"),
"expected pubkey validation error, got: {message}"
);
let entry = handle
.get_member(&AgentIdentity::from("l-zero"))
.await
.unwrap()
.expect("member should exist");
assert!(
!entry
.external_peer_specs
.contains_key(&AgentIdentity::from("external-worker")),
"failed external binding must not project an external edge"
);
let trusted = service.trusted_peer_names(&sid_l).await;
assert!(
!trusted.iter().any(|name| name == "external-worker"),
"failed external binding must not install comms trust"
);
}
#[tokio::test]
async fn test_resume_seeds_mob_machine_topology_from_event_projection() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let definition = sample_definition();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-resume"),
None,
)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-resume"),
None,
)
.await
.expect("spawn worker");
handle
.wire(
AgentIdentity::from("l-resume"),
AgentIdentity::from("w-resume"),
)
.await
.expect("wire local peers");
let external = test_trusted_peer_descriptor(
"remote-mob/worker/resume-agent",
"inproc://remote-mob/worker/resume-agent",
);
handle
.wire(
AgentIdentity::from("l-resume"),
PeerTarget::External(external.clone()),
)
.await
.expect("wire external peer");
handle.stop().await.expect("stop before resume");
let restarted_service = Arc::new(service.cold_restart_preserving_durable_state().await);
let _ = restarted_service.enable_runtime_adapter();
drop(handle);
drop(service);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(restarted_service)
.resume()
.await
.expect("resume mob");
let dsl = resumed
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
let local_edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-resume"),
crate::machines::mob_machine::AgentIdentity::from("w-resume"),
);
assert!(
dsl.wiring_edges.contains(&local_edge),
"resume must seed machine-owned local topology from event projection"
);
let external_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-resume"),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&external),
);
assert!(
dsl.external_peer_edges.contains(&external_edge),
"resume must seed machine-owned external topology from event projection"
);
}
#[tokio::test]
async fn test_wire_emits_peers_wired_event() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let events = handle.events().replay_all().await.expect("replay");
let wired = events
.iter()
.find(|e| matches!(e.kind, MobEventKind::MembersWired { .. }));
assert!(wired.is_some(), "should have MembersWired event");
}
#[tokio::test]
async fn test_wire_members_batch_materializes_dense_topology_once() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker one")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn worker two");
let report = handle
.wire_members_batch([
(AgentIdentity::from("l-1"), AgentIdentity::from("w-1")),
(AgentIdentity::from("w-2"), AgentIdentity::from("l-1")),
(AgentIdentity::from("l-1"), AgentIdentity::from("w-1")),
])
.await
.expect("batch wire");
assert_eq!(report.requested, 3);
assert_eq!(report.wired.len(), 2);
assert!(report.already_wired.is_empty());
let lead = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead projected");
assert_eq!(lead.wired_to.len(), 2);
assert!(lead.wired_to.contains(&AgentIdentity::from("w-1")));
assert!(lead.wired_to.contains(&AgentIdentity::from("w-2")));
let worker = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("worker projected");
assert!(worker.wired_to.contains(&AgentIdentity::from("l-1")));
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
assert!(
trusted_by_lead
.iter()
.any(|name| name.as_str() == test_comms_name("worker", "w-1"))
);
assert!(
trusted_by_lead
.iter()
.any(|name| name.as_str() == test_comms_name("worker", "w-2"))
);
let trusted_by_worker = service.trusted_peer_names(&sid_w1).await;
assert!(
trusted_by_worker
.iter()
.any(|name| name.as_str() == test_comms_name("lead", "l-1"))
);
let events = handle.events().replay_all().await.expect("replay");
assert_eq!(
events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. }))
.count(),
1,
"batch topology materialization should emit one compact event"
);
assert_eq!(
events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWired { .. }))
.count(),
0,
"batch topology materialization must not fan out per-edge events"
);
let second = handle
.wire_members_batch([
(AgentIdentity::from("l-1"), AgentIdentity::from("w-1")),
(AgentIdentity::from("l-1"), AgentIdentity::from("w-2")),
])
.await
.expect("idempotent batch wire");
assert_eq!(second.requested, 2);
assert_eq!(second.already_wired.len(), 2);
assert!(second.wired.is_empty());
}
#[tokio::test]
async fn test_wire_members_batch_append_failure_does_not_publish_machine_topology() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MembersWiredBatch").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let result = handle
.wire_members_batch([(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))])
.await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"batch wire should surface append failure"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::AgentIdentity::from("w-1"),
);
assert!(
!dsl.wiring_edges.contains(&edge),
"failed batch append must not publish MobMachine wiring"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead projected");
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("worker projected");
assert!(
entry_l.wired_to.is_empty() && entry_w.wired_to.is_empty(),
"failed batch append must not update roster wiring"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed batch append must cleanup lead trust"
);
assert!(
!trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed batch append must cleanup worker trust"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. })),
"failed batch append must not persist MembersWiredBatch"
);
}
#[tokio::test]
async fn test_wire_members_batch_trust_failure_surfaces_error_and_retry_repairs() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_add_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.wire_members_batch([(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))])
.await
.expect_err("post-commit trust delivery failure must not report batch wire success");
assert!(
error.to_string().contains("mock add_trusted_peer failure"),
"batch wire should surface the generated-authority trust mutation failure, got: {error}"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("debug dsl snapshot");
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from("l-1"),
crate::machines::mob_machine::AgentIdentity::from("w-1"),
);
assert!(
dsl.wiring_edges.contains(&edge),
"trust delivery is attempted only after the generated edge commits"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. })),
"durable batch marker must exist before trust delivery is attempted"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed batch trust delivery should rollback lead-side trust installed before the reciprocal failure"
);
assert!(
!trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"configured worker trust failure should not install worker-side trust"
);
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior::default(),
)
.await;
let retry = handle
.wire_members_batch([(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))])
.await
.expect("retry should repair already-wired batch trust through generated authority");
assert!(retry.wired.is_empty());
assert_eq!(
retry.already_wired.len(),
1,
"retry should classify the topology edge as already wired while repairing trust"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"retry should restore worker trust on the lead runtime"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"retry should restore lead trust on the worker runtime"
);
let batch_events = handle
.events()
.replay_all()
.await
.expect("replay")
.into_iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. }))
.count();
assert_eq!(
batch_events, 1,
"retry repair must not append a replacement MembersWiredBatch event"
);
}
#[tokio::test]
async fn test_wire_members_batch_repair_failure_preserves_preexisting_trust() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire_members_batch([(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))])
.await
.expect("initial batch wire");
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_add_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.wire_members_batch([(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))])
.await
.expect_err("failed repair must still surface trust mutation failure");
assert!(
error.to_string().contains("mock add_trusted_peer failure"),
"repair should surface the generated-authority trust mutation failure, got: {error}"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"repair rollback must not remove lead trust that existed before this attempt"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed repair must preserve worker trust that existed before this attempt"
);
let batch_events = handle
.events()
.replay_all()
.await
.expect("replay")
.into_iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. }))
.count();
assert_eq!(
batch_events, 1,
"failed repair must not append another MembersWiredBatch event"
);
}
#[cfg_attr(
target_os = "linux",
ignore = "tracked in #904: run the dense topology stress on a reliable dedicated lane"
)]
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_wire_members_batch_materializes_300_by_150_dense_topology_in_seconds() {
const AGENTS: usize = 300;
const PEERS_PER_AGENT: usize = 150;
const EDGE_OFFSETS: usize = PEERS_PER_AGENT / 2;
const EXPECTED_EDGES: usize = AGENTS * PEERS_PER_AGENT / 2;
let (handle, _service) = create_test_mob(sample_definition()).await;
let identities = (0..AGENTS)
.map(|index| AgentIdentity::from(format!("agent-{index:03}")))
.collect::<Vec<_>>();
let specs = identities
.iter()
.map(|identity| SpawnMemberSpec::new("worker", identity.as_str()))
.collect::<Vec<_>>();
let spawn_started = std::time::Instant::now();
let spawned = handle
.spawn_many(specs)
.await
.expect("spawn_many dense topology authority");
let spawn_elapsed = spawn_started.elapsed();
assert_eq!(spawned.len(), AGENTS);
for result in spawned {
result.expect("spawn_many dense topology member");
}
let mut edges = Vec::with_capacity(EXPECTED_EDGES);
for index in 0..AGENTS {
for offset in 1..=EDGE_OFFSETS {
edges.push((
identities[index].clone(),
identities[(index + offset) % AGENTS].clone(),
));
}
}
assert_eq!(edges.len(), EXPECTED_EDGES);
let wire_started = std::time::Instant::now();
let report = handle
.wire_members_batch(edges)
.await
.expect("batch wire dense topology");
let wire_elapsed = wire_started.elapsed();
assert_eq!(report.requested, EXPECTED_EDGES);
assert_eq!(report.wired.len(), EXPECTED_EDGES);
assert!(report.already_wired.is_empty());
for identity in &identities {
let member = handle
.get_member(identity)
.await
.unwrap()
.expect("dense topology member projected");
assert_eq!(
member.wired_to.len(),
PEERS_PER_AGENT,
"{identity} should have exactly {PEERS_PER_AGENT} peers"
);
}
let events = handle.events().replay_all().await.expect("replay");
let batch_events = events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWiredBatch { .. }))
.count();
let batch_event_edges = events
.iter()
.filter_map(|event| match &event.kind {
MobEventKind::MembersWiredBatch { edges } => Some(edges.len()),
_ => None,
})
.sum::<usize>();
let single_edge_events = events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MembersWired { .. }))
.count();
assert_eq!(batch_events, 1);
assert_eq!(
batch_event_edges, EXPECTED_EDGES,
"dense topology materialization should publish every new edge through the compact batch event"
);
assert_eq!(
single_edge_events, 0,
"dense topology materialization must not emit per-edge wire events"
);
// GitHub-hosted x86 runners are materially slower than the isolated
// BuildBuddy lane (observed at roughly 239s versus 145s). Keep the test
// isolated through nextest and allow a portable budget that still catches
// a greater-than-2x regression from the isolated baseline.
let max_wire_elapsed = std::time::Duration::from_secs(300);
assert!(
wire_elapsed < max_wire_elapsed,
"300x150 topology materialization should stay inside the generated-authority stress budget of {max_wire_elapsed:?} (spawn={spawn_elapsed:?}, wire={wire_elapsed:?})"
);
eprintln!(
"dense topology stress: agents={AGENTS}, edges={EXPECTED_EDGES}, spawn={spawn_elapsed:?}, wire={wire_elapsed:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_retire_fanout_notifies_150_peers_with_bounded_parallelism() {
const PEERS: usize = 150;
const PER_NOTIFICATION_DELAY_MS: u64 = 20;
let (handle, service) = create_test_mob(sample_definition()).await;
let retiring = AgentIdentity::from("retiring");
let retiring_sid = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("retiring"),
None,
)
.await
.expect("spawn retiring member")
.bridge_session_id()
.expect("session-backed retiring member")
.clone();
let peer_identities = (0..PEERS)
.map(|index| AgentIdentity::from(format!("peer-{index:03}")))
.collect::<Vec<_>>();
let specs = peer_identities
.iter()
.map(|identity| SpawnMemberSpec::new("worker", identity.as_str()))
.collect::<Vec<_>>();
let spawned = handle
.spawn_many(specs)
.await
.expect("spawn_many peer authority");
for result in spawned {
result.expect("spawn peer");
}
let mut peer_session_ids = Vec::with_capacity(PEERS);
for identity in &peer_identities {
peer_session_ids.push(
handle
.resolve_bridge_session_id(identity)
.await
.expect("peer bridge session id"),
);
}
handle
.wire_members_batch(
peer_identities
.iter()
.map(|peer| (retiring.clone(), peer.clone())),
)
.await
.expect("wire retiring member to peers");
service
.set_comms_behavior(
&test_comms_name("worker", "retiring"),
MockCommsBehavior {
peer_lifecycle_delay_ms: PER_NOTIFICATION_DELAY_MS,
..MockCommsBehavior::default()
},
)
.await;
let started = Instant::now();
handle
.retire(retiring.clone())
.await
.expect("retire high-degree member");
let elapsed = started.elapsed();
let max_concurrent = service
.max_concurrent_peer_lifecycle_sends(&retiring_sid)
.await;
assert!(
max_concurrent > 1,
"150 peer-retired notifications with {PER_NOTIFICATION_DELAY_MS}ms send delay should overlap; max observed concurrent sends: {max_concurrent}"
);
assert!(
max_concurrent <= super::actor::RETIRE_LOCAL_TRUST_CLEANUP_CONCURRENCY as u64,
"retire notification/trust cleanup concurrency must stay bounded at {} jobs; max observed concurrent sends: {max_concurrent}",
super::actor::RETIRE_LOCAL_TRUST_CLEANUP_CONCURRENCY
);
assert!(
elapsed < Duration::from_secs(10),
"150 peer-retired notifications with {PER_NOTIFICATION_DELAY_MS}ms send delay should stay comfortably below a serialized stress timeout (elapsed={elapsed:?}, max_concurrent={max_concurrent})"
);
eprintln!(
"retire fanout stress: peers={PEERS}, per_notification_delay_ms={PER_NOTIFICATION_DELAY_MS}, retire={elapsed:?}, max_concurrent={max_concurrent}"
);
let retiring_intents = service.sent_intents(&retiring_sid).await;
assert_eq!(
retiring_intents
.iter()
.filter(|intent| intent.as_str() == "mob.peer_retired")
.count(),
PEERS,
"retiring member must send one peer-retired notice per wired peer"
);
for (identity, session_id) in peer_identities.iter().zip(peer_session_ids.iter()) {
let member = handle
.get_member(identity)
.await
.unwrap()
.expect("peer should remain active");
assert!(
!member.wired_to.contains(&retiring),
"{identity} should no longer project the retired peer"
);
let trusted = service.trusted_peer_names(session_id).await;
assert!(
!trusted
.iter()
.any(|name| name == &test_comms_name("worker", "retiring")),
"{identity} should remove trust for the retired peer"
);
}
}
#[tokio::test]
async fn test_wire_is_idempotent_and_emits_single_pair_event() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("first wire");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("second wire should reconcile as idempotent");
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert_eq!(
entry_l.wired_to.len(),
1,
"idempotent wire must preserve one canonical edge on the lead"
);
assert_eq!(
entry_w.wired_to.len(),
1,
"idempotent wire must preserve one canonical edge on the worker"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert_eq!(
trusted_by_lead
.iter()
.filter(|name| name.as_str() == test_comms_name("worker", "w-1"))
.count(),
1,
"idempotent wire must not duplicate trusted-peer state on the lead"
);
assert_eq!(
trusted_by_worker
.iter()
.filter(|name| name.as_str() == test_comms_name("lead", "l-1"))
.count(),
1,
"idempotent wire must not duplicate trusted-peer state on the worker"
);
let events = handle.events().replay_all().await.expect("replay");
let pair_wired_events = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MembersWired { a, b }
if (a == &AgentIdentity::from("l-1") && b == &AgentIdentity::from("w-1"))
|| (a == &AgentIdentity::from("w-1") && b == &AgentIdentity::from("l-1"))
)
})
.count();
assert_eq!(
pair_wired_events, 1,
"idempotent wire must emit one logical MembersWired event for one canonical edge"
);
}
#[tokio::test]
async fn test_rewire_repairs_local_trust_for_existing_machine_edge() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("initial wire");
service.force_remove_trust(&sid_l, &sid_w).await;
service.force_remove_trust(&sid_w, &sid_l).await;
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("rewire should repair both local trust edges");
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead
.iter()
.any(|name| name.as_str() == test_comms_name("worker", "w-1")),
"rewire should restore worker trust on the lead runtime"
);
assert!(
trusted_by_worker
.iter()
.any(|name| name.as_str() == test_comms_name("lead", "l-1")),
"rewire should restore lead trust on the worker runtime"
);
let events = handle.events().replay_all().await.expect("replay");
let pair_wired_events = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MembersWired { a, b }
if (a == &AgentIdentity::from("l-1") && b == &AgentIdentity::from("w-1"))
|| (a == &AgentIdentity::from("w-1") && b == &AgentIdentity::from("l-1"))
)
})
.count();
assert_eq!(
pair_wired_events, 1,
"trust repair for an existing machine edge must not duplicate the logical wire event"
);
}
#[tokio::test]
async fn test_rewire_local_repair_failure_rolls_back_first_trust_side() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("initial wire");
service.force_remove_trust(&sid_l, &sid_w).await;
service.force_remove_trust(&sid_w, &sid_l).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_add_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
result.is_err(),
"repair should surface reciprocal trust install failure"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted_by_lead
.iter()
.any(|name| name.as_str() == test_comms_name("worker", "w-1")),
"failed repair must rollback trust installed on the first side"
);
assert!(
!trusted_by_worker
.iter()
.any(|name| name.as_str() == test_comms_name("lead", "l-1")),
"failed repair must not leave reciprocal trust on the failing side"
);
let events = handle.events().replay_all().await.expect("replay");
let pair_wired_events = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MembersWired { a, b }
if (a == &AgentIdentity::from("l-1") && b == &AgentIdentity::from("w-1"))
|| (a == &AgentIdentity::from("w-1") && b == &AgentIdentity::from("l-1"))
)
})
.count();
assert_eq!(
pair_wired_events, 1,
"failed trust repair must not append a replacement MembersWired event"
);
}
#[tokio::test]
async fn test_rewire_local_repair_failure_preserves_preexisting_trust() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("initial wire");
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_add_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
result.is_err(),
"repair should surface reciprocal trust install failure"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead
.iter()
.any(|name| name.as_str() == test_comms_name("worker", "w-1")),
"failed repair must preserve lead trust that existed before this attempt"
);
assert!(
trusted_by_worker
.iter()
.any(|name| name.as_str() == test_comms_name("lead", "l-1")),
"failed repair must preserve worker trust that existed before this attempt"
);
}
#[tokio::test]
async fn test_rewire_local_machine_edge_without_roster_projection_repairs_trust_only() {
let (handle, service) = create_test_mob(sample_definition()).await;
let left = AgentIdentity::from("machine-repair-left");
let right = AgentIdentity::from("machine-repair-right");
let sid_left = handle
.spawn(ProfileName::from("lead"), left.clone(), None)
.await
.expect("spawn left")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_right = handle
.spawn(ProfileName::from("worker"), right.clone(), None)
.await
.expect("spawn right")
.bridge_session_id()
.expect("session-backed")
.clone();
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from(left.as_str()),
crate::machines::mob_machine::AgentIdentity::from(right.as_str()),
);
handle
.project_machine_input(crate::machines::mob_machine::MobMachineInput::WireMembers { edge })
.await
.expect("seed machine-owned edge without roster projection");
handle
.wire(
AgentIdentity::from(left.as_str()),
PeerTarget::Local(AgentIdentity::from(right.as_str())),
)
.await
.expect("machine-edge rewire should repair trust only");
let trusted_by_left = service.trusted_peer_names(&sid_left).await;
let trusted_by_right = service.trusted_peer_names(&sid_right).await;
assert!(
trusted_by_left
.iter()
.any(|name| name.as_str() == test_comms_name("worker", right.as_str())),
"machine-edge rewire should install right trust from generated repair authority"
);
assert!(
trusted_by_right
.iter()
.any(|name| name.as_str() == test_comms_name("lead", left.as_str())),
"machine-edge rewire should install left trust from generated repair authority"
);
let left_entry = handle
.get_member(&AgentIdentity::from(left.as_str()))
.await
.unwrap()
.expect("left member");
let right_entry = handle
.get_member(&AgentIdentity::from(right.as_str()))
.await
.unwrap()
.expect("right member");
assert!(
left_entry
.wired_to
.contains(&AgentIdentity::from(right.as_str())),
"trust repair must expose machine-owned left projection"
);
assert!(
right_entry
.wired_to
.contains(&AgentIdentity::from(left.as_str())),
"trust repair must expose machine-owned right projection"
);
let status = handle
.member_status(&AgentIdentity::from(left.as_str()))
.await
.expect("left member status");
let meerkat_contracts::WirePeerConnectivity::Known {
snapshot: connectivity,
} = status
.peer_connectivity
.expect("left member should have live peer connectivity")
else {
panic!("left member should resolve a Known peer-connectivity snapshot");
};
assert_eq!(
connectivity.unknown_peer_count, 1,
"peer connectivity projection must count machine-owned wiring"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|event| matches!(event.kind, MobEventKind::MembersWired { .. })),
"trust repair must not append MembersWired without a generated graph-change effect"
);
}
#[tokio::test]
async fn test_retire_cleans_machine_edge_without_roster_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let lead = AgentIdentity::from("retire-machine-lead");
let worker = AgentIdentity::from("retire-machine-worker");
let lead_sid = handle
.spawn(ProfileName::from("lead"), lead.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed lead")
.clone();
let worker_sid = handle
.spawn(ProfileName::from("worker"), worker.clone(), None)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed worker")
.clone();
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from(lead.as_str()),
crate::machines::mob_machine::AgentIdentity::from(worker.as_str()),
);
handle
.project_machine_input(crate::machines::mob_machine::MobMachineInput::WireMembers {
edge: edge.clone(),
})
.await
.expect("seed machine-owned edge without roster projection");
handle
.wire(
AgentIdentity::from(lead.as_str()),
PeerTarget::Local(AgentIdentity::from(worker.as_str())),
)
.await
.expect("machine-edge rewire should repair trust only");
let events_before_retire = handle.events().replay_all().await.expect("replay");
assert!(
!events_before_retire
.iter()
.any(|event| matches!(event.kind, MobEventKind::MembersWired { .. })),
"test setup must not create a roster wiring event"
);
assert!(
service
.trusted_peer_names(&lead_sid)
.await
.contains(&test_comms_name("worker", worker.as_str())),
"setup should install worker trust on lead"
);
assert!(
service
.trusted_peer_names(&worker_sid)
.await
.contains(&test_comms_name("lead", lead.as_str())),
"setup should install lead trust on worker"
);
handle
.retire(AgentIdentity::from(worker.as_str()))
.await
.expect("retire worker");
let lead_trust_after_retire = service.trusted_peer_names(&lead_sid).await;
assert!(
!lead_trust_after_retire.contains(&test_comms_name("worker", worker.as_str())),
"retire cleanup must derive peer trust removal from MobMachine wiring, not roster projection"
);
let dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("dsl snapshot after retire");
assert!(
!dsl.wiring_edges.contains(&edge),
"retire must remove the machine-owned edge"
);
}
#[tokio::test]
async fn test_wire_unknown_meerkat_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
let result = handle
.wire(
AgentIdentity::from("w-1"),
AgentIdentity::from("nonexistent"),
)
.await;
assert!(matches!(result, Err(MobError::MemberNotFound(_))));
}
#[tokio::test]
async fn test_wire_fails_when_comms_runtime_missing_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
let _sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_missing_comms_runtime(&sid_w).await;
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(matches!(result, Err(MobError::WiringError(_))));
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty(),
"failed wire must not update roster"
);
assert!(
entry_w.wired_to.is_empty(),
"failed wire must not update roster"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersWired { .. })),
"failed wire must not emit MembersWired"
);
}
#[tokio::test]
async fn test_wire_fails_when_public_key_missing_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(matches!(result, Err(MobError::WiringError(_))));
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty(),
"failed wire must not update roster"
);
assert!(
entry_w.wired_to.is_empty(),
"failed wire must not update roster"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersWired { .. })),
"failed wire must not emit MembersWired"
);
}
#[tokio::test]
async fn test_wire_fails_when_peer_added_notification_fails_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_send_peer_added: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::CommsError(_))),
"wire should fail when required notification fails"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty(),
"failed wire must not mutate roster"
);
assert!(
entry_w.wired_to.is_empty(),
"failed wire must not mutate roster"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed wire must rollback lead trust"
);
assert!(
!trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed wire must rollback worker trust"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersWired { .. })),
"failed wire must not emit MembersWired"
);
}
#[tokio::test]
async fn test_wire_establishes_comms_trust() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire should succeed");
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"lead runtime must trust worker runtime"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"worker runtime must trust lead runtime"
);
let intents_lead = service.sent_intents(&sid_l).await;
let intents_worker = service.sent_intents(&sid_w).await;
assert!(
intents_lead.iter().any(|intent| intent == "mob.peer_added"),
"lead should send mob.peer_added during wire"
);
assert!(
intents_worker
.iter()
.any(|intent| intent == "mob.peer_added"),
"worker should send mob.peer_added during wire"
);
}
#[tokio::test]
async fn test_wire_append_failure_rolls_back_runtime_state() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MembersWired").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let result = handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"wire should surface append failure"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty(),
"wire append failure must rollback roster wiring"
);
assert!(
entry_w.wired_to.is_empty(),
"wire append failure must rollback roster wiring"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"wire append failure must rollback trust on lead"
);
assert!(
!trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"wire append failure must rollback trust on worker"
);
let recorded = handle.events().replay_all().await.expect("replay");
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersWired { .. })),
"failed wire append must not persist MembersWired"
);
}
// -----------------------------------------------------------------------
// P1-T07: unwire() removes bidirectional trust
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_unwire_removes_bidirectional() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("unwire");
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
assert!(entry_l.wired_to.is_empty());
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(entry_w.wired_to.is_empty());
}
#[tokio::test]
async fn test_unwire_emits_peers_unwired_event() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("unwire");
let events = handle.events().replay_all().await.expect("replay");
let unwired = events
.iter()
.find(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. }));
assert!(unwired.is_some(), "should have MembersUnwired event");
}
#[tokio::test]
async fn test_unwire_is_idempotent_and_emits_single_pair_event() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("first unwire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("second unwire should be idempotent");
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty() && entry_w.wired_to.is_empty(),
"idempotent unwire must preserve a symmetric empty wiring projection"
);
let events = handle.events().replay_all().await.expect("replay");
let pair_unwired_events = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MembersUnwired { a, b }
if (a == &AgentIdentity::from("l-1") && b == &AgentIdentity::from("w-1"))
|| (a == &AgentIdentity::from("w-1") && b == &AgentIdentity::from("l-1"))
)
})
.count();
assert_eq!(
pair_unwired_events, 1,
"idempotent unwire must emit one logical MembersUnwired event for one canonical edge removal"
);
}
#[tokio::test]
async fn test_unwire_fails_when_comms_runtime_missing_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
service.set_missing_comms_runtime(&sid_w).await;
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(matches!(result, Err(MobError::WiringError(_))));
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must not mutate roster"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must not mutate roster"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire must not emit MembersUnwired"
);
}
#[tokio::test]
async fn test_unwire_fails_when_public_key_missing_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
service.clear_public_key(&sid_w).await;
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(matches!(result, Err(MobError::WiringError(_))));
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must not mutate roster"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must not mutate roster"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire must not emit MembersUnwired"
);
}
#[tokio::test]
async fn test_unwire_second_trust_removal_failure_restores_first_side() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_remove_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::CommsError(_))),
"unwire should surface second trust-removal failure"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must keep roster wiring on lead"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must keep roster wiring on worker"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed unwire must restore first trust removal on lead"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed unwire must keep worker trust unchanged"
);
}
#[tokio::test]
async fn test_unwire_fails_when_peer_unwired_notification_fails_without_side_effects() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_unwired: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::CommsError(_))),
"unwire should fail when required notification fails"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must not mutate roster"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must not mutate roster"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire must not emit MembersUnwired"
);
}
#[tokio::test]
async fn test_unwire_second_notification_failure_compensates_and_preserves_state() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_send_peer_unwired: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let peer_added_count = |intents: &[String]| {
intents
.iter()
.filter(|intent| intent.as_str() == "mob.peer_added")
.count()
};
let lead_peer_added_before = peer_added_count(&service.sent_intents(&sid_l).await);
let worker_peer_added_before = peer_added_count(&service.sent_intents(&sid_w).await);
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::CommsError(_))),
"unwire should fail when second required notification fails"
);
let lead_peer_added_after = peer_added_count(&service.sent_intents(&sid_l).await);
let worker_peer_added_after = peer_added_count(&service.sent_intents(&sid_w).await);
assert_eq!(
lead_peer_added_after,
lead_peer_added_before + 1,
"second-notification failure should compensate with mob.peer_added from lead"
);
assert_eq!(
worker_peer_added_after, worker_peer_added_before,
"worker should not emit extra mob.peer_added on second-notification failure"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must preserve lead wiring"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must preserve worker wiring"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed unwire must keep lead trust"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed unwire must keep worker trust"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire must not emit MembersUnwired"
);
}
#[tokio::test]
async fn test_unwire_first_trust_removal_failure_compensates_and_preserves_state() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_remove_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let peer_added_count = |intents: &[String]| {
intents
.iter()
.filter(|intent| intent.as_str() == "mob.peer_added")
.count()
};
let lead_peer_added_before = peer_added_count(&service.sent_intents(&sid_l).await);
let worker_peer_added_before = peer_added_count(&service.sent_intents(&sid_w).await);
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::CommsError(_))),
"unwire should surface first trust-removal failure"
);
let lead_peer_added_after = peer_added_count(&service.sent_intents(&sid_l).await);
let worker_peer_added_after = peer_added_count(&service.sent_intents(&sid_w).await);
assert_eq!(
lead_peer_added_after,
lead_peer_added_before + 1,
"first-removal failure should compensate with mob.peer_added from lead"
);
assert_eq!(
worker_peer_added_after,
worker_peer_added_before + 1,
"first-removal failure should compensate with mob.peer_added from worker"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"failed unwire must preserve lead wiring"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"failed unwire must preserve worker wiring"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"failed unwire must keep lead trust"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"failed unwire must keep worker trust"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire must not emit MembersUnwired"
);
}
#[tokio::test]
async fn test_unwire_append_failure_restores_runtime_state() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MembersUnwired").await;
let (handle, service) = create_test_mob_with_events(sample_definition(), events).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let result = handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"unwire should surface append failure"
);
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"unwire append failure must restore roster wiring"
);
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"unwire append failure must restore roster wiring"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
let trusted_by_worker = service.trusted_peer_names(&sid_w).await;
assert!(
trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"unwire append failure must restore trust on lead"
);
assert!(
trusted_by_worker.contains(&test_comms_name("lead", "l-1")),
"unwire append failure must restore trust on worker"
);
let recorded = handle.events().replay_all().await.expect("replay");
assert!(
!recorded
.iter()
.any(|e| matches!(e.kind, MobEventKind::MembersUnwired { .. })),
"failed unwire append must not persist MembersUnwired"
);
}
// -----------------------------------------------------------------------
// P1-T08: auto_wire_orchestrator on spawn
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_auto_wire_orchestrator() {
let (handle, _service) = create_test_mob(sample_definition_with_auto_wire()).await;
// Spawn orchestrator first
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
// Spawn worker — should be auto-wired to orchestrator
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
// Verify wiring
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.contains(&AgentIdentity::from("w-1")),
"orchestrator should be wired to worker"
);
let entry_w = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_w.wired_to.contains(&AgentIdentity::from("l-1")),
"worker should be wired to orchestrator"
);
}
#[tokio::test]
async fn test_auto_wire_orchestrator_updates_real_comms_peers() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_real_comms(sample_definition_with_auto_wire()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let comms_l = service.real_comms(&sid_l).await.expect("comms for l-1");
let comms_w = service.real_comms(&sid_w).await.expect("comms for w-1");
let peers_l = CoreCommsRuntime::peers(&*comms_l).await;
let peers_w = CoreCommsRuntime::peers(&*comms_w).await;
assert!(
peers_l
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-1")),
"auto-wire should add worker to lead peers()"
);
assert!(
peers_w
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("lead", "l-1")),
"auto-wire should add lead to worker peers()"
);
}
#[tokio::test]
async fn test_auto_wire_orchestrator_not_wired_to_self() {
let (handle, _service) = create_test_mob(sample_definition_with_auto_wire()).await;
// Spawn orchestrator — should NOT wire to itself
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let entry_l = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.unwrap();
assert!(
entry_l.wired_to.is_empty(),
"orchestrator should not be wired to itself"
);
}
#[tokio::test]
async fn test_auto_wire_orchestrator_real_comms_does_not_surface_self_peer() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_real_comms(sample_definition_with_auto_wire()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let comms_l = service.real_comms(&sid_l).await.expect("comms for l-1");
let peers_l = CoreCommsRuntime::peers(&*comms_l).await;
assert!(
!peers_l
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("lead", "l-1")),
"auto-wire must not surface the spawned member itself in peers()"
);
}
#[tokio::test]
async fn test_auto_wire_parent_uses_spawning_member_not_orchestrator() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn first worker")
.bridge_session_id()
.expect("session-backed")
.clone();
{
let w1 = AgentIdentity::from("w-1");
let mut snapshot = handle.roster.read().await.snapshot();
snapshot.get_mut(&w1).expect("w-1 roster entry").member_ref =
crate::event::MemberRef::from_bridge_session_id(SessionId::new());
*handle.roster.write().await =
super::roster_authority::RosterAuthority::from_roster(snapshot);
}
let mut spec = super::handle::SpawnMemberSpec::new("worker", "w-2");
spec.auto_wire_parent = true;
let owner_context = handle
.generated_ops_owner_context_for_test(sid_w1.clone())
.await
.expect("generated owner context for w-1");
let receipt = handle
.spawn_spec_receipt_with_owner_context(spec, owner_context)
.await
.expect("spawn second worker from worker owner context");
let sid_w2 = receipt
.member_ref
.bridge_session_id()
.expect("session-backed")
.clone();
let comms_l = service.real_comms(&sid_l).await.expect("comms for l-1");
let comms_w1 = service.real_comms(&sid_w1).await.expect("comms for w-1");
let comms_w2 = service.real_comms(&sid_w2).await.expect("comms for w-2");
let peers_w2 = CoreCommsRuntime::peers(&*comms_w2).await;
assert!(
peers_w2
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-1")),
"auto_wire_parent should wire the spawned member to its actual spawner"
);
assert!(
!peers_w2
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("lead", "l-1")),
"auto_wire_parent must not fall back to wiring the orchestrator"
);
assert!(
!peers_w2
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-2")),
"auto_wire_parent must not surface the spawned member itself in peers()"
);
let peers_w1 = CoreCommsRuntime::peers(&*comms_w1).await;
assert!(
peers_w1
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-2")),
"spawner should see the newly spawned member as a peer"
);
let peers_l = CoreCommsRuntime::peers(&*comms_l).await;
assert!(
!peers_l
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-2")),
"orchestrator should stay uninvolved when a non-orchestrator member spawns with auto_wire_parent"
);
}
#[tokio::test]
async fn test_spawn_skips_broken_orchestrator_in_auto_wire_selection() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition_with_auto_wire();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle.stop().await.expect("stop");
service.archive(&sid_l).await.expect("archive orchestrator");
service.delete_persisted_session(&sid_l).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("partial resume should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"cold reconstruction must not classify missing session actors before explicit resume"
);
resumed
.resume()
.await
.expect("explicit resume should classify the missing orchestrator");
let lead = resumed
.member_status(&AgentIdentity::from("l-1"))
.await
.expect("broken orchestrator");
assert_eq!(lead.status, crate::runtime::handle::MobMemberStatus::Broken);
let sid_w = resumed
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker should ignore broken orchestrator")
.bridge_session_id()
.expect("session-backed")
.clone();
let worker = resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("spawned worker entry");
assert!(
worker.wired_to.is_empty(),
"auto-wire must not target broken orchestrators"
);
let trusted = service.trusted_peer_names(&sid_w).await;
assert!(
!trusted.contains(&test_comms_name("lead", "l-1")),
"spawn must not install trust to a broken orchestrator"
);
}
#[tokio::test]
async fn test_auto_wire_failure_is_returned_to_spawn_caller() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"spawn must fail when auto-wire fails"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"failed spawn should rollback roster entry"
);
let lead_entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead should remain in roster");
assert!(
lead_entry.wired_to.is_empty(),
"failed spawn should not leave partial wiring on lead"
);
}
#[tokio::test]
async fn test_auto_wire_failure_after_partial_wire_cleans_peer_trust_via_spawn_rollback() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_added: true,
fail_remove_trust_once: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"spawn must surface wiring failure after rollback cleanup, got {result:?}"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"failed spawn should rollback worker roster entry"
);
let lead_entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead should remain in roster");
assert!(
lead_entry.wired_to.is_empty(),
"failed spawn should not leave partial roster wiring on lead"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"spawn rollback must remove leaked trust from lead"
);
}
#[tokio::test]
async fn test_spawn_rollback_retirement_start_failure_precedes_cleanup_side_effects() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, service) =
create_test_mob_with_events(sample_definition_with_auto_wire(), events.clone()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_added: true,
..MockCommsBehavior::default()
},
)
.await;
events.fail_appends_for("MemberRetirementStarted").await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::Internal(_))),
"rollback must surface the failed durable start before cleanup: {result:?}"
);
let worker_session = service
.session_id_for_comms_name(&test_comms_name("worker", "w-1"))
.await
.expect("failed spawn session remains available behind retry anchor");
assert!(
!service
.sent_intents(&worker_session)
.await
.iter()
.any(|intent| intent == "mob.peer_retired"),
"rollback must not notify peers before MemberRetirementStarted is durable"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("query failed spawn anchor")
.is_some(),
"failed start append retains the exact member/session tuple for retry"
);
let recorded = events.replay_all().await.expect("replay failed rollback");
assert!(
recorded
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetirementStarted { .. })),
"faulted start append must not fabricate the retry carrier"
);
assert!(
recorded
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. })),
"cleanup must not publish final retirement without its durable start"
);
}
#[tokio::test]
async fn test_spawn_rollback_reconciles_lost_member_retired_append_ack() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_after_append_for("MemberRetired").await;
let (handle, service) =
create_test_mob_with_events(sample_definition_with_auto_wire(), events.clone()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_added: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"a lost terminal-event acknowledgement must reconcile to the original wiring failure, got {result:?}"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("query rolled-back worker")
.is_none(),
"lost terminal-event acknowledgement must not retain the failed spawn"
);
let recorded = events.replay_all().await.expect("replay rollback events");
let retired_count = recorded
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::MemberRetired { agent_identity, .. }
if agent_identity == &AgentIdentity::from("w-1")
)
})
.count();
assert_eq!(
retired_count, 1,
"wrote-then-error reconciliation must retain exactly one final carrier"
);
let trusted_by_lead = service.trusted_peer_names(&sid_l).await;
assert!(
!trusted_by_lead.contains(&test_comms_name("worker", "w-1")),
"lost terminal-event acknowledgement must still clean leaked trust"
);
}
#[tokio::test]
async fn test_members_unwired_lost_ack_reconciles_only_new_cursor_occurrence() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let mob_id = MobId::from("lost-unwire-ack");
let desired = MobEventKind::MembersUnwired {
a: AgentIdentity::from("a"),
b: AgentIdentity::from("b"),
};
let old = events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
.expect("seed an identical earlier unwire cycle");
let cursor_floor = events.latest_cursor().await.expect("cursor floor");
assert_eq!(cursor_floor, old.cursor);
events.fail_after_append_for("MembersUnwired").await;
let append_error = events
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
.expect_err("fault store writes then loses the acknowledgement");
assert!(
append_error
.to_string()
.contains("lost append acknowledgement")
);
let reconciled = super::actor::reconcile_exact_mob_event_after_cursor(
events.as_ref(),
&mob_id,
cursor_floor,
&desired,
)
.await
.expect("cursor reconciliation")
.expect("the new exact occurrence proves commit");
assert!(
reconciled.cursor > cursor_floor,
"an identical event from the older cycle must not authorize this unwire"
);
}
#[test]
fn test_cold_replayed_retiring_edge_suppresses_local_repair_and_remote_install() {
use crate::machines::mob_machine as mob_dsl;
// This is the machine snapshot cold recovery sees after replaying a
// durable MemberRetirementStarted carrier: topology is still present,
// one endpoint is Retiring, and the surviving endpoint is remote.
let retiring = mob_dsl::AgentIdentity::from("retiring-local");
let survivor = mob_dsl::AgentIdentity::from("surviving-remote");
let runtime_id = mob_dsl::AgentRuntimeId::from("retiring-local:1");
let host = mob_dsl::HostId::from("bound-host");
let edge = mob_dsl::WiringEdge::new(retiring.clone(), survivor.clone());
let mut state = mob_dsl::MobMachineState::default();
state.wiring_edges.insert(edge.clone());
state
.identity_to_runtime
.insert(retiring, runtime_id.clone());
state
.member_state_markers
.insert(runtime_id, mob_dsl::MobMemberState::Retiring);
state.member_placement.insert(survivor, host.clone());
state
.host_bind_phase
.insert(host, mob_dsl::HostBindPhase::Bound);
assert!(
!super::recovery_member_edge_trust_is_desired(&state, &edge),
"cold local trust recovery must not repair an edge incident to Retiring"
);
assert!(
super::derive_install_obligations(&state, None).is_empty(),
"cold route derivation must not reinstall the surviving remote lane"
);
}
#[tokio::test]
#[cfg(not(target_arch = "wasm32"))]
async fn test_sqlite_members_unwired_lost_ack_reconciles_across_bounded_pages() {
let dir = tempfile::tempdir().expect("sqlite reconcile tempdir");
let store = SqliteWriteThenErrorMobEventStore::new(
SqliteMobStores::open(dir.path().join("mob.sqlite"))
.expect("open sqlite mob stores")
.event_store(),
);
let mob_id = MobId::from("sqlite-lost-unwire-ack");
let desired = MobEventKind::MembersUnwired {
a: AgentIdentity::from("a"),
b: AgentIdentity::from("b"),
};
store
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
.expect("seed identical older unwire");
let cursor_floor = store.latest_cursor().await.expect("cursor floor");
for index in 0..=super::actor::EXACT_EVENT_RECONCILE_PAGE_SIZE {
store
.append(NewMobEvent {
mob_id: MobId::from(format!("sqlite-filler-{index}")),
timestamp: None,
kind: MobEventKind::MobCompleted,
})
.await
.expect("append page-crossing filler");
}
store.fail_next_members_unwired_after_write();
let append_error = store
.append(NewMobEvent {
mob_id: mob_id.clone(),
timestamp: None,
kind: desired.clone(),
})
.await
.expect_err("SQLite wrapper commits then loses the append acknowledgement");
assert!(
append_error
.to_string()
.contains("SQLite wrote-then-error MembersUnwired")
);
let written_without_ack_cursor = store
.latest_cursor()
.await
.expect("committed SQLite cursor after lost acknowledgement");
assert!(
written_without_ack_cursor
> cursor_floor + super::actor::EXACT_EVENT_RECONCILE_PAGE_SIZE as u64,
"target must sit beyond the first bounded SQLite page"
);
let reconciled = super::actor::reconcile_exact_mob_event_after_cursor(
&store,
&mob_id,
cursor_floor,
&desired,
)
.await
.expect("bounded SQLite reconciliation")
.expect("new exact event found across page boundary");
assert_eq!(reconciled.cursor, written_without_ack_cursor);
}
#[tokio::test]
async fn test_retire_all_start_append_failure_does_not_cancel_other_pending_spawn() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("committed"),
None,
)
.await
.expect("committed member");
service.set_create_session_delay_ms(500);
let pending = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("still-pending"),
None,
)
.await
})
};
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if handle
.debug_orchestrator_snapshot()
.await
.expect("pending snapshot")
.pending_spawn_count
== 1
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("pending spawn becomes visible");
events.fail_appends_for("MemberRetirementStarted").await;
let error = handle
.retire_all()
.await
.expect_err("bulk retirement must stop before global drain");
assert!(
error.to_string().contains("before pending-spawn drain"),
"typed failure names the ordering boundary: {error}"
);
assert_eq!(
handle
.debug_orchestrator_snapshot()
.await
.expect("post-failure snapshot")
.pending_spawn_count,
1,
"another identity's pending spawn must remain owned and uncancelled"
);
assert!(
!pending.is_finished(),
"pending spawn must not receive global cancellation without every durable Started anchor"
);
events.allow_appends_for("MemberRetirementStarted").await;
pending
.await
.expect("pending spawn join")
.expect("pending spawn remains able to commit");
handle.retire_all().await.expect("cleanup retry converges");
}
#[tokio::test]
async fn test_spawn_rollback_ignores_missing_comms_for_non_wired_planned_targets() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_missing_comms_runtime(&sid_l).await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"spawn should surface original wiring failure instead of rollback failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"rollback should still remove failed worker entry"
);
}
#[tokio::test]
async fn test_spawn_rollback_ignores_missing_comms_for_non_wired_planned_targets_with_spawned_key()
{
let (handle, service) = create_test_mob(sample_definition_with_role_wiring()).await;
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_send_peer_added: true,
fail_remove_trust_once: true,
..MockCommsBehavior::default()
},
)
.await;
service.set_missing_comms_runtime(&sid_w2).await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-3"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"spawn should surface role-wiring failure, not rollback failure"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-3"))
.await
.unwrap()
.is_none(),
"failed spawn should not keep w-3 in roster"
);
let trusted_by_w1 = service.trusted_peer_names(&sid_w1).await;
assert!(
!trusted_by_w1.contains(&test_comms_name("worker", "w-3")),
"rollback should remove leaked trust for first failed planned target"
);
}
#[tokio::test]
async fn test_spawn_rollback_archive_failure_closes_projection_and_persists_started_event() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_added: true,
..MockCommsBehavior::default()
},
)
.await;
service
.set_archive_failure_for_comms_name(&test_comms_name("worker", "w-1"))
.await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::Internal(_))),
"spawn should surface rollback archive failure"
);
assert!(
handle
.list_members()
.await
.iter()
.all(|member| member.agent_identity != "w-1"),
"generated retirement authority must close the active member projection even when archive cleanup fails"
);
let retained = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read retained rollback anchor")
.expect("archive failure must retain the durable rollback cleanup anchor");
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|e| matches!(
&e.kind,
MobEventKind::MemberSpawned(event)
if event.agent_identity == "w-1"
)),
"test must exercise post-spawn rollback after generated spawn authority accepts"
);
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"rollback must persist MemberRetirementStarted after generated retirement admission"
);
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"rollback archive failure must not mint terminal MemberRetired"
);
}
#[tokio::test]
async fn test_spawn_rollback_archive_failure_retains_cleanup_authority_until_retry() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-1"),
MockCommsBehavior {
fail_send_peer_added: true,
..MockCommsBehavior::default()
},
)
.await;
service
.set_archive_failure_for_comms_name(&test_comms_name("worker", "w-1"))
.await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::Internal(_))),
"spawn should surface rollback archive failure"
);
assert!(
handle
.list_members()
.await
.iter()
.all(|member| member.agent_identity != "w-1"),
"generated retirement authority must close the active member projection even when archive cleanup fails"
);
let retained = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read retained rollback anchor")
.expect("archive failure must retain the durable rollback cleanup anchor");
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|e| matches!(
&e.kind,
MobEventKind::MemberSpawned(event)
if event.agent_identity == "w-1"
)),
"test must exercise post-spawn rollback after generated spawn authority accepts"
);
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"rollback must persist retirement-start authority before cleanup"
);
assert!(
events
.iter()
.all(|e| !matches!(e.kind, MobEventKind::MemberRetired { .. })),
"rollback must not publish MemberRetired before archive cleanup succeeds"
);
let session_id = retained
.bridge_session_id()
.expect("retained rollback anchor remains session-correlated")
.clone();
service.clear_archive_failure(&session_id).await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retry must complete the retained rollback cleanup");
let completed_events = handle
.events()
.replay_all()
.await
.expect("replay completion");
assert!(
completed_events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"retry must publish MemberRetired after archive cleanup succeeds"
);
}
#[tokio::test]
async fn test_fault_injected_lifecycle_operations_preserve_transactional_invariants() {
// Spawn rollback invariants.
let (spawn_handle, spawn_service) = create_test_mob(sample_definition_with_auto_wire()).await;
spawn_service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = spawn_handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let spawn_err = spawn_handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect_err("spawn should fail under injected auto-wire fault");
assert!(
matches!(
spawn_err,
MobError::WiringError(_) | MobError::StorageError(_)
),
"spawn fault should surface as wiring/storage error"
);
assert!(
spawn_handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"spawn rollback should keep failed member out of roster"
);
assert!(
!spawn_service
.trusted_peer_names(&sid_l)
.await
.contains(&test_comms_name("worker", "w-1")),
"spawn rollback should remove leaked trust edges"
);
// Retire cleanup invariants: archive failure is surfaced and the retiring
// roster entry remains as a non-routable retry anchor.
let (retire_handle, retire_service) = create_test_mob(sample_definition()).await;
let sid_r1 = retire_handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("r-1"),
None,
)
.await
.expect("spawn r-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let _sid_r2 = retire_handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("r-2"),
None,
)
.await
.expect("spawn r-2");
retire_handle
.wire(AgentIdentity::from("r-1"), AgentIdentity::from("r-2"))
.await
.expect("wire");
retire_service.set_archive_failure(&sid_r1).await;
let retire_result = retire_handle.retire(AgentIdentity::from("r-1")).await;
assert!(
retire_result.is_err(),
"retire must return Err when ArchiveSession fails"
);
assert!(
retire_handle
.get_member(&AgentIdentity::from("r-1"))
.await
.unwrap()
.is_some(),
"archive failure must retain the exact retirement retry anchor"
);
let r2 = retire_handle
.get_member(&AgentIdentity::from("r-2"))
.await
.unwrap()
.expect("r-2 should remain");
assert!(
!r2.wired_to.contains(&AgentIdentity::from("r-1")),
"roster.remove should clean r-1 from r-2's wired_to set"
);
}
// -----------------------------------------------------------------------
// P1-T09: role_wiring fan-out on spawn
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_role_wiring_fan_out() {
let (handle, _service) = create_test_mob(sample_definition_with_role_wiring()).await;
// Spawn 3 workers
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-3"),
None,
)
.await
.expect("spawn w-3");
// w-1 was spawned first (no other workers existed), so no wiring for it at spawn time.
// w-2 was spawned second, rule worker<->worker matches, wired to w-1.
// w-3 was spawned third, rule worker<->worker matches, wired to w-1 and w-2.
let entry_1 = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
let entry_2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.unwrap();
let entry_3 = handle
.get_member(&AgentIdentity::from("w-3"))
.await
.unwrap()
.unwrap();
// w-2 should be wired to w-1 (and vice versa)
assert!(entry_2.wired_to.contains(&AgentIdentity::from("w-1")));
assert!(entry_1.wired_to.contains(&AgentIdentity::from("w-2")));
// w-3 should be wired to both w-1 and w-2
assert!(entry_3.wired_to.contains(&AgentIdentity::from("w-1")));
assert!(entry_3.wired_to.contains(&AgentIdentity::from("w-2")));
// w-1 should be wired to both w-2 and w-3
// Need to re-read since roster may have been updated after w-3 spawn
let entry_1 = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap();
assert!(entry_1.wired_to.contains(&AgentIdentity::from("w-3")));
}
#[tokio::test]
async fn test_spawn_skips_broken_role_peers_in_role_wiring_selection() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let mut definition = sample_definition_with_role_wiring();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
handle.stop().await.expect("stop");
service.archive(&sid_w1).await.expect("archive w-1");
service.delete_persisted_session(&sid_w1).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("partial resume should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"cold reconstruction must not classify missing session actors before explicit resume"
);
resumed
.resume()
.await
.expect("explicit resume should classify the missing role peer");
let broken = resumed
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("broken member status");
assert_eq!(
broken.status,
crate::runtime::handle::MobMemberStatus::Broken
);
let sid_w2 = resumed
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn should ignore broken role peers")
.bridge_session_id()
.expect("session-backed")
.clone();
let worker = resumed
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("spawned worker");
assert!(
!worker.wired_to.contains(&AgentIdentity::from("w-1")),
"role wiring must not target broken peers"
);
let trusted = service.trusted_peer_names(&sid_w2).await;
assert!(
!trusted.contains(&test_comms_name("worker", "w-1")),
"spawn must not install trust to broken role peers"
);
}
#[tokio::test]
async fn test_role_wiring_cross_role_fans_out_to_three_existing_targets() {
let (handle, _service) = create_test_mob(sample_definition_with_cross_role_wiring()).await;
for worker_id in ["w-1", "w-2", "w-3"] {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(worker_id),
None,
)
.await
.expect("spawn worker");
}
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let lead = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead should be in roster");
assert_eq!(
lead.wired_to.len(),
3,
"new role_x meerkat should fan out to all 3 existing role_y peers"
);
for worker_id in ["w-1", "w-2", "w-3"] {
assert!(
lead.wired_to.contains(&AgentIdentity::from(worker_id)),
"lead should be wired to {worker_id}"
);
let worker = handle
.get_member(&AgentIdentity::from(worker_id))
.await
.unwrap()
.expect("worker should remain in roster");
assert!(
worker.wired_to.contains(&AgentIdentity::from("l-1")),
"{worker_id} should be wired back to lead"
);
}
}
#[tokio::test]
async fn test_spawn_wiring_deduplicates_overlapping_orchestrator_and_role_edges() {
let (handle, _service) =
create_test_mob(sample_definition_with_overlapping_orchestrator_and_role_wiring()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let lead = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("lead should exist");
assert_eq!(
lead.wired_to.len(),
1,
"overlapping auto-wire + role rule should produce one trust edge"
);
assert!(lead.wired_to.contains(&AgentIdentity::from("w-1")));
}
#[tokio::test]
async fn test_role_wiring_failure_is_returned_to_spawn_caller() {
let (handle, service) = create_test_mob(sample_definition_with_role_wiring()).await;
service
.set_comms_behavior(
&test_comms_name("worker", "w-2"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::WiringError(_))),
"spawn must fail when role-wiring fan-out fails"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.is_none(),
"failed spawn should rollback roster entry"
);
let entry_w1 = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("w-1 should remain in roster");
assert!(
!entry_w1.wired_to.contains(&AgentIdentity::from("w-2")),
"failed spawn should not leave partial role wiring"
);
}
// -----------------------------------------------------------------------
// P1-T10: external_turn enforces addressability
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_external_turn_addressable_succeeds() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead (external_addressable=true)");
let result = handle
.member(&AgentIdentity::from("l-1"))
.await
.expect("member handle")
.send(
"Hello from outside",
meerkat_core::types::HandlingMode::Queue,
)
.await;
assert!(
result.is_ok(),
"external_turn to addressable meerkat should succeed"
);
}
#[tokio::test]
async fn test_member_handle_send_accepts_multimodal_content() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead (external_addressable=true)");
let member = handle
.member(&AgentIdentity::from("l-1"))
.await
.expect("member handle");
let result = member
.send(
meerkat_core::types::ContentInput::Blocks(vec![
meerkat_core::types::ContentBlock::Text {
text: "look at this".to_string(),
},
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".into(),
},
]),
meerkat_core::types::HandlingMode::Queue,
)
.await;
assert!(
result.is_ok(),
"member-directed send should accept multimodal content"
);
}
#[tokio::test]
async fn test_external_turn_not_addressable_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker (external_addressable=false)");
let result = handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.send(
"Hello from outside",
meerkat_core::types::HandlingMode::Queue,
)
.await;
assert!(
matches!(result, Err(MobError::NotExternallyAddressable(_))),
"external_turn to non-addressable meerkat should fail"
);
}
#[tokio::test]
async fn test_external_turn_unknown_meerkat_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let result = handle
.external_turn_for_member(
AgentIdentity::from("nonexistent"),
"Hello".to_string().into(),
meerkat_core::types::HandlingMode::Queue,
None,
None,
super::handle::MemberTurnObservers::default(),
)
.await;
assert!(matches!(result, Err(MobError::MemberNotFound(_))));
}
#[tokio::test]
async fn test_internal_turn_bypasses_external_addressable_check() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let result = handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn("internal message")
.await;
assert!(
result.is_ok(),
"internal_turn should bypass external_addressable=false"
);
}
#[tokio::test]
async fn test_internal_turn_unknown_meerkat_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let result = match handle.member(&AgentIdentity::from("nonexistent")).await {
Err(error) => Err(error),
Ok(member) => member.internal_turn("Hello").await,
};
assert!(matches!(result, Err(MobError::MemberNotFound(_))));
}
// -----------------------------------------------------------------------
// Phase 0 CHOKE-001 red test:
// autonomous dispatch must route via injector (currently not wired yet).
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_external_turn_autonomous_mode_uses_injector_dispatch() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("l-autonomous"),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous lead");
handle
.member(&AgentIdentity::from("l-autonomous"))
.await
.expect("member handle")
.send("inject me", meerkat_core::types::HandlingMode::Queue)
.await
.expect("external turn should execute");
// Runtime adapter path: spawn uses accept_input_with_completion (1 keep-alive),
// send() uses inject (1).
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.inject_call_count(),
1,
"autonomous dispatch must use event injector for send() (1 send)"
);
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"autonomous spawn uses keep-alive start_turn via runtime adapter"
);
}
#[tokio::test]
async fn test_external_turn_autonomous_mode_keeps_injector_dispatch_after_kickoff_completion() {
let (handle, service) = create_test_mob_with_runtime_adapter(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("l-idle-autonomous"),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous lead");
tokio::time::sleep(Duration::from_millis(25)).await;
let baseline_keep_alive_turns = service.keep_alive_start_turn_call_count();
let baseline_injects = service.inject_call_count();
handle
.member(&AgentIdentity::from("l-idle-autonomous"))
.await
.expect("member handle")
.send(
"inject after kickoff",
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("external turn should execute");
assert_eq!(
service.keep_alive_start_turn_call_count(),
baseline_keep_alive_turns,
"idle autonomous dispatch must not restart a new kickoff turn after the original one completed"
);
assert!(
service.inject_call_count() > baseline_injects,
"idle autonomous dispatch should still route through the live injector"
);
}
#[tokio::test]
async fn test_external_turn_turn_driven_mode_uses_start_turn_dispatch() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("l-turn-driven"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let baseline_start_turn_calls = service.start_turn_call_count();
handle
.member(&AgentIdentity::from("l-turn-driven"))
.await
.expect("member handle")
.send(
"turn-driven message",
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("external turn should execute");
assert_eq!(
service.inject_call_count(),
0,
"turn-driven external dispatch should not use injector"
);
wait_for_start_turn_call_count(
service.as_ref(),
baseline_start_turn_calls + 1,
"turn-driven external dispatch should eventually issue start_turn after ingress admission",
)
.await;
assert_eq!(
service.start_turn_call_count(),
baseline_start_turn_calls + 1,
"turn-driven external dispatch should issue start_turn"
);
}
#[tokio::test]
async fn test_runtime_backed_turn_driven_dispatch_returns_after_ingress_admission() {
let (handle, service) = create_test_mob_with_runtime_adapter(sample_definition()).await;
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("l-runtime-fail"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let baseline_start_turn_calls = service.start_turn_call_count();
service.set_fail_start_turn(true);
let result = handle
.member(&AgentIdentity::from("l-runtime-fail"))
.await
.expect("member handle")
.send("turn should fail", meerkat_core::types::HandlingMode::Queue)
.await;
let debug = format!("{result:?}");
assert!(
result.is_ok(),
"runtime-backed turn dispatch should return after ingress admission, got: {debug}"
);
let deadline = Instant::now() + Duration::from_secs(2);
while service.start_turn_call_count() < baseline_start_turn_calls + 1 {
assert!(
Instant::now() < deadline,
"runtime-backed dispatch should still attempt exactly one turn"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
#[cfg(feature = "runtime-adapter")]
async fn provision_runtime_backed_disposal_fixture(
provisioner: &super::provisioner::SessionBackend,
session: Session,
fixture_name: &str,
) -> MemberSpawnReceipt {
let session_id = session.id().clone();
provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: format!("{fixture_name} disposal fixture").into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(session),
comms_name: Some(format!("test-mob/worker/{fixture_name}")),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding(fixture_name),
peer_name: fixture_name.to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(session_id),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.unwrap_or_else(|error| {
panic!("provision exact disposal fixture '{fixture_name}': {error}")
})
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_abort_member_provision_retires_runtime_before_absent_cleanup_unregisters() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
provision_runtime_backed_disposal_fixture(&provisioner, session, "abort-absent-cleanup").await;
provisioner
.abort_member_provision(
&MemberRef::from_bridge_session_id(session_id.clone()),
&meerkat_core::ops::OperationId::new(),
"cleanup absent operation",
)
.await
.expect("absent operation cleanup should archive then unregister");
assert!(
!adapter.contains_session(&session_id).await,
"successful cleanup should unregister only after archive authority succeeds"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"runtime lifecycle must be durably retired before registration cleanup"
);
assert!(
!service
.has_live_session(&session_id)
.await
.expect("read live service state"),
"successful cleanup should archive the session projection"
);
}
/// Runtime retirement can move the captured attachment into the machine-owned
/// Draining teardown while the durable archive is still committing. The
/// serving-witness projection is absent in that state, but the exact captured
/// attachment remains the only authority that may join its teardown.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_joins_exact_attachment_teardown_started_during_archive() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
service.set_archive_delay_ms(100);
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let receipt = provision_runtime_backed_disposal_fixture(
&provisioner,
session,
"archive-draining-attachment",
)
.await;
provisioner
.retire_member(&receipt.member_ref)
.await
.expect("disposal must join the exact attachment teardown started by archive retirement");
assert!(
!adapter.contains_session(&session_id).await,
"joining exact teardown must remove the old terminal registration"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load retired runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"exact teardown joining must preserve the durable retired terminal"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("read live actor state after exact teardown"),
"the attachment-local live actor must leave with the retired attachment"
);
}
/// A transient service cleanup failure occurs after canonical unregister has
/// hidden the attachment from the serving-witness projection. The next
/// ReleaseMember attempt must recapture the retained exact sidecar and join
/// that same Draining saga instead of stranding the durable retry anchor.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_retry_recaptures_draining_attachment_after_actor_cleanup_failure() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let receipt =
provision_runtime_backed_disposal_fixture(&provisioner, session, "draining-cleanup-retry")
.await;
service.fail_next_exact_actor_discard();
let first_error = provisioner
.retire_member(&receipt.member_ref)
.await
.expect_err("the injected post-stop actor cleanup must fail once");
assert!(
first_error
.to_string()
.contains("synthetic transient exact actor discard failure"),
"the exact cleanup error must remain visible to the retry owner: {first_error:?}"
);
assert!(
adapter.contains_session(&session_id).await,
"failed cleanup must retain the machine registration retry anchor"
);
assert!(
adapter
.current_executor_attachment_witness(&session_id)
.await
.is_none(),
"Draining must hide the attachment from the serving projection"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("read actor state after injected cleanup failure"),
"the durable archive authority may remove its actor before the retained machine cleanup reports failure"
);
provisioner
.retire_member(&receipt.member_ref)
.await
.expect("retry must recapture and join the exact Draining attachment");
assert!(
!adapter.contains_session(&session_id).await,
"successful retry must remove the exact terminal registration"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("read actor registry after cleanup retry"),
"the retry must discard the retained exact actor"
);
}
/// Ask 21d regression (meerkat-studio, identity-first construction): the
/// archive authority CLAIMS the session (ownership probe true) yet the
/// archive itself resolves NotFound, and the runtime is already RETIRED by
/// the pre-archive retire. The old escalation ("NotFound for registered
/// runtime session") fired on the mere registration — but a terminal-phase
/// registration has no live obligations to protect; it is un-unregistered
/// residue. Disposal must complete instead of stranding the member.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_completes_when_archive_notfounds_a_terminal_registered_runtime() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
// The session is persisted (the ownership probe claims it) but the
// archive authority NotFounds — the identity-first wiring disagreement.
let session = Session::new();
let session_id = session.id().clone();
let receipt =
provision_runtime_backed_disposal_fixture(&provisioner, session, "identity-first-worker")
.await;
service.set_archive_not_found(&session_id).await;
provisioner
.retire_member(&receipt.member_ref)
.await
.expect(
"archive-NotFound on a terminal-phase registered runtime must complete disposal, not escalate",
);
assert!(
!adapter.contains_session(&session_id).await,
"disposal must unregister the terminal runtime session"
);
}
/// Ordinary Stop tears down the exact executor attachment and its paired live
/// actor while retaining the stopped machine registration as the durable
/// resume anchor. If the archive authority then reports NotFound, disposal may
/// remove that exact terminal registration; there is no attachment-local
/// authority left to preserve.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_removes_stopped_unattached_registration_on_archive_notfound() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let receipt =
provision_runtime_backed_disposal_fixture(&provisioner, session, "stopped-worker").await;
service.set_archive_not_found(&session_id).await;
adapter
.stop_runtime_executor(&session_id, "torn shutdown")
.await
.expect("stop runtime executor");
assert!(
adapter.contains_session(&session_id).await,
"ordinary stop must retain the durable machine registration"
);
assert!(
adapter
.current_executor_attachment_witness(&session_id)
.await
.is_none(),
"ordinary stop must remove the serving executor attachment"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("observe stopped actor registry"),
"the live actor is attachment-local and must leave with the stopped attachment"
);
let disposal = provisioner
.retire_member(&receipt.member_ref)
.await
.expect("archive-NotFound may dispose an exact stopped unattached registration");
assert_eq!(
disposal,
crate::machines::mob_machine::MemberSessionDisposal::Archived,
"the provisioner folds owned archive-NotFound into the machine's durable Archived terminal"
);
assert!(
!adapter.contains_session(&session_id).await,
"disposal must compare-and-remove the exact stopped registration"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("observe actor registry after disposal"),
"disposal must not reconstruct an actor for terminal registration cleanup"
);
}
/// K1 regression (meerkat-studio P0): retire on a SESSION-OWNED member — a
/// runtime-registered session the mob archive authority never had a record
/// for (e.g. materialized by an embedder's session service) — used to fail
/// deterministically with "mob archive authority returned NotFound for
/// registered runtime session", leaving the runtime registered so every
/// retry failed identically and the member could be neither respawned nor
/// removed. Authority-NotFound must complete the disposal pipeline instead:
/// retire the runtime session and drop the binding.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_session_owned_member_completes_disposal_on_archive_authority_not_found() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
// Materialize through the production transaction so the machine and
// sidecar form an exact pair. Then model the independent product fact:
// this live attachment's durable document belongs to another host, so
// the mob archive authority must not claim it.
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let receipt =
provision_runtime_backed_disposal_fixture(&provisioner, session, "host-owned-session")
.await;
service.set_archive_authority_unknown(&session_id).await;
service.set_archive_not_found(&session_id).await;
let disposal = provisioner.retire_member(&receipt.member_ref).await.expect(
"retire of a session-owned member must complete disposal, not escalate authority NotFound",
);
assert_eq!(
disposal,
crate::machines::mob_machine::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned,
"host-owned disposal must be classified as runtime-released-only, not laundered into a durable archive claim (§19.L4)",
);
assert!(
!adapter.contains_session(&session_id).await,
"disposal must unregister the runtime session (a registered leftover made every retry fail identically)"
);
let retry_disposal = provisioner.retire_member(&receipt.member_ref).await.expect(
"retry after runtime unregister must preserve the host-owned disposal classification",
);
assert_eq!(
retry_disposal,
crate::machines::mob_machine::MemberSessionDisposal::RuntimeReleasedOnlyHostOwned,
"host-owned durable ownership is stable across retries; missing runtime registration must not become AlreadyArchived",
);
assert!(
!adapter.contains_session(&session_id).await,
"idempotent host-owned disposal retry must leave the runtime unregistered"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"the runtime lifecycle must be durably retired even though the archive authority had no record"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn mob_runtime_executor_forwards_exact_profile_boundary_acknowledgements() {
use meerkat_core::lifecycle::core_executor::{
CoreApplyOutput, CoreExecutor, CoreExecutorError,
};
use meerkat_core::lifecycle::run_primitive::RunPrimitive;
struct IdleExecutor;
#[async_trait::async_trait]
impl CoreExecutor for IdleExecutor {
async fn apply(
&mut self,
_run_id: meerkat_core::RunId,
_primitive: RunPrimitive,
) -> Result<CoreApplyOutput, CoreExecutorError> {
Err(CoreExecutorError::Internal(
"boundary acknowledgement forwarding fixture must not run".to_string(),
))
}
async fn cancel_after_boundary(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
async fn stop_runtime_executor(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
}
let service = Arc::new(MockSessionService::new());
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
let session_id = SessionId::new();
adapter
.register_session(session_id.clone())
.await
.expect("register boundary-ack forwarding fixture");
let pending = match adapter
.ensure_session_with_executor_factory(session_id.clone(), |_| Box::new(IdleExecutor))
.await
.expect("prepare boundary-ack forwarding fixture")
{
meerkat_runtime::EnsureRuntimeExecutorAttachment::Pending(pending) => pending,
meerkat_runtime::EnsureRuntimeExecutorAttachment::Existing(witness) => {
panic!("fresh boundary-ack forwarding fixture unexpectedly found {witness:?}")
}
};
let state = Arc::new(
super::provisioner::RuntimeSessionState::for_attachment_without_actor_witness(
pending.witness().clone(),
),
);
let session_service: Arc<dyn super::session_service::MobSessionService> = service.clone();
let mut executor = super::provisioner::MobSessionRuntimeExecutor::new(
session_service,
Arc::clone(&adapter),
None,
session_id.clone(),
state,
Arc::new(tokio::sync::RwLock::new(HashMap::new())),
);
let authorities = [
meerkat_core::CommittedSessionBoundaryAuthority::WholeBlob {
session_id: session_id.clone(),
committed_store_revision: 41,
committed_blob_sha256: "sha256:whole-blob-exact".to_string(),
},
meerkat_core::CommittedSessionBoundaryAuthority::HeadCanonical {
session_id: session_id.clone(),
committed_head_token: "head-cas-exact".to_string(),
},
meerkat_core::CommittedSessionBoundaryAuthority::Provisional {
session_id: session_id.clone(),
committed_store_revision: 42,
committed_authority_token: "provisional-authority-exact".to_string(),
},
];
for authority in &authorities {
CoreExecutor::acknowledge_committed_session_boundary(&mut executor, authority)
.await
.expect("forward exact committed-session boundary authority");
}
assert_eq!(
*service.runtime_boundary_acknowledgements.read().await,
vec![
RuntimeBoundaryAcknowledgement::WholeBlob {
session_id: session_id.clone(),
store_revision: 41,
blob_sha256: "sha256:whole-blob-exact".to_string(),
},
RuntimeBoundaryAcknowledgement::HeadCanonical {
session_id: session_id.clone(),
head_token: "head-cas-exact".to_string(),
},
RuntimeBoundaryAcknowledgement::Provisional {
session_id,
store_revision: 42,
authority_token: "provisional-authority-exact".to_string(),
},
],
"every CoreExecutor profile hook must forward the exact bridge-session authority payload"
);
pending
.abort()
.await
.expect("abort boundary-ack forwarding fixture");
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn staged_runtime_sidecar_activation_rolls_back_until_finalized() {
use meerkat_core::lifecycle::core_executor::{
CoreApplyOutput, CoreExecutor, CoreExecutorError,
};
use meerkat_core::lifecycle::run_primitive::RunPrimitive;
struct IdleExecutor;
#[async_trait::async_trait]
impl CoreExecutor for IdleExecutor {
async fn apply(
&mut self,
_run_id: meerkat_core::RunId,
_primitive: RunPrimitive,
) -> Result<CoreApplyOutput, CoreExecutorError> {
Err(CoreExecutorError::Internal(
"staged sidecar test executor must not run".to_string(),
))
}
async fn cancel_after_boundary(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
async fn stop_runtime_executor(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
}
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
let session_id = SessionId::new();
adapter
.register_session(session_id.clone())
.await
.expect("register staged-sidecar fixture");
let pending = match adapter
.ensure_session_with_executor_factory(session_id.clone(), |_| Box::new(IdleExecutor))
.await
.expect("prepare staged-sidecar attachment")
{
meerkat_runtime::EnsureRuntimeExecutorAttachment::Pending(pending) => pending,
meerkat_runtime::EnsureRuntimeExecutorAttachment::Existing(witness) => {
panic!("fresh staged-sidecar fixture unexpectedly found {witness:?}")
}
};
let state = Arc::new(
super::provisioner::RuntimeSessionState::for_attachment_without_actor_witness(
pending.witness().clone(),
),
);
let staged = state
.stage_attachment_activation(pending.witness())
.expect("stage exact sidecar activation");
assert!(state.attachment_is_active(pending.witness()));
drop(staged);
assert!(state.attachment_is_pending(pending.witness()));
state
.stage_attachment_activation(pending.witness())
.expect("restage exact sidecar activation")
.finalize();
assert!(state.attachment_is_active(pending.witness()));
pending
.abort()
.await
.expect("abort staged-sidecar fixture attachment");
}
// A session id is not actor cleanup authority. Synthetic services without an
// actor-incarnation registry must fail closed and retain every retry anchor;
// production-created sidecars always carry the service-minted exact witness.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn cleanup_without_exact_actor_witness_fails_closed_with_retry_anchors() {
use meerkat_core::lifecycle::core_executor::{
CoreApplyOutput, CoreExecutor, CoreExecutorError,
};
use meerkat_core::lifecycle::run_primitive::RunPrimitive;
struct IdleExecutor;
#[async_trait::async_trait]
impl CoreExecutor for IdleExecutor {
async fn apply(
&mut self,
_run_id: meerkat_core::RunId,
_primitive: RunPrimitive,
) -> Result<CoreApplyOutput, CoreExecutorError> {
Err(CoreExecutorError::Internal(
"idle cleanup-ordering executor must not run".to_string(),
))
}
async fn cancel_after_boundary(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
async fn stop_runtime_executor(
&mut self,
_reason: String,
) -> Result<(), CoreExecutorError> {
Ok(())
}
}
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let session = Session::new();
let session_id = session.id().clone();
service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "idle cleanup ordering".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(session),
comms_name: Some("test-mob/worker/idle-cleanup-ordering".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create runtime-backed cleanup session");
adapter
.register_session(session_id.clone())
.await
.expect("register session");
assert!(
adapter.contains_session(&session_id).await,
"precondition: session registered in the runtime adapter"
);
let runtime_sessions = Arc::new(tokio::sync::RwLock::new(HashMap::new()));
let mut boundary = super::provisioner::RuntimeTurnFinalizationBoundaryLease::acquire(
&(service.clone() as Arc<dyn super::session_service::MobSessionService>),
&session_id,
)
.await
.expect("acquire exact cleanup-ordering service boundary");
let state_slot = Arc::new(std::sync::Mutex::new(None));
let factory_state_slot = Arc::clone(&state_slot);
let ensured = adapter
.ensure_session_with_executor_factory(session_id.clone(), move |witness| {
let state = Arc::new(
super::provisioner::RuntimeSessionState::for_attachment_without_actor_witness(
witness,
),
);
*factory_state_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(state);
Box::new(IdleExecutor)
})
.await
.expect("prepare exact cleanup-ordering attachment");
let meerkat_runtime::EnsureRuntimeExecutorAttachment::Pending(pending) = ensured else {
panic!("cleanup-ordering test unexpectedly found an existing attachment");
};
let state = state_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.expect("executor factory should publish its exact state");
super::provisioner::commit_pending_runtime_session_state(
&adapter,
&session_id,
&runtime_sessions,
Arc::clone(&state),
None,
pending,
&mut boundary,
|_| Ok(()),
)
.await
.expect("commit exact cleanup-ordering attachment");
let cleanup = super::provisioner::MobSessionRuntimePostStopCleanupHandle::new(
service.clone(),
session_id.clone(),
Arc::clone(&state),
Arc::clone(&runtime_sessions),
);
let error = meerkat_core::lifecycle::core_executor::CoreExecutorPostStopCleanupHandle::cleanup_after_runtime_stop_terminalized(
&cleanup,
)
.await
.expect_err("missing exact live actor witness must fail closed");
assert!(
error
.to_string()
.contains("has no exact live actor witness")
);
assert!(adapter.contains_session(&session_id).await);
assert!(
service
.has_live_session(&session_id)
.await
.expect("read live service state after failed cleanup")
);
assert!(
runtime_sessions
.read()
.await
.get(&session_id)
.is_some_and(|current| Arc::ptr_eq(current, &state)),
"failed discard must not erase the provisioner retry anchor"
);
// Explicit test teardown uses the fake service's id-only seam. The
// production cleanup path above must never fall back to it.
super::session_service::MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard synthetic live session during test teardown");
runtime_sessions.write().await.remove(&session_id);
adapter
.unregister_session(&session_id)
.await
.expect("remove synthetic runtime during test teardown");
// End state: torn down on both sides and out of the runtime_sessions map.
assert!(
!adapter.contains_session(&session_id).await,
"session should be unregistered from the runtime adapter after cleanup"
);
assert!(
!service
.has_live_session(&session_id)
.await
.expect("read live service state"),
"session should be discarded from the session-service after cleanup"
);
assert!(
runtime_sessions.read().await.get(&session_id).is_none(),
"session should be removed from the runtime_sessions map after cleanup"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_abort_member_provision_archive_failure_keeps_runtime_binding_for_retry() {
let service = Arc::new(MockSessionService::new());
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_for_machine: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
runtime_store_for_machine,
));
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
provision_runtime_backed_disposal_fixture(&provisioner, session, "abort-archive-failure").await;
service.set_archive_failure(&session_id).await;
let err = provisioner
.abort_member_provision(
&MemberRef::from_bridge_session_id(session_id.clone()),
&meerkat_core::ops::OperationId::new(),
"cleanup absent operation",
)
.await
.expect_err("archive failure should block runtime unregister cleanup");
assert!(
err.to_string().contains("mock archive failure"),
"archive failure should surface to caller, got {err:?}"
);
assert!(
adapter.contains_session(&session_id).await,
"failed archive projection must keep the runtime binding for retry"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"archive authority may retire durably before projection cleanup fails"
);
assert!(
service
.has_live_session(&session_id)
.await
.expect("read live service state"),
"failed archive projection should retain the live session for retry"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retire_member_waits_for_active_runtime_turn_before_unregister() {
struct BlockingExecutor {
apply_started: Arc<tokio::sync::Notify>,
allow_finish: Arc<tokio::sync::Notify>,
}
#[async_trait::async_trait]
impl meerkat_core::lifecycle::core_executor::CoreExecutor for BlockingExecutor {
async fn apply(
&mut self,
run_id: meerkat_core::RunId,
primitive: meerkat_core::lifecycle::run_primitive::RunPrimitive,
) -> Result<
meerkat_core::lifecycle::core_executor::CoreApplyOutput,
meerkat_core::lifecycle::core_executor::CoreExecutorError,
> {
self.apply_started.notify_waiters();
self.allow_finish.notified().await;
Ok(
meerkat_core::lifecycle::core_executor::CoreApplyOutput::with_untyped_snapshot(
meerkat_core::lifecycle::run_receipt::RunBoundaryReceiptDraft {
run_id,
boundary:
meerkat_core::lifecycle::run_primitive::RunApplyBoundary::RunStart,
contributing_input_ids: primitive.contributing_input_ids().to_vec(),
conversation_digest: None,
message_count: 0,
},
None,
None,
),
)
}
async fn cancel_after_boundary(
&mut self,
_reason: String,
) -> Result<(), meerkat_core::lifecycle::core_executor::CoreExecutorError> {
Ok(())
}
async fn stop_runtime_executor(
&mut self,
_reason: String,
) -> Result<(), meerkat_core::lifecycle::core_executor::CoreExecutorError> {
Ok(())
}
}
let service = Arc::new(MockSessionService::new());
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
service.set_runtime_adapter(adapter.clone());
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let apply_started = Arc::new(tokio::sync::Notify::new());
let allow_finish = Arc::new(tokio::sync::Notify::new());
service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "retire active runtime turn".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(session),
comms_name: Some("test-mob/worker/retire-active-turn".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create runtime-backed active-turn session");
adapter
.ensure_session_with_executor(
session_id.clone(),
Box::new(BlockingExecutor {
apply_started: Arc::clone(&apply_started),
allow_finish: Arc::clone(&allow_finish),
}),
)
.await
.expect("runtime executor registration should succeed");
let input = meerkat_runtime::Input::Prompt(meerkat_runtime::PromptInput::new(
"active turn before retire",
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
handling_mode: Some(HandlingMode::Steer),
..Default::default()
},
),
));
let apply_started_wait = apply_started.notified();
tokio::pin!(apply_started_wait);
apply_started_wait.as_mut().enable();
let (outcome, completion) = adapter
.accept_input_with_completion(&session_id, input)
.await
.expect("active steered prompt should be accepted");
assert!(outcome.is_accepted());
let completion = completion.expect("active prompt should register a completion waiter");
tokio::time::timeout(Duration::from_secs(1), &mut apply_started_wait)
.await
.expect("runtime executor should enter active apply before retire");
let member_ref = MemberRef::from_bridge_session_id(session_id.clone());
let bindings = adapter
.prepare_local_session_bindings(session_id.clone())
.await
.expect("prepare generated runtime bindings for retire test");
assert!(
meerkat_runtime::session_runtime_bindings_have_machine_authority(&bindings),
"retire test operation binding must come from MeerkatMachine authority"
);
provisioner
.bind_member_owner_context(
&member_ref,
session_id.clone(),
Arc::clone(bindings.ops_lifecycle()),
)
.await
.expect("bind generated owner context for retire test");
let retire_task = tokio::spawn(async move { provisioner.retire_member(&member_ref).await });
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!retire_task.is_finished(),
"retire_member must not archive/unregister while the runtime authority still has an active turn"
);
assert!(
adapter.contains_session(&session_id).await,
"runtime session must remain registered while the active turn can still emit turn-state transitions"
);
allow_finish.notify_waiters();
match completion
.wait()
.await
.expect("active runtime turn should complete before retire cleanup")
{
meerkat_runtime::CompletionOutcome::CompletedWithoutResult => {}
other => {
panic!("expected active runtime turn to complete before retire cleanup: {other:?}")
}
}
retire_task
.await
.expect("retire task should not panic")
.expect("retire_member should finish after active turn drains");
assert!(
!adapter.contains_session(&session_id).await,
"retire_member should unregister after the active turn drains and archive succeeds"
);
assert!(
!service
.has_live_session(&session_id)
.await
.expect("read live service state"),
"retire_member should archive the session after runtime drain"
);
}
#[tokio::test]
async fn test_builder_rejects_runtime_adapter_with_mismatched_persistence_authority() {
let service = Arc::new(MockSessionService::new());
let service_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let service_adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent_without_blobs(
service_store,
));
*service
.runtime_adapter
.lock()
.expect("runtime_adapter mutex") = Some(service_adapter);
let builder_adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
let err = match MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.with_runtime_adapter(builder_adapter)
.create()
.await
{
Ok(_) => panic!("mismatched runtime persistence authority should fail closed"),
Err(err) => err,
};
assert!(
err.to_string().contains("runtime persistence authority"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_autonomous_host_loop_uses_builder_runtime_adapter_for_comms_drain() {
let service = Arc::new(MockSessionService::new());
let service_adapter = service.enable_runtime_adapter();
let builder_adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.with_runtime_adapter(builder_adapter.clone())
.create()
.await
.expect("create mob");
let session_id = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-override"),
None,
)
.await
.expect("spawn autonomous lead")
.bridge_session_id()
.expect("session-backed")
.clone();
tokio::time::sleep(Duration::from_millis(25)).await;
assert!(
builder_adapter.contains_session(&session_id).await,
"the builder-selected runtime adapter should own the autonomous member session"
);
assert!(
!service_adapter.contains_session(&session_id).await,
"the session service's default adapter must stay unused when an explicit mob runtime adapter override is provided"
);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timed out")
.expect("stop");
}
#[tokio::test]
async fn test_flow_dispatch_autonomous_mode_uses_injector_and_avoids_non_host_start_turn() {
let (handle, service) =
create_test_mob(sample_definition_with_dispatch_mode(DispatchMode::OneToOne)).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let baseline_non_host_start_turn = service
.start_turn_call_count()
.saturating_sub(service.keep_alive_start_turn_call_count());
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(
terminal.status,
MobRunStatus::Completed,
"peer-only external flow dispatch failed: failures={:?} steps={:?}",
terminal.failure_ledger,
terminal.step_ledger
);
let non_host_start_turn = service
.start_turn_call_count()
.saturating_sub(service.keep_alive_start_turn_call_count());
assert_eq!(
non_host_start_turn, baseline_non_host_start_turn,
"autonomous flow dispatch should avoid non-host start_turn calls"
);
assert!(
service.inject_call_count() > 0,
"autonomous flow dispatch should use injector routing"
);
}
#[tokio::test]
async fn test_flow_dispatch_turn_driven_mode_uses_machine_bound_session() {
let mut definition = sample_definition_with_dispatch_mode(DispatchMode::OneToOne);
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|profile| profile.as_inline_mut())
.expect("worker profile")
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let machine_session = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("machine binding for turn-driven worker");
// The autonomous lead's keep-alive turn is admitted in the background and
// may race this snapshot under suite load. Scope the assertion to the
// worker binding so it detects duplicate flow dispatches, not unrelated
// kickoff scheduling.
let baseline_machine_session_turns = service
.recorded_start_turn_prompts()
.await
.iter()
.filter(|(session_id, _)| session_id == &machine_session)
.count();
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(
terminal.status,
MobRunStatus::Completed,
"turn-driven flow dispatch failed: failures={:?} steps={:?}",
terminal.failure_ledger,
terminal.step_ledger
);
let prompts = service.recorded_start_turn_prompts().await;
assert_eq!(
prompts
.iter()
.filter(|(session_id, _)| session_id == &machine_session)
.count(),
baseline_machine_session_turns + 1,
"turn-driven flow dispatch must issue exactly one start_turn through the MobMachine session binding"
);
}
#[tokio::test]
async fn test_flow_dispatch_autonomous_mode_with_overlay_rejects() {
let mut definition = sample_definition_with_dispatch_mode(DispatchMode::OneToOne);
let flow = definition
.flows
.get_mut(&flow_id("dispatch"))
.expect("dispatch flow exists");
let step = flow
.steps
.get_mut(&step_id("dispatch"))
.expect("dispatch step exists");
step.allowed_tools = Some(vec!["alpha".to_string()]);
step.blocked_tools = Some(vec!["beta".to_string()]);
let (handle, _service) = create_test_mob(definition).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(
terminal.status,
MobRunStatus::Failed,
"flow should fail because tool overlay cannot be enforced for autonomous host members"
);
}
#[tokio::test]
async fn test_internal_turn_mode_routing_uses_injector_for_autonomous_and_start_turn_for_turn_driven()
{
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-auto"),
None,
)
.await
.expect("spawn autonomous lead");
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("l-turn"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
// Allow background start_turn from autonomous spawn to register.
tokio::time::sleep(Duration::from_millis(10)).await;
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&AgentIdentity::from("l-auto"))
.await
.expect("member handle")
.internal_turn("internal autonomous")
.await
.expect("autonomous internal turn");
handle
.member(&AgentIdentity::from("l-turn"))
.await
.expect("member handle")
.internal_turn("internal turn-driven")
.await
.expect("turn-driven internal turn");
// Runtime adapter path: autonomous spawn uses accept_input_with_completion (already in baseline).
// internal_turn for autonomous uses inject (1). internal_turn for turn-driven uses start_turn (+1).
assert_eq!(
service.inject_call_count(),
1,
"autonomous internal turn should route via injector (1 internal_turn only)"
);
assert_eq!(
service.start_turn_call_count(),
baseline_start_turn + 1,
"turn-driven internal turn should route via start_turn"
);
}
#[tokio::test]
async fn test_force_cancel_member_routes_boundary_cancel_without_retiring_member() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("cancel-target");
let member_ref = handle
.spawn_with_options(
// `send` is the external-admission seam used below to establish
// one exact in-flight turn before the cancellation assertion.
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn target");
let session_id = member_ref
.bridge_session_id()
.cloned()
.expect("session-backed target");
// An idle attachment accepts a machine-owned queued cancel and applies
// the executor callback asynchronously. Hold one real turn open so this
// test exercises the synchronous active-run boundary-cancel contract
// instead of racing the idle effect queue.
service.set_start_turn_delay_ms(600_000);
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&member_id)
.await
.expect("member handle")
.send("hold active turn", HandlingMode::Queue)
.await
.expect("active turn admission");
wait_for_start_turn_call_count(
service.as_ref(),
baseline_start_turn + 1,
"force-cancel target should reach the live session boundary",
)
.await;
let baseline_boundary = service.cancel_after_boundary_call_count();
let baseline_interrupts = service.interrupt_call_count();
handle
.force_cancel_member(AgentIdentity::from(member_id.as_str()))
.await
.expect("force cancel should succeed");
assert!(
service.cancel_after_boundary_call_count() > baseline_boundary,
"force-cancel must request cooperative boundary cancel"
);
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"force-cancel must not use hard session interrupt authority"
);
let entry = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("member remains");
assert_eq!(
entry.member_ref.bridge_session_id().cloned(),
Some(session_id.clone()),
"force-cancel must not rewrite the member's canonical session binding"
);
assert_eq!(
service.active_session_count().await,
1,
"force-cancel must not archive the backing session"
);
assert!(
service.read(&session_id).await.is_ok(),
"force-cancel must leave the backing session reachable"
);
// Release the deliberately blocked test turn after sampling the hard-
// interrupt counter so teardown cannot weaken the assertion above.
SessionService::interrupt(service.as_ref(), &session_id)
.await
.expect("release blocked force-cancel test turn");
}
/// Defect regression (HomeCore report, defect C): the operator remedy for a
/// wedged run must itself never wedge. Force-cancel of a member whose turn is
/// mid-provider-call (a `start_turn` that never completes) must transition and
/// interrupt, and a second force-cancel while the first cancellation is still
/// converging must be an idempotent no-op success — never
/// `invalid state transition: Running -> Running`.
#[tokio::test]
async fn test_force_cancel_member_mid_provider_call_is_idempotent() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("cancel-wedged");
let member_ref = handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn target");
let session_id = member_ref
.bridge_session_id()
.cloned()
.expect("session-backed target");
// Wedge the member: the provider call (start_turn) never returns.
service.set_start_turn_delay_ms(600_000);
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&member_id)
.await
.expect("member handle")
.send("wedge mid-provider-call", HandlingMode::Queue)
.await
.expect("active turn admission");
wait_for_start_turn_call_count(
service.as_ref(),
baseline_start_turn + 1,
"wedged turn should reach the live session boundary",
)
.await;
let baseline_boundary = service.cancel_after_boundary_call_count();
handle
.force_cancel_member(AgentIdentity::from(member_id.as_str()))
.await
.expect("force cancel of a mid-provider-call member must succeed");
assert!(
service.cancel_after_boundary_call_count() > baseline_boundary,
"force-cancel must request cooperative boundary cancel"
);
// Idempotency: a second force-cancel while the first cancellation is
// still in flight converges as success rather than an error.
handle
.force_cancel_member(AgentIdentity::from(member_id.as_str()))
.await
.expect("repeated force cancel must be an idempotent success");
// The member stays in the roster with its canonical binding intact.
let entry = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("member remains after repeated force-cancel");
assert_eq!(
entry.member_ref.bridge_session_id().cloned(),
Some(session_id.clone()),
"repeated force-cancel must not rewrite the member's session binding"
);
// Release the deliberately blocked test turn.
SessionService::interrupt(service.as_ref(), &session_id)
.await
.expect("release blocked force-cancel test turn");
}
/// Retirement must quiesce a provider turn before ingress detach. A provider
/// future that never returns used to retain runtime authority and wedge the
/// retire command indefinitely.
#[tokio::test]
async fn test_retire_quiesces_mid_provider_call_before_ingress_detach() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("retire-wedged");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn retirement target");
service.set_start_turn_delay_ms(600_000);
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&member_id)
.await
.expect("member handle")
.send("wedge before retire", HandlingMode::Queue)
.await
.expect("active turn admission");
wait_for_start_turn_call_count(
service.as_ref(),
baseline_start_turn + 1,
"retirement target should reach the provider call",
)
.await;
tokio::time::timeout(
Duration::from_secs(10),
handle.retire(AgentIdentity::from(member_id.as_str())),
)
.await
.expect("retirement must not wedge behind ingress detach")
.expect("retirement should succeed after hard-cancel escalation");
assert!(
handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.expect("read roster")
.is_none(),
"retirement must remove the quiesced member"
);
}
#[tokio::test]
async fn test_runtime_adapter_cancel_all_work_rejects_unsupported_boundary_cancel() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("runtime-unsupported-boundary");
let session_id = handle
.spawn_with_options(
// `send` is the ingress-acknowledged external admission seam, so
// use the fixture's externally addressable profile. The worker
// profile deliberately rejects external-origin work before a run
// can reach the boundary-cancel path under test.
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn runtime-backed target")
.bridge_session_id()
.cloned()
.expect("session-backed target");
let entry = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("member exists");
// Boundary-handle failures are synchronous only for an exact current
// run. An idle attachment legitimately accepts the machine-owned queued
// cancel effect and reports any later executor failure asynchronously.
// Hold one real run open so this test exercises the supported rejection
// contract rather than racing an idle admission acknowledgement.
service.set_start_turn_delay_ms(600_000);
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&member_id)
.await
.expect("member handle")
.send("hold active turn", HandlingMode::Queue)
.await
.expect("active turn admission");
wait_for_start_turn_call_count(
service.as_ref(),
baseline_start_turn + 1,
"runtime-backed active turn should reach the session boundary",
)
.await;
service.set_cancel_after_boundary_supported(false);
let baseline_boundary = service.cancel_after_boundary_call_count();
let baseline_interrupts = service.interrupt_call_count();
let error = handle
.cancel_all_work(entry.agent_runtime_id.clone(), entry.fence_token)
.await
.expect_err("unsupported runtime-backed boundary cancel must not report success");
assert!(
matches!(&error, MobError::Internal(message) if message.contains("cancel_after_boundary")),
"runtime-backed unsupported boundary cancel should surface through the adapter path, got {error:?}"
);
assert_eq!(
service.cancel_after_boundary_call_count(),
baseline_boundary + 1,
"runtime-backed cancel_all_work should attempt exactly one live boundary cancel"
);
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"runtime-backed unsupported boundary cancel must not fall back to hard interrupt"
);
// Test teardown: release the deliberately blocked run after all
// cooperative-cancel assertions have sampled their counters.
SessionService::interrupt(service.as_ref(), &session_id)
.await
.expect("release blocked test turn");
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_provision_member_uses_local_bindings_before_routed_runtime_bound() {
type RecordingSignalFields = Vec<(
meerkat_machine_schema::identity::FieldId,
meerkat_runtime::composition::OwnedFieldValue,
)>;
type RecordingMobSignalLog = Vec<(
meerkat_machine_schema::identity::SignalVariantId,
RecordingSignalFields,
)>;
#[derive(Default)]
struct RecordingMobSignalSurface {
log: tokio::sync::Mutex<RecordingMobSignalLog>,
}
#[async_trait]
impl meerkat_runtime::composition::SignalConsumerSurface for RecordingMobSignalSurface {
fn instance_id(&self) -> &meerkat_machine_schema::identity::MachineInstanceId {
static ID: std::sync::OnceLock<meerkat_machine_schema::identity::MachineInstanceId> =
std::sync::OnceLock::new();
ID.get_or_init(|| {
meerkat_machine_schema::identity::MachineInstanceId::parse("mob")
.expect("canonical mob instance id")
})
}
async fn receive_signal(
&self,
variant: meerkat_machine_schema::identity::SignalVariantId,
projected_fields: RecordingSignalFields,
) -> Result<(), meerkat_runtime::composition::ConsumerError> {
self.log.lock().await.push((variant, projected_fields));
Ok(())
}
}
let service = Arc::new(MockSessionService::new());
let adapter = Arc::new(meerkat_runtime::MeerkatMachine::ephemeral());
let signal_surface = Arc::new(RecordingMobSignalSurface::default());
let schema = meerkat_machine_schema::catalog::meerkat_mob_seam_composition();
let table =
meerkat_runtime::composition::RouteTable::from_schema(&schema).expect("catalog routes");
let dispatcher: meerkat_runtime::composition::CatalogCompositionSignalDispatcher<
meerkat_runtime::meerkat_machine::composition::MeerkatSeamSignal,
> = meerkat_runtime::composition::CatalogCompositionSignalDispatcher::new(
schema.name.clone(),
table,
)
.with_consumer(signal_surface.clone());
adapter.set_composition_signal_dispatcher(Arc::new(dispatcher));
let provisioner = super::provisioner::SessionBackend::new(service, Some(adapter.clone()), None);
let bridge_session = Session::new();
let bridge_session_id = bridge_session.id().clone();
provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "local binding provision".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(bridge_session),
comms_name: Some("test-mob/worker/local-binding".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("local-binding-worker"),
peer_name: "local-binding-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(bridge_session_id.clone()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect("member provision should install local runtime resources");
assert!(
adapter.contains_session(&bridge_session_id).await,
"local binding preparation should still register the bridge session"
);
assert!(
signal_surface.log.lock().await.is_empty(),
"member provisioning must not publish RuntimeBound with the session runtime id; the routed mob binding owns that signal"
);
}
#[cfg(feature = "runtime-adapter")]
struct FencedRetirementSessionStore {
inner: Arc<MemoryStore>,
presented_fence: AtomicU64,
current_fence: AtomicU64,
content_projection_attempts: AtomicU64,
stale_content_projection_attempts: AtomicU64,
}
#[cfg(feature = "runtime-adapter")]
impl FencedRetirementSessionStore {
fn new(fence: u64) -> Self {
Self {
inner: Arc::new(MemoryStore::new()),
presented_fence: AtomicU64::new(fence),
current_fence: AtomicU64::new(fence),
content_projection_attempts: AtomicU64::new(0),
stale_content_projection_attempts: AtomicU64::new(0),
}
}
fn rotate_fence(&self, fence: u64) {
self.current_fence.store(fence, Ordering::SeqCst);
}
fn content_projection_attempts(&self) -> u64 {
self.content_projection_attempts.load(Ordering::SeqCst)
}
fn stale_content_projection_attempts(&self) -> u64 {
self.stale_content_projection_attempts
.load(Ordering::SeqCst)
}
fn authorize_content_projection(&self) -> Result<(), meerkat_store::SessionStoreError> {
self.content_projection_attempts
.fetch_add(1, Ordering::SeqCst);
let presented = self.presented_fence.load(Ordering::SeqCst);
let current = self.current_fence.load(Ordering::SeqCst);
if presented == current {
return Ok(());
}
self.stale_content_projection_attempts
.fetch_add(1, Ordering::SeqCst);
Err(meerkat_store::SessionStoreError::Internal(format!(
"stale fencing token: presented {presented}, current {current}"
)))
}
}
#[cfg(feature = "runtime-adapter")]
#[async_trait]
impl SessionStore for FencedRetirementSessionStore {
async fn save(&self, session: &Session) -> Result<(), meerkat_store::SessionStoreError> {
self.authorize_content_projection()?;
SessionStore::save(self.inner.as_ref(), session).await
}
async fn save_transcript_rewrite(
&self,
session: &Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<(), meerkat_store::SessionStoreError> {
self.authorize_content_projection()?;
SessionStore::save_transcript_rewrite(self.inner.as_ref(), session, commit).await
}
async fn save_authoritative_projection(
&self,
session: &Session,
) -> Result<(), meerkat_store::SessionStoreError> {
self.authorize_content_projection()?;
SessionStore::save_authoritative_projection(self.inner.as_ref(), session).await
}
async fn save_authoritative_projection_if_current_revision(
&self,
session: &Session,
expected_current_revision: Option<String>,
) -> Result<(), meerkat_store::SessionStoreError> {
self.authorize_content_projection()?;
SessionStore::save_authoritative_projection_if_current_revision(
self.inner.as_ref(),
session,
expected_current_revision,
)
.await
}
async fn load(
&self,
id: &SessionId,
) -> Result<Option<Session>, meerkat_store::SessionStoreError> {
SessionStore::load(self.inner.as_ref(), id).await
}
async fn list(
&self,
filter: meerkat_core::session_store::SessionFilter,
) -> Result<Vec<meerkat_core::session::SessionMeta>, meerkat_store::SessionStoreError> {
SessionStore::list(self.inner.as_ref(), filter).await
}
async fn delete(&self, id: &SessionId) -> Result<(), meerkat_store::SessionStoreError> {
SessionStore::delete(self.inner.as_ref(), id).await
}
async fn delete_if_current_revision(
&self,
id: &SessionId,
expected_current_revision: &str,
) -> Result<bool, meerkat_store::SessionStoreError> {
SessionStore::delete_if_current_revision(self.inner.as_ref(), id, expected_current_revision)
.await
}
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_successful_retire_commits_runtime_terminal_before_content_projection_quiescence() {
let fenced_store = Arc::new(FencedRetirementSessionStore::new(1));
let session_store: Arc<dyn SessionStore> = fenced_store.clone();
let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> =
Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
PersistentMockBuilder,
16,
session_store,
runtime_store.clone(),
blob_store,
));
let adapter = service
.runtime_adapter()
.expect("persistent service runtime adapter");
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create persistent mob");
let identity = AgentIdentity::from("w-fenced-retirement");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle
.spawn_spec(spec)
.await
.expect("spawn fenced persistent member");
let session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("session-backed fenced member");
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
handle
.retire(identity.clone())
.await
.expect("retire while the admitted write fence remains valid");
assert!(
handle
.get_member(&identity)
.await
.expect("read roster after fenced retirement")
.is_none(),
"successful retirement must remove the roster anchor"
);
assert!(
!adapter.contains_session(&session_id).await,
"successful retirement must unregister the exact runtime session"
);
assert!(
!service
.has_live_session(&session_id)
.await
.expect("read live session after fenced retirement"),
"successful retirement must remove the live session actor"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read runtime lifecycle after fenced retirement"),
Some(meerkat_runtime::RuntimeState::Retired),
"successful retirement must commit the RuntimeStore-owned lifecycle terminal before returning"
);
// SessionStore is a content projection, not archive authority. Rotate its
// fixture fence only after the RuntimeStore terminal is observed; any late
// content writer must then be visible as a stale projection attempt.
let attempts_at_retirement = fenced_store.content_projection_attempts();
fenced_store.rotate_fence(2);
handle
.shutdown()
.await
.expect("whole-mob shutdown after fenced retirement");
tokio::task::yield_now().await;
assert_eq!(
fenced_store.content_projection_attempts(),
attempts_at_retirement,
"no SessionStore projection may begin after successful retirement"
);
assert_eq!(
fenced_store.stale_content_projection_attempts(),
0,
"rotating the content-projection fence after successful retirement must observe no old-fence writer"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_fresh_provision_failure_preserves_resumable_document_and_quiesces_incarnation() {
let memory_store = Arc::new(MemoryStore::new());
let session_store: Arc<dyn SessionStore> = memory_store.clone();
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_dyn: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
OverlayProbeSessionAgentBuilder {
provider_visible_tools: Arc::new(Mutex::new(Vec::new())),
provider_turn_overlays: Arc::new(Mutex::new(Vec::new())),
},
16,
session_store,
runtime_store_dyn,
blob_store,
));
let adapter = service
.runtime_adapter()
.expect("persistent service runtime adapter");
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let error = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "fresh provision cancellation".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(session),
comms_name: Some("test-mob/worker/fresh-cancelled".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("fresh-cancelled-worker"),
peer_name: "fresh-cancelled-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(SessionId::new()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect_err("post-create owner mismatch must fail before receipt publication");
assert!(
error
.to_string()
.contains("did not match created bridge session"),
"unexpected fresh rollback failure: {error:?}"
);
let durable = service
.load_authoritative_session(&session_id)
.await
.expect("load authoritative fresh provision document")
.expect("runtime-authoritative session remains discoverable after response loss");
assert_ne!(
durable.lifecycle_terminal(),
Some(meerkat_core::session::SessionLifecycleTerminal::Archived),
"post-create cancellation must not pretend the durable create never happened"
);
assert!(
service
.load_persisted_session(&session_id)
.await
.expect("read preserved durable session")
.is_some(),
"the preserved session must remain eligible for explicit resume"
);
assert!(!service.has_live_session(&session_id).await.unwrap());
assert!(!adapter.contains_session(&session_id).await);
assert!(
!provisioner
.has_runtime_session_sidecar_for_test(&session_id)
.await
);
let runtime_state =
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load quiesced fresh provision runtime");
assert!(
!matches!(
runtime_state,
Some(meerkat_runtime::RuntimeState::Attached | meerkat_runtime::RuntimeState::Running)
),
"failed publication must leave no serving executor incarnation: {runtime_state:?}"
);
let resumed = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "explicitly resume preserved provision".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(durable),
comms_name: Some("test-mob/worker/fresh-cancelled".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("fresh-cancelled-worker"),
peer_name: "fresh-cancelled-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(session_id.clone()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect("explicit resume should reconstruct the preserved session");
assert!(service.has_live_session(&session_id).await.unwrap());
assert!(adapter.contains_session(&session_id).await);
assert!(
provisioner
.has_runtime_session_sidecar_for_test(&session_id)
.await
);
provisioner
.retire_member(&resumed.member_ref)
.await
.expect("retire resumed test member");
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_retired_session_revival_failure_preserves_resumable_document() {
let provider_visible_tools = Arc::new(Mutex::new(Vec::new()));
let provider_turn_overlays = Arc::new(Mutex::new(Vec::new()));
let memory_store = Arc::new(MemoryStore::new());
let session_store: Arc<dyn SessionStore> = memory_store.clone();
let runtime_store = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let runtime_store_dyn: Arc<dyn meerkat_runtime::RuntimeStore> = runtime_store.clone();
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
OverlayProbeSessionAgentBuilder {
provider_visible_tools,
provider_turn_overlays,
},
16,
session_store,
runtime_store_dyn,
blob_store,
));
let adapter = service
.runtime_adapter()
.expect("persistent service runtime adapter");
let provisioner =
super::provisioner::SessionBackend::new(service.clone(), Some(adapter.clone()), None);
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let receipt = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::Fresh,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "retired revival rollback seed".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(session),
comms_name: Some("test-mob/worker/revival-rollback".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("revival-rollback-worker"),
peer_name: "revival-rollback-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(session_id.clone()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect("seed session provision");
provisioner
.retire_member(&receipt.member_ref)
.await
.expect("retire seed session");
let archived = service
.load_authoritative_session(&session_id)
.await
.expect("load authoritative archived seed")
.expect("archived seed remains durable under runtime authority");
assert!(
service
.load_persisted_session(&session_id)
.await
.expect("ordinary read of retired seed session")
.is_none(),
"Retired runtime authority must make the seed session archived to ordinary reads"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load retired seed runtime"),
Some(meerkat_runtime::RuntimeState::Retired)
);
// Product-level explicit resume first fences the retired attachment-local
// plumbing. In this retired/no-attachment case that compare-removes the
// exact terminal ops-registry binding left by the old incarnation; the
// lower-level provision call must never overwrite it with a new registry.
assert!(
provisioner
.prepare_member_session_for_explicit_resume(&session_id)
.await
.expect("prepare retired seed for explicit resume"),
"a retired seed session must be rebuilt by explicit resume"
);
// The generated owner mismatch is deliberately validated after durable
// archived-session promotion. The response is therefore an ambiguous
// post-commit outcome: volatile state must quiesce, while the promoted
// document remains discoverable and resumable.
let mismatched_owner = SessionId::new();
let error = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "retired revival rollback resume".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(archived),
comms_name: Some("test-mob/worker/revival-rollback".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("revival-rollback-worker"),
peer_name: "revival-rollback-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(mismatched_owner),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect_err("post-attachment validation must fail");
assert!(
error
.to_string()
.contains("did not match created bridge session"),
"unexpected post-attachment failure: {error:?}"
);
let preserved = service
.load_authoritative_session(&session_id)
.await
.expect("load authoritative preserved session")
.expect("authorized session remains durable under runtime authority");
// RuntimeStore owns lifecycle truth. The portable Session body need not
// mirror an Active marker; ordinary visibility and explicit resume below
// prove that the store-owned reset survived the lost response.
assert!(
service
.load_persisted_session(&session_id)
.await
.expect("ordinary read after failed authorized revival")
.is_some(),
"the authorized session must remain eligible for explicit resume"
);
assert!(
!service
.has_live_session(&session_id)
.await
.expect("live-session lookup after failed authorized revival"),
"failed receipt publication must quiesce the unpublished actor incarnation"
);
assert!(
!adapter.contains_session(&session_id).await,
"failed receipt publication must remove the unpublished runtime incarnation"
);
assert!(
!provisioner
.has_runtime_session_sidecar_for_test(&session_id)
.await,
"failed receipt publication must remove the unpublished attachment sidecar"
);
let runtime_state =
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load quiesced failed-revival runtime");
assert!(
!matches!(
runtime_state,
Some(meerkat_runtime::RuntimeState::Attached | meerkat_runtime::RuntimeState::Running)
),
"failed receipt publication must leave no serving executor incarnation: {runtime_state:?}"
);
let resumed = provisioner
.provision_member(super::provisioner::ProvisionMemberRequest {
session_origin: super::provisioner::ProvisionSessionOrigin::ResumedDurable,
create_session: CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "explicitly resume preserved promoted session"
.to_string()
.into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(preserved),
comms_name: Some("test-mob/worker/revival-rollback".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
},
binding: test_external_binding("revival-rollback-worker"),
peer_name: "revival-rollback-worker".to_string(),
owner_bridge_session_id: None,
ops_registry: None,
generated_self_owned_operation_owner: Some(session_id.clone()),
runtime_revival_intent: super::provisioner::RuntimeRevivalIntent::None,
})
.await
.expect("explicit resume should reconstruct the preserved promoted session");
assert!(service.has_live_session(&session_id).await.unwrap());
assert!(adapter.contains_session(&session_id).await);
provisioner
.retire_member(&resumed.member_ref)
.await
.expect("retire explicitly resumed test member");
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_cancel_all_work_without_adapter_uses_boundary_cancel_not_hard_interrupt() {
let service = Arc::new(MockSessionService::new());
let session = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "no-adapter boundary".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-boundary".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create session-backed member");
let provisioner = super::provisioner::SessionBackend::new(service.clone(), None, None);
let member_ref = MemberRef::from_bridge_session_id(session.session_id.clone());
let wake = service
.keep_alive_notifiers
.read()
.await
.get(&session.session_id)
.cloned()
.expect("keep-alive session should install a cooperative wake notifier");
let notified = wake.notified();
let baseline_boundary = service.cancel_after_boundary_call_count();
let baseline_interrupts = service.interrupt_call_count();
provisioner
.interrupt_member(&member_ref, None)
.await
.expect("no-adapter member interrupt should request cooperative boundary cancel");
tokio::time::timeout(Duration::from_secs(1), notified)
.await
.expect("no-adapter cooperative boundary cancel should wake the waiting turn");
assert_eq!(
service.cancel_after_boundary_call_count(),
baseline_boundary + 1,
"no-adapter member interrupt must request cooperative boundary cancel"
);
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"no-adapter member interrupt must not use hard session interrupt"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_interrupt_member_without_adapter_rejects_unsupported_boundary_cancel() {
let service = Arc::new(MockSessionService::new());
service.set_cancel_after_boundary_supported(false);
let session = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "no-adapter unsupported boundary".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-unsupported-boundary".to_string()),
keep_alive: true,
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create session-backed member");
let provisioner = super::provisioner::SessionBackend::new(service.clone(), None, None);
let member_ref = MemberRef::from_bridge_session_id(session.session_id.clone());
let baseline_boundary = service.cancel_after_boundary_call_count();
let baseline_interrupts = service.interrupt_call_count();
let error = provisioner
.interrupt_member(&member_ref, None)
.await
.expect_err("unsupported boundary cancel must not be reported as success");
assert!(
matches!(
error,
MobError::SessionError(SessionError::Unsupported(ref operation))
if operation == "cancel_after_boundary"
),
"unsupported cooperative boundary cancel should surface as a session error, got {error:?}"
);
assert_eq!(
service.cancel_after_boundary_call_count(),
baseline_boundary + 1,
"no-adapter interrupt should attempt cooperative boundary cancel before failing"
);
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"unsupported cooperative boundary cancel must not fall back to hard interrupt"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_explicit_hard_cancel_member_without_adapter_is_rejected() {
let service = Arc::new(MockSessionService::new());
let session = service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "no-adapter hard cancel".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
comms_name: Some("test-mob/worker/w-hard-cancel".to_string()),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create session-backed member");
let provisioner = super::provisioner::SessionBackend::new(service.clone(), None, None);
let member_ref = MemberRef::from_bridge_session_id(session.session_id.clone());
let baseline_boundary = service.cancel_after_boundary_call_count();
let baseline_interrupts = service.interrupt_call_count();
let error = provisioner
.hard_cancel_member(&member_ref, "explicit no-adapter force cancel")
.await
.expect_err("no-adapter hard_cancel_member must not bypass runtime authority");
assert!(
matches!(error, MobError::Internal(ref message) if message.contains("requires MeerkatMachine runtime authority")),
"no-adapter hard cancel should fail closed on missing runtime authority, got {error:?}"
);
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"explicit no-adapter force cancel must not use hard session interrupt"
);
assert_eq!(
service.cancel_after_boundary_call_count(),
baseline_boundary,
"explicit force cancel must not be downgraded to boundary cancel"
);
}
#[tokio::test]
async fn test_external_backend_turn_driven_mode_uses_start_turn_dispatch() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-turn-driven",
);
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob(definition).await;
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "lead", "l-ext-turn")).await;
{
let mut spec = SpawnMemberSpec::new("lead", "l-ext-turn");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
spec.binding = Some(external.binding());
spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn external turn-driven lead");
}
let baseline_start_turn = service.start_turn_call_count();
handle
.member(&AgentIdentity::from("l-ext-turn"))
.await
.expect("member handle")
.send(
"external turn-driven",
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("external turn should execute");
assert_eq!(
service.inject_call_count(),
0,
"peer-only turn-driven external dispatch should not use injector"
);
assert_eq!(
service.start_turn_call_count(),
baseline_start_turn,
"peer-only turn-driven external dispatch should avoid local bridge start_turn"
);
}
/// Phase-6 supersession of the peer-only failure pin (§18.11:1038 /
/// design-flow-spine §5.4). An UNPLACED legacy peer-only EXTERNAL member's
/// flow step STILL fails — `ClassifyFlowStepDispatch` classifies a
/// placement-less member `Local`, the local TurnDriven path routes to the
/// peer-only `start_turn`, and its tracked-turn (`event_tx`) reject surfaces
/// as a TYPED PER-STEP ledger failure at dispatch (never a whole-run
/// mid-flight surprise). The WORKING remote path is placed members via
/// `RemoteTurnDirective` — covered by the cross_host_flows.rs battery
/// (T-F1/T-F3); making unplaced peer-only steps work is an explicit
/// non-goal.
#[tokio::test]
async fn test_unplaced_external_flow_step_fails_typed_per_step_at_dispatch() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_dispatch_mode(DispatchMode::OneToOne),
"external-autonomous-flow",
);
definition.backend.external = Some(crate::definition::ExternalBackendConfig {
address_base: "https://backend.example.invalid/mesh".to_string(),
supervisor_bridge: None,
});
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob(definition).await;
let external_lead =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "lead", "l-ext-auto")).await;
let external_worker =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext-auto")).await;
{
let mut spec = SpawnMemberSpec::new("lead", "l-ext-auto");
spec.runtime_mode = Some(crate::MobRuntimeMode::AutonomousHost);
spec.binding = Some(external_lead.binding());
spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn external autonomous lead");
}
{
let mut spec = SpawnMemberSpec::new("worker", "w-ext-auto");
spec.runtime_mode = Some(crate::MobRuntimeMode::AutonomousHost);
spec.binding = Some(external_worker.binding());
spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn external autonomous worker");
}
let baseline_non_host_start_turn = service
.start_turn_call_count()
.saturating_sub(service.keep_alive_start_turn_call_count());
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(
terminal.status,
MobRunStatus::Failed,
"unplaced peer-only flow dispatch keeps the typed failure shape: failures={:?} steps={:?}",
terminal.failure_ledger,
terminal.step_ledger
);
// The failure is a PER-STEP ledger entry recorded at dispatch: the
// failure ledger names the step, and the reason is the typed peer-only
// tracked-turn reject (UnsupportedForMode), not a generic run collapse.
assert!(
!terminal.failure_ledger.is_empty(),
"dispatch failure lands in the per-step failure ledger"
);
assert!(
terminal
.failure_ledger
.iter()
.all(|entry| !entry.step_id.as_str().is_empty()),
"every failure ledger entry names its step: {:?}",
terminal.failure_ledger
);
let failure_reasons = terminal
.failure_ledger
.iter()
.map(|entry| entry.reason.as_str())
.collect::<Vec<_>>();
assert!(
failure_reasons
.iter()
.any(|message| message.contains("tracked turn event streams are not supported")),
"the reason is the typed peer-only tracked-turn reject, got: {failure_reasons:?}"
);
let non_host_start_turn = service
.start_turn_call_count()
.saturating_sub(service.keep_alive_start_turn_call_count());
assert_eq!(
non_host_start_turn, baseline_non_host_start_turn,
"peer-only external flow dispatch should avoid non-host start_turn calls"
);
assert_eq!(
service.inject_call_count(),
0,
"peer-only external flow dispatch should not rely on local injector routing"
);
}
#[tokio::test]
async fn test_subscribe_all_agent_events_keeps_session_backed_members_in_mixed_roster() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"mixed-subscribe-all",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-local"),
None,
)
.await
.expect("spawn local lead");
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext-sub")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext-sub"),
None,
external.binding(),
)
.await
.expect("spawn external worker");
let streams = handle
.subscribe_all_agent_events()
.await
.expect("mixed roster should still expose session-backed event streams");
assert_eq!(streams.len(), 1);
assert_eq!(streams[0].0, AgentIdentity::from("l-local"));
}
#[tokio::test]
async fn test_subscribe_all_agent_events_surfaces_session_subscription_failure() {
// Regression: a session-backed member whose subscribe_session_events fails
// must propagate the error, not be silently dropped. Previously the loop
// used `if let Ok(stream) = ...` which turned real subscription errors
// into partial/empty lists and masked real regressions.
let definition = with_unique_mob_id(sample_definition(), "subscribe-all-failure-propagates");
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-local"),
None,
)
.await
.expect("spawn local lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-local"),
None,
)
.await
.expect("spawn local worker");
let broken_session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-local"))
.await
.expect("worker session should have a bridge session id");
service.fail_subscribe_for_session(broken_session_id).await;
match handle.subscribe_all_agent_events().await {
Ok(_) => panic!("broken session subscription must surface as an error"),
Err(MobError::Internal(message)) => assert!(
message.contains("failed to subscribe to agent events") && message.contains("w-local"),
"internal error should cite the failing member, got: {message}"
),
Err(other) => {
panic!("expected MobError::Internal propagating subscribe failure, got: {other:?}")
}
}
}
#[tokio::test]
async fn test_external_backend_lifecycle_and_turn_policy() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-lifecycle-policy",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external_lead =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "lead", "l-ext")).await;
let external_worker =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext")).await;
handle
.spawn_with_binding(
ProfileName::from("lead"),
AgentIdentity::from("l-ext"),
None,
external_lead.binding(),
)
.await
.expect("spawn external lead");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext"),
None,
external_worker.binding(),
)
.await
.expect("spawn external worker");
handle
.wire(AgentIdentity::from("l-ext"), AgentIdentity::from("w-ext"))
.await
.expect("wire external members");
handle
.unwire(AgentIdentity::from("l-ext"), AgentIdentity::from("w-ext"))
.await
.expect("unwire external members");
handle
.member(&AgentIdentity::from("l-ext"))
.await
.expect("member handle")
.send(
"outside hello".to_string(),
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("external lead should accept external turns");
let denied = handle
.member(&AgentIdentity::from("w-ext"))
.await
.expect("member handle")
.send(
"outside hello".to_string(),
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect_err("worker external turn should be denied by profile policy");
assert!(matches!(denied, MobError::NotExternallyAddressable(_)));
handle
.retire(AgentIdentity::from("w-ext"))
.await
.expect("retire external member");
assert!(
handle
.get_member(&AgentIdentity::from("w-ext"))
.await
.unwrap()
.is_none(),
"retire should remove external backend member"
);
}
#[tokio::test]
async fn test_rewire_repairs_remote_trust_for_existing_peer_only_edge() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"rewire-peer-only-trust",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external_a =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "lead", "l-ext-rewire")).await;
let external_b =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-ext-rewire")).await;
handle
.spawn_with_binding(
ProfileName::from("lead"),
AgentIdentity::from("l-ext-rewire"),
None,
external_a.binding(),
)
.await
.expect("spawn external lead");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-ext-rewire"),
None,
external_b.binding(),
)
.await
.expect("spawn external worker");
handle
.wire(
AgentIdentity::from("l-ext-rewire"),
AgentIdentity::from("w-ext-rewire"),
)
.await
.expect("initial wire");
let crate::RuntimeBinding::External {
peer_id: lead_peer_id,
..
} = external_a.binding()
else {
panic!("expected external lead binding");
};
let crate::RuntimeBinding::External {
peer_id: worker_peer_id,
..
} = external_b.binding()
else {
panic!("expected external worker binding");
};
external_a.remove_trusted_peer(&worker_peer_id).await;
external_b.remove_trusted_peer(&lead_peer_id).await;
handle
.wire(
AgentIdentity::from("l-ext-rewire"),
AgentIdentity::from("w-ext-rewire"),
)
.await
.expect("rewire should reconcile both peer-only trust edges");
let lead_name = test_comms_name_for(&mob_id, "lead", "l-ext-rewire");
let worker_name = test_comms_name_for(&mob_id, "worker", "w-ext-rewire");
assert!(
external_a.trusted_peer_names().await.contains(&worker_name),
"rewire should restore worker trust on the lead peer-only runtime"
);
assert!(
external_b.trusted_peer_names().await.contains(&lead_name),
"rewire should restore lead trust on the worker peer-only runtime"
);
}
#[tokio::test]
async fn test_external_turn_prefers_effective_profile_override_addressability() {
let definition = sample_definition();
let mut override_profile = definition
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.expect("worker profile should be inline")
.clone();
override_profile.external_addressable = true;
let (handle, _service) = create_test_mob(definition).await;
let mut spec = SpawnMemberSpec::new("worker", AgentIdentity::from("w-override"));
spec.override_profile = Some(override_profile);
handle
.spawn_spec(spec)
.await
.expect("spawn override worker");
handle
.member(&AgentIdentity::from("w-override"))
.await
.expect("member handle")
.send(
"outside hello".to_string(),
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("override profile should permit external turns");
}
// -----------------------------------------------------------------------
// P1-T11: MobCreated event stores definition
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_mob_created_event_stores_definition() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let events = handle.events().replay_all().await.expect("replay");
// First event should be MobCreated
assert!(!events.is_empty(), "should have at least one event");
match &events[0].kind {
MobEventKind::MobCreated { definition } => {
assert_eq!(definition.id.as_str(), "test-mob");
assert_eq!(definition.profiles.len(), 2);
assert!(definition.profiles.contains_key(&ProfileName::from("lead")));
assert!(
definition
.profiles
.contains_key(&ProfileName::from("worker"))
);
}
other => panic!("first event should be MobCreated, got: {other:?}"),
}
}
#[tokio::test]
async fn test_mob_created_definition_roundtrips_through_json() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let events = handle.events().replay_all().await.expect("replay");
if let MobEventKind::MobCreated { definition } = &events[0].kind {
let json = serde_json::to_string(definition).expect("serialize");
let parsed: MobDefinition = serde_json::from_str(&json).expect("deserialize");
assert_eq!(**definition, parsed);
}
}
// -----------------------------------------------------------------------
// CHOKE-MOB-005: MobHandle parallel spawn provisioning
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_concurrent_spawns_parallelize_provisioning() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_create_session_delay_ms(120);
// Spawn 10 workers concurrently
let mut join_handles = Vec::new();
for i in 0..10 {
let h = handle.clone();
let jh = tokio::spawn(async move {
h.spawn(
ProfileName::from("worker"),
AgentIdentity::from(format!("w-{i}")),
None,
)
.await
});
join_handles.push(jh);
}
for jh in join_handles {
jh.await.expect("join").expect("spawn");
}
let meerkats = handle.list_members().await;
assert_eq!(
meerkats.len(),
10,
"all 10 concurrent spawns should succeed"
);
let ids = meerkats
.iter()
.map(|entry| entry.agent_identity.as_str().to_string())
.collect::<HashSet<_>>();
assert_eq!(
ids.len(),
10,
"serialized actor path must avoid duplicate IDs"
);
for expected in [
"w-0", "w-1", "w-2", "w-3", "w-4", "w-5", "w-6", "w-7", "w-8", "w-9",
] {
assert!(ids.contains(expected), "missing spawned meerkat {expected}");
}
let events = handle.events().replay_all().await.expect("replay");
let spawned_count = events
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberSpawned(..)))
.count();
assert_eq!(
spawned_count, 10,
"parallel spawn path should emit exactly one MemberSpawned per request"
);
assert!(
service.max_concurrent_create_session_calls() > 1,
"spawn provisioning should run in parallel (max observed concurrent create_session calls: {})",
service.max_concurrent_create_session_calls()
);
}
#[tokio::test]
async fn test_spawn_many_member_refs_returns_results_in_input_order() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_create_session_delay_ms(100);
let specs = vec![
SpawnMemberSpec::new("worker", "w-a"),
SpawnMemberSpec::new("worker", "w-b"),
SpawnMemberSpec::new("worker", "w-c"),
];
let results = handle
.spawn_many(specs)
.await
.expect("spawn_many result authority");
assert_eq!(results.len(), 3);
for (idx, result) in results.into_iter().enumerate() {
let member = result.expect("spawn_many result");
assert!(
!member.agent_identity.as_str().is_empty(),
"spawn_many result {idx} should include agent identity"
);
}
assert!(
service.max_concurrent_create_session_calls() > 1,
"spawn_many should use parallel provisioning under the hood"
);
}
#[tokio::test]
async fn test_spawn_many_parallel_finalize_deduplicates_worker_pair_wiring() {
let (handle, service) = create_test_mob(sample_definition_with_role_wiring()).await;
service.set_create_session_delay_ms(120);
let specs = vec![
SpawnMemberSpec::new("worker", "w-a"),
SpawnMemberSpec::new("worker", "w-b"),
];
let results = handle
.spawn_many(specs)
.await
.expect("spawn_many member ref authority");
for result in results {
result.expect("spawn_many member ref");
}
let a = handle
.get_member(&AgentIdentity::from("w-a"))
.await
.unwrap()
.expect("w-a should exist");
let b = handle
.get_member(&AgentIdentity::from("w-b"))
.await
.unwrap()
.expect("w-b should exist");
assert_eq!(
a.wired_to.len(),
1,
"parallel spawn finalization must dedupe the w-a<->w-b edge"
);
assert_eq!(
b.wired_to.len(),
1,
"parallel spawn finalization must dedupe the w-a<->w-b edge"
);
assert!(a.wired_to.contains(&AgentIdentity::from("w-b")));
assert!(b.wired_to.contains(&AgentIdentity::from("w-a")));
}
#[tokio::test]
async fn test_spawn_many_parallel_finalize_tolerates_overlapping_role_wiring_targets() {
let (handle, service) =
create_test_mob(sample_definition_with_overlapping_orchestrator_and_role_wiring()).await;
service.set_create_session_delay_ms(120);
let specs = vec![
SpawnMemberSpec::new("lead", "l-a"),
SpawnMemberSpec::new("worker", "w-a"),
];
let results = handle
.spawn_many(specs)
.await
.expect("spawn_many overlapping wiring authority");
for result in results {
result.expect("spawn_many member ref");
}
let lead = handle
.get_member(&AgentIdentity::from("l-a"))
.await
.unwrap()
.expect("lead should exist");
let worker = handle
.get_member(&AgentIdentity::from("w-a"))
.await
.unwrap()
.expect("worker should exist");
assert_eq!(
lead.wired_to.len(),
1,
"overlapping batch fan-out must treat an already-created edge as satisfied"
);
assert_eq!(
worker.wired_to.len(),
1,
"overlapping batch fan-out must treat an already-created edge as satisfied"
);
assert!(lead.wired_to.contains(&AgentIdentity::from("w-a")));
assert!(worker.wired_to.contains(&AgentIdentity::from("l-a")));
}
#[tokio::test]
async fn test_concurrent_spawn_and_retire_same_meerkat_is_serialized() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let h1 = handle.clone();
let h2 = handle.clone();
let spawn = tokio::spawn(async move {
h1.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
});
let retire = tokio::spawn(async move { h2.retire(AgentIdentity::from("w-1")).await });
let _ = spawn.await.expect("spawn join");
retire.await.expect("retire join").expect("retire");
let roster = handle.list_members().await;
assert!(
roster.is_empty() || (roster.len() == 1 && roster[0].agent_identity.as_str() == "w-1"),
"serialized spawn/retire should never corrupt roster"
);
}
#[tokio::test]
async fn test_retiring_member_is_not_routable_before_disposal_completes() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_archive_delay_ms(250);
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
let retire_handle = {
let handle = handle.clone();
tokio::spawn(async move { handle.retire(AgentIdentity::from("w-1")).await })
};
tokio::time::sleep(Duration::from_millis(25)).await;
let all_members = handle.list_all_members().await;
assert_eq!(
all_members.len(),
1,
"retiring member should remain observable"
);
assert_eq!(all_members[0].agent_identity.as_str(), "w-1");
let retiring_listed = handle.list_members_including_retiring().await;
assert_eq!(
retiring_listed.len(),
1,
"retiring member should remain visible with machine-owned Retiring status"
);
assert_eq!(
retiring_listed[0].status,
crate::runtime::handle::MobMemberStatus::Retiring
);
let active_members = handle.list_members().await;
assert!(
active_members.is_empty(),
"retiring member must leave the active roster before disposal completes"
);
let start_turn_calls_before = service.start_turn_call_count();
let external_turn = handle
.submit_work(
all_members[0].agent_runtime_id.clone(),
all_members[0].fence_token,
WorkRef::new(),
WorkSpec::new("still there?".to_string(), WorkOrigin::External),
)
.await;
assert!(matches!(external_turn, Err(MobError::MemberNotFound(id)) if id.as_str() == "w-1"));
let internal_turn = match handle.member(&AgentIdentity::from("w-1")).await {
Err(error) => error,
Ok(member) => member
.internal_turn("still there?".to_string())
.await
.expect_err("retiring member must reject new internal work"),
};
assert!(matches!(internal_turn, MobError::MemberNotFound(id) if id.as_str() == "w-1"));
assert_eq!(
service.start_turn_call_count(),
start_turn_calls_before,
"blocked turns must not reach the backing session"
);
retire_handle
.await
.expect("retire join")
.expect("retire completes");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"member should be removed once retirement completes"
);
assert_eq!(
service.active_session_count().await,
0,
"completed retirement must archive the backing session"
);
assert!(
service.read(&session_id).await.is_err(),
"retired backing session should no longer resolve as active in the owner test harness"
);
}
#[tokio::test]
async fn test_spawn_rejects_duplicate_id_while_first_spawn_is_pending() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_create_session_delay_ms(180);
let first = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-dup"),
None,
)
.await
})
};
tokio::time::sleep(Duration::from_millis(25)).await;
let duplicate = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-dup"),
None,
)
.await
.expect_err("duplicate pending spawn should fail immediately");
assert!(matches!(duplicate, MobError::MemberAlreadyExists(_)));
first
.await
.expect("spawn join")
.expect("first spawn succeeds");
assert_eq!(service.active_session_count().await, 1);
}
#[tokio::test]
async fn test_stop_fails_pending_spawns_and_cleans_up_provisioned_session() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_create_session_delay_ms(220);
let pending_spawn = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-stop"),
None,
)
.await
})
};
tokio::time::sleep(Duration::from_millis(20)).await;
handle.stop().await.expect("stop should succeed");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
let spawn_error = pending_spawn
.await
.expect("spawn join")
.expect_err("pending spawn should fail once stop begins");
match spawn_error {
MobError::Internal(message) => {
assert!(
message.contains("mob is stopping"),
"expected stop cancellation message, got: {message}"
);
}
other => panic!("expected internal pending-spawn cancellation, got: {other}"),
}
tokio::time::sleep(Duration::from_millis(260)).await;
assert_eq!(
service.active_session_count().await,
0,
"canceled pending spawn should retire any provisioned session during cleanup"
);
}
#[tokio::test]
async fn test_stop_pending_spawn_archive_failure_retains_cleanup_anchor_for_retry() {
let (handle, service) = create_test_mob(sample_definition()).await;
let agent_identity = AgentIdentity::from("w-stop-archive-fail");
let committed_session_id = handle
.spawn(ProfileName::from("worker"), agent_identity.clone(), None)
.await
.expect("spawn committed member for exact pending-spawn fixture")
.bridge_session_id()
.expect("committed member must be session-backed")
.clone();
// Stage the exact post-provision capability directly. The preceding test
// covers cancellation before provisioning settles; this test specifically
// proves Stop's archive-failure retention and retry contract after the
// pending spawn owns both its session and operation identifiers.
let pending_session = Session::new();
let pending_session_id = pending_session.id().clone();
service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "stop retained pending cleanup".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(pending_session),
comms_name: Some(test_comms_name("worker", "w-stop-archive-fail-anchor")),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create provisioned pending-spawn session");
service.set_archive_failure(&pending_session_id).await;
handle
.debug_stage_pending_spawn_for_retire(
agent_identity,
pending_session_id.clone(),
meerkat_core::ops::OperationId::new(),
)
.await
.expect("stage exact pending-spawn capability for Stop");
let stop_error = handle
.stop()
.await
.expect_err("stop should fail closed while pending spawn archive cleanup is ambiguous");
assert!(
stop_error
.to_string()
.contains("pending spawn cleanup incomplete"),
"stop should expose pending spawn cleanup failure, got: {stop_error}"
);
assert_eq!(
handle.status().await.unwrap(),
MobState::Running,
"failed stop must not commit the mob into Stopped"
);
assert!(
service
.has_live_session(&pending_session_id)
.await
.expect("read retained pending session"),
"archive failure must retain the exact provisioned session for retry"
);
assert_eq!(
service.active_session_count().await,
2,
"failed Stop must retain both the committed member and pending cleanup session"
);
service.clear_archive_failure(&pending_session_id).await;
handle
.stop()
.await
.expect("retrying stop should drain retained pending-spawn cleanup anchor");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
assert!(
!service
.has_live_session(&pending_session_id)
.await
.expect("read cleaned pending session"),
"retry must archive the exact pending-spawn cleanup session"
);
assert!(
service
.has_live_session(&committed_session_id)
.await
.expect("read committed member session"),
"Stop must preserve the committed member session"
);
assert_eq!(
service.active_session_count().await,
1,
"Stop preserves the committed member session after pending cleanup converges"
);
}
#[tokio::test]
async fn test_retire_retry_drains_exact_failed_pending_cleanup_before_classifying_later_spawn() {
let (handle, service) = create_test_mob(sample_definition()).await;
let agent_identity = AgentIdentity::from("w-retire-pending-cleanup-retry");
let committed_session_id = handle
.spawn(ProfileName::from("worker"), agent_identity.clone(), None)
.await
.expect("spawn committed member")
.bridge_session_id()
.expect("session-backed committed member")
.clone();
// Materialize the provisioned session that the pending-spawn abort owns.
// The test-only actor seam stages only the real pending shell capability;
// the public Retire below performs classification, CancelPendingSpawn,
// abort/archive, and exact anchor retention through production code.
let pending_session = Session::new();
let pending_session_id = pending_session.id().clone();
service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "retained pending cleanup".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(pending_session),
comms_name: Some(test_comms_name("worker", "w-retire-pending-cleanup-anchor")),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create provisioned pending-spawn session");
service.set_archive_failure(&pending_session_id).await;
handle
.debug_stage_pending_spawn_for_retire(
agent_identity.clone(),
pending_session_id.clone(),
meerkat_core::ops::OperationId::new(),
)
.await
.expect("stage exact pending-spawn capability for committed member");
let first_error = handle
.retire(agent_identity.clone())
.await
.expect_err("first exact pending-spawn abort must retain its failed cleanup anchor");
assert!(
first_error.to_string().contains("mock archive failure"),
"archive refusal should surface from the first abort, got: {first_error}"
);
let after_first_failure = handle
.query_machine_state()
.await
.expect("machine state after first failed abort");
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from(agent_identity.as_str());
assert!(
!after_first_failure
.pending_spawn_sessions
.contains_key(&dsl_identity),
"CancelPendingSpawn must commit before the mechanical cleanup failure"
);
assert!(
service
.has_live_session(&pending_session_id)
.await
.expect("read retained pending session"),
"failed abort must retain the exact provisioned session for retry"
);
// A distinct later pending incarnation may appear before the caller
// retries. While the old exact cleanup still fails, Retire must stop before
// classification and leave this current machine-owned pending session
// untouched.
let later_pending_session = Session::new();
let later_pending_session_id = later_pending_session.id().clone();
service
.create_session(CreateSessionRequest {
injected_context: Vec::new(),
model: "claude-sonnet-4-5".to_string(),
prompt: "later pending incarnation".to_string().into(),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
build: Some(meerkat_core::service::SessionBuildOptions {
resume_session: Some(later_pending_session),
comms_name: Some(test_comms_name("worker", "w-retire-pending-cleanup-later")),
..Default::default()
}),
initial_turn: meerkat_core::service::InitialTurnPolicy::Defer,
deferred_prompt_policy: meerkat_core::service::DeferredPromptPolicy::Discard,
labels: None,
})
.await
.expect("create later provisioned pending-spawn session");
handle
.debug_stage_pending_spawn_for_retire(
agent_identity.clone(),
later_pending_session_id.clone(),
meerkat_core::ops::OperationId::new(),
)
.await
.expect("stage later pending incarnation");
let retry_error = handle
.retire(agent_identity.clone())
.await
.expect_err("retry must fail before classifying while exact older cleanup is refused");
assert!(
retry_error.to_string().contains("mock archive failure"),
"retry should surface the retained exact cleanup failure, got: {retry_error}"
);
let after_failed_retry = handle
.query_machine_state()
.await
.expect("machine state after failed retry");
assert_eq!(
after_failed_retry
.pending_spawn_sessions
.get(&dsl_identity)
.map(|session_id| session_id.0.clone()),
Some(later_pending_session_id.to_string()),
"a failed older-anchor retry must not classify or cancel a later pending incarnation"
);
// Clear the original cleanup refusal. The next public retire must drain
// the retained exact session first, then classify the distinct current
// pending incarnation through MobMachine and clean it through the normal
// cancellation path before retiring the committed member.
service.clear_archive_failure(&pending_session_id).await;
handle
.retire(agent_identity.clone())
.await
.expect("retry should clean the exact retained anchor then retire the member");
assert!(
!service
.has_live_session(&pending_session_id)
.await
.expect("read cleaned pending session"),
"successful retry must archive the exact retained pending session"
);
assert!(
!service
.has_live_session(&later_pending_session_id)
.await
.expect("read cleaned later pending session"),
"post-anchor classification must cancel the distinct current pending incarnation"
);
assert!(
!service
.has_live_session(&committed_session_id)
.await
.expect("read retired committed session"),
"normal machine classification must continue after exact anchor cleanup"
);
assert!(
handle
.get_member(&agent_identity)
.await
.expect("read retired member")
.is_none(),
"retry must complete retirement after exact cleanup converges"
);
}
#[tokio::test]
async fn test_retire_absent_does_not_cancel_later_pending_spawn_incarnation() {
let _delay_guard = SpawnProvisionedCommandDelayGuard::set(500);
let (handle, service) = create_test_mob(sample_definition()).await;
let agent_identity = AgentIdentity::from("w-retire-absent-pending-spawn");
let comms_name = test_comms_name("worker", agent_identity.as_str());
let pending_spawn = {
let handle = handle.clone();
let agent_identity = agent_identity.clone();
tokio::spawn(async move {
handle
.spawn(ProfileName::from("worker"), agent_identity, None)
.await
})
};
let _session_id = wait_for_session_id_for_comms_name(&service, &comms_name).await;
let before_retire = handle
.debug_dsl_t2_snapshot()
.await
.expect("pending spawn machine snapshot before absent retire");
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from(agent_identity.as_str());
let pending_session_before = before_retire
.pending_spawn_sessions
.get(&dsl_identity)
.cloned()
.expect("MobMachine should own the pending spawn session incarnation");
handle
.retire(AgentIdentity::from(agent_identity.as_str()))
.await
.expect("RetireAbsent is an inert idempotent success");
let after_retire = handle
.debug_dsl_t2_snapshot()
.await
.expect("pending spawn machine snapshot after absent retire");
assert_eq!(
after_retire.pending_spawn_sessions.get(&dsl_identity),
Some(&pending_session_before),
"machine-owned absent-retire verdict must preserve the exact pending session incarnation"
);
pending_spawn
.await
.expect("spawn join")
.expect("an absent retire must not cancel a later pending incarnation");
assert_eq!(
service.active_session_count().await,
1,
"the new incarnation must remain live"
);
}
#[tokio::test]
async fn test_stop_clears_pending_spawn_count_and_failed_member_projection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let initial = handle
.debug_orchestrator_snapshot()
.await
.expect("initial orchestrator snapshot");
assert_eq!(initial.pending_spawn_count, 0);
service.set_create_session_delay_ms(220);
let pending_spawn = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-stop-lineage"),
None,
)
.await
})
};
tokio::time::sleep(Duration::from_millis(20)).await;
let staged = handle
.debug_orchestrator_snapshot()
.await
.expect("staged orchestrator snapshot");
assert_eq!(
staged.pending_spawn_count, 1,
"pending spawn must contribute to orchestrator-owned lineage while provisioning is in flight"
);
let staged_dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("staged dsl snapshot");
assert!(
staged_dsl
.pending_spawn_sessions
.keys()
.any(|identity| identity.0 == "w-stop-lineage"),
"pending spawn admission must name the member identity before provisioning completes"
);
handle.stop().await.expect("stop should succeed");
let _ = pending_spawn
.await
.expect("spawn join")
.expect_err("pending spawn should fail once stop begins");
tokio::time::sleep(Duration::from_millis(260)).await;
let stopped = handle
.debug_orchestrator_snapshot()
.await
.expect("stopped orchestrator snapshot");
assert_eq!(
stopped.pending_spawn_count, 0,
"stop must clear orchestrator-owned pending spawn lineage"
);
let stopped_dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("stopped dsl snapshot");
assert!(
stopped_dsl.pending_spawn_sessions.is_empty(),
"stop must clear machine-owned pending spawn admissions"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-stop-lineage"))
.await
.unwrap()
.is_none(),
"stop must not leave a canonical roster projection for a canceled pending spawn"
);
assert_eq!(
service.active_session_count().await,
0,
"stop must retire any provisioned session for the canceled pending spawn"
);
}
#[tokio::test]
async fn test_stop_rejects_active_flow_and_schema_requires_no_active_runs() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(60_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
tokio::time::sleep(Duration::from_millis(50)).await;
let error = handle
.stop()
.await
.expect_err("stop should reject active flows");
assert!(matches!(
error,
MobError::InvalidTransition {
from: MobState::Running,
to: MobState::Stopped,
}
));
assert_eq!(handle.status().await.unwrap(), MobState::Running);
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow after rejected stop");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
let schema = schema_mob_machine();
let stop = schema
.transitions
.iter()
.find(|transition| transition.name.as_str() == "StopRunning")
.expect("StopRunning transition");
assert!(
stop.guards
.iter()
.any(|guard| guard.name == "no_active_runs"),
"StopRunning should require no active runs"
);
}
#[tokio::test]
async fn test_failed_spawn_clears_pending_spawn_count_and_failed_roster_entry() {
let (handle, service) = create_test_mob(sample_definition_with_auto_wire()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let _initial = handle
.debug_orchestrator_snapshot()
.await
.expect("initial orchestrator snapshot");
service.set_create_session_delay_ms(150);
service
.set_comms_behavior(
&test_comms_name("worker", "w-pending"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
let spawn = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-pending"),
None,
)
.await
})
};
tokio::time::sleep(Duration::from_millis(30)).await;
let staged = handle
.debug_orchestrator_snapshot()
.await
.expect("staged orchestrator snapshot");
assert_eq!(
staged.pending_spawn_count, 1,
"pending spawn admission must be reflected in orchestrator-owned pending count while provisioning is in flight"
);
// topology_revision is orchestrator shell metadata not tracked by the DSL authority.
// The meaningful assertion is pending_spawn_count (checked above).
let error = spawn
.await
.expect("spawn join")
.expect_err("spawn should fail during auto-wire setup");
assert!(
matches!(error, MobError::WiringError(_) | MobError::Internal(_)),
"failed pending spawn should surface wiring/internal failure, got: {error}"
);
let settled = handle
.debug_orchestrator_snapshot()
.await
.expect("settled orchestrator snapshot");
assert_eq!(
settled.pending_spawn_count, 0,
"failed spawn finalization must clear orchestrator-owned pending spawn count"
);
// topology_revision is orchestrator shell metadata not tracked by the DSL authority.
assert!(
handle
.get_member(&AgentIdentity::from("w-pending"))
.await
.unwrap()
.is_none(),
"failed spawn must not leave a canonical roster entry behind"
);
}
#[tokio::test]
async fn test_failed_role_wiring_spawn_preserves_survivor_wiring_symmetry() {
let (handle, service) = create_test_mob(sample_definition_with_role_wiring()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
service
.set_comms_behavior(
&test_comms_name("worker", "w-2"),
MockCommsBehavior {
missing_public_key: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-3"),
None,
)
.await
.expect_err("spawn should fail during role-wiring fan-out");
assert!(
matches!(error, MobError::WiringError(_) | MobError::Internal(_)),
"failed spawn should surface wiring/internal failure, got: {error}"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-3"))
.await
.unwrap()
.is_none(),
"failed spawn must not leave the new member in the canonical roster"
);
let w1 = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("w-1 remains active");
let w2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 remains active");
assert!(
w1.wired_to.contains(&AgentIdentity::from("w-2")),
"rollback must preserve the pre-existing worker pair edge on w-1"
);
assert!(
w2.wired_to.contains(&AgentIdentity::from("w-1")),
"rollback must preserve the pre-existing worker pair edge on w-2"
);
assert!(
!w1.wired_to.contains(&AgentIdentity::from("w-3"))
&& !w2.wired_to.contains(&AgentIdentity::from("w-3")),
"rollback must not leave dangling projections to the failed spawn target"
);
}
// -----------------------------------------------------------------------
// Flow runtime lifecycle and orchestration behavior
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_run_flow_persists_before_reply_and_is_queryable() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(2_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_delay_ms(200);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({ "input": "x" }))
.await
.expect("run flow");
let immediate = handle
.flow_status(run_id.clone())
.await
.expect("flow status call should succeed");
assert!(
immediate.is_some(),
"run must be persisted before run_flow returns"
);
let run = immediate.expect("run should exist");
assert_eq!(run.flow_id, FlowId::from("demo"));
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert!(
terminal
.step_ledger
.iter()
.any(|entry| entry.status == crate::run::StepRunStatus::Dispatched),
"runtime flow path should persist dispatched ledger entries"
);
assert!(
terminal.step_ledger.iter().any(|entry| {
entry.step_id.as_str() == "start"
&& entry.status == crate::run::StepRunStatus::Completed
}),
"runtime flow path should persist completed step ledger entries"
);
assert!(
terminal.failure_ledger.is_empty(),
"successful flow should not persist failure ledger entries"
);
}
#[tokio::test]
async fn external_delivery_claim_failure_compensates_pending_run_and_delivery() {
let definition = with_unique_mob_id(
sample_definition_with_single_step_flow(2_000, 8),
"external-delivery-claim-compensation",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service, runs, _specs, _runtime_metadata, _identity, _identity_status) =
create_test_mob_with_recoverable_fault_events(definition, events.clone()).await;
let identity = crate::store::MobExternalDeliveryIdentity::new(
"binding-claim-compensation",
Uuid::new_v4().to_string(),
)
.expect("external delivery identity");
let intent = crate::store::MobExternalDeliveryIntent::new(
mob_id.clone(),
identity.clone(),
crate::store::MobExternalDeliveryTargetKind::Flow,
b"flow:demo",
)
.expect("external delivery intent");
handle
.begin_external_delivery(&intent)
.await
.expect("begin external delivery");
events.fail_next_external_delivery_claim();
let outcome = handle
.run_flow_with_external_delivery(
FlowId::from("demo"),
serde_json::json!({ "case": "claim-compensation" }),
&intent,
)
.await
.expect("claim failure must converge to an observable terminal delivery");
assert!(
matches!(
&outcome,
crate::store::MobExternalFlowLaunchOutcome::ExistingTerminal(
crate::store::MobExternalDeliveryTerminal::Failed { .. }
)
),
"claim compensation returned {outcome:?}"
);
let run_id =
RunId::for_external_delivery(&mob_id, &FlowId::from("demo"), &identity.idempotency_key);
let run = runs
.get_run(&run_id)
.await
.expect("read compensated run")
.expect("compensated run must remain durable");
assert_eq!(run.status, MobRunStatus::Failed);
let record = handle
.load_external_delivery(&identity.idempotency_key)
.await
.expect("read compensated delivery")
.expect("delivery must remain durable");
assert!(matches!(
record.phase,
crate::store::MobExternalDeliveryPhase::Terminal {
terminal: crate::store::MobExternalDeliveryTerminal::Failed { .. }
}
));
assert_eq!(
handle
.debug_flow_tracker_counts()
.await
.expect("flow tracker counts"),
(0, 0),
"compensation must not leave an executable Flow tracker"
);
let replay = events
.replay_all()
.await
.expect("replay compensation events");
assert_eq!(
replay
.iter()
.filter(|event| matches!(&event.kind, MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id))
.count(),
1,
"compensation must emit exactly one durable Flow failure"
);
assert!(
!replay.iter().any(|event| matches!(&event.kind, MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id)),
"the compensated Flow must never dispatch a step"
);
handle
.status()
.await
.expect("proven Begun compensation must not fail-stop the actor");
}
#[tokio::test]
async fn external_delivery_claim_and_reread_failure_fail_stops_with_pending_run() {
let definition = with_unique_mob_id(
sample_definition_with_single_step_flow(2_000, 8),
"external-delivery-claim-quarantine",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service, runs, _specs, _runtime_metadata, _identity, _identity_status) =
create_test_mob_with_recoverable_fault_events(definition, events.clone()).await;
let identity = crate::store::MobExternalDeliveryIdentity::new(
"binding-claim-quarantine",
Uuid::new_v4().to_string(),
)
.expect("external delivery identity");
let intent = crate::store::MobExternalDeliveryIntent::new(
mob_id.clone(),
identity.clone(),
crate::store::MobExternalDeliveryTargetKind::Flow,
b"flow:demo",
)
.expect("external delivery intent");
handle
.begin_external_delivery(&intent)
.await
.expect("begin external delivery");
events.fail_next_external_delivery_claim_and_reread();
let outcome = handle
.run_flow_with_external_delivery(
FlowId::from("demo"),
serde_json::json!({ "case": "claim-quarantine" }),
&intent,
)
.await
.expect("ambiguous claim must return a typed uncertain launch");
assert!(matches!(
outcome,
crate::store::MobExternalFlowLaunchOutcome::Uncertain { .. }
));
let run_id =
RunId::for_external_delivery(&mob_id, &FlowId::from("demo"), &identity.idempotency_key);
let run = runs
.get_run(&run_id)
.await
.expect("read quarantined run")
.expect("quarantined pending run must remain durable");
assert_eq!(run.status, MobRunStatus::Pending);
let record = handle
.load_external_delivery(&identity.idempotency_key)
.await
.expect("reread delivery after one-shot fault")
.expect("begun delivery must remain durable");
assert!(matches!(
record.phase,
crate::store::MobExternalDeliveryPhase::Begun { .. }
));
let actor_stopped = tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.status().await.is_err() {
break;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(
actor_stopped.is_ok(),
"unreadable claim outcome must crash-quiesce the actor for cold recovery"
);
}
#[tokio::test]
async fn external_delivery_post_start_compensation_failure_fail_stops() {
let definition = with_unique_mob_id(
sample_definition_with_single_step_flow(2_000, 8),
"external-delivery-post-start-failure",
);
let mob_id = definition.id.clone();
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service, runs, _specs, _runtime_metadata, _identity, _identity_status) =
create_test_mob_with_recoverable_fault_events(definition, events.clone()).await;
let identity = crate::store::MobExternalDeliveryIdentity::new(
"binding-post-start-failure",
Uuid::new_v4().to_string(),
)
.expect("external delivery identity");
let intent = crate::store::MobExternalDeliveryIntent::new(
mob_id.clone(),
identity.clone(),
crate::store::MobExternalDeliveryTargetKind::Flow,
b"flow:demo",
)
.expect("external delivery intent");
handle
.begin_external_delivery(&intent)
.await
.expect("begin external delivery");
events.fail_appends_for("FlowFailed").await;
events.fail_next_external_delivery_claim();
let outcome = handle
.run_flow_with_external_delivery(
FlowId::from("demo"),
serde_json::json!({ "case": "post-start-failure" }),
&intent,
)
.await
.expect("failed compensation must return an uncertain launch");
assert!(matches!(
outcome,
crate::store::MobExternalFlowLaunchOutcome::Uncertain { .. }
));
let run_id =
RunId::for_external_delivery(&mob_id, &FlowId::from("demo"), &identity.idempotency_key);
let run = runs
.get_run(&run_id)
.await
.expect("read post-start run")
.expect("post-start run must remain durable");
assert_eq!(run.status, MobRunStatus::Failed);
let replay = events.replay_all().await.expect("replay post-start events");
assert!(
!replay.iter().any(|event| matches!(&event.kind, MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id)),
"post-start compensation failure must never dispatch a step"
);
assert!(
!replay.iter().any(|event| matches!(&event.kind, MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id)),
"the injected projection failure must remain observable rather than being fabricated"
);
let record = handle
.load_external_delivery(&identity.idempotency_key)
.await
.expect("read delivery after failed compensation")
.expect("delivery must remain durable");
assert!(matches!(
record.phase,
crate::store::MobExternalDeliveryPhase::Begun { .. }
));
let actor_stopped = tokio::time::timeout(Duration::from_secs(1), async {
loop {
if handle.status().await.is_err() {
break;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(
actor_stopped.is_ok(),
"post-StartRun compensation failure must crash-quiesce the actor"
);
}
#[tokio::test]
async fn test_run_flow_provisions_missing_role_targets_before_dispatch() {
let (handle, _service) =
create_test_mob(sample_definition_with_single_step_flow(2_000, 8)).await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({ "input": "x" }))
.await
.expect("run flow should provision the required worker target");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, crate::run::MobRunStatus::Completed);
assert!(
terminal.step_ledger.iter().any(|entry| {
entry.step_id.as_str() == "start"
&& entry.status == crate::run::StepRunStatus::Completed
}),
"runtime flow path should dispatch and complete the step after provisioning"
);
let roster = handle.roster().await;
assert!(
roster
.by_profile(&ProfileName::from("worker"))
.any(|entry| entry.agent_identity.as_str().starts_with("__flow_worker_")),
"run-scoped provisioning should create a worker member"
);
}
#[tokio::test]
async fn test_parallel_targets_complete_concurrently() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(5_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
service.set_flow_turn_delay_ms(400);
let run_id = handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({ "mode": "parallel" }),
)
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
assert!(
service.max_concurrent_flow_turns() > 1,
"two target turns should overlap; max observed concurrent flow turns: {}",
service.max_concurrent_flow_turns()
);
}
#[tokio::test]
async fn test_fanout_emits_per_target_and_aggregate_completion_events() {
let (handle, _service) =
create_test_mob(sample_definition_with_single_step_flow(2_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
let run_id = handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({ "case": "fanout-events" }),
)
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
let per_target_completed = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepTargetCompleted { run_id: id, step_id, .. }
if id == &run_id && step_id.as_str() == "start"
)
})
.count();
let aggregate_completed = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "start"
)
})
.count();
assert_eq!(
per_target_completed, 2,
"fanout should emit one StepTargetCompleted per target"
);
assert_eq!(
aggregate_completed, 1,
"fanout should emit a single aggregate StepCompleted event"
);
}
#[tokio::test]
async fn test_run_snapshots_are_captured_per_run_and_remain_immutable() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(5_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_delay_ms(400);
let run_a = handle
.run_flow(FlowId::from("demo"), serde_json::json!({"ticket":"A"}))
.await
.expect("run A");
let snap_a_initial = handle
.flow_status(run_a.clone())
.await
.expect("status A")
.expect("run A exists");
assert_eq!(
snap_a_initial.activation_params,
serde_json::json!({"ticket":"A"})
);
let run_b = handle
.run_flow(FlowId::from("demo"), serde_json::json!({"ticket":"B"}))
.await
.expect("run B");
let snap_b = handle
.flow_status(run_b.clone())
.await
.expect("status B")
.expect("run B exists");
assert_eq!(snap_b.activation_params, serde_json::json!({"ticket":"B"}));
let snap_a_after = handle
.flow_status(run_a.clone())
.await
.expect("status A after")
.expect("run A still exists");
assert_eq!(
snap_a_after.activation_params,
serde_json::json!({"ticket":"A"}),
"run A activation snapshot must remain unchanged by run B"
);
let _ = handle.cancel_flow(run_a).await;
let _ = handle.cancel_flow(run_b).await;
}
#[tokio::test]
async fn test_handle_list_flows_returns_definition_flow_ids() {
let mut definition = sample_definition_with_single_step_flow(500, 8);
let mut extra_steps = IndexMap::new();
extra_steps.insert(step_id("start"), flow_step("worker", "Secondary"));
definition.flows.insert(
flow_id("secondary"),
FlowSpec::new(Some("secondary flow".to_string()), extra_steps, None),
);
let (handle, _service) = create_test_mob(definition).await;
let flows = handle.list_flows();
assert_eq!(flows, vec![flow_id("demo"), flow_id("secondary")]);
}
#[tokio::test]
async fn test_dispatch_mode_one_to_one_dispatches_single_target() {
let (handle, _service) =
create_test_mob(sample_definition_with_dispatch_mode(DispatchMode::OneToOne)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
let dispatched = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
})
.count();
assert_eq!(dispatched, 1, "one_to_one should dispatch one target");
}
#[tokio::test]
async fn test_dispatch_mode_fan_in_dispatches_all_targets() {
let (handle, _service) =
create_test_mob(sample_definition_with_dispatch_mode(DispatchMode::FanIn)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
let dispatched = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
})
.count();
assert_eq!(dispatched, 2, "fan_in should dispatch all matching targets");
}
#[tokio::test]
async fn test_fan_in_aggregate_output_is_deterministic_array_shape() {
let (handle, _service) = create_test_mob(sample_definition_with_dispatch_mode_and_policy(
DispatchMode::FanIn,
CollectionPolicy::All,
))
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let aggregate_output = terminal
.step_ledger
.iter()
.find(|entry| {
entry.step_id.as_str() == "dispatch"
&& entry.agent_identity == crate::runtime::flow_system_member_id()
&& entry.status == StepRunStatus::Completed
})
.and_then(|entry| entry.output.clone())
.expect("aggregate dispatch output should be persisted");
assert_eq!(
aggregate_output,
serde_json::json!([
{"target":"w-1","output":"Turn completed"},
{"target":"w-2","output":"Turn completed"}
]),
"fan_in aggregate shape and ordering should be deterministic"
);
}
#[tokio::test]
async fn test_max_step_retries_controls_dispatch_attempts() {
let (handle, service) = create_test_mob(sample_definition_with_retry_flow(2)).await;
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_flow_turn_fail_for_session(&sid_w1, true).await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
let dispatched = events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, step_id, target }
if id == &run_id
&& step_id.as_str() == "start"
&& target.identity.as_str() == "w-1"
)
})
.count();
assert_eq!(
dispatched, 3,
"max_step_retries=2 should dispatch three attempts"
);
}
#[tokio::test]
#[ignore = "Row #320 folded only the GLOBAL orphan budget into MobMachine state \
(seeded once via SeedOrphanBudget, decremented at \
ClassifyTurnTimeoutDisposition). The per-run sub-cap fairness \
(orphan_budget_max/2 + per-run accounting) was intentionally NOT \
folded; restore this test once a per-run orphan sub-cap input/state \
is added to the mob catalog."]
async fn test_orphan_budget_fairness_prevents_single_run_monopoly() {
let (handle, service) = create_test_mob(sample_definition_with_single_step_flow(20, 2)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
service.set_flow_turn_never_terminal(true);
let run_id_1 = handle
.run_flow(FlowId::from("demo"), serde_json::json!({"run": 1}))
.await
.expect("run flow 1");
let _terminal_1 = wait_for_run_terminal(&handle, &run_id_1, Duration::from_secs(3)).await;
let run_id_2 = handle
.run_flow(FlowId::from("demo"), serde_json::json!({"run": 2}))
.await
.expect("run flow 2");
let terminal_2 = wait_for_run_terminal(&handle, &run_id_2, Duration::from_secs(3)).await;
assert_eq!(terminal_2.status, MobRunStatus::Failed);
assert!(
terminal_2
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("timeout after")),
"second run should still obtain timeout orphan capacity instead of complete starvation"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_branch_winner_is_selected_only_after_success_allowing_fallback() {
let (handle, service) = create_test_mob(sample_definition_with_branch_fallback_flow()).await;
let sid_fail = handle
.spawn(
ProfileName::from("worker_fail"),
AgentIdentity::from("w-fail"),
None,
)
.await
.expect("spawn failing worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker_ok"),
AgentIdentity::from("w-ok"),
None,
)
.await
.expect("spawn healthy worker");
service
.set_flow_turn_fail_for_session(&sid_fail, true)
.await;
let run_id = handle
.run_flow(
FlowId::from("branch_fallback"),
serde_json::json!({"try_fallback": true}),
)
.await
.expect("run branch fallback flow");
// This path exercises several generated frame transitions and has taken
// 20-50s locally under an unoptimized test build. Loaded Linux CI has
// exceeded the helper's generic 60s floor even though the exact observed
// failure and fallback both complete. Keep a bounded lane-specific margin;
// the assertions below still require the real terminal cause and join.
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(120)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
let first_failure = events.iter().find_map(|event| match &event.kind {
MobEventKind::StepFailed {
run_id: id,
step_id,
reason,
} if id == &run_id && step_id.as_str() == "candidate_first" => Some(reason.as_str()),
_ => None,
});
assert_eq!(
first_failure,
Some("mock flow turn failure"),
"the fallback must be triggered by the observed failed terminal, not merely a deadline"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "candidate_second"
)
}),
"fallback branch candidate should execute and complete"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "join"
)
}),
"downstream join should complete after fallback branch success"
);
}
#[tokio::test]
async fn test_malformed_templates_fail_flow_explicitly() {
for template in ["Bad {{", "Bad }}", "{{ }}"] {
let (handle, _service) =
create_test_mob(sample_definition_with_template_message(template)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
terminal.failure_ledger.iter().any(|entry| {
entry.reason.contains("invalid template syntax")
&& entry.reason.contains("line")
&& entry.reason.contains("column")
&& entry.reason.contains("byte")
}),
"malformed template '{template}' should fail with deterministic syntax diagnostics"
);
}
}
#[tokio::test]
async fn test_template_syntax_diagnostics_are_deterministic_for_multiline_inputs() {
let bad_template = "line1\nline2 }}";
let (handle, _service) =
create_test_mob(sample_definition_with_template_message(bad_template)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let reason = terminal
.failure_ledger
.iter()
.find(|entry| entry.reason.contains("invalid template syntax"))
.map(|entry| entry.reason.clone())
.expect("template error reason");
assert!(
reason.contains("line 2, column 7"),
"diagnostic location should be deterministic for multiline templates: {reason}"
);
}
#[tokio::test]
async fn test_shared_path_resolver_is_used_for_conditions_and_templates() {
let (handle, _service) =
create_test_mob(sample_definition_with_shared_path_resolution_flow()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("shared_paths"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
assert!(
terminal.step_ledger.iter().any(|entry| {
entry.step_id.as_str() == "follow" && entry.status == StepRunStatus::Completed
}),
"follow step should execute using shared path semantics in both condition and template"
);
}
#[tokio::test]
async fn test_flow_finished_cleans_tracking_maps() {
let (handle, _service) = create_test_mob(sample_definition_with_single_step_flow(500, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let deadline = Instant::now() + Duration::from_secs(2);
loop {
let (tasks, tokens) = handle
.debug_flow_tracker_counts()
.await
.expect("tracker counts");
if tasks == 0 && tokens == 0 {
break;
}
assert!(
Instant::now() < deadline,
"timed out waiting for flow trackers to drain (tasks={tasks}, tokens={tokens})"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
#[tokio::test]
async fn test_mob_schema_initial_orchestrator_projection_matches_runtime() {
let schema = schema_mob_machine();
assert_eq!(schema.state.init.phase.as_str(), "Running");
let coordinator_bound = schema
.state
.init
.fields
.iter()
.find(|field| field.field.as_str() == "coordinator_bound")
.expect("coordinator_bound init");
assert_eq!(coordinator_bound.expr, Expr::Bool(true));
let (handle, _service) = create_test_mob(sample_definition()).await;
let initial = handle
.debug_orchestrator_snapshot()
.await
.expect("initial orchestrator snapshot");
assert_eq!(initial.phase, MobState::Running);
assert!(initial.coordinator_bound);
assert_eq!(initial.pending_spawn_count, 0);
}
#[tokio::test]
async fn test_mob_runtime_parity_field_evaluator_covers_formal_state_fields() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let snapshot = mob_runtime_parity_snapshot_summary(&handle)
.await
.expect("runtime parity snapshot");
let schema = schema_mob_machine();
let missing = schema
.state
.fields
.iter()
.filter(|field| mob_runtime_parity_field_value(&snapshot, field.name.as_str()).is_none())
.map(|field| field.name.as_str().to_string())
.collect::<Vec<_>>();
assert!(
missing.is_empty(),
"runtime parity field evaluator is missing schema fields: {missing:?}"
);
}
#[test]
fn test_bridge_protocol_types_live_in_contracts_not_runtime() {
// Verify the canonical protocol types are re-exported from contracts,
// not defined in meerkat-runtime. This is a compile-time structural
// assertion — if the re-export chain breaks, this won't compile.
let _cmd: meerkat_contracts::BridgeCommand = meerkat_contracts::BridgeCommand::ObserveMember(
meerkat_contracts::BridgeSupervisorPayload {
supervisor: meerkat_contracts::BridgePeerSpec {
name: "test".into(),
peer_id: "ed25519:test".into(),
address: "inproc://test".into(),
pubkey: [0u8; 32],
},
epoch: 1,
protocol_version:
meerkat_contracts::wire::supervisor_bridge::SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
},
);
}
#[test]
fn test_top_level_mob_schema_does_not_duplicate_bridge_protocol_effects() {
let schema = schema_mob_machine();
let forbidden_effects = [
"BridgeBindMember",
"BridgeAuthorizeSupervisor",
"BridgeRevokeSupervisor",
"BridgeDeliverMemberInput",
"BridgeObserveMember",
"BridgeInterruptMember",
"BridgeHardCancelMember",
"BridgeRetireMember",
"BridgeDestroyMember",
"BridgeWireMember",
"BridgeUnwireMember",
];
let schema_effect_names: Vec<_> = schema
.effects
.variants
.iter()
.map(|v| v.name.as_str())
.collect();
let duplicated: Vec<_> = forbidden_effects
.iter()
.filter(|name| schema_effect_names.contains(name))
.collect();
assert!(
duplicated.is_empty(),
"top-level mob schema must not duplicate bridge protocol effects: {duplicated:?}"
);
}
#[test]
fn test_top_level_mob_schema_avoids_shadow_bridge_state_fields() {
let schema = schema_mob_machine();
let forbidden_fields = [
"remote_member_count",
"supervisor_authorized",
"supervisor_rotating",
"rotation_pending_acks",
];
let field_names: Vec<_> = schema
.state
.fields
.iter()
.map(|f| f.name.as_str())
.collect();
let shadowed: Vec<_> = forbidden_fields
.iter()
.filter(|name| field_names.contains(name))
.collect();
assert!(
shadowed.is_empty(),
"top-level mob schema must not carry shadow bridge state fields: {shadowed:?}"
);
}
#[test]
fn test_top_level_mob_schema_does_not_model_schema_only_rotation_transitions() {
let schema = schema_mob_machine();
let transition_names: Vec<_> = schema.transitions.iter().map(|t| t.name.as_str()).collect();
assert!(
!transition_names.contains(&"RotateSupervisorRunning"),
"top-level mob schema must not model schema-only RotateSupervisor transitions"
);
assert!(
!transition_names.contains(&"AckRotationRunning"),
"top-level mob schema must not model schema-only AckRotation transitions"
);
let manifest = crate::mob_machine::MobMachineCommand::command_manifest();
assert!(
!manifest.contains(&"RotateSupervisor"),
"canonical mob machine command surface must not include schema-only RotateSupervisor"
);
assert!(
!manifest.contains(&"AckRotation"),
"canonical mob machine command surface must not include schema-only AckRotation"
);
}
#[tokio::test]
async fn test_orchestrator_snapshot_tracks_pending_spawn_ownership_and_revision() {
let (handle, service) = create_test_mob(sample_definition()).await;
let initial = handle
.debug_orchestrator_snapshot()
.await
.expect("orchestrator snapshot");
assert!(initial.coordinator_bound);
assert_eq!(initial.pending_spawn_count, 0);
service.set_create_session_delay_ms(150);
let spawn_handle = {
let handle = handle.clone();
tokio::spawn(async move {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
})
};
tokio::time::sleep(Duration::from_millis(30)).await;
let staged = handle
.debug_orchestrator_snapshot()
.await
.expect("staged snapshot");
assert_eq!(staged.pending_spawn_count, 1);
let staged_dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("staged dsl snapshot");
assert_eq!(
staged_dsl.pending_spawn_sessions.len(),
1,
"exactly one machine-owned pending spawn admission should be visible while provisioning is blocked"
);
assert!(
staged_dsl
.pending_spawn_sessions
.keys()
.any(|identity| identity.0 == "w-1"),
"pending admission should carry the member identity, not just an anonymous count"
);
// topology_revision is orchestrator shell metadata not tracked by the DSL authority.
spawn_handle
.await
.expect("spawn join")
.expect("spawn worker");
let settled = handle
.debug_orchestrator_snapshot()
.await
.expect("settled snapshot");
assert_eq!(settled.pending_spawn_count, 0);
let settled_dsl = handle
.debug_dsl_t2_snapshot()
.await
.expect("settled dsl snapshot");
assert!(
settled_dsl.pending_spawn_sessions.is_empty(),
"successful spawn completion should clear pending admission"
);
let events = handle.events().replay_all().await.expect("replay events");
assert!(events.iter().any(|event| matches!(
&event.kind,
MobEventKind::MemberSpawned(member_spawned)
if member_spawned.agent_identity.as_str() == "w-1"
)));
}
#[tokio::test]
async fn test_flow_tracker_maps_remain_coherent_under_concurrent_run_and_cancel_commands() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(5_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_delay_ms(150);
let mut run_ids = Vec::new();
for _ in 0..8 {
run_ids.push(
handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow"),
);
}
let mut cancel_tasks = Vec::new();
for run_id in run_ids.clone() {
let handle = handle.clone();
cancel_tasks.push(tokio::spawn(async move {
handle.cancel_flow(run_id).await.expect("cancel flow");
}));
}
for task in cancel_tasks {
task.await.expect("cancel join");
}
for run_id in &run_ids {
let terminal = wait_for_run_terminal(&handle, run_id, Duration::from_secs(8)).await;
assert!(
matches!(
terminal.status,
MobRunStatus::Canceled | MobRunStatus::Failed | MobRunStatus::Completed
),
"concurrent run/cancel pressure must still converge to terminal runs"
);
}
let deadline = Instant::now() + Duration::from_secs(3);
loop {
let (tasks, tokens) = handle
.debug_flow_tracker_counts()
.await
.expect("tracker counts");
if tasks == 0 && tokens == 0 {
break;
}
assert!(
Instant::now() < deadline,
"timed out waiting for actor-owned flow trackers to drain (tasks={tasks}, tokens={tokens})"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
#[tokio::test]
async fn test_orchestrator_snapshot_tracks_flow_activation_and_lifecycle_transitions() {
let (handle, service) = create_test_mob(sample_definition_with_supervisor_threshold(1)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_delay_ms(150);
let run_id = handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({"source":"orchestrator-test"}),
)
.await
.expect("run flow");
// Poll for the flow to register as active rather than asserting after a
// fixed sleep: the flow turn is delayed 150ms, but under parallel test load
// a fixed 30ms wall-clock is unreliable (the active window can be missed in
// either direction), so we poll until the machine-owned active count is
// observed (mirrors the wait_for_run_terminal pattern used below).
let active_deadline = Instant::now() + Duration::from_secs(2);
let active = loop {
let snap = handle
.debug_orchestrator_snapshot()
.await
.expect("active snapshot");
if snap.active_flow_count == 1 {
break snap;
}
assert!(
Instant::now() < active_deadline,
"flow never registered active: active_flow_count={}",
snap.active_flow_count
);
tokio::time::sleep(Duration::from_millis(5)).await;
};
assert_eq!(active.active_flow_count, 1);
assert!(active.coordinator_bound);
// supervisor_active is orchestrator shell metadata not tracked by the DSL authority.
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let deadline = Instant::now() + Duration::from_secs(2);
let settled = loop {
let snapshot = handle
.debug_orchestrator_snapshot()
.await
.expect("settled snapshot");
if snapshot.active_flow_count == 0 {
break snapshot;
}
assert!(
Instant::now() < deadline,
"timed out waiting for orchestrator active-flow count to drain"
);
tokio::time::sleep(Duration::from_millis(20)).await;
};
assert!(settled.coordinator_bound);
// supervisor_active is orchestrator shell metadata not tracked by the DSL authority.
handle.stop().await.expect("stop mob");
let stopped = handle
.debug_orchestrator_snapshot()
.await
.expect("stopped snapshot");
assert!(!stopped.coordinator_bound);
// supervisor_active is orchestrator shell metadata not tracked by the DSL authority.
handle.resume().await.expect("resume mob");
let resumed = handle
.debug_orchestrator_snapshot()
.await
.expect("resumed snapshot");
assert!(resumed.coordinator_bound);
assert!(resumed.supervisor_active);
let events = handle.events().replay_all().await.expect("replay events");
assert!(events.iter().any(|event| matches!(
&event.kind,
MobEventKind::FlowStarted { run_id: event_run_id, .. } if event_run_id == &run_id
)));
assert!(events.iter().any(|event| matches!(
&event.kind,
MobEventKind::FlowCompleted { run_id: event_run_id, .. } if event_run_id == &run_id
)));
}
#[tokio::test]
async fn test_failed_flow_run_drains_orchestrator_and_tracker_state() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(2_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
service.set_flow_turn_delay_ms(150);
let run_id = handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({"mode":"fail-cleanup"}),
)
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let deadline = Instant::now() + Duration::from_secs(2);
loop {
let snapshot = handle
.debug_orchestrator_snapshot()
.await
.expect("orchestrator snapshot");
let (tasks, tokens) = handle
.debug_flow_tracker_counts()
.await
.expect("flow tracker counts");
if snapshot.active_flow_count == 0 && tasks == 0 && tokens == 0 {
break;
}
assert!(
Instant::now() < deadline,
"failed flow must fail closed by draining orchestrator/tracker state (active_flows={}, tasks={}, tokens={})",
snapshot.active_flow_count,
tasks,
tokens
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
#[tokio::test]
async fn test_complete_cancels_inflight_flow_run() {
let (handle, service) =
create_test_mob(sample_definition_with_single_step_flow(60_000, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
tokio::time::sleep(Duration::from_millis(50)).await;
handle.complete().await.expect("complete mob");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
}
#[tokio::test]
async fn test_destroy_cancels_inflight_flow_run() {
let run_store = Arc::new(RecordingRunStore::new());
let (handle, service) = create_test_mob_with_run_store(
sample_definition_with_single_step_flow(60_000, 8),
run_store.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
tokio::time::sleep(Duration::from_millis(50)).await;
handle.destroy().await.expect("destroy mob");
let terminal = run_store
.get_run(&run_id)
.await
.expect("read run after destroy")
.expect("run should remain queryable in run store after destroy");
assert_eq!(terminal.status, MobRunStatus::Canceled);
}
#[tokio::test]
async fn test_run_flow_store_admission_failure_does_not_commit_mob_machine_authority() {
let run_store = Arc::new(RecordingRunStore::new());
let definition = sample_definition_with_single_step_flow(60_000, 8);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob_with_run_store(definition, run_store.clone()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker before flow admission");
run_store.fail_next_create_run();
let error = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect_err("fault-injected create_run failure should fail RunFlow admission");
assert!(
error
.to_string()
.contains("fault-injected create_run failure"),
"admission error should surface run-store failure, got: {error}"
);
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state after failed admission");
assert_eq!(
machine_state.active_run_count, 0,
"RunFlow store failure must not leave MobMachine active_run_count advanced"
);
assert!(
machine_state.run_status.is_empty(),
"RunFlow store failure must not leave MobMachine run projections behind"
);
let persisted_runs = run_store
.list_runs(&mob_id, Some(&FlowId::from("demo")))
.await
.expect("list persisted runs");
assert!(
persisted_runs.is_empty(),
"fault-injected admission failure must not persist a run"
);
}
#[test]
fn test_flow_cleanup_uses_terminal_projection_not_authority_clone_probe() {
let source = include_str!("actor.rs");
let start = source
.find("async fn handle_flow_cleanup")
.expect("handle_flow_cleanup exists");
let end = source[start..]
.find("async fn handle_cancel_flow")
.expect("handle_cancel_flow follows cleanup");
let body = &source[start..start + end];
assert!(
!body.contains(
"MobMachineAuthority::recover_from_state(self.dsl_authority.state().clone())"
),
"flow cleanup must use run terminality and live authority state directly, not clone-probe MobMachine"
);
assert!(
!body.contains("authorities_expect_completion"),
"flow cleanup must not infer terminality by probing whether authority signals would be accepted"
);
}
#[test]
fn test_flow_canceled_cleanup_requires_terminalization_proof() {
let actor = include_str!("actor.rs");
let state = include_str!("state.rs");
assert!(
state
.contains("FlowCanceledCleanup {\n run_id: RunId,\n terminalized: bool,"),
"FlowCanceledCleanup must carry spawned-path terminalization proof"
);
assert!(
actor.contains("if !terminalized"),
"actor must fail closed before cleanup when terminalization proof is false"
);
assert!(
actor.contains("terminalized = false"),
"spawned cancellation path must downgrade the proof on settle/terminalization errors"
);
assert!(
actor.contains("terminalized,"),
"FlowCanceledCleanup sends must include the proof field"
);
}
#[test]
fn test_lifecycle_commands_admit_on_live_authority_not_clone_probe() {
let source = include_str!("actor.rs");
let start = source
.find("MobCommand::Stop { reply_tx }")
.expect("Stop command arm exists");
let end = source[start..]
.find("MobCommand::RotateSupervisor")
.expect("RotateSupervisor command arm follows lifecycle command arms");
let lifecycle_arms = &source[start..start + end];
for command in ["Stop", "ResumeLifecycle", "Complete", "Reset"] {
let command_start = lifecycle_arms
.find(&format!("MobCommand::{command}"))
.unwrap_or_else(|| panic!("{command} command arm exists"));
let command_end = [
"MobCommand::ResumeLifecycle",
"MobCommand::Complete",
"MobCommand::Destroy",
"MobCommand::Reset",
"MobCommand::RotateSupervisor",
]
.into_iter()
.filter_map(|marker| {
lifecycle_arms[command_start + 1..]
.find(marker)
.map(|offset| command_start + 1 + offset)
})
.min()
.unwrap_or(lifecycle_arms.len());
let arm = &lifecycle_arms[command_start..command_end];
assert!(
!arm.contains("probe_mob_machine_input"),
"{command} lifecycle admission must not be decided by cloned MobMachine probe truth"
);
}
}
#[test]
fn test_flow_frame_store_plan_commits_authority_with_store_in_actor() {
let source = include_str!("flow_frame_engine.rs");
let start = source
.find("async fn execute_store_plan")
.expect("execute_store_plan exists");
let end = source[start..]
.find("async fn project_store_plan")
.expect("project_store_plan follows execute_store_plan");
let body = &source[start..start + end];
assert!(
body.contains("commit_flow_frame_store_plan"),
"frame loop store plans must be committed through the actor authority/store seam"
);
assert!(
!body.contains("commit_mob_machine_inputs(machine_inputs)"),
"frame loop store plans must not persist store state before separately committing MobMachine inputs"
);
}
#[test]
fn test_flow_frame_kernel_mutations_route_store_and_authority_through_actor() {
let source = include_str!("flow_frame_engine.rs");
let start = source
.find("impl FlowFrameMutator for FlowFrameKernel")
.expect("FlowFrameKernel mutator impl exists");
let end = source[start..]
.find("// ─── FlowFrameEngine")
.expect("FlowFrameEngine section follows kernel mutator impl");
let body = &source[start..start + end];
for disallowed in [
".cas_frame_state(",
".cas_complete_step_and_record_output(",
".cas_run_snapshot(",
"commit_mob_machine_inputs(",
] {
assert!(
!body.contains(disallowed),
"FlowFrameKernel store mutation '{disallowed}' must be committed through the actor authority/store seam"
);
}
assert!(
body.contains("commit_flow_frame_store_plan"),
"FlowFrameKernel frame store mutations must route through actor-owned prepared authority commits"
);
}
#[test]
fn test_flow_frame_scheduler_start_run_routes_through_actor_authority() {
let source = include_str!("flow_frame_engine.rs");
let start = source
.find("async fn ensure_scheduler_state_initialized")
.expect("ensure_scheduler_state_initialized exists");
let end = source[start..]
.find("async fn heal_orphaned_running_nodes")
.expect("heal helper follows scheduler initialization");
let body = &source[start..start + end];
assert!(
body.contains("commit_flow_run_command"),
"scheduler initialization must use the actor-owned flow run command seam"
);
for disallowed in [".cas_run_snapshot(", "commit_mob_machine_inputs("] {
assert!(
!body.contains(disallowed),
"scheduler initialization must not persist run state before a separate MobMachine authority commit"
);
}
}
fn serialized_flow_authority_inputs(run: &MobRun) -> Vec<serde_json::Value> {
let run_json = serde_json::to_value(run).expect("serialize MobRun");
run_json
.get("flow_authority_inputs")
.and_then(serde_json::Value::as_array)
.cloned()
.unwrap_or_else(|| panic!("MobRun must persist flow_authority_inputs with projections"))
}
fn flow_authority_log_text(run: &MobRun) -> String {
serde_json::to_string(&serialized_flow_authority_inputs(run))
.expect("serialize flow authority log")
}
fn assert_flow_authority_log_contains(run: &MobRun, needle: &str) {
let log = flow_authority_log_text(run);
assert!(
log.contains(needle),
"flow authority log should contain {needle}; log={log}"
);
}
fn apply_authority_input_for_test(
authority: &mut crate::machines::mob_machine::MobMachineAuthority,
input: crate::machines::mob_machine::MobMachineInput,
) {
let transition = crate::machines::mob_machine::MobMachineMutator::apply(authority, input)
.expect("test authority input should be accepted");
let _ = transition;
}
fn authority_backed_root_frame_run(
include_loop_seed: bool,
) -> (MobRun, crate::ids::FrameId, crate::ids::LoopInstanceId) {
use crate::ids::{FlowNodeId, LoopId, RunId};
let definition = sample_definition_with_single_step_flow(500, 8);
let config = crate::run::FlowRunConfig::from_definition(FlowId::from("demo"), &definition)
.expect("flow config");
let run_id = RunId::new();
let create_run = MobRun::create_run_seed_input(&run_id, &config).expect("create run seed");
let loop_node_id = FlowNodeId::from("loop-node");
let loop_id = LoopId::from("retry");
let frame_id = crate::ids::FrameId::from(format!("{run_id}-root").as_str());
let create_frame = crate::machines::mob_machine::MobMachineInput::CreateFrameSeed {
run_id: crate::machines::mob_machine::RunId::from(run_id.to_string()),
frame_id: crate::machines::mob_machine::FrameId::from(frame_id.as_str()),
frame_scope: crate::machines::mob_machine::FrameScope::Root,
loop_instance_id: None,
iteration: 0,
tracked_nodes: [crate::machines::mob_machine::FlowNodeId::from(
loop_node_id.as_str(),
)]
.into_iter()
.collect(),
ordered_nodes: vec![crate::machines::mob_machine::FlowNodeId::from(
loop_node_id.as_str(),
)],
node_kind: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
crate::machines::mob_machine::FlowNodeKind::Loop,
)]
.into_iter()
.collect(),
node_dependencies: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
Vec::new(),
)]
.into_iter()
.collect(),
node_dependency_modes: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
crate::machines::mob_machine::DependencyMode::All,
)]
.into_iter()
.collect(),
node_branches: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
None,
)]
.into_iter()
.collect(),
node_step_ids: BTreeMap::new(),
node_loop_ids: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
crate::machines::mob_machine::LoopId::from(loop_id.as_str()),
)]
.into_iter()
.collect(),
node_status: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
crate::machines::mob_machine::NodeRunStatus::Ready,
)]
.into_iter()
.collect(),
ready_queue: vec![crate::machines::mob_machine::FlowNodeId::from(
loop_node_id.as_str(),
)],
output_recorded: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
false,
)]
.into_iter()
.collect(),
node_condition_results: [(
crate::machines::mob_machine::FlowNodeId::from(loop_node_id.as_str()),
None,
)]
.into_iter()
.collect(),
last_admitted_node: None,
};
let loop_instance_id =
crate::ids::LoopInstanceId::from(format!("{frame_id}::{loop_node_id}").as_str());
let create_loop = MobRun::create_loop_seed_input_for_start(
&loop_instance_id,
&frame_id,
&loop_node_id,
&loop_id,
1,
3,
);
let mut authority = crate::machines::mob_machine::MobMachineAuthority::new();
apply_authority_input_for_test(&mut authority, create_run.clone());
apply_authority_input_for_test(&mut authority, create_frame.clone());
if include_loop_seed {
apply_authority_input_for_test(&mut authority, create_loop.clone());
}
let mut run = MobRun::pending_with_run_id(
run_id.clone(),
definition.id.clone(),
FlowId::from("demo"),
MobRun::flow_state_for_config_with_authority(&run_id, &config, authority.state())
.expect("project flow state"),
serde_json::json!({}),
);
let mut inputs = vec![create_run, create_frame];
if include_loop_seed {
inputs.push(create_loop);
}
run.append_flow_authority_inputs(inputs)
.expect("append authority inputs");
run.frames.insert(
frame_id.clone(),
crate::run::FrameSnapshot {
kernel_state: crate::run::project_flow_frame_authority_state_from_machine(
authority.state(),
&frame_id,
)
.expect("project frame state"),
},
);
if include_loop_seed {
run.flow_state.ready_frames = vec![frame_id.clone()];
run.flow_state.ready_frame_membership = [frame_id.clone()].into_iter().collect();
run.flow_state.pending_body_frame_loops.clear();
run.flow_state.pending_body_frame_loop_membership.clear();
}
(run, frame_id, loop_instance_id)
}
fn running_authority_backed_run(definition: &MobDefinition) -> MobRun {
use crate::ids::RunId;
use crate::run::{
MobMachineFlowAuthorityToken, MobMachineFlowRunCommand, apply_mob_machine_flow_run_command,
flow_run,
};
let flow_id = FlowId::from("demo");
let config = crate::run::FlowRunConfig::from_definition(flow_id.clone(), definition)
.expect("flow config");
let run_id = RunId::new();
let run_flow = MobRun::run_flow_input(&run_id, &config).expect("run flow input");
let mut authority = crate::machines::mob_machine::MobMachineAuthority::new();
apply_authority_input_for_test(&mut authority, run_flow.clone());
let mut run = MobRun::pending_with_run_id(
run_id.clone(),
definition.id.clone(),
flow_id,
MobRun::flow_state_for_config_with_authority(&run_id, &config, authority.state())
.expect("project admitted flow state"),
serde_json::json!({}),
);
run.append_flow_authority_inputs(vec![run_flow])
.expect("append RunFlow authority input");
let start = MobMachineFlowRunCommand::StartRun(flow_run::inputs::StartRun {});
let start_input = start.authority_input(&run_id);
apply_authority_input_for_test(&mut authority, start_input.clone());
let authority_token =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&start_input)
.expect("start authority token");
let outcome = apply_mob_machine_flow_run_command(
&run.flow_state,
authority.state(),
&run_id,
start,
authority_token,
&[],
)
.expect("project started flow state");
run.status = MobRunStatus::Running;
run.flow_state = outcome.next_state;
run.append_flow_authority_inputs(vec![start_input])
.expect("append StartRun authority input");
run
}
#[tokio::test]
async fn test_flow_frame_store_plan_persists_authority_input_with_projection() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::{FlowNodeId, FrameId};
use crate::runtime::flow_frame_engine::{FlowFrameKernel, FlowFrameMutator};
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create run");
let (handle, _) = create_test_mob_with_run_store(sample_definition(), store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let node_id = FlowNodeId::from("start-node");
let mut nodes = IndexMap::new();
nodes.insert(
node_id,
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
let root_spec = FrameSpec { nodes };
let frame_id = FrameId::from(format!("{run_id}-root").as_str());
let frame_kernel = FlowFrameKernel::new(store.clone(), handle);
frame_kernel
.start_frame(&run_id, &frame_id, &root_spec)
.await
.expect("start root frame");
frame_kernel
.admit_next_ready_node_with_retry(&run_id, &frame_id, 5)
.await
.expect("admit next ready node")
.expect("admission effects");
let persisted = store
.get_run(&run_id)
.await
.expect("get run")
.expect("run exists");
let authority_inputs = serialized_flow_authority_inputs(&persisted);
assert!(
authority_inputs.len() >= 2,
"frame seed and admit projection commits must both record authority inputs"
);
assert_flow_authority_log_contains(&persisted, "CreateFrameSeed");
assert_flow_authority_log_contains(&persisted, "AuthorizeFlowFrameReducerCommand");
assert_flow_authority_log_contains(&persisted, "AdmitNextReadyNode");
}
#[tokio::test]
async fn test_flow_run_start_and_completion_persist_authority_inputs_with_projection() {
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create run");
let (handle, _) = create_test_mob_with_run_store(sample_definition(), store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let effects = handle
.commit_flow_run_command(
&run_id,
crate::run::MobMachineFlowRunCommand::StartRun(
crate::run::flow_run::inputs::StartRun {},
),
"test_start_run_authority_log",
)
.await
.expect("start run command")
.expect("start run should transition");
assert!(!effects.is_empty(), "start run should emit effects");
let started = store
.get_run(&run_id)
.await
.expect("get started run")
.expect("started run exists");
assert_eq!(started.status, MobRunStatus::Running);
assert_flow_authority_log_contains(&started, "StartRun");
let terminal = handle
.commit_flow_terminalization(
run_id.clone(),
FlowId::from("test-flow"),
super::terminalization::TerminalizationTarget::Completed {
structured_output: None,
},
crate::run::MobMachineFlowRunCommand::TerminalizeCompleted(
crate::run::flow_run::inputs::TerminalizeCompleted {},
),
"test_completed_run_authority_log",
)
.await
.expect("terminalize completed");
assert_eq!(
terminal,
super::terminalization::TerminalizationOutcome::Transitioned
);
let completed = store
.get_run(&run_id)
.await
.expect("get completed run")
.expect("completed run exists");
assert_eq!(completed.status, MobRunStatus::Completed);
assert_flow_authority_log_contains(&completed, "TerminalizeCompleted");
}
#[tokio::test]
async fn test_recovery_rejects_flow_authority_log_projection_divergence() {
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
let seed_input = run
.flow_authority_inputs
.first()
.expect("seed authority record")
.to_machine_input();
store.create_run(run).await.expect("create run");
let (handle, _) = create_test_mob_with_run_store(sample_definition(), store.clone()).await;
handle
.project_machine_input(seed_input)
.await
.expect("seed run in MobMachine");
handle
.commit_flow_run_command(
&run_id,
crate::run::MobMachineFlowRunCommand::StartRun(
crate::run::flow_run::inputs::StartRun {},
),
"test_start_run_recovery_authority_log",
)
.await
.expect("start run command")
.expect("start run should transition");
let mut persisted = store
.get_run(&run_id)
.await
.expect("get run")
.expect("run exists");
crate::runtime::recovery::reconcile_run_state(&mut persisted)
.expect("fresh persisted run should validate");
persisted.flow_state.phase = crate::run::flow_run::Phase::Pending;
let error = crate::runtime::recovery::reconcile_run_state(&mut persisted)
.expect_err("tampered projection must not validate against authority log");
assert!(
matches!(
error,
crate::runtime::recovery::RestoreIncompatible::FlowAuthorityProjectionMismatch { .. }
),
"unexpected recovery error: {error}"
);
}
#[test]
fn test_recovery_rejects_authority_created_frame_missing_from_projection() {
let (mut run, frame_id, _) = authority_backed_root_frame_run(false);
run.frames.remove(&frame_id);
let error = crate::runtime::recovery::reconcile_run_state(&mut run)
.expect_err("authority-created frame missing from persisted projections must fail");
assert!(
matches!(
error,
crate::runtime::recovery::RestoreIncompatible::FlowAuthorityProjectionMismatch { .. }
),
"unexpected recovery error: {error}"
);
}
#[test]
fn test_recovery_rejects_authority_created_loop_missing_from_projection() {
let (mut run, _, loop_instance_id) = authority_backed_root_frame_run(true);
assert!(
!run.loops.contains_key(&loop_instance_id),
"test fixture intentionally omits the authority-created loop projection"
);
let error = crate::runtime::recovery::reconcile_run_state(&mut run)
.expect_err("authority-created loop missing from persisted projections must fail");
assert!(
matches!(
error,
crate::runtime::recovery::RestoreIncompatible::FlowAuthorityProjectionMismatch { .. }
),
"unexpected recovery error: {error}"
);
}
#[tokio::test]
async fn test_resume_converges_untracked_active_flow_runs() {
let definition = sample_definition_with_single_step_flow(60_000, 8);
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runs = storage.runs.clone();
let specs = storage.specs.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let identity = storage.identity.clone();
let identity_status = storage.identity_status.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle.shutdown().await.expect("shutdown actor");
let active = running_authority_backed_run(&definition);
let run_id = active.run_id.clone();
runs.create_run(active)
.await
.expect("persist active run as crash residue");
let resumed = MobBuilder::for_resume(MobStorage::custom_with_runtime_metadata(
events,
runs.clone(),
specs,
runtime_metadata,
identity,
identity_status,
))
.with_session_service(service)
.resume()
.await
.expect("resume mob");
let terminal = wait_for_run_terminal(&resumed, &run_id, Duration::from_secs(2)).await;
assert_eq!(
terminal.status,
MobRunStatus::Canceled,
"resume must converge persisted active runs that have no actor trackers"
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query machine state");
assert_eq!(
machine_state.active_run_count, 0,
"recovered active flow must not leave MobMachine active_run_count blocking stop"
);
resumed
.stop()
.await
.expect("stop should not be blocked by an untracked recovered flow");
}
#[test]
fn test_resume_path_replays_persisted_flow_authority_inputs() {
let builder = include_str!("builder.rs");
assert!(
builder.contains("seed_mob_authority_sync_from_flow_runs"),
"resume must seed MobMachine flow authority from persisted run authority inputs"
);
let recovery = include_str!("recovery.rs");
assert!(
recovery.contains("validate_flow_authority_projection"),
"recovery must validate projection snapshots against the persisted MobMachine authority input log"
);
}
#[test]
fn test_actor_owned_terminalization_paths_do_not_reenter_actor_mailbox() {
let source = include_str!("actor.rs");
let run_flow_start = source
.find("async fn handle_run_flow")
.expect("handle_run_flow exists");
let run_flow_pre_spawn_end = source[run_flow_start..]
.find("let cancel_token = tokio_util::sync::CancellationToken::new();")
.expect("handle_run_flow creates cancel token after admission");
let run_flow_pre_spawn = &source[run_flow_start..run_flow_start + run_flow_pre_spawn_end];
assert!(
!run_flow_pre_spawn.contains(".flow_engine\n .terminalize_failed("),
"actor-owned run admission failure must terminalize through the in-actor helper, not by sending to its own actor mailbox"
);
assert!(
run_flow_pre_spawn.contains("terminalize_failed_in_actor"),
"actor-owned run admission failure must use the in-actor failed terminalization helper"
);
let cancel_start = source
.find("async fn handle_cancel_flow")
.expect("handle_cancel_flow exists");
let cancel_body_end = source[cancel_start..]
.find("let flow_engine = self.flow_engine.clone();")
.expect("handle_cancel_flow spawns async cancellation cleanup after no-handle path");
let cancel_no_handle_body = &source[cancel_start..cancel_start + cancel_body_end];
assert!(
!cancel_no_handle_body.contains(".flow_engine\n .terminalize_canceled("),
"actor-owned no-task cancel path must terminalize through the in-actor helper, not by sending to its own actor mailbox"
);
assert!(
cancel_no_handle_body.contains("terminalize_canceled_in_actor"),
"actor-owned no-task cancel path must use the in-actor canceled terminalization helper"
);
}
#[tokio::test]
async fn test_stale_flow_frame_store_plan_loses_cas_before_authority_prepare() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::{FlowNodeId, FrameId};
use crate::runtime::flow_frame_engine::{FlowFrameKernel, FlowFrameLoopStorePlan};
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create run");
let (handle, _) = create_test_mob_with_run_store(sample_definition(), store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let node_id = FlowNodeId::from("start-node");
let mut nodes = IndexMap::new();
nodes.insert(
node_id.clone(),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
let root_spec = FrameSpec { nodes };
let frame_id = FrameId::from(format!("{run_id}-root").as_str());
let frame_kernel = FlowFrameKernel::new(store.clone(), handle.clone());
let initial_frame = frame_kernel
.start_frame(&run_id, &frame_id, &root_spec)
.await
.expect("start root frame");
frame_kernel
.admit_next_ready_node_with_retry(&run_id, &frame_id, 5)
.await
.expect("admit next ready node")
.expect("admission effects");
let advanced_frame = store
.get_run(&run_id)
.await
.expect("get run")
.expect("run exists")
.frames
.get(&frame_id)
.cloned()
.expect("advanced frame exists");
assert_ne!(
advanced_frame, initial_frame,
"admission should advance the persisted frame before replaying stale plan"
);
let authority_log_len_before_stale_replay = serialized_flow_authority_inputs(
&store
.get_run(&run_id)
.await
.expect("get run before stale replay")
.expect("run exists before stale replay"),
)
.len();
let stale_admit = crate::run::MobMachineFlowFrameCommand::AdmitNextReadyNode(
crate::run::flow_frame::inputs::AdmitNextReadyNode {
node_id,
ready_queue: Vec::new(),
},
);
let won = handle
.commit_flow_frame_store_plan(
&run_id,
FlowFrameLoopStorePlan::FrameState {
machine_inputs: vec![stale_admit.authority_input(&frame_id)],
frame_id: frame_id.clone(),
expected_frame: initial_frame,
next_frame: advanced_frame.clone(),
},
)
.await
.expect("stale frame plan should lose through store expectation, not authority prepare");
assert!(
!won,
"stale frame plan must return a clean CAS miss so callers can retry/revisit"
);
let after_replay = store
.get_run(&run_id)
.await
.expect("get run after replay")
.expect("run exists after replay")
.frames
.get(&frame_id)
.cloned()
.expect("frame exists after replay");
assert_eq!(
after_replay, advanced_frame,
"losing stale plan must not mutate the persisted frame"
);
let authority_log_len_after_stale_replay = serialized_flow_authority_inputs(
&store
.get_run(&run_id)
.await
.expect("get run after stale replay")
.expect("run exists after stale replay"),
)
.len();
assert_eq!(
authority_log_len_after_stale_replay, authority_log_len_before_stale_replay,
"losing stale plan must not persist a MobMachine authority input"
);
}
#[test]
fn test_supervisor_private_trust_realizes_generated_publish_obligation() {
let source = include_str!("actor.rs");
let start = source
.find("async fn install_supervisor_private_trust_for_session")
.expect("install_supervisor_private_trust_for_session exists");
let end = source[start..]
.find("async fn cleanup_supervisor_private_trust_for_session")
.expect("cleanup helper follows install helper");
let body = &source[start..start + end];
assert!(
body.contains("protocol_supervisor_trust_publish::extract_obligations"),
"supervisor private trust publication must realize the generated supervisor_trust_publish handoff obligation"
);
assert!(
body.contains("Box::pin(self.install_supervisor_private_trust_for_session_authority"),
"the supervisor private-trust authority future must be boxed before awaiting so the spawn finalization chain does not live on the worker stack"
);
assert!(
body.contains("stage_supervisor_trust_publish_request"),
"already-bound private trust publication must request a generated publish obligation before touching trust state"
);
assert!(
!body.contains("(None, spec.clone())"),
"already-bound private trust publication must not fall back to a locally reconstructed trust descriptor"
);
assert!(
!body.contains(
"stage_supervisor_trust_published(session_id, next_peer_id.clone(), next_epoch)"
),
"publish feedback must use the generated obligation fields, not locally reconstructed peer/epoch atoms"
);
assert!(
body.contains("supervisor private trust publication ack rejected")
&& body.contains("self.cleanup_supervisor_private_trust_publish_attempt"),
"rejected supervisor_trust_publish ack must clean up any attempted private trust before returning"
);
}
#[tokio::test]
async fn test_cancel_flow_cooperative_path_finishes_before_fallback_window() {
let (handle, service) = create_test_mob(sample_definition_with_two_step_flow(5_000)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_delay_ms(200);
let run_id = handle
.run_flow(FlowId::from("two_step"), serde_json::json!({}))
.await
.expect("run flow");
tokio::time::sleep(Duration::from_millis(20)).await;
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let start = Instant::now();
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
assert!(
start.elapsed() < Duration::from_secs(2),
"cooperative cancel path should finalize before fallback timeout window"
);
}
#[tokio::test]
async fn test_cancel_flow_fallback_marks_run_canceled_when_turn_stalls() {
let (handle, service) = create_test_mob(with_cancel_grace_timeout(
sample_definition_with_single_step_flow(60_000, 8),
25,
))
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowCanceled { run_id: id, .. } if id == &run_id
)
}),
"expected FlowCanceled event for canceled run"
);
}
#[tokio::test]
async fn test_cancel_flow_fallback_uses_configured_grace_timeout() {
let mut definition = sample_definition_with_single_step_flow(60_000, 8);
definition.limits = Some(LimitsSpec {
max_flow_duration_ms: None,
max_step_retries: None,
max_orphaned_turns: Some(8),
cancel_grace_timeout_ms: Some(25),
..Default::default()
});
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let start = Instant::now();
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
assert!(
start.elapsed() < Duration::from_millis(500),
"configured cancel grace timeout should be honored by fallback cancellation"
);
}
#[tokio::test]
async fn test_cancel_fallback_uses_direct_pending_to_terminal_cas_attempts() {
let run_store = Arc::new(RecordingRunStore::new());
let (handle, service) = create_test_mob_with_run_store(
with_cancel_grace_timeout(sample_definition_with_single_step_flow(60_000, 8), 25),
run_store.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
let history = run_store.snapshot_cas_history().await;
assert!(
history.iter().any(|(_, from, to)| {
matches!(from, MobRunStatus::Pending | MobRunStatus::Running)
&& *to == MobRunStatus::Canceled
}),
"fallback should attempt direct active->Canceled snapshot CAS transitions"
);
assert!(
history.iter().any(|(_, from, to)| {
matches!(from, MobRunStatus::Pending | MobRunStatus::Running)
&& matches!(
to,
MobRunStatus::Completed | MobRunStatus::Failed | MobRunStatus::Canceled
)
}),
"terminalization authority must use direct active->terminal snapshot CAS semantics"
);
}
#[tokio::test]
async fn test_concurrent_fail_and_cancel_resolve_single_terminal_state_without_raw_ledger_fallback()
{
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("FlowFailed").await;
events.fail_appends_for("FlowCanceled").await;
let (handle, service, runs, specs, runtime_metadata, identity, identity_status) =
create_test_mob_with_recoverable_fault_events(
sample_definition_with_single_step_flow(60_000, 8),
events.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
service.set_flow_turn_delay_ms(120);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let cancel_handle = {
let handle = handle.clone();
let run_id = run_id.clone();
tokio::spawn(async move { handle.cancel_flow(run_id).await })
};
let terminal =
wait_for_run_terminal_in_store(runs.as_ref(), &run_id, Duration::from_secs(8)).await;
let cancel_result = cancel_handle.await.expect("cancel join");
assert!(
cancel_result.is_ok()
|| matches!(
&cancel_result,
Err(MobError::ActorCommandChannelClosed | MobError::ActorReplyChannelClosed)
),
"cancel either linearizes first or observes the required fail-stop: {cancel_result:?}"
);
assert!(
matches!(
terminal.status,
MobRunStatus::Canceled | MobRunStatus::Failed
),
"concurrent fail/cancel race should resolve to one terminal status"
);
wait_for_actor_fail_stop(&handle).await;
let recorded_events = events.replay_all().await.expect("replay");
let terminal_events = recorded_events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::FlowCompleted { run_id: id, .. }
| MobEventKind::FlowFailed { run_id: id, .. }
| MobEventKind::FlowCanceled { run_id: id, .. }
if id == &run_id
)
})
.count();
assert!(
terminal_events <= 1,
"terminal race must never produce multiple terminal events for one run"
);
if terminal_events == 0 {
assert!(
!terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("append failed")),
"terminal append failures must not mutate failure ledger outside machine authority"
);
}
events.allow_appends_for("FlowFailed").await;
events.allow_appends_for("FlowCanceled").await;
let resumed = cold_resume_after_terminal_append_fail_stop(
events,
runs,
specs,
runtime_metadata,
identity,
identity_status,
service,
)
.await;
let recovered_run = resumed
.flow_status(run_id.clone())
.await
.expect("cold actor should read the durable terminal run")
.expect("cold-recovered terminal run should remain queryable");
assert!(
matches!(
recovered_run.status,
MobRunStatus::Canceled | MobRunStatus::Failed
),
"cold recovery must preserve the exact durable terminal winner"
);
let recovered_machine = resumed
.query_machine_state()
.await
.expect("cold actor should expose recovered machine state");
assert_eq!(recovered_machine.active_run_count, 0);
let repaired_events = resumed.events().replay_all().await.expect("replay repair");
assert_eq!(
repaired_events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, .. }
| MobEventKind::FlowCanceled { run_id: id, .. }
if id == &run_id
)
})
.count(),
1,
"cold recovery must repair exactly one carrier for the durable winner"
);
}
#[tokio::test]
async fn test_timeout_maps_to_step_failed_reason_and_run_failed() {
let (handle, service) = create_test_mob(sample_definition_with_single_step_flow(20, 1)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("timeout after")),
"timeout path should persist failure ledger reason"
);
assert_eq!(
terminal
.failure_ledger
.iter()
.filter(|entry| entry.reason.contains("timeout after"))
.count(),
1,
"timeout path should persist exactly one timeout failure ledger entry"
);
assert!(
terminal
.step_ledger
.iter()
.any(|entry| entry.status == crate::run::StepRunStatus::Failed),
"timeout path should persist failed step ledger entries"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepTargetFailed { run_id: id, reason, .. }
if id == &run_id && reason.contains("timeout after")
)
}),
"expected timeout StepTargetFailed reason"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id
)
}),
"expected FlowFailed event for timed out run"
);
}
#[tokio::test]
async fn test_timeout_budget_exhaustion_fails_run() {
let (handle, service) = create_test_mob(sample_definition_with_single_step_flow(20, 0)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("timeout + canceled after")),
"budget exhaustion should abort the timed-out turn and persist the machine-projected step failure"
);
}
#[tokio::test]
async fn test_flow_turn_failure_records_target_failure_reason() {
let (handle, service) = create_test_mob(sample_definition_with_single_step_flow(500, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepTargetFailed { run_id: id, reason, .. }
if id == &run_id && reason.contains("mock flow turn failure")
)
}),
"expected StepTargetFailed to include provisioner failure reason"
);
}
#[tokio::test]
async fn test_malformed_turn_output_fails_without_json_coercion() {
let (handle, service) = create_test_mob(sample_definition_with_single_step_flow(500, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_completed_result("not-json").await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
terminal.failure_ledger.iter().any(|entry| {
entry.reason.contains("malformed JSON output")
&& entry.reason.contains("raw_output=\"not-json\"")
}),
"malformed output should fail run with parse diagnostics and raw payload excerpt"
);
}
#[tokio::test]
async fn test_plain_text_step_output_can_skip_json_parsing() {
let mut definition = sample_definition_with_single_step_flow(500, 8);
let step = definition
.flows
.get_mut(&FlowId::from("demo"))
.and_then(|flow| flow.steps.get_mut(&crate::StepId::from("start")))
.expect("start step exists");
step.output_format = Some(StepOutputFormat::Text);
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_completed_result("not-json").await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
assert!(
terminal.step_ledger.iter().any(|entry| {
entry.step_id == "start"
&& entry.status == StepRunStatus::Completed
&& entry.output == Some(serde_json::json!({"w-1": "not-json"}))
}),
"completed start ledger entry should persist plain-text output in the canonical aggregate shape"
);
}
#[tokio::test]
async fn test_spawn_rejects_reserved_flow_member_prefix() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let reserved_member_identity = format!("{}worker_demo", crate::runtime::FLOW_MEMBER_ID_PREFIX);
let err = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from(reserved_member_identity.as_str()),
None,
)
.await
.expect_err("reserved system member prefix should be rejected");
assert!(
err.to_string()
.contains("reserved system identifier namespace"),
"error should clearly explain reserved id namespace: {err}"
);
}
#[tokio::test]
async fn test_flow_failed_append_failure_does_not_write_raw_failure_ledger_entry() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("FlowFailed").await;
let (handle, service, runs, specs, runtime_metadata, identity, identity_status) =
create_test_mob_with_recoverable_fault_events(
sample_definition_with_single_step_flow(500, 8),
events.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal =
wait_for_run_terminal_in_store(runs.as_ref(), &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
!terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("FlowFailed append failed")),
"failed terminal event append must not write failure ledger without machine authority"
);
wait_for_actor_fail_stop(&handle).await;
events.allow_appends_for("FlowFailed").await;
let resumed = cold_resume_after_terminal_append_fail_stop(
events,
runs,
specs,
runtime_metadata,
identity,
identity_status,
service,
)
.await;
let recovered_run = resumed
.flow_status(run_id.clone())
.await
.expect("query cold-recovered durable run after failed append")
.expect("failed run should remain queryable after cold recovery");
assert_eq!(
recovered_run.status,
MobRunStatus::Failed,
"cold recovery must preserve Failed after the durable run CAS wins before terminal event append"
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query cold-recovered MobMachine state after failed append");
assert_eq!(
machine_state.active_run_count, 0,
"failed terminal append failure must not leave MobMachine active_run_count running"
);
let repaired_events = resumed.events().replay_all().await.expect("replay repair");
assert!(
repaired_events.iter().any(
|event| matches!(&event.kind, MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id)
),
"cold recovery must repair the missing FlowFailed carrier"
);
}
#[tokio::test]
async fn test_flow_completed_append_failure_does_not_write_raw_failure_ledger_entry() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("FlowCompleted").await;
let (handle, service, runs, specs, runtime_metadata, identity, identity_status) =
create_test_mob_with_recoverable_fault_events(
sample_definition_with_single_step_flow(500, 8),
events.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal =
wait_for_run_terminal_in_store(runs.as_ref(), &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
assert!(
!terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("FlowCompleted append failed")),
"failed terminal event append must not write failure ledger without machine authority"
);
wait_for_actor_fail_stop(&handle).await;
let events_after_failed_append = events.replay_all().await.expect("replay events");
assert!(
!events_after_failed_append
.iter()
.any(|event| matches!(&event.kind, MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id)),
"fallback error handling must not append FlowFailed for a run persisted as Completed"
);
events.allow_appends_for("FlowCompleted").await;
let resumed = cold_resume_after_terminal_append_fail_stop(
events.clone(),
runs,
specs,
runtime_metadata,
identity,
identity_status,
service,
)
.await;
let recovered_run = resumed
.flow_status(run_id.clone())
.await
.expect("query cold-recovered durable run after completed append failure")
.expect("completed run should remain queryable after cold recovery");
assert_eq!(
recovered_run.status,
MobRunStatus::Completed,
"cold recovery must preserve Completed after the durable run CAS wins before terminal event append"
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query cold-recovered MobMachine state after terminal append failure");
assert_eq!(
machine_state.active_run_count, 0,
"terminal append failure must not leave MobMachine active_run_count running"
);
let events = events.replay_all().await.expect("replay events");
assert!(
!events
.iter()
.any(|event| matches!(&event.kind, MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id)),
"completed repair must not leave a duplicate wrong FlowFailed event"
);
let completed = events.iter().find_map(|event| match &event.kind {
MobEventKind::FlowCompleted {
run_id: event_run_id,
structured_output,
..
} if event_run_id == &run_id => Some(structured_output),
_ => None,
});
assert_eq!(
completed,
Some(&Some(serde_json::json!({
"steps": {
"start": {
"w-1": "Turn completed"
}
}
}))),
"cold recovery should repair the missing FlowCompleted event from the canonical fan-out aggregate"
);
}
#[tokio::test]
async fn test_flow_canceled_append_failure_does_not_write_raw_failure_ledger_entry() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("FlowCanceled").await;
let (handle, service, runs, specs, runtime_metadata, identity, identity_status) =
create_test_mob_with_recoverable_fault_events(
with_cancel_grace_timeout(sample_definition_with_single_step_flow(60_000, 8), 25),
events.clone(),
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let terminal =
wait_for_run_terminal_in_store(runs.as_ref(), &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
assert!(
!terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("FlowCanceled append failed")),
"failed terminal event append must not write failure ledger without machine authority"
);
wait_for_actor_fail_stop(&handle).await;
events.allow_appends_for("FlowCanceled").await;
let resumed = cold_resume_after_terminal_append_fail_stop(
events,
runs,
specs,
runtime_metadata,
identity,
identity_status,
service,
)
.await;
let recovered_run = resumed
.flow_status(run_id.clone())
.await
.expect("query cold-recovered durable run after canceled append failure")
.expect("canceled run should remain queryable after cold recovery");
assert_eq!(
recovered_run.status,
MobRunStatus::Canceled,
"cold recovery must preserve Canceled after the durable run CAS wins before terminal event append"
);
let machine_state = resumed
.query_machine_state()
.await
.expect("query cold-recovered MobMachine state after canceled append failure");
assert_eq!(
machine_state.active_run_count, 0,
"canceled terminal append failure must not leave MobMachine active_run_count running"
);
let repaired_events = resumed.events().replay_all().await.expect("replay repair");
assert!(
repaired_events.iter().any(
|event| matches!(&event.kind, MobEventKind::FlowCanceled { run_id: id, .. } if id == &run_id)
),
"cold recovery must repair the missing FlowCanceled carrier"
);
}
#[tokio::test]
async fn test_topology_strict_denial_fails_before_dispatch() {
let (handle, _service) = create_test_mob(sample_definition_with_strict_topology_deny()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id
)
}),
"expected FlowFailed event under strict topology denial"
);
assert!(
!events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
}),
"strict topology denial should block dispatch"
);
}
#[tokio::test]
async fn test_topology_advisory_emits_violation_and_continues() {
let (handle, _service) = create_test_mob(sample_definition_with_advisory_topology_deny()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|event| { matches!(event.kind, MobEventKind::TopologyViolation { .. }) }),
"advisory topology should emit violation event"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
}),
"advisory topology should still dispatch steps"
);
}
#[tokio::test]
async fn test_topology_strict_wildcard_denial_fails_before_dispatch() {
let (handle, _service) =
create_test_mob(sample_definition_with_strict_topology_wildcard_deny()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, .. } if id == &run_id
)
}),
"expected FlowFailed event under strict wildcard topology denial"
);
assert!(
!events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
}),
"strict wildcard topology denial should block dispatch"
);
}
#[tokio::test]
async fn test_topology_advisory_wildcard_emits_violation_and_continues() {
let (handle, _service) =
create_test_mob(sample_definition_with_advisory_topology_wildcard_deny()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|event| { matches!(event.kind, MobEventKind::TopologyViolation { .. }) }),
"advisory wildcard topology should emit violation event"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, .. } if id == &run_id
)
}),
"advisory wildcard topology should still dispatch steps"
);
}
#[tokio::test]
async fn test_collection_policy_any_succeeds_with_single_target_success() {
let (handle, service) = create_test_mob(sample_definition_with_collection_policy(
CollectionPolicy::Any,
))
.await;
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let _sid_w2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_flow_turn_fail_for_session(&sid_w1, true).await;
let run_id = handle
.run_flow(FlowId::from("collect"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
}
#[tokio::test]
async fn test_collection_policy_quorum_requires_threshold_successes() {
let (handle, service) = create_test_mob(sample_definition_with_collection_policy(
CollectionPolicy::Quorum { n: 2 },
))
.await;
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let _sid_w2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_flow_turn_fail_for_session(&sid_w1, true).await;
let run_id = handle
.run_flow(FlowId::from("collect"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
}
#[tokio::test]
async fn test_impossible_quorum_fails_before_dispatching_any_targets() {
let (handle, _service) = create_test_mob(sample_definition_with_collection_policy(
CollectionPolicy::Quorum { n: 3 },
))
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
let run_id = handle
.run_flow(FlowId::from("collect"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
terminal.failure_ledger.iter().any(|entry| {
entry.step_id.as_str() == "collect" && entry.reason.contains("insufficient targets")
}) || events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, reason, .. }
if id == &run_id && reason.contains("insufficient targets")
)
}),
"impossible quorum should surface an explicit insufficient-targets reason"
);
assert_eq!(
events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepDispatched { run_id: id, step_id, .. }
if id == &run_id && step_id.as_str() == "collect"
)
})
.count(),
0,
"impossible quorum should fail before dispatching any target turns"
);
}
#[tokio::test]
async fn test_collection_policy_all_uses_map_shape_for_single_target() {
let (handle, _service) = create_test_mob(sample_definition_with_collection_policy(
CollectionPolicy::All,
))
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
let run_id = handle
.run_flow(FlowId::from("collect"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let aggregate_output = terminal
.step_ledger
.iter()
.find(|entry| {
entry.step_id.as_str() == "collect"
&& entry.agent_identity == crate::runtime::flow_system_member_id()
&& entry.status == StepRunStatus::Completed
})
.and_then(|entry| entry.output.clone())
.expect("aggregate collect output should be persisted");
assert_eq!(
aggregate_output,
serde_json::json!({"w-1": "Turn completed"})
);
}
#[tokio::test]
async fn test_max_flow_duration_limit_is_enforced() {
let mut definition = sample_definition_with_single_step_flow(60_000, 8);
definition.limits = Some(LimitsSpec {
max_flow_duration_ms: Some(25),
max_step_retries: None,
max_orphaned_turns: Some(8),
cancel_grace_timeout_ms: None,
..Default::default()
});
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let start = Instant::now();
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
start.elapsed() < Duration::from_secs(1),
"global flow duration limit should fail the run quickly"
);
}
#[tokio::test]
async fn test_schema_ref_file_path_validation_passes_and_fails() {
let mut schema_ok = NamedTempFile::new().expect("create schema file");
write!(
schema_ok,
"{{\"type\":\"object\",\"additionalProperties\":{{\"type\":\"string\"}}}}"
)
.expect("write schema");
let ok_path = schema_ok.path().to_string_lossy().to_string();
let (ok_handle, _ok_service) =
create_test_mob(sample_definition_with_schema_ref(&ok_path)).await;
ok_handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let ok_run_id = ok_handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let ok_terminal = wait_for_run_terminal(&ok_handle, &ok_run_id, Duration::from_secs(8)).await;
assert_eq!(ok_terminal.status, MobRunStatus::Completed);
let mut schema_bad = NamedTempFile::new().expect("create schema file");
write!(
schema_bad,
"{{\"type\":\"object\",\"additionalProperties\":{{\"type\":\"integer\"}}}}"
)
.expect("write schema");
let bad_path = schema_bad.path().to_string_lossy().to_string();
let (bad_handle, _bad_service) =
create_test_mob(sample_definition_with_schema_ref(&bad_path)).await;
bad_handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bad_run_id = bad_handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let bad_terminal =
wait_for_run_terminal(&bad_handle, &bad_run_id, Duration::from_secs(8)).await;
assert_eq!(bad_terminal.status, MobRunStatus::Failed);
}
#[tokio::test]
async fn test_supervisor_escalation_forces_reset_via_retire_path() {
let (handle, service) = create_test_mob(sample_definition_with_supervisor_threshold(1)).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead supervisor");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::SupervisorEscalation { run_id: id, .. } if id == &run_id
)
}),
"expected supervisor escalation event after threshold breach"
);
let remaining = handle.list_members().await;
assert!(
remaining.is_empty(),
"force_reset should retire active meerkats via handle.retire()"
);
}
#[tokio::test]
async fn test_cleanup_fail_step_routes_generated_supervisor_escalation_effect() {
use crate::machines::mob_machine as mob_dsl;
use crate::run::{
MobMachineFlowAuthorityToken, MobMachineFlowRunCommand, apply_mob_machine_flow_run_command,
flow_run,
};
let definition = sample_definition_with_supervisor_threshold(1);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let run_store: Arc<dyn MobRunStore> = Arc::new(InMemoryMobRunStore::new());
let event_store: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let storage = MobStorage {
events: event_store.clone(),
runs: run_store.clone(),
specs: Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata: Arc::new(InMemoryMobRuntimeMetadataStore::new()),
identity: Arc::new(InMemoryMobIdentityStore::new()),
identity_member: None,
identity_status: Arc::new(InMemoryMobIdentityStatusStore::new()),
realm_profiles: None,
};
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let config = crate::run::FlowRunConfig::from_definition(flow_id("demo"), &definition)
.expect("demo flow config");
let run_id = RunId::new();
let seed_input =
MobRun::create_run_seed_input(&run_id, &config).expect("create run seed input");
let mut authority = mob_dsl::MobMachineAuthority::new();
mob_dsl::MobMachineMutator::apply(&mut authority, seed_input.clone())
.expect("seed input accepted");
let flow_state =
MobRun::flow_state_for_config_with_authority(&run_id, &config, authority.state())
.expect("project flow state");
let mut run = MobRun::pending_with_run_id(
run_id.clone(),
handle.mob_id().clone(),
config.flow_id.clone(),
flow_state,
serde_json::json!({}),
);
run.append_flow_authority_inputs(vec![seed_input.clone()])
.expect("append seed authority");
let start_command = MobMachineFlowRunCommand::StartRun(flow_run::inputs::StartRun {});
let start_input = start_command.authority_input(&run_id);
let start_transition = mob_dsl::MobMachineMutator::apply(&mut authority, start_input.clone())
.expect("start input accepted");
let start_authority =
MobMachineFlowAuthorityToken::from_accepted_mob_machine_input(&start_input)
.expect("start authority token");
let start_outcome = apply_mob_machine_flow_run_command(
&run.flow_state,
authority.state(),
&run_id,
start_command,
start_authority,
start_transition.effects(),
)
.expect("project start");
run.flow_state = start_outcome.next_state;
run.status = MobRunStatus::Running;
run.append_flow_authority_inputs(vec![start_input.clone()])
.expect("append start authority");
run_store.create_run(run).await.expect("create run");
handle
.project_machine_input(seed_input)
.await
.expect("project seed into actor machine");
handle
.project_machine_input(start_input)
.await
.expect("project start into actor machine");
let engine = FlowEngine::new(
Arc::new(UnusedFlowTurnExecutor),
handle,
run_store.clone(),
event_store,
Arc::new(super::topology::MobTopologyService::new(
definition.topology.clone(),
)),
);
let escalation_steps = engine
.fail_unfinished_steps(&run_id, "cleanup failure")
.await
.expect("fail unfinished steps");
assert_eq!(escalation_steps, vec![step_id("start")]);
let stored = run_store
.get_run(&run_id)
.await
.expect("load run")
.expect("run exists");
assert_eq!(
stored
.step_status_snapshot()
.unwrap()
.get(&step_id("start")),
Some(&StepRunStatus::Failed)
);
assert_eq!(stored.step_ledger.len(), 1);
assert_eq!(stored.step_ledger[0].status, StepRunStatus::Failed);
assert_eq!(stored.failure_ledger.len(), 1);
assert_eq!(stored.failure_ledger[0].step_id, step_id("start"));
assert_eq!(stored.failure_ledger[0].reason, "cleanup failure");
}
#[tokio::test]
async fn test_supervisor_escalation_times_out_when_turn_hangs() {
let mut definition = sample_definition_with_supervisor_threshold(1);
definition
.supervisor
.as_mut()
.expect("supervisor configured")
.escalation_turn_timeout_ms = Some(25);
let lead = definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap();
lead.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead supervisor");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
// Model a genuinely hung supervisor turn. A short finite delay can become
// ready alongside the timeout when a loaded CI worker is descheduled,
// allowing the mock completion to win a wall-clock polling race.
service.set_start_turn_delay_ms(600_000);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
terminal
.failure_ledger
.iter()
.any(|entry| entry.reason.contains("supervisor escalation timed out"))
|| events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, reason, .. }
if id == &run_id && reason.contains("supervisor escalation timed out")
)
}),
"supervisor escalation timeout should fail with explicit bounded-time reason"
);
}
#[tokio::test]
async fn test_supervisor_force_reset_reports_aggregate_retire_errors() {
let events = Arc::new(FaultInjectedMobEventStore::new());
events.fail_appends_for("MemberRetired").await;
let (handle, _service) = create_test_mob_with_events(sample_definition(), events).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
let supervisor = super::supervisor::Supervisor::new(
handle.clone(),
super::events::MobEventEmitter::new(
Arc::new(InMemoryMobEventStore::new()),
handle.mob_id().clone(),
),
);
let error = supervisor
.force_reset()
.await
.expect_err("force_reset should aggregate retire errors");
assert!(
error
.to_string()
.contains("force_reset encountered 2 retirement error(s)"),
"force_reset should continue through all retire failures and return aggregate count"
);
}
#[tokio::test]
async fn test_branch_winner_and_join_any_behavior() {
let (handle, _service) = create_test_mob(sample_definition_with_branch_flow()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(
FlowId::from("branching"),
serde_json::json!({ "severity": "critical" }),
)
.await
.expect("run branch flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "fix_critical"
)
}),
"expected winning branch step to complete"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepSkipped { run_id: id, step_id, .. }
if id == &run_id && step_id.as_str() == "fix_minor"
)
}),
"expected losing branch step to be skipped"
);
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "summarize"
)
}),
"expected join step to run with depends_on_mode=any"
);
}
#[tokio::test]
async fn test_cancel_after_completion_is_noop_without_flow_canceled_event() {
let (handle, _service) = create_test_mob(sample_definition_with_single_step_flow(500, 8)).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(2)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
tokio::time::sleep(Duration::from_millis(50)).await;
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel completed run should be noop");
let status = handle
.flow_status(run_id.clone())
.await
.expect("flow status call")
.expect("run exists");
assert_eq!(status.status, MobRunStatus::Completed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowCanceled { run_id: id, .. } if id == &run_id
)
}),
"cancel after completion should not emit FlowCanceled"
);
}
// -----------------------------------------------------------------------
// Additional: MobState
// -----------------------------------------------------------------------
#[test]
fn test_mob_state_as_str() {
assert_eq!(MobState::Creating.as_str(), "Creating");
assert_eq!(MobState::Running.as_str(), "Running");
assert_eq!(MobState::Stopped.as_str(), "Stopped");
assert_eq!(MobState::Completed.as_str(), "Completed");
assert_eq!(MobState::Destroyed.as_str(), "Destroyed");
}
// -----------------------------------------------------------------------
// P1-T04b: spawn() with initial_message uses custom message
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_spawn_with_custom_initial_message() {
let (handle, service) = create_test_mob(sample_definition()).await;
let custom_msg = "You are the orchestrator. Begin planning.";
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-1"),
Some(custom_msg.to_string().into()),
)
.await
.expect("spawn with custom message");
// Verify the custom message was passed to create_session
let prompts = service.recorded_prompts().await;
assert_eq!(prompts.len(), 1, "should have exactly one recorded prompt");
assert_eq!(
prompts[0].1, custom_msg,
"spawn should use the custom initial_message, not the default"
);
// Runtime adapter path: autonomous spawn uses accept_input_with_completion,
// which delegates to start_turn and increments keep_alive_start_turn_call_count.
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"autonomous spawn must issue exactly one keep-alive start_turn"
);
assert_eq!(
service.inject_call_count(),
0,
"autonomous spawn uses start_turn via runtime adapter, not inject"
);
}
#[tokio::test]
async fn test_spawn_without_initial_message_uses_default() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn with default message");
let prompts = service.recorded_prompts().await;
assert_eq!(prompts.len(), 1);
assert!(
prompts[0].1.contains("You have been spawned as 'w-1'"),
"default message should contain meerkat id, got: '{}'",
prompts[0].1
);
assert!(
prompts[0].1.contains("role: worker"),
"default message should contain role, got: '{}'",
prompts[0].1
);
assert!(
prompts[0].1.contains("mob 'test-mob'"),
"default message should contain mob id, got: '{}'",
prompts[0].1
);
// Runtime adapter path: autonomous spawn uses accept_input_with_completion,
// which delegates to start_turn and increments keep_alive_start_turn_call_count.
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"autonomous spawn must issue exactly one keep-alive start_turn"
);
assert_eq!(
service.inject_call_count(),
0,
"autonomous spawn uses start_turn via runtime adapter, not inject"
);
}
#[tokio::test]
async fn test_autonomous_host_without_adapter_rejected_at_build_time() {
// §8: AutonomousHost requires a runtime adapter. The builder must reject
// at create() time, not defer to spawn() time where the Option<adapter>
// would hide the ownership requirement.
let result = try_create_test_mob_without_adapter(sample_definition()).await;
assert!(
result.is_err(),
"AutonomousHost without adapter must fail at build time"
);
}
#[tokio::test]
async fn test_turn_driven_without_adapter_accepted_at_build_time() {
// TurnDriven profiles don't require a runtime adapter — the Option is
// genuinely optional for them.
let mut def = sample_definition();
for profile in def.profiles.values_mut().filter_map(|b| b.as_inline_mut()) {
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let result = try_create_test_mob_without_adapter(def).await;
assert!(result.is_ok(), "TurnDriven without adapter must succeed");
}
async fn wait_for_start_turn_call_count(
service: &MockSessionService,
expected: u64,
context: &str,
) {
let deadline = Instant::now() + Duration::from_secs(2);
while service.start_turn_call_count() < expected {
assert!(Instant::now() < deadline, "{context}");
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
#[tokio::test]
async fn test_turn_driven_spawn_with_initial_message_starts_initial_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
let initial_message = "Run the requested review task exactly once.";
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-turn-initial"),
Some(initial_message.to_string().into()),
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker with initial message");
assert_eq!(
service.keep_alive_start_turn_call_count(),
0,
"turn-driven initial prompt must not start an autonomous host loop"
);
wait_for_start_turn_call_count(
&service,
1,
"turn-driven initial prompt should reach the runtime executor",
)
.await;
assert_eq!(
service.start_turn_call_count(),
1,
"turn-driven spawn with initial_message must execute exactly one initial turn"
);
let prompts = service.recorded_start_turn_prompts().await;
assert_eq!(
prompts,
vec![(
service.recorded_prompts().await[0].0.clone(),
initial_message.to_string()
)],
"turn-driven spawn must deliver the caller's initial_message to start_turn"
);
}
#[tokio::test]
async fn test_turn_driven_spawn_without_initial_message_does_not_synthesize_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-turn-idle"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker without initial message");
assert_eq!(
service.start_turn_call_count(),
0,
"turn-driven spawn without initial_message should remain idle until an explicit turn"
);
}
#[tokio::test]
async fn test_turn_driven_respawn_with_initial_message_starts_initial_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("w-turn-respawn");
let initial_message = "Resume the queued review after replacement.";
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original turn-driven worker");
assert_eq!(
service.start_turn_call_count(),
0,
"test setup should begin with an idle turn-driven member"
);
let receipt = handle
.respawn(
AgentIdentity::from(member_id.as_str()),
Some(initial_message.into()),
)
.await
.expect("respawn turn-driven worker with initial message");
wait_for_start_turn_call_count(
&service,
1,
"turn-driven respawn initial prompt should reach the runtime executor",
)
.await;
assert_eq!(
service.start_turn_call_count(),
1,
"turn-driven respawn with initial_message must execute exactly one initial turn"
);
let snapshot = handle
.member_status(&receipt.identity)
.await
.expect("respawned member snapshot");
let bridge_session_id = snapshot
.current_bridge_session_id
.expect("respawned turn-driven member should have a bridge session");
assert_eq!(
service.recorded_start_turn_prompts().await,
vec![(bridge_session_id, initial_message.to_string())],
"turn-driven respawn must deliver the caller's initial_message to the replacement session"
);
}
#[tokio::test]
async fn test_turn_driven_respawn_without_initial_message_does_not_synthesize_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("w-turn-respawn-idle");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn original turn-driven worker");
handle
.respawn(AgentIdentity::from(member_id.as_str()), None)
.await
.expect("respawn turn-driven worker without initial message");
assert_eq!(
service.start_turn_call_count(),
0,
"turn-driven respawn without initial_message should remain idle until an explicit turn"
);
}
#[tokio::test]
async fn test_spawn_autonomous_surfaces_immediate_host_loop_start_failure() {
// Runtime adapter path: accept_input_with_completion queues the input
// successfully. The start_turn failure surfaces asynchronously through
// the runtime loop's executor, which completes the initial turn with
// a non-success outcome.
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_fail_start_turn(true);
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-fail"),
None,
)
.await;
// With the runtime adapter path, the spawn itself succeeds because
// accept_input_with_completion returns Ok (input accepted into the
// driver queue). The start_turn failure surfaces asynchronously
// through the completion handle in the background task.
assert!(
result.is_ok(),
"runtime-backed spawn should accept input even when start_turn will fail: {result:?}"
);
}
#[tokio::test]
async fn test_retire_interrupts_autonomous_host_loop() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
// Runtime adapter path: autonomous spawn uses accept_input_with_completion.
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"spawn should issue keep-alive start_turn"
);
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire worker");
// With the runtime adapter path, interrupt goes through the runtime-owned
// hard-cancel path, not direct session_service.interrupt().
// The behavioral guarantee is that retire succeeds (above).
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"retire must remove the member from the roster"
);
}
#[tokio::test]
async fn test_stop_resume_host_loop_lifecycle_is_mode_aware() {
let (handle, service) = create_test_mob(sample_definition()).await;
let adapter = service.enable_runtime_adapter();
let lead_session_id = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn autonomous lead")
.bridge_session_id()
.cloned()
.expect("autonomous lead is session-backed");
let lead_attachment_before_stop = adapter
.current_executor_attachment_witness(&lead_session_id)
.await
.expect("autonomous lead has an exact attachment");
let turn_driven_session_id = handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-td"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker")
.bridge_session_id()
.cloned()
.expect("turn-driven worker is session-backed");
let turn_driven_attachment_before_stop = adapter
.current_executor_attachment_witness(&turn_driven_session_id)
.await
.expect("turn-driven worker has an exact attachment");
// Runtime adapter path: autonomous spawn uses accept_input_with_completion,
// which delegates to start_turn. No injects from spawn.
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"only autonomous members should issue keep-alive start_turn"
);
assert_eq!(
service.inject_call_count(),
0,
"no injects from spawn via runtime adapter path"
);
handle.stop().await.expect("stop");
// With the runtime adapter path, interrupt goes through the runtime-owned
// hard-cancel path, not direct session_service.interrupt().
// The behavioral guarantee is that stop succeeds (above) and the
// mob transitions to Stopped.
assert_eq!(
handle.status().await.unwrap(),
MobState::Stopped,
"stop should transition mob to Stopped"
);
// Stop notifies the orchestrator (autonomous lead) via inject (+1).
// Total inject count: 0 (no spawn inject) + 1 (stop notification) = 1.
assert_eq!(
service.inject_call_count(),
1,
"stop should have notified the orchestrator via inject"
);
// A fail-closed service discard can remove the exact actor while leaving
// its machine attachment and this provisioner's sidecar behind. Explicit
// resume must recognize that the full incarnation is missing, retire the
// stale attachment, and rebuild it; the healthy lead remains reusable.
MobSessionService::discard_live_session(service.as_ref(), &turn_driven_session_id)
.await
.expect("discard stopped turn-driven actor");
assert!(
!service
.live_session_actor_registered(&turn_driven_session_id)
.await
.expect("observe exact actor registry"),
"discard removes the service actor"
);
assert_eq!(
adapter
.current_executor_attachment_witness(&turn_driven_session_id)
.await,
Some(turn_driven_attachment_before_stop.clone()),
"the stale machine attachment remains until explicit resume"
);
handle.resume().await.expect("resume");
let lead_attachment_after_resume = adapter
.current_executor_attachment_witness(&lead_session_id)
.await
.expect("same-handle resume retains an exact attachment");
assert_eq!(
lead_attachment_after_resume, lead_attachment_before_stop,
"same-handle resume must reuse its exact sidecar rather than replacing it"
);
let repaired_turn_driven = handle
.member_status(&AgentIdentity::from("w-td"))
.await
.expect("repaired turn-driven member status");
assert_eq!(
repaired_turn_driven.status,
crate::runtime::handle::MobMemberStatus::Active,
"missing-actor repair must not classify the member Broken: {repaired_turn_driven:?}"
);
assert!(
service
.live_session_actor_registered(&turn_driven_session_id)
.await
.expect("observe repaired actor registry"),
"explicit resume must restore the exact service actor: {repaired_turn_driven:?}"
);
let turn_driven_attachment_after_resume = adapter
.current_executor_attachment_witness(&turn_driven_session_id)
.await
.expect("explicit resume rebuilds the missing actor attachment");
assert_ne!(
turn_driven_attachment_after_resume, turn_driven_attachment_before_stop,
"an attachment whose exact actor is missing must be replaced"
);
handle
.member(&AgentIdentity::from("w-td"))
.await
.expect("resumed turn-driven member")
.internal_turn("post-resume actor repair")
.await
.expect("rebuilt turn-driven member accepts input");
// Resume does not replay a member prompt, but the default lifecycle
// contract does inject one informational coordinator notification.
assert_eq!(
service.inject_call_count(),
2,
"resume should add exactly one coordinator lifecycle notification"
);
}
#[tokio::test]
async fn test_stop_notifies_every_active_orchestrator_profile_member() {
let (handle, service) = create_test_mob(sample_definition()).await;
for identity in ["lead-one", "lead-two"] {
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from(identity),
None,
)
.await
.expect("spawn orchestrator-profile member");
}
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("orchestrator-profile members ready");
tokio::time::sleep(Duration::from_millis(50)).await;
let baseline = service.inject_call_count();
handle.stop().await.expect("stop");
tokio::time::timeout(Duration::from_secs(2), async {
while service.inject_call_count() < baseline + 2 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("stop notification should reach every active orchestrator-profile member");
assert_eq!(
service.inject_call_count(),
baseline + 2,
"machine-owned profile membership must fan lifecycle delivery out to every active member"
);
}
#[tokio::test]
async fn test_destroy_does_not_open_orchestrator_turn_before_archive() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
// Runtime adapter path: autonomous members use accept_input_with_completion,
// which delegates to start_turn. No injects from spawn.
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.keep_alive_start_turn_call_count(),
2,
"both autonomous members should issue keep-alive start_turn"
);
assert_eq!(
service.inject_call_count(),
0,
"no injects from spawn via runtime adapter path"
);
handle.destroy().await.expect("destroy");
assert_eq!(
service.inject_call_count(),
0,
"destroy must not create a member turn immediately before retiring that same member"
);
// With the runtime adapter path, interrupt goes through the runtime-owned
// hard-cancel path, not direct session_service.interrupt().
// The behavioral guarantee is that destroy succeeds (above) and
// transitions to Destroyed.
assert_eq!(
handle.status().await.unwrap(),
MobState::Destroyed,
"destroy must transition mob to Destroyed"
);
}
#[tokio::test]
async fn test_complete_does_not_open_orchestrator_turn_before_archive() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
tokio::time::sleep(Duration::from_millis(10)).await;
assert_eq!(
service.inject_call_count(),
0,
"no lifecycle injection should precede completion"
);
handle.complete().await.expect("complete");
assert_eq!(
service.inject_call_count(),
0,
"complete must not create a member turn immediately before retiring that same member"
);
assert_eq!(
handle.status().await.unwrap(),
MobState::Completed,
"complete must retain its terminal machine transition"
);
}
#[tokio::test]
async fn test_resume_from_events_restarts_autonomous_host_loops_from_runtime_mode() {
let storage = MobStorage::in_memory();
let storage_for_resume = MobStorage {
events: storage.events.clone(),
runs: storage.runs.clone(),
specs: storage.specs.clone(),
runtime_metadata: storage.runtime_metadata.clone(),
identity: storage.identity.clone(),
identity_member: storage.identity_member.clone(),
identity_status: storage.identity_status.clone(),
realm_profiles: storage.realm_profiles.clone(),
};
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let turn_driven_session_id = handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("w-td"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker")
.bridge_session_id()
.cloned()
.expect("turn-driven worker has a bridge session");
tokio::time::timeout(std::time::Duration::from_secs(2), handle.stop())
.await
.expect("stop original runtime timed out")
.expect("stop original runtime");
let original_attachment = adapter
.current_executor_attachment_witness(&turn_driven_session_id)
.await
.expect("stopped mob retains the exact member attachment");
let resumed = tokio::time::timeout(
std::time::Duration::from_secs(2),
MobBuilder::for_resume(storage_for_resume)
.with_session_service(service.clone())
.notify_orchestrator_on_resume(false)
.resume(),
)
.await
.expect("resume mob timed out")
.expect("resume mob");
assert_eq!(
resumed.status().await.unwrap(),
MobState::Stopped,
"event reconstruction preserves the durable stopped lifecycle"
);
resumed
.resume()
.await
.expect("explicit lifecycle resume should start the reconstructed mob");
assert_eq!(
resumed.status().await.unwrap(),
MobState::Running,
"explicit lifecycle resume should transition the reconstructed mob"
);
let resumed_attachment = adapter
.current_executor_attachment_witness(&turn_driven_session_id)
.await
.expect("explicit resume mints a serving replacement attachment");
assert_ne!(
resumed_attachment, original_attachment,
"explicit resume must replace, never adopt, the prior attachment"
);
resumed
.member(&AgentIdentity::from("w-td"))
.await
.expect("resumed turn-driven member handle")
.internal_turn("post-resume delivery")
.await
.expect("replacement attachment accepts member input");
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
// The original spawn issued 1 keep-alive start_turn via the runtime adapter.
// Resume ensures autonomous runtime readiness but does NOT fire additional
// kickoff start_turn calls. Counter stays at 1 from the original spawn.
assert_eq!(
service.keep_alive_start_turn_call_count(),
1,
"only the original spawn should have issued a keep-alive start_turn — resume must not add more"
);
}
#[tokio::test]
async fn test_mob_resume_fails_when_runtime_metadata_store_unavailable() {
let definition = sample_definition();
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
runtime_metadata.set_fail_load_supervisor(true);
let storage = MobStorage::custom_with_runtime_metadata(
events.clone(),
Arc::new(InMemoryMobRunStore::new()),
Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata,
Arc::new(InMemoryMobIdentityStore::new()),
Arc::new(InMemoryMobIdentityStatusStore::new()),
);
events
.append(NewMobEvent {
mob_id: definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition.clone()),
},
})
.await
.expect("append mob created");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let error = match MobBuilder::for_resume(storage)
.with_session_service(service)
.resume()
.await
{
Ok(_) => panic!("resume must fail when runtime metadata cannot be loaded"),
Err(error) => error,
};
assert!(
error
.to_string()
.contains("fault-injected runtime metadata load failure"),
"resume should surface runtime metadata load failures, got: {error}"
);
}
#[tokio::test]
async fn test_mob_resume_seeds_missing_supervisor_runtime_metadata() {
let definition = sample_definition();
let events: Arc<dyn MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let storage = MobStorage::custom_with_runtime_metadata(
events.clone(),
Arc::new(InMemoryMobRunStore::new()),
Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata.clone(),
Arc::new(InMemoryMobIdentityStore::new()),
Arc::new(InMemoryMobIdentityStatusStore::new()),
);
events
.append(NewMobEvent {
mob_id: definition.id.clone(),
timestamp: None,
kind: MobEventKind::MobCreated {
definition: Box::new(definition.clone()),
},
})
.await
.expect("append mob created");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let resumed = MobBuilder::for_resume(storage)
.with_session_service(service)
.resume()
.await
.expect("resume should seed missing supervisor runtime metadata");
assert_eq!(resumed.status().await.unwrap(), MobState::Running);
let supervisor_authority = runtime_metadata
.load_supervisor_authority(&definition.id)
.await
.expect("load supervisor authority")
.expect("resume should seed supervisor authority");
assert!(
!supervisor_authority.public_peer_id.is_empty(),
"seeded supervisor authority should have a real public peer id"
);
}
#[tokio::test]
async fn test_resume_startup_keep_alive_loop_failure_enters_stopped_state() {
// TODO: This test exercises condemned direct host-loop failure behavior.
// Needs rewrite for runtime-backed keep-alive path.
if true {
return;
}
let service = Arc::new(MockSessionService::new());
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn autonomous lead");
handle.stop().await.expect("stop");
service.set_fail_start_turn(true);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.allow_ephemeral_sessions(true)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume handle should still be constructed");
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
assert_eq!(
resumed.status().await.unwrap(),
MobState::Stopped,
"startup host-loop failure should force runtime into Stopped state"
);
}
#[tokio::test]
async fn test_resume_skips_broken_autonomous_member_in_host_loop_startup() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn autonomous lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle.stop().await.expect("stop");
service.archive(&sid_l).await.expect("archive lead");
service.delete_persisted_session(&sid_l).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("stopped mob reconstruction should succeed");
assert_eq!(
resumed.status().await.expect("reconstructed mob status"),
MobState::Stopped,
"reconstruction must not implicitly recover autonomous member sessions"
);
resumed
.resume()
.await
.expect("explicit resume should keep recovery partial");
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
assert_eq!(
resumed.status().await.unwrap(),
MobState::Running,
"Broken autonomous members must be skipped during host-loop startup so partial resume stays running"
);
let snapshot = resumed
.member_status(&AgentIdentity::from("l-1"))
.await
.expect("broken member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Broken
);
}
// -----------------------------------------------------------------------
// Behavioral wire test: wire() enables PeerRequest communication
//
// Uses real CommsRuntime (inproc_only) to prove add_trusted_peer was
// actually called by verifying that a PeerRequest sent from A arrives
// in B's inbox.
// -----------------------------------------------------------------------
/// Mock session service that creates sessions with REAL CommsRuntime
/// instances (inproc_only) instead of MockCommsRuntime. This enables
/// actual PeerRequest delivery between wired meerkats.
struct RealCommsSessionService {
sessions: RwLock<HashMap<SessionId, Arc<meerkat_comms::CommsRuntime>>>,
/// Durable documents are separate from the live comms/actor map. This
/// test service advertises persistent-session support, so exact attachment
/// replacement must be able to discard a live actor and rebuild it from
/// the same typed snapshot as production.
persisted_sessions: RwLock<HashMap<SessionId, Session>>,
archived_session_ids: RwLock<HashSet<SessionId>>,
actor_registry: meerkat_session::LiveSessionActorRegistry,
/// Production session-scoped comms reloads the persisted signing identity.
/// Keep the same invariant in this in-memory harness so restarting comms
/// does not silently create a new endpoint for the same member generation.
session_keypairs: RwLock<HashMap<SessionId, meerkat_comms::Keypair>>,
keep_alive_notifiers: RwLock<HashMap<SessionId, Arc<tokio::sync::Notify>>>,
session_comms_names: RwLock<HashMap<SessionId, String>>,
session_counter: AtomicU64,
runtime_adapter: Arc<meerkat_runtime::MeerkatMachine>,
}
impl RealCommsSessionService {
fn new() -> Self {
Self {
sessions: RwLock::new(HashMap::new()),
persisted_sessions: RwLock::new(HashMap::new()),
archived_session_ids: RwLock::new(HashSet::new()),
actor_registry: meerkat_session::LiveSessionActorRegistry::default(),
session_keypairs: RwLock::new(HashMap::new()),
keep_alive_notifiers: RwLock::new(HashMap::new()),
session_comms_names: RwLock::new(HashMap::new()),
session_counter: AtomicU64::new(0),
runtime_adapter: Arc::new(meerkat_runtime::MeerkatMachine::ephemeral()),
}
}
async fn create_session_with_actor_witness_slot(
&self,
req: CreateSessionRequest,
actor_witness_slot: Option<&meerkat_session::LiveSessionActorWitnessSlot>,
) -> Result<RunResult, SessionError> {
let actor_materialization_permit = begin_test_runtime_actor_materialization(&req)?;
let mut session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
let session_id = session.id().clone();
let n = self.session_counter.fetch_add(1, Ordering::Relaxed);
let comms_name = req
.build
.as_ref()
.and_then(|b| b.comms_name.clone())
.unwrap_or_else(|| format!("real-comms-session-{n}"));
let keypair = self
.session_keypairs
.write()
.await
.entry(session_id.clone())
.or_insert_with(meerkat_comms::Keypair::generate)
.clone();
let comms = Arc::new(
meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
&comms_name,
None,
keypair,
Arc::new(HashSet::new()),
)
.expect("create inproc CommsRuntime"),
);
if !install_session_owned_peer_request_response_authority(&req, &comms)? {
install_machine_peer_request_response_authority(
&self.runtime_adapter,
&comms,
&session_id,
)
.await?;
}
let transient_turn_context_state = test_actor_transient_turn_context_state();
let local_actor_witness_slot = meerkat_session::LiveSessionActorWitnessSlot::default();
let actor_witness_slot = actor_witness_slot.unwrap_or(&local_actor_witness_slot);
let actor_witness = {
let mut sessions = self.sessions.write().await;
if sessions.contains_key(&session_id) {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
));
}
let actor_witness = self.actor_registry.insert_and_publish(
actor_witness_slot,
session_id.clone(),
transient_turn_context_state,
)?;
sessions.insert(session_id.clone(), Arc::clone(&comms));
actor_witness
};
let is_keep_alive = req.build.as_ref().map(|b| b.keep_alive).unwrap_or(false);
if is_keep_alive {
self.keep_alive_notifiers
.write()
.await
.insert(session_id.clone(), Arc::new(tokio::sync::Notify::new()));
}
self.session_comms_names
.write()
.await
.insert(session_id.clone(), comms_name.clone());
if let Some(system_prompt) = req.system_prompt.as_set_prompt() {
session.append_system_message(system_prompt.to_string());
}
if session.session_metadata().is_none() {
let build = req.build.as_ref();
session
.set_session_metadata(SessionMetadata {
schema_version: meerkat_core::session_metadata_schema_version(),
model: req.model.clone(),
max_tokens: req.max_tokens.unwrap_or(4096),
structured_output_retries: 2,
provider: build
.and_then(|options| options.provider)
.unwrap_or(Provider::Anthropic),
self_hosted_server_id: None,
provider_params: build.and_then(|options| options.provider_params.clone()),
tooling: SessionTooling {
builtins: build
.map(|options| options.override_builtins)
.unwrap_or(ToolCategoryOverride::from_effective(true)),
shell: build
.map(|options| options.override_shell)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
comms: ToolCategoryOverride::from_effective(
build
.and_then(|options| options.comms_name.as_ref())
.is_some(),
),
mob: build
.map(|options| options.override_mob)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
memory: build
.map(|options| options.override_memory)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
schedule: build
.map(|options| options.override_schedule)
.unwrap_or(ToolCategoryOverride::Inherit),
workgraph: build
.map(|options| options.override_workgraph)
.unwrap_or(ToolCategoryOverride::Inherit),
image_generation: build
.map(|options| options.override_image_generation)
.unwrap_or(ToolCategoryOverride::from_effective(false)),
web_search: build
.map(|options| options.override_web_search)
.unwrap_or(ToolCategoryOverride::Inherit),
tool_access_policy: build
.and_then(|options| options.tool_access_policy.clone()),
active_skills: build.and_then(|options| options.preload_skills.clone()),
},
keep_alive: build.map(|options| options.keep_alive).unwrap_or(false),
comms_name: build.and_then(|options| options.comms_name.clone()),
peer_meta: build.and_then(|options| options.peer_meta.clone()),
realm_id: build.and_then(|options| options.realm_id.clone()),
instance_id: build.and_then(|options| options.instance_id.clone()),
backend: build
.and_then(|options| options.backend.map(|kind| kind.as_str().to_string())),
config_generation: build.and_then(|options| options.config_generation),
auth_binding: None,
mob_member_binding: build
.and_then(|options| options.mob_member_binding.clone()),
})
.map_err(|error| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to seed real-comms session metadata: {error}"
)))
})?;
}
self.archived_session_ids.write().await.remove(&session_id);
self.persisted_sessions
.write()
.await
.insert(session_id.clone(), session);
if let Some(permit) = actor_materialization_permit
&& let Err(error) = permit.commit()
{
let cleanup = self.discard_exact_actor(&actor_witness).await;
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(match cleanup {
Ok(removed) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor removed: {removed}"
),
Err(cleanup_error) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor cleanup also failed: {cleanup_error}"
),
}),
));
}
Ok(mock_run_result(session_id, "Session created".to_string()))
}
async fn discard_exact_actor(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
let mut sessions = self.sessions.write().await;
if self.actor_registry.current(witness.session_id()).as_ref() != Some(witness)
|| !sessions.contains_key(witness.session_id())
{
return Ok(false);
}
let registry_removed = self.actor_registry.compare_remove(witness);
debug_assert!(registry_removed);
sessions.remove(witness.session_id());
let session_id = witness.session_id();
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(session_id) {
notifier.notify_waiters();
}
self.session_comms_names.write().await.remove(session_id);
drop(sessions);
Ok(true)
}
}
#[async_trait]
impl SessionService for RealCommsSessionService {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, None).await
}
async fn start_turn(
&self,
id: &SessionId,
_req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
// Block only for keep-alive sessions (notifier registered at create time).
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notified().await;
return Ok(mock_run_result(
id.clone(),
"Host loop interrupted".to_string(),
));
}
Ok(mock_run_result(id.clone(), "Turn completed".to_string()))
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
}
Ok(())
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
return Ok(());
}
Err(SessionError::NotRunning { id: id.clone() })
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
Ok(SessionView {
state: SessionInfo {
session_id: id.clone(),
created_at: SystemTime::now(),
updated_at: SystemTime::now(),
message_count: 0,
is_active: false,
model: "claude-sonnet-4-5".to_string(),
provider: Provider::Anthropic,
last_assistant_text: None,
labels: Default::default(),
},
billing: SessionUsage {
total_tokens: 0,
usage: Usage::default(),
},
})
}
async fn list(&self, _query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
let sessions = self.sessions.read().await;
let persisted = self.persisted_sessions.read().await;
Ok(persisted
.keys()
.map(|id| SessionSummary {
session_id: id.clone(),
created_at: SystemTime::now(),
updated_at: SystemTime::now(),
message_count: 0,
total_tokens: 0,
is_active: sessions.contains_key(id),
labels: Default::default(),
})
.collect())
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
Ok(self.sessions.read().await.contains_key(id))
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
let mut sessions = self.sessions.write().await;
if sessions.remove(id).is_some() {
self.actor_registry.remove_current(id);
}
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(id) {
notifier.notify_waiters();
}
self.session_comms_names.write().await.remove(id);
self.persisted_sessions.write().await.remove(id);
self.archived_session_ids.write().await.insert(id.clone());
Ok(())
}
}
#[async_trait]
impl SessionServiceCommsExt for RealCommsSessionService {
async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CoreCommsRuntime>> {
let sessions = self.sessions.read().await;
sessions
.get(session_id)
.map(|c| Arc::clone(c) as Arc<dyn CoreCommsRuntime>)
}
}
#[async_trait]
impl meerkat_core::service::SessionServiceHistoryExt for RealCommsSessionService {
async fn read_history(
&self,
id: &SessionId,
query: meerkat_core::service::SessionHistoryQuery,
) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
Ok(meerkat_core::service::SessionHistoryPage::from_messages(
id.clone(),
&[],
query,
))
}
}
#[async_trait]
impl SessionServiceControlExt for RealCommsSessionService {
async fn append_system_context(
&self,
id: &SessionId,
_req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() }.into());
}
Ok(AppendSystemContextResult {
status: AppendSystemContextStatus::Applied,
})
}
}
#[async_trait]
impl MobSessionService for RealCommsSessionService {
async fn prepare_session_for_resume(
&self,
_session_id: &SessionId,
) -> Result<(), SessionError> {
Ok(())
}
async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
&self,
_session_id: &SessionId,
_authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
Err(SessionError::Unsupported(
"real-comms test service has no store-owned boundary authority".to_string(),
))
}
/// Test double: nothing durable survives archive here, so the two-read
/// composition is the exact truth — `ArchivedNotRevivable` cannot exist.
async fn load_session_for_resume(
&self,
session_id: &SessionId,
) -> Result<ResumeSessionLoad, SessionError> {
if let Some(session) = self.load_persisted_session(session_id).await? {
return Ok(ResumeSessionLoad::Active(Box::new(session)));
}
if let Some(session) = self.load_revivable_retired_session(session_id).await? {
return Ok(ResumeSessionLoad::Revivable(Box::new(session)));
}
Ok(ResumeSessionLoad::Absent)
}
async fn create_session_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
) -> Result<RunResult, SessionError> {
SessionService::create_session(self, req).await
}
async fn create_session_with_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, Some(actor_witness_slot))
.await
}
async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.archive_with_mob_lifecycle_authority(session_id).await
}
async fn discard_live_session_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.discard_live_session(session_id).await
}
fn supports_persistent_sessions(&self) -> bool {
true
}
async fn live_session_actor_registered(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
Ok(self.actor_registry.contains(session_id))
}
fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
Some(Arc::clone(&self.runtime_adapter))
}
async fn interrupt_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::interrupt(self, session_id).await
}
async fn cancel_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_expected_run_id: &meerkat_core::RunId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn cancel_current_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn archive_with_mob_lifecycle_authority(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
let session_present = self.sessions.read().await.contains_key(session_id);
retire_test_runtime_archive(&self.runtime_adapter, session_id, session_present).await?;
SessionService::archive(self, session_id).await
}
async fn session_belongs_to_mob(&self, session_id: &SessionId, mob_id: &MobId) -> bool {
if self
.session_comms_names
.read()
.await
.get(session_id)
.is_some_and(|name| name.starts_with(&format!("{mob_id}/")))
{
return true;
}
self.persisted_sessions
.read()
.await
.get(session_id)
.and_then(Session::session_metadata)
.is_some_and(|metadata| {
metadata
.comms_name
.as_deref()
.is_some_and(|name| name.starts_with(&format!("{mob_id}/")))
})
}
async fn load_persisted_session(
&self,
session_id: &SessionId,
) -> Result<Option<Session>, SessionError> {
if self.archived_session_ids.read().await.contains(session_id) {
return Ok(None);
}
Ok(self
.persisted_sessions
.read()
.await
.get(session_id)
.cloned())
}
async fn session_known_to_archive_authority(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
if self.archived_session_ids.read().await.contains(session_id) {
return Ok(true);
}
if self
.persisted_sessions
.read()
.await
.contains_key(session_id)
{
return Ok(true);
}
Ok(self.sessions.read().await.contains_key(session_id))
}
async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
let mut sessions = self.sessions.write().await;
if sessions.remove(session_id).is_some() {
self.actor_registry.remove_current(session_id);
}
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(session_id) {
notifier.notify_waiters();
}
self.session_comms_names.write().await.remove(session_id);
drop(sessions);
Ok(())
}
async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
self.discard_exact_actor(witness).await
}
}
impl RealCommsSessionService {
/// Get the real CommsRuntime for a session (for test inspection).
async fn real_comms(&self, session_id: &SessionId) -> Option<Arc<meerkat_comms::CommsRuntime>> {
let sessions = self.sessions.read().await;
sessions.get(session_id).cloned()
}
async fn restart_comms_for_session(
&self,
session_id: &SessionId,
comms_name: &str,
) -> Arc<meerkat_comms::CommsRuntime> {
let keypair = self
.session_keypairs
.write()
.await
.entry(session_id.clone())
.or_insert_with(meerkat_comms::Keypair::generate)
.clone();
let comms = Arc::new(
meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
comms_name,
None,
keypair,
Arc::new(HashSet::new()),
)
.expect("restart inproc CommsRuntime"),
);
install_machine_peer_request_response_authority(&self.runtime_adapter, &comms, session_id)
.await
.expect("install restarted peer-comms authority");
self.sessions
.write()
.await
.insert(session_id.clone(), Arc::clone(&comms));
self.session_comms_names
.write()
.await
.insert(session_id.clone(), comms_name.to_string());
comms
}
}
/// Create a MobHandle with real comms for wiring/trust behavior tests.
///
/// This harness intentionally forces all profiles to `TurnDriven` so peer/trust
/// assertions are not coupled to autonomous keep-alive kickoff behavior. Tests
/// that need autonomous ingress should use
/// `create_test_mob_with_runtime_backed_real_comms(...)` instead.
async fn create_test_mob_with_real_comms(
mut definition: MobDefinition,
) -> (MobHandle, Arc<RealCommsSessionService>) {
for profile in definition
.profiles
.values_mut()
.filter_map(|b| b.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let service = Arc::new(RealCommsSessionService::new());
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
/// Real comms + runtime-backed session service for autonomous idle-delivery regressions.
///
/// Uses real inproc comms delivery, a real MeerkatMachine, and records
/// runtime-applied prompts so tests can prove peer inbox traffic still reaches
/// `apply_runtime_turn()` after the initial autonomous kickoff turn has exited.
#[derive(Clone)]
struct RecordingSessionLlmReconfigureHost {
current_identity: Arc<std::sync::Mutex<meerkat_core::SessionLlmIdentity>>,
current_visibility: Arc<std::sync::Mutex<meerkat_core::SessionToolVisibilityState>>,
live_apply_count: Arc<AtomicU64>,
fail_live_apply: Arc<AtomicBool>,
boundary_acquire_count: Arc<AtomicU64>,
turn_finalization_gate: Option<Arc<tokio::sync::Mutex<()>>>,
}
impl RecordingSessionLlmReconfigureHost {
fn new() -> Self {
let current_visibility = meerkat_core::SessionToolVisibilityState {
capability_base_filter: meerkat_core::capability_base_filter_for_image_tool_results(
false,
),
..Default::default()
};
Self {
current_identity: Arc::new(std::sync::Mutex::new(meerkat_core::SessionLlmIdentity {
model: "baseline-model".to_string(),
provider: Provider::Anthropic,
self_hosted_server_id: None,
provider_params: Some(Default::default()),
auth_binding: None,
})),
current_visibility: Arc::new(std::sync::Mutex::new(current_visibility)),
live_apply_count: Arc::new(AtomicU64::new(0)),
fail_live_apply: Arc::new(AtomicBool::new(false)),
boundary_acquire_count: Arc::new(AtomicU64::new(0)),
turn_finalization_gate: None,
}
}
fn with_turn_finalization_gate(gate: Arc<tokio::sync::Mutex<()>>) -> Self {
Self {
turn_finalization_gate: Some(gate),
..Self::new()
}
}
fn current_identity(&self) -> meerkat_core::SessionLlmIdentity {
self.current_identity
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
fn set_fail_live_apply(&self, enabled: bool) {
self.fail_live_apply.store(enabled, Ordering::SeqCst);
}
}
fn test_session_llm_capabilities() -> meerkat_runtime::SessionLlmCapabilitySurface {
meerkat_runtime::SessionLlmCapabilitySurface {
supports_temperature: true,
supports_thinking: true,
supports_reasoning: true,
inline_video: false,
vision: false,
image_input: false,
image_tool_results: false,
supports_web_search: false,
image_generation: false,
realtime: false,
call_timeout_secs: None,
}
}
#[async_trait]
impl meerkat_runtime::SessionLlmReconfigureHost for RecordingSessionLlmReconfigureHost {
async fn acquire_turn_finalization_boundary(
&self,
_session_id: &SessionId,
) -> Result<
Box<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationGuard>,
meerkat_runtime::RuntimeDriverError,
> {
self.boundary_acquire_count.fetch_add(1, Ordering::SeqCst);
match self.turn_finalization_gate.as_ref() {
Some(gate) => Ok(Box::new(Arc::clone(gate).lock_owned().await)),
None => Ok(Box::new(())),
}
}
async fn hydrate_session_llm_state(
&self,
_session_id: &SessionId,
) -> Result<meerkat_runtime::HydratedSessionLlmState, meerkat_runtime::RuntimeDriverError> {
Ok(meerkat_runtime::HydratedSessionLlmState {
current_identity: self.current_identity(),
current_visibility_state: self
.current_visibility
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone(),
current_capability_surface: Some(test_session_llm_capabilities()),
capability_surface_status: meerkat_runtime::SessionLlmCapabilitySurfaceStatus::Resolved,
base_tool_names: Default::default(),
})
}
async fn resolve_target_session_llm_identity(
&self,
request: &meerkat_runtime::SessionLlmReconfigureRequest,
current_identity: &meerkat_core::SessionLlmIdentity,
) -> Result<meerkat_runtime::ResolvedSessionLlmReconfigure, meerkat_runtime::RuntimeDriverError>
{
let provider = request
.provider
.as_deref()
.map(|provider| {
Provider::parse_strict(provider).ok_or_else(|| {
meerkat_runtime::RuntimeDriverError::ValidationFailed {
reason: format!("unknown test provider '{provider}'"),
}
})
})
.transpose()?
.unwrap_or(current_identity.provider);
let provider_params = match request.provider_params.as_ref() {
Some(meerkat_core::lifecycle::run_primitive::TurnMetadataOverride::Set(value)) => {
Some(value.clone())
}
Some(meerkat_core::lifecycle::run_primitive::TurnMetadataOverride::Clear) => None,
None => current_identity.provider_params.clone(),
};
let auth_binding = match request.auth_binding.as_ref() {
Some(meerkat_core::lifecycle::run_primitive::TurnMetadataOverride::Set(value)) => {
Some(value.clone())
}
Some(meerkat_core::lifecycle::run_primitive::TurnMetadataOverride::Clear) => None,
None => current_identity.auth_binding.clone(),
};
Ok(meerkat_runtime::ResolvedSessionLlmReconfigure {
target_identity: meerkat_core::SessionLlmIdentity {
model: request
.model
.clone()
.unwrap_or_else(|| current_identity.model.clone()),
provider,
self_hosted_server_id: if provider == Provider::SelfHosted {
request
.self_hosted_server_id
.clone()
.or_else(|| current_identity.self_hosted_server_id.clone())
} else {
None
},
provider_params,
auth_binding,
},
target_capability_surface: test_session_llm_capabilities(),
})
}
async fn apply_live_session_llm_identity(
&self,
_session_id: &SessionId,
identity: &meerkat_core::SessionLlmIdentity,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
if self.fail_live_apply.load(Ordering::SeqCst) {
return Err(meerkat_runtime::RuntimeDriverError::ValidationFailed {
reason: "injected live LLM apply failure".to_string(),
});
}
*self
.current_identity
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = identity.clone();
self.live_apply_count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
async fn apply_live_session_tool_visibility_state(
&self,
_session_id: &SessionId,
visibility_state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
*self
.current_visibility
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
visibility_state.unwrap_or_default();
Ok(())
}
async fn persist_live_session(
&self,
_session_id: &SessionId,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
Ok(())
}
async fn discard_live_session(
&self,
_session_id: &SessionId,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
Ok(())
}
}
#[derive(Clone)]
struct RuntimeTurnContentBarrier {
content_substring: String,
entered: Arc<tokio::sync::Notify>,
}
struct RuntimeBackedRealCommsSessionService {
sessions: RwLock<HashMap<SessionId, Arc<meerkat_comms::CommsRuntime>>>,
actor_registry: meerkat_session::LiveSessionActorRegistry,
keep_alive_notifiers: RwLock<HashMap<SessionId, Arc<tokio::sync::Notify>>>,
session_comms_names: RwLock<HashMap<SessionId, String>>,
session_counter: AtomicU64,
runtime_adapter: Arc<meerkat_runtime::MeerkatMachine>,
keep_alive_turns_complete_immediately: std::sync::atomic::AtomicBool,
append_system_context_delay_ms: AtomicU64,
block_runtime_turns: AtomicBool,
fail_runtime_turns: AtomicBool,
fail_runtime_boundary_acknowledgement: AtomicBool,
return_extraction_failure: AtomicBool,
runtime_turn_started: tokio::sync::Notify,
runtime_turn_content_barriers: RwLock<HashMap<SessionId, RuntimeTurnContentBarrier>>,
release_runtime_turns: tokio::sync::Notify,
block_session_reads: AtomicBool,
session_read_entered: tokio::sync::Notify,
release_session_reads: tokio::sync::Notify,
applied_runtime_prompts: RwLock<HashMap<SessionId, Vec<ContentInput>>>,
applied_runtime_system_appends: RwLock<HashMap<SessionId, Vec<AppendSystemContextRequest>>>,
applied_runtime_render_metadata:
RwLock<HashMap<SessionId, Vec<Option<meerkat_core::types::RenderMetadata>>>>,
applied_runtime_turn_metadata: RwLock<
HashMap<
SessionId,
Vec<Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>>,
>,
>,
runtime_llm_identity_probe:
std::sync::RwLock<Option<Arc<std::sync::Mutex<meerkat_core::SessionLlmIdentity>>>>,
observed_runtime_llm_identities:
RwLock<HashMap<SessionId, Vec<meerkat_core::SessionLlmIdentity>>>,
emit_runtime_event_before_completion: AtomicBool,
runtime_event_delta_count: AtomicU64,
applied_runtime_contributing_input_ids:
RwLock<HashMap<SessionId, Vec<Vec<meerkat_core::InputId>>>>,
turn_finalization_gate: std::sync::RwLock<Option<Arc<tokio::sync::Mutex<()>>>>,
active_runtime_runs: RwLock<HashMap<SessionId, meerkat_core::RunId>>,
}
impl RuntimeBackedRealCommsSessionService {
fn new() -> Self {
Self::with_runtime_adapter(Arc::new(meerkat_runtime::MeerkatMachine::ephemeral()))
}
fn with_runtime_adapter(runtime_adapter: Arc<meerkat_runtime::MeerkatMachine>) -> Self {
Self {
sessions: RwLock::new(HashMap::new()),
actor_registry: meerkat_session::LiveSessionActorRegistry::default(),
keep_alive_notifiers: RwLock::new(HashMap::new()),
session_comms_names: RwLock::new(HashMap::new()),
session_counter: AtomicU64::new(0),
runtime_adapter,
keep_alive_turns_complete_immediately: std::sync::atomic::AtomicBool::new(false),
append_system_context_delay_ms: AtomicU64::new(0),
block_runtime_turns: AtomicBool::new(false),
fail_runtime_turns: AtomicBool::new(false),
fail_runtime_boundary_acknowledgement: AtomicBool::new(false),
return_extraction_failure: AtomicBool::new(false),
runtime_turn_started: tokio::sync::Notify::new(),
runtime_turn_content_barriers: RwLock::new(HashMap::new()),
release_runtime_turns: tokio::sync::Notify::new(),
block_session_reads: AtomicBool::new(false),
session_read_entered: tokio::sync::Notify::new(),
release_session_reads: tokio::sync::Notify::new(),
applied_runtime_prompts: RwLock::new(HashMap::new()),
applied_runtime_system_appends: RwLock::new(HashMap::new()),
applied_runtime_render_metadata: RwLock::new(HashMap::new()),
applied_runtime_turn_metadata: RwLock::new(HashMap::new()),
runtime_llm_identity_probe: std::sync::RwLock::new(None),
observed_runtime_llm_identities: RwLock::new(HashMap::new()),
emit_runtime_event_before_completion: AtomicBool::new(false),
runtime_event_delta_count: AtomicU64::new(1),
applied_runtime_contributing_input_ids: RwLock::new(HashMap::new()),
turn_finalization_gate: std::sync::RwLock::new(None),
active_runtime_runs: RwLock::new(HashMap::new()),
}
}
async fn create_session_with_actor_witness_slot(
&self,
req: CreateSessionRequest,
actor_witness_slot: Option<&meerkat_session::LiveSessionActorWitnessSlot>,
) -> Result<RunResult, SessionError> {
let actor_materialization_permit = begin_test_runtime_actor_materialization(&req)?;
let session_id = requested_test_session_id(&req);
let n = self.session_counter.fetch_add(1, Ordering::Relaxed);
let comms_name = req
.build
.as_ref()
.and_then(|b| b.comms_name.clone())
.unwrap_or_else(|| format!("real-runtime-comms-session-{n}"));
let comms = Arc::new(
meerkat_comms::CommsRuntime::inproc_only(&comms_name)
.expect("create inproc CommsRuntime"),
);
if !install_session_owned_peer_request_response_authority(&req, &comms)? {
install_machine_peer_request_response_authority(
&self.runtime_adapter,
&comms,
&session_id,
)
.await?;
}
let transient_turn_context_state = test_actor_transient_turn_context_state();
let local_actor_witness_slot = meerkat_session::LiveSessionActorWitnessSlot::default();
let actor_witness_slot = actor_witness_slot.unwrap_or(&local_actor_witness_slot);
let actor_witness = {
let mut sessions = self.sessions.write().await;
if sessions.contains_key(&session_id) {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::SessionIdentityInUse(session_id),
));
}
let actor_witness = self.actor_registry.insert_and_publish(
actor_witness_slot,
session_id.clone(),
transient_turn_context_state,
)?;
sessions.insert(session_id.clone(), Arc::clone(&comms));
actor_witness
};
let is_keep_alive = req.build.as_ref().map(|b| b.keep_alive).unwrap_or(false);
if is_keep_alive {
self.keep_alive_notifiers
.write()
.await
.insert(session_id.clone(), Arc::new(tokio::sync::Notify::new()));
}
self.session_comms_names
.write()
.await
.insert(session_id.clone(), comms_name);
if let Some(permit) = actor_materialization_permit
&& let Err(error) = permit.commit()
{
let cleanup = self.discard_exact_actor(&actor_witness).await;
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(match cleanup {
Ok(removed) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor removed: {removed}"
),
Err(cleanup_error) => format!(
"runtime actor materialization commit failed for session {session_id}: {error}; exact actor cleanup also failed: {cleanup_error}"
),
}),
));
}
Ok(mock_run_result(session_id, "Session created".to_string()))
}
async fn discard_exact_actor(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
let mut sessions = self.sessions.write().await;
if self.actor_registry.current(witness.session_id()).as_ref() != Some(witness)
|| !sessions.contains_key(witness.session_id())
{
return Ok(false);
}
let registry_removed = self.actor_registry.compare_remove(witness);
debug_assert!(registry_removed);
sessions.remove(witness.session_id());
let session_id = witness.session_id();
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(session_id) {
notifier.notify_waiters();
}
self.session_comms_names.write().await.remove(session_id);
self.applied_runtime_prompts
.write()
.await
.remove(session_id);
self.applied_runtime_system_appends
.write()
.await
.remove(session_id);
self.applied_runtime_render_metadata
.write()
.await
.remove(session_id);
self.applied_runtime_turn_metadata
.write()
.await
.remove(session_id);
self.observed_runtime_llm_identities
.write()
.await
.remove(session_id);
self.applied_runtime_contributing_input_ids
.write()
.await
.remove(session_id);
self.active_runtime_runs.write().await.remove(session_id);
self.runtime_turn_content_barriers
.write()
.await
.remove(session_id);
drop(sessions);
Ok(true)
}
fn set_keep_alive_turns_complete_immediately(&self, enabled: bool) {
self.keep_alive_turns_complete_immediately
.store(enabled, Ordering::Relaxed);
}
async fn reset_keep_alive_notifier(&self, session_id: &SessionId) {
self.keep_alive_notifiers
.write()
.await
.insert(session_id.clone(), Arc::new(tokio::sync::Notify::new()));
}
fn set_append_system_context_delay_ms(&self, delay_ms: u64) {
self.append_system_context_delay_ms
.store(delay_ms, Ordering::Relaxed);
}
fn set_block_runtime_turns(&self, enabled: bool) {
self.block_runtime_turns.store(enabled, Ordering::Relaxed);
}
fn set_fail_runtime_turns(&self, enabled: bool) {
self.fail_runtime_turns.store(enabled, Ordering::Relaxed);
}
fn set_fail_runtime_boundary_acknowledgement(&self, enabled: bool) {
self.fail_runtime_boundary_acknowledgement
.store(enabled, Ordering::Relaxed);
}
fn set_return_extraction_failure(&self, enabled: bool) {
self.return_extraction_failure
.store(enabled, Ordering::Relaxed);
}
fn observe_runtime_llm_identity_from(&self, host: &RecordingSessionLlmReconfigureHost) {
*self
.runtime_llm_identity_probe
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(Arc::clone(&host.current_identity));
}
fn release_runtime_turns(&self) {
self.release_runtime_turns.notify_waiters();
}
fn release_one_runtime_turn(&self) {
self.release_runtime_turns.notify_one();
}
async fn arm_runtime_turn_content_barrier(
&self,
session_id: &SessionId,
content_substring: impl Into<String>,
) -> Arc<tokio::sync::Notify> {
let entered = Arc::new(tokio::sync::Notify::new());
self.runtime_turn_content_barriers.write().await.insert(
session_id.clone(),
RuntimeTurnContentBarrier {
content_substring: content_substring.into(),
entered: Arc::clone(&entered),
},
);
entered
}
async fn clear_runtime_turn_content_barrier(&self, session_id: &SessionId) {
self.runtime_turn_content_barriers
.write()
.await
.remove(session_id);
}
fn set_emit_runtime_event_before_completion(&self, enabled: bool) {
self.emit_runtime_event_before_completion
.store(enabled, Ordering::Relaxed);
}
fn set_runtime_event_delta_count(&self, count: u64) {
self.runtime_event_delta_count
.store(count, Ordering::Relaxed);
}
fn install_non_reentrant_turn_finalization_gate(&self) -> Arc<tokio::sync::Mutex<()>> {
let gate = Arc::new(tokio::sync::Mutex::new(()));
*self
.turn_finalization_gate
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::clone(&gate));
gate
}
fn set_block_session_reads(&self, enabled: bool) {
self.block_session_reads.store(enabled, Ordering::Relaxed);
}
fn release_session_reads(&self) {
self.release_session_reads.notify_waiters();
}
async fn real_comms(&self, session_id: &SessionId) -> Option<Arc<meerkat_comms::CommsRuntime>> {
self.sessions.read().await.get(session_id).cloned()
}
async fn applied_runtime_prompts(&self, session_id: &SessionId) -> Vec<ContentInput> {
self.applied_runtime_prompts
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn applied_runtime_contributing_input_ids(
&self,
session_id: &SessionId,
) -> Vec<Vec<meerkat_core::InputId>> {
self.applied_runtime_contributing_input_ids
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn applied_runtime_system_appends(
&self,
session_id: &SessionId,
) -> Vec<AppendSystemContextRequest> {
self.applied_runtime_system_appends
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn applied_runtime_render_metadata(
&self,
session_id: &SessionId,
) -> Vec<Option<meerkat_core::types::RenderMetadata>> {
self.applied_runtime_render_metadata
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn applied_runtime_turn_metadata(
&self,
session_id: &SessionId,
) -> Vec<Option<meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata>> {
self.applied_runtime_turn_metadata
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
async fn observed_runtime_llm_identities(
&self,
session_id: &SessionId,
) -> Vec<meerkat_core::SessionLlmIdentity> {
self.observed_runtime_llm_identities
.read()
.await
.get(session_id)
.cloned()
.unwrap_or_default()
}
}
fn provider_visible_prompt_from_start_turn_request(req: &StartTurnRequest) -> ContentInput {
let mut blocks = if req.runtime.typed_turn_appends.is_empty() {
req.prompt.clone().into_blocks()
} else {
Vec::new()
};
for append in &req.runtime.typed_turn_appends {
if !matches!(
append.role,
meerkat_core::lifecycle::run_primitive::ConversationAppendRole::System
| meerkat_core::lifecycle::run_primitive::ConversationAppendRole::SystemNotice
| meerkat_core::lifecycle::run_primitive::ConversationAppendRole::User
| meerkat_core::lifecycle::run_primitive::ConversationAppendRole::InjectedContext
) {
continue;
}
match &append.content {
meerkat_core::lifecycle::run_primitive::CoreRenderable::Blocks {
blocks: append_blocks,
} => blocks.extend(append_blocks.iter().cloned()),
meerkat_core::lifecycle::run_primitive::CoreRenderable::SystemNotice {
kind,
body,
blocks: notice_blocks,
} => {
let projection =
SystemNoticeMessage::with_blocks(*kind, body.clone(), notice_blocks.clone())
.model_projection_text();
blocks.push(ContentBlock::Text { text: projection });
for notice_block in notice_blocks {
let content = match notice_block {
SystemNoticeBlock::Comms { content, .. }
| SystemNoticeBlock::ExternalEvent { content, .. } => content,
_ => continue,
};
blocks.extend(content.iter().filter_map(|block| match block {
ContentBlock::Image { .. } | ContentBlock::Video { .. } => {
Some(block.clone())
}
ContentBlock::Text { .. } | _ => None,
}));
}
}
content => blocks.push(ContentBlock::Text {
text: content.render_text(),
}),
}
}
if blocks.len() == 1
&& let ContentBlock::Text { text } = blocks.remove(0)
{
return ContentInput::Text(text);
}
ContentInput::Blocks(blocks)
}
fn is_incoming_terminal_peer_response(
append: &AppendSystemContextRequest,
expected_peer_id: &str,
expected_display_name: &str,
expected_request_id: &str,
expected_payload: &serde_json::Value,
) -> bool {
let meerkat_core::lifecycle::run_primitive::CoreRenderable::SystemNotice {
kind: SystemNoticeKind::Comms,
blocks,
..
} = &append.content
else {
return false;
};
blocks.iter().any(|block| {
matches!(
block,
SystemNoticeBlock::Comms {
kind: CommsNoticeKind::ResponseTerminal,
direction: SystemNoticeDirection::Incoming,
peer: Some(peer),
request_id: Some(request_id),
status: Some(status),
payload: Some(payload),
..
} if peer.id.to_string() == expected_peer_id
&& peer.display_name.as_deref() == Some(expected_display_name)
&& request_id == expected_request_id
&& status
== meerkat_core::PeerResponseTerminalProjectionStatus::Completed.label()
&& payload == expected_payload
)
})
}
#[async_trait]
impl SessionService for RuntimeBackedRealCommsSessionService {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, None).await
}
async fn start_turn(
&self,
id: &SessionId,
_req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
if self
.keep_alive_turns_complete_immediately
.load(Ordering::Relaxed)
{
return Ok(mock_run_result(
id.clone(),
"Autonomous kickoff completed".to_string(),
));
}
notifier.notified().await;
return Ok(mock_run_result(
id.clone(),
"Host loop interrupted".to_string(),
));
}
Ok(mock_run_result(id.clone(), "Turn completed".to_string()))
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
}
Ok(())
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if let Some(notifier) = self.keep_alive_notifiers.read().await.get(id).cloned() {
notifier.notify_waiters();
return Ok(());
}
Err(SessionError::NotRunning { id: id.clone() })
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
let sessions = self.sessions.read().await;
if !sessions.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(sessions);
if self.block_session_reads.load(Ordering::Relaxed) {
self.session_read_entered.notify_waiters();
self.release_session_reads.notified().await;
}
Ok(SessionView {
state: SessionInfo {
session_id: id.clone(),
created_at: SystemTime::now(),
updated_at: SystemTime::now(),
message_count: 0,
is_active: false,
model: "claude-sonnet-4-5".to_string(),
provider: Provider::Anthropic,
last_assistant_text: None,
labels: Default::default(),
},
billing: SessionUsage {
total_tokens: 0,
usage: Usage::default(),
},
})
}
async fn list(&self, _query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
let sessions = self.sessions.read().await;
Ok(sessions
.keys()
.map(|id| SessionSummary {
session_id: id.clone(),
created_at: SystemTime::now(),
updated_at: SystemTime::now(),
message_count: 0,
total_tokens: 0,
is_active: false,
labels: Default::default(),
})
.collect())
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
if self.block_session_reads.load(Ordering::Relaxed) {
self.session_read_entered.notify_waiters();
self.release_session_reads.notified().await;
}
Ok(self.sessions.read().await.contains_key(id))
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
let mut sessions = self.sessions.write().await;
if sessions.remove(id).is_some() {
self.actor_registry.remove_current(id);
}
if let Some(notifier) = self.keep_alive_notifiers.write().await.remove(id) {
notifier.notify_waiters();
}
self.session_comms_names.write().await.remove(id);
self.applied_runtime_prompts.write().await.remove(id);
self.applied_runtime_system_appends.write().await.remove(id);
drop(sessions);
Ok(())
}
}
#[async_trait]
impl SessionServiceCommsExt for RuntimeBackedRealCommsSessionService {
async fn comms_runtime(&self, session_id: &SessionId) -> Option<Arc<dyn CoreCommsRuntime>> {
self.sessions
.read()
.await
.get(session_id)
.map(|c| Arc::clone(c) as Arc<dyn CoreCommsRuntime>)
}
}
#[async_trait]
impl meerkat_core::service::SessionServiceHistoryExt for RuntimeBackedRealCommsSessionService {
async fn read_history(
&self,
id: &SessionId,
query: meerkat_core::service::SessionHistoryQuery,
) -> Result<meerkat_core::service::SessionHistoryPage, SessionError> {
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() });
}
Ok(meerkat_core::service::SessionHistoryPage::from_messages(
id.clone(),
&[],
query,
))
}
}
#[async_trait]
impl SessionServiceControlExt for RuntimeBackedRealCommsSessionService {
async fn append_system_context(
&self,
id: &SessionId,
req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
if !self.sessions.read().await.contains_key(id) {
return Err(SessionError::NotFound { id: id.clone() }.into());
}
let delay_ms = self.append_system_context_delay_ms.load(Ordering::Relaxed);
if delay_ms > 0 {
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
}
let mut appends = self.applied_runtime_system_appends.write().await;
let session_appends = appends.entry(id.clone()).or_default();
if let Some(key) = req.idempotency_key.as_ref()
&& let Some(existing) = session_appends
.iter()
.find(|append| append.idempotency_key.as_ref() == Some(key))
{
if existing == &req {
return Ok(AppendSystemContextResult {
status: AppendSystemContextStatus::Duplicate,
});
}
return Err(SessionControlError::Conflict {
id: id.clone(),
key: key.clone(),
});
}
session_appends.push(req);
Ok(AppendSystemContextResult {
status: AppendSystemContextStatus::Applied,
})
}
}
#[async_trait]
impl MobSessionService for RuntimeBackedRealCommsSessionService {
async fn prepare_session_for_resume(
&self,
_session_id: &SessionId,
) -> Result<(), SessionError> {
Ok(())
}
async fn acknowledge_committed_runtime_session_boundary_under_turn_finalization_boundary(
&self,
_session_id: &SessionId,
_authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
if self
.fail_runtime_boundary_acknowledgement
.load(Ordering::Relaxed)
{
Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(
"injected runtime boundary acknowledgement failure".to_string(),
),
))
} else {
Ok(())
}
}
/// Test double: nothing durable survives archive here, so the two-read
/// composition is the exact truth — `ArchivedNotRevivable` cannot exist.
async fn load_session_for_resume(
&self,
session_id: &SessionId,
) -> Result<ResumeSessionLoad, SessionError> {
if let Some(session) = self.load_persisted_session(session_id).await? {
return Ok(ResumeSessionLoad::Active(Box::new(session)));
}
if let Some(session) = self.load_revivable_retired_session(session_id).await? {
return Ok(ResumeSessionLoad::Revivable(Box::new(session)));
}
Ok(ResumeSessionLoad::Absent)
}
async fn create_session_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
) -> Result<RunResult, SessionError> {
SessionService::create_session(self, req).await
}
async fn create_session_with_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
actor_witness_slot: &meerkat_session::LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.create_session_with_actor_witness_slot(req, Some(actor_witness_slot))
.await
}
async fn archive_with_mob_lifecycle_authority_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.archive_with_mob_lifecycle_authority(session_id).await
}
async fn discard_live_session_under_runtime_turn_boundary(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
self.discard_live_session(session_id).await
}
fn supports_persistent_sessions(&self) -> bool {
true
}
fn supports_runtime_turn_apply(&self) -> bool {
true
}
async fn acquire_runtime_turn_finalization_guard(
&self,
_session_id: &SessionId,
) -> Result<Box<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationGuard>, SessionError>
{
let gate = self
.turn_finalization_gate
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
match gate {
Some(gate) => Ok(Box::new(gate.lock_owned().await)),
None => Ok(Box::new(())),
}
}
async fn live_session_actor_registered(
&self,
session_id: &SessionId,
) -> Result<bool, SessionError> {
Ok(self.actor_registry.contains(session_id))
}
fn runtime_adapter(&self) -> Option<Arc<meerkat_runtime::MeerkatMachine>> {
Some(Arc::clone(&self.runtime_adapter))
}
async fn interrupt_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::interrupt(self, session_id).await
}
async fn cancel_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_expected_run_id: &meerkat_core::RunId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn cancel_current_after_boundary_with_machine_authority(
&self,
session_id: &SessionId,
_authority: meerkat_runtime::MachineSessionControlAuthority,
) -> Result<(), SessionError> {
SessionService::cancel_after_boundary(self, session_id).await
}
async fn archive_with_mob_lifecycle_authority(
&self,
session_id: &SessionId,
) -> Result<(), SessionError> {
let session_present = self.sessions.read().await.contains_key(session_id);
retire_test_runtime_archive(&self.runtime_adapter, session_id, session_present).await?;
SessionService::archive(self, session_id).await
}
async fn session_belongs_to_mob(&self, session_id: &SessionId, mob_id: &MobId) -> bool {
let names = self.session_comms_names.read().await;
names
.get(session_id)
.is_some_and(|name| name.starts_with(&format!("{mob_id}/")))
}
async fn apply_runtime_turn(
&self,
session_id: &SessionId,
run_id: meerkat_core::RunId,
req: StartTurnRequest,
boundary: meerkat_core::lifecycle::run_primitive::RunApplyBoundary,
contributing_input_ids: Vec<meerkat_core::InputId>,
) -> Result<meerkat_core::lifecycle::core_executor::CoreApplyOutput, SessionError> {
let event_tx = req.event_tx.clone();
let applied_turn_metadata = req.runtime.turn_metadata.clone();
let provider_visible_prompt = provider_visible_prompt_from_start_turn_request(&req);
let content_barrier = self
.runtime_turn_content_barriers
.read()
.await
.get(session_id)
.filter(|barrier| {
provider_visible_prompt
.text_content()
.contains(&barrier.content_substring)
|| req.runtime.typed_turn_appends.iter().any(|append| {
append
.content
.render_text()
.contains(&barrier.content_substring)
})
})
.cloned();
self.applied_runtime_system_appends
.write()
.await
.entry(session_id.clone())
.or_default()
.extend(
req.runtime
.typed_turn_appends
.iter()
.filter(|append| {
matches!(
append.role,
meerkat_core::lifecycle::run_primitive::ConversationAppendRole::System
| meerkat_core::lifecycle::run_primitive::ConversationAppendRole::SystemNotice
)
})
.map(|append| AppendSystemContextRequest {
content: append.content.clone(),
source: append
.identity
.as_ref()
.and_then(|identity| identity.source.clone()),
idempotency_key: append
.identity
.as_ref()
.and_then(|identity| identity.idempotency_key.clone()),
}),
);
let identity_probe = {
self.runtime_llm_identity_probe
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
};
if let Some(identity_probe) = identity_probe {
let identity = identity_probe
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
self.observed_runtime_llm_identities
.write()
.await
.entry(session_id.clone())
.or_default()
.push(identity);
}
self.applied_runtime_prompts
.write()
.await
.entry(session_id.clone())
.or_default()
.push(provider_visible_prompt);
self.applied_runtime_render_metadata
.write()
.await
.entry(session_id.clone())
.or_default()
.push(
req.runtime
.turn_metadata
.as_ref()
.and_then(|meta| meta.render_metadata.clone()),
);
self.applied_runtime_turn_metadata
.write()
.await
.entry(session_id.clone())
.or_default()
.push(applied_turn_metadata);
self.active_runtime_runs
.write()
.await
.insert(session_id.clone(), run_id.clone());
self.applied_runtime_contributing_input_ids
.write()
.await
.entry(session_id.clone())
.or_default()
.push(contributing_input_ids.clone());
if self
.emit_runtime_event_before_completion
.load(Ordering::Relaxed)
&& let Some(event_tx) = event_tx
{
let delta_count = self.runtime_event_delta_count.load(Ordering::Relaxed);
for seq in 1..=delta_count {
let _ = event_tx
.send(EventEnvelope::new_session(
session_id.clone(),
seq,
None,
AgentEvent::TextDelta {
delta: format!("runtime event {seq} before commit"),
},
))
.await;
}
let terminal = if self.return_extraction_failure.load(Ordering::Relaxed) {
AgentEvent::ExtractionFailed {
session_id: session_id.clone(),
last_output: "runtime terminal before commit".to_string(),
attempts: 1,
reason: "injected extraction failure".to_string(),
}
} else {
AgentEvent::RunCompleted {
session_id: session_id.clone(),
result: "runtime terminal before commit".to_string(),
structured_output: None,
extraction_required: false,
usage: Usage::default(),
terminal_cause_kind: None,
}
};
let _ = event_tx
.send(EventEnvelope::new_session(
session_id.clone(),
delta_count + 1,
None,
terminal,
))
.await;
let _ = event_tx
.send(EventEnvelope::new_session(
session_id.clone(),
delta_count + 2,
None,
AgentEvent::InteractionComplete {
interaction_id: meerkat_core::interaction::InteractionId(
uuid::Uuid::new_v4(),
),
result: "runtime interaction terminal before commit".to_string(),
structured_output: None,
},
))
.await;
}
let block_all_runtime_turns = self.block_runtime_turns.load(Ordering::Relaxed);
if block_all_runtime_turns {
self.runtime_turn_started.notify_waiters();
}
if let Some(barrier) = &content_barrier {
barrier.entered.notify_one();
}
if block_all_runtime_turns || content_barrier.is_some() {
self.release_runtime_turns.notified().await;
}
if self.fail_runtime_turns.load(Ordering::Relaxed) {
self.active_runtime_runs.write().await.remove(session_id);
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(
"injected runtime turn failure".to_string(),
),
));
}
if let Some(notifier) = self
.keep_alive_notifiers
.read()
.await
.get(session_id)
.cloned()
&& !self
.keep_alive_turns_complete_immediately
.load(Ordering::Relaxed)
{
notifier.notified().await;
}
self.active_runtime_runs.write().await.remove(session_id);
let inject_boundary_acknowledgement_failure = self
.fail_runtime_boundary_acknowledgement
.load(Ordering::Relaxed);
let receipt = meerkat_core::lifecycle::run_receipt::RunBoundaryReceiptDraft {
run_id,
boundary,
contributing_input_ids,
conversation_digest: inject_boundary_acknowledgement_failure.then(|| {
// Canonical accumulator digest (`sha256:<hex>`) of the
// empty message list in the valid synthetic Session below.
// The committed-boundary witness
// validation recomputes this exact format.
"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"
.to_string()
}),
message_count: 0,
};
let output = if self.return_extraction_failure.load(Ordering::Relaxed) {
let mut result = mock_run_result(
session_id.clone(),
"runtime main turn completed".to_string(),
);
result.extraction_error = Some(meerkat_core::types::ExtractionError {
last_output: result.text.clone(),
attempts: 1,
reason: "injected extraction failure".to_string(),
});
meerkat_core::lifecycle::core_executor::CoreApplyOutput::with_run_result(
receipt, None, result,
)
} else {
meerkat_core::lifecycle::core_executor::CoreApplyOutput::with_untyped_snapshot(
receipt, None, None,
)
};
if inject_boundary_acknowledgement_failure {
output
.with_session(Arc::new(Session::with_id(session_id.clone())))
.map_err(|error| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to seal injected committed session boundary: {error}"
)))
})
} else {
Ok(output)
}
}
async fn checkpoint_committed_runtime_session_snapshot(
&self,
_session_id: &SessionId,
_session_snapshot: Arc<Vec<u8>>,
) -> Result<(), SessionError> {
Ok(())
}
async fn discard_live_session(&self, session_id: &SessionId) -> Result<(), SessionError> {
let mut sessions = self.sessions.write().await;
if sessions.remove(session_id).is_some() {
self.actor_registry.remove_current(session_id);
}
self.keep_alive_notifiers.write().await.remove(session_id);
self.session_comms_names.write().await.remove(session_id);
self.applied_runtime_prompts
.write()
.await
.remove(session_id);
self.applied_runtime_system_appends
.write()
.await
.remove(session_id);
self.applied_runtime_render_metadata
.write()
.await
.remove(session_id);
self.applied_runtime_turn_metadata
.write()
.await
.remove(session_id);
self.observed_runtime_llm_identities
.write()
.await
.remove(session_id);
self.active_runtime_runs.write().await.remove(session_id);
self.runtime_turn_content_barriers
.write()
.await
.remove(session_id);
drop(sessions);
Ok(())
}
async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &meerkat_session::LiveSessionActorWitness,
) -> Result<bool, SessionError> {
self.discard_exact_actor(witness).await
}
}
async fn create_test_mob_with_runtime_backed_real_comms(
definition: MobDefinition,
) -> (MobHandle, Arc<RuntimeBackedRealCommsSessionService>) {
let service = Arc::new(RuntimeBackedRealCommsSessionService::new());
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
async fn create_test_mob_with_persistent_runtime_backed_real_comms(
definition: MobDefinition,
) -> (MobHandle, Arc<RuntimeBackedRealCommsSessionService>) {
let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> =
Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let runtime_adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
runtime_store,
blob_store,
));
let service = Arc::new(RuntimeBackedRealCommsSessionService::with_runtime_adapter(
runtime_adapter,
));
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
(handle, service)
}
fn mcp_tool_context_for(comms: &Arc<meerkat_comms::CommsRuntime>) -> meerkat_comms::ToolContext {
let runtime: Arc<dyn CoreCommsRuntime> = comms.clone();
meerkat_comms::ToolContext {
router: comms.router_arc(),
trusted_peers: comms.trusted_peers_shared(),
runtime: Some(meerkat_comms::RuntimeCommsCommandHandle::new(runtime)),
}
}
async fn mcp_peer_id_for(comms: &Arc<meerkat_comms::CommsRuntime>, name: &str) -> String {
let ctx = mcp_tool_context_for(comms);
let peers = meerkat_comms::handle_tools_call(&ctx, "peers", &serde_json::json!({}))
.await
.expect("peers tool should succeed");
let peer = peers["peers"]
.as_array()
.expect("peers should be an array")
.iter()
.find(|peer| peer["name"].as_str() == Some(name))
.unwrap_or_else(|| panic!("MCP peer not found for {name}; peers={peers}"));
peer["peer_id"]
.as_str()
.expect("peer_id should be a string")
.to_string()
}
fn peer_request_notice_request_id(text: &str) -> uuid::Uuid {
let request_id = text
.split("Request ID: ")
.nth(1)
.and_then(|tail| tail.split_whitespace().next())
.expect("peer request notice should include a Request ID");
uuid::Uuid::parse_str(request_id).expect("request ID should be a UUID")
}
fn assert_peer_response_terminal_consumed(
snapshot: &meerkat_runtime::MeerkatMachineSpineSnapshot,
expected_handling_mode: HandlingMode,
) {
let terminal = snapshot
.inputs
.admission_order
.iter()
.find(|entry| {
entry.content_shape.as_ref().is_some_and(|shape| {
shape.kind() == meerkat_runtime::identifiers::InputKind::PeerResponseTerminal
})
})
.unwrap_or_else(|| {
panic!(
"requester snapshot should include peer_response_terminal admission: {snapshot:?}"
)
});
assert_eq!(terminal.handling_mode, Some(expected_handling_mode));
assert!(
matches!(
terminal.lifecycle,
Some(meerkat_runtime::input_state::InputLifecycleState::Consumed)
),
"peer_response_terminal should be consumed after the runtime context append apply: {terminal:?}"
);
}
#[tokio::test]
async fn test_peer_message_delivery_backpressure_does_not_block_actor_mailbox() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-backpressure"),
MockCommsBehavior {
peer_message_delay_ms: 500,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-backpressure"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-backpressure"),
None,
)
.await
.expect("spawn worker");
handle
.wire(
AgentIdentity::from("l-backpressure"),
AgentIdentity::from("w-backpressure"),
)
.await
.expect("wire should succeed");
let send_handle = handle.clone();
let delivery = tokio::spawn(async move {
send_handle
.send_peer_message(
AgentIdentity::from("l-backpressure"),
AgentIdentity::from("w-backpressure"),
"slow peer message",
meerkat_core::types::HandlingMode::Queue,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if service
.sent_intents(&sid_l)
.await
.iter()
.any(|intent| intent == "peer_message_started")
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("peer message delivery should enter the slow comms send path");
assert!(
!delivery.is_finished(),
"mock peer delivery should still be waiting on artificial backpressure"
);
let members = tokio::time::timeout(Duration::from_millis(100), handle.list_members())
.await
.expect("actor mailbox commands must not queue behind peer delivery backpressure");
assert_eq!(members.len(), 2);
let receipt = tokio::time::timeout(Duration::from_secs(2), delivery)
.await
.expect("peer delivery should finish after the artificial delay")
.expect("delivery task should join")
.expect("peer message should succeed");
assert_eq!(receipt.from, AgentIdentity::from("l-backpressure"));
assert_eq!(receipt.to, AgentIdentity::from("w-backpressure"));
}
/// The host's outbound content-taint declaration reaches the MEMBER's own
/// comms runtime (where every outbound envelope is stamped), addressed by
/// stable AgentIdentity — the host never touches the runtime directly.
#[tokio::test]
async fn declare_member_outbound_taint_installs_on_member_comms_runtime() {
let (handle, service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("l-taint");
let sid = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.declare_member_outbound_taint(
identity.clone(),
Some(meerkat_core::comms::SenderContentTaint::Tainted),
)
.await
.expect("declare tainted");
assert_eq!(
service.outbound_content_taint(&sid).await,
Some(meerkat_core::comms::SenderContentTaint::Tainted),
"the declaration must land on the member's comms runtime"
);
// Clearing goes through the per-member wrapper.
handle
.member(&identity)
.await
.expect("member handle")
.declare_outbound_taint(None)
.await
.expect("clear declaration");
assert_eq!(service.outbound_content_taint(&sid).await, None);
let err = handle
.declare_member_outbound_taint(AgentIdentity::from("ghost"), None)
.await
.expect_err("unknown member must fail typed");
assert!(
matches!(err, MobError::MemberNotFound(_)),
"expected MemberNotFound, got {err:?}"
);
}
#[tokio::test]
async fn test_peer_message_delivery_canceled_when_mob_lifecycle_stops() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-stop-delivery"),
MockCommsBehavior {
peer_message_delay_ms: 5_000,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-stop-delivery"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-stop-delivery"),
None,
)
.await
.expect("spawn worker");
handle
.wire(
AgentIdentity::from("l-stop-delivery"),
AgentIdentity::from("w-stop-delivery"),
)
.await
.expect("wire should succeed");
let send_handle = handle.clone();
let delivery = tokio::spawn(async move {
send_handle
.send_peer_message(
AgentIdentity::from("l-stop-delivery"),
AgentIdentity::from("w-stop-delivery"),
"pending peer message",
meerkat_core::types::HandlingMode::Queue,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if service
.sent_intents(&sid_l)
.await
.iter()
.any(|intent| intent == "peer_message_started")
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("peer message delivery should enter the slow comms send path");
handle.stop().await.expect("stop should succeed");
let result = tokio::time::timeout(Duration::from_secs(1), delivery)
.await
.expect("delivery should be released by lifecycle cancellation")
.expect("delivery task should join");
assert!(
matches!(result, Err(MobError::Internal(ref message)) if message.contains("topology or lifecycle changed")),
"pending peer delivery should fail through the lifecycle cancellation path: {result:?}"
);
assert!(
!service
.sent_intents(&sid_l)
.await
.iter()
.any(|intent| intent == "peer_message"),
"pending peer delivery must not commit after the mob stops"
);
}
#[tokio::test]
async fn test_peer_message_delivery_canceled_when_members_unwire() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-unwire-delivery"),
MockCommsBehavior {
peer_message_delay_ms: 5_000,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-unwire-delivery"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-unwire-delivery"),
None,
)
.await
.expect("spawn worker");
handle
.wire(
AgentIdentity::from("l-unwire-delivery"),
AgentIdentity::from("w-unwire-delivery"),
)
.await
.expect("wire should succeed");
let send_handle = handle.clone();
let delivery = tokio::spawn(async move {
send_handle
.send_peer_message(
AgentIdentity::from("l-unwire-delivery"),
AgentIdentity::from("w-unwire-delivery"),
"pending peer message",
meerkat_core::types::HandlingMode::Queue,
)
.await
});
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if service
.sent_intents(&sid_l)
.await
.iter()
.any(|intent| intent == "peer_message_started")
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("peer message delivery should enter the slow comms send path");
handle
.unwire(
AgentIdentity::from("l-unwire-delivery"),
AgentIdentity::from("w-unwire-delivery"),
)
.await
.expect("unwire should succeed");
let result = tokio::time::timeout(Duration::from_secs(1), delivery)
.await
.expect("delivery should be released by topology cancellation")
.expect("delivery task should join");
assert!(
matches!(result, Err(MobError::Internal(ref message)) if message.contains("topology or lifecycle changed")),
"pending peer delivery should fail through the topology cancellation path: {result:?}"
);
assert!(
!service
.sent_intents(&sid_l)
.await
.iter()
.any(|intent| intent == "peer_message"),
"pending peer delivery must not commit after the members unwire"
);
}
#[tokio::test]
async fn test_peer_message_delivery_lane_is_bounded() {
let (handle, service) = create_test_mob(sample_definition()).await;
service
.set_comms_behavior(
&test_comms_name("lead", "l-bounded-delivery"),
MockCommsBehavior {
peer_message_delay_ms: 5_000,
..MockCommsBehavior::default()
},
)
.await;
let sid_l = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-bounded-delivery"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-bounded-delivery"),
None,
)
.await
.expect("spawn worker");
handle
.wire(
AgentIdentity::from("l-bounded-delivery"),
AgentIdentity::from("w-bounded-delivery"),
)
.await
.expect("wire should succeed");
let mut deliveries = Vec::new();
for _ in 0..super::actor::MAX_PENDING_PEER_DELIVERIES {
let send_handle = handle.clone();
deliveries.push(tokio::spawn(async move {
send_handle
.send_peer_message(
AgentIdentity::from("l-bounded-delivery"),
AgentIdentity::from("w-bounded-delivery"),
"pending peer message",
meerkat_core::types::HandlingMode::Queue,
)
.await
}));
}
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let started = service
.sent_intents(&sid_l)
.await
.iter()
.filter(|intent| intent.as_str() == "peer_message_started")
.count();
if started == super::actor::MAX_PENDING_PEER_DELIVERIES {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("all delivery-lane permits should be occupied");
let saturated = handle
.send_peer_message(
AgentIdentity::from("l-bounded-delivery"),
AgentIdentity::from("w-bounded-delivery"),
"one too many peer messages",
meerkat_core::types::HandlingMode::Queue,
)
.await;
assert!(
matches!(saturated, Err(MobError::Internal(ref message)) if message.contains("delivery lane saturated")),
"extra peer delivery should fail instead of spawning unbounded work: {saturated:?}"
);
handle
.stop()
.await
.expect("stop should cancel pending sends");
for delivery in deliveries {
let result = tokio::time::timeout(Duration::from_secs(1), delivery)
.await
.expect("delivery should be released by stop")
.expect("delivery task should join");
assert!(
matches!(result, Err(MobError::Internal(ref message)) if message.contains("topology or lifecycle changed")),
"pending peer delivery should fail through lifecycle cancellation: {result:?}"
);
}
}
#[tokio::test]
async fn test_peer_message_reaches_idle_autonomous_member_after_kickoff_completion() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
let unwired = handle
.send_peer_message(
AgentIdentity::from("l-1"),
AgentIdentity::from("w-1"),
"unwired peer send should fail",
meerkat_core::types::HandlingMode::Queue,
)
.await;
assert!(
matches!(
unwired,
Err(MobError::WiringError(ref message))
if message.contains("requires wired members")
),
"sender-aware peer send must enforce mob wiring before delivery: {unwired:?}"
);
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire should succeed");
tokio::time::sleep(Duration::from_millis(50)).await;
let baseline_prompts = service.applied_runtime_prompts(&sid_b).await.len();
let receipt = handle
.send_peer_message(
AgentIdentity::from("l-1"),
AgentIdentity::from("w-1"),
meerkat_core::types::ContentInput::Blocks(vec![
meerkat_core::types::ContentBlock::Text {
text: "caption: this block text should survive".to_string(),
},
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".into(),
},
]),
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("MobHandle peer message should succeed");
assert_eq!(receipt.from, AgentIdentity::from("l-1"));
assert_eq!(receipt.to, AgentIdentity::from("w-1"));
assert_eq!(
receipt.handling_mode,
meerkat_core::types::HandlingMode::Queue
);
let delivered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_b).await;
if let Some(prompt) = prompts
.iter()
.skip(baseline_prompts)
.find(|prompt| {
let text = prompt.text_content();
prompt.has_images() || text.contains("caption: this block text should survive")
})
.cloned()
{
break prompt;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("peer message should reach runtime apply path");
let delivered_text = delivered.text_content();
assert!(
delivered_text.contains("Peer message from test-mob/lead/l-1"),
"peer message should carry typed comms source projection: {delivered_text:?}"
);
assert!(
delivered_text.contains("caption: this block text should survive"),
"peer message block text should survive runtime delivery: {delivered_text:?}"
);
assert!(
delivered.has_images(),
"peer message image block should survive runtime delivery: {delivered:?}"
);
}
#[tokio::test]
#[ignore = "exploratory runtime-backed kickoff contract; follow-up hardening pending"]
async fn test_wait_for_ready_blocks_until_autonomous_kickoff_resolves() {
let _serial = lock_real_comms_tests();
let (handle, _service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-ready"),
None,
)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-ready"),
None,
)
.await
.expect("spawn worker");
let kickoff = tokio::time::timeout(
Duration::from_millis(250),
handle.wait_for_members_kickoff_complete(
&[
AgentIdentity::from("l-ready"),
AgentIdentity::from("w-ready"),
],
None,
),
)
.await;
assert!(
kickoff.is_err(),
"the harness should still have pending kickoff turns before they are explicitly released"
);
let ready = tokio::time::timeout(Duration::from_millis(250), handle.wait_for_ready(None)).await;
assert!(
ready.is_err(),
"wait_for_ready must not resolve while autonomous kickoff turns are still pending"
);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after readiness assertion")
.expect("stop after readiness assertion");
}
#[tokio::test]
#[ignore = "exploratory runtime-backed active-member delivery probe; smoke lane covers the end-to-end contract"]
async fn test_peer_message_reaches_ready_autonomous_member_before_kickoff_settles() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-prekickoff"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-prekickoff"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-prekickoff"),
AgentIdentity::from("w-prekickoff"),
)
.await
.expect("wire should succeed");
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("wait_for_ready should resolve for startup-ready autonomous members");
let baseline_prompts = service.applied_runtime_prompts(&sid_b).await.len();
let comms_a = service
.real_comms(&sid_a)
.await
.expect("comms for l-prekickoff");
let receipt = CoreCommsRuntime::send(
&*comms_a,
CommsCommand::PeerMessage {
objective_id: None,
content_taint: None,
to: test_peer_route(&*comms_a, &test_comms_name("worker", "w-prekickoff")).await,
body: "body: immediate orchestration after wait_for_ready".to_string(),
blocks: Some(vec![
meerkat_core::types::ContentBlock::Text {
text: "caption: startup-ready multimodal comms must still deliver".to_string(),
},
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".into(),
},
]),
handling_mode: meerkat_core::types::HandlingMode::Steer,
},
)
.await
.expect("PeerMessage should succeed");
assert!(
matches!(receipt, SendReceipt::PeerMessageSent { .. }),
"expected PeerMessageSent receipt, got: {receipt:?}"
);
let delivered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_b).await;
if let Some(prompt) = prompts
.iter()
.skip(baseline_prompts)
.find(|prompt| {
let text = prompt.text_content();
prompt.has_images()
|| text.contains("body: immediate orchestration after wait_for_ready")
|| text
.contains("caption: startup-ready multimodal comms must still deliver")
})
.cloned()
{
break prompt;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("peer message should reach a startup-ready autonomous member");
let delivered_text = delivered.text_content();
assert!(
delivered_text.contains("Peer message from test-mob/lead/l-prekickoff"),
"peer message should carry typed comms source projection: {delivered_text:?}"
);
assert!(
delivered_text.contains("caption: startup-ready multimodal comms must still deliver"),
"peer message text block should survive runtime delivery: {delivered_text:?}"
);
assert!(
delivered.has_images(),
"peer message image block should survive runtime delivery: {delivered:?}"
);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after direct peer delivery assertion")
.expect("stop after direct peer delivery assertion");
}
#[tokio::test]
#[ignore = "exploratory runtime-backed multi-recipient delivery probe; smoke lane covers the end-to-end contract"]
async fn test_peer_messages_reach_all_ready_autonomous_members_before_kickoff_settles() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let sid_lead = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-multi"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let targets = [
("w-multi-1", "worker"),
("w-multi-2", "worker"),
("w-multi-3", "worker"),
];
let mut target_sessions = Vec::new();
for (agent_identity, profile) in targets {
let sid = handle
.spawn(
ProfileName::from(profile),
AgentIdentity::from(agent_identity),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-multi"),
AgentIdentity::from(agent_identity),
)
.await
.expect("wire should succeed");
target_sessions.push((agent_identity.to_string(), sid));
}
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("wait_for_ready should resolve for startup-ready autonomous members");
let baselines = futures::future::join_all(
target_sessions
.iter()
.map(|(_, sid)| service.applied_runtime_prompts(sid)),
)
.await
.into_iter()
.map(|prompts| prompts.len())
.collect::<Vec<_>>();
let comms_lead = service.real_comms(&sid_lead).await.expect("comms for lead");
for (agent_identity, _) in &target_sessions {
let receipt = CoreCommsRuntime::send(
&*comms_lead,
CommsCommand::PeerMessage {
objective_id: None,
content_taint: None,
to: test_peer_route(&*comms_lead, &test_comms_name("worker", agent_identity)).await,
body: format!("body: fanout to {agent_identity}"),
blocks: Some(vec![
meerkat_core::types::ContentBlock::Text {
text: format!("caption: fanout should reach {agent_identity}"),
},
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".into(),
},
]),
handling_mode: meerkat_core::types::HandlingMode::Steer,
},
)
.await
.expect("PeerMessage should succeed");
assert!(
matches!(receipt, SendReceipt::PeerMessageSent { .. }),
"expected PeerMessageSent receipt, got: {receipt:?}"
);
}
for ((agent_identity, sid), baseline_prompts) in target_sessions.iter().zip(baselines.iter()) {
let delivered = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(sid).await;
if let Some(prompt) = prompts
.iter()
.skip(*baseline_prompts)
.find(|prompt| {
let text = prompt.text_content();
prompt.has_images()
|| text.contains(&format!("body: fanout to {agent_identity}"))
|| text
.contains(&format!("caption: fanout should reach {agent_identity}"))
})
.cloned()
{
break prompt;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.unwrap_or_else(|_| panic!("peer message should reach {agent_identity}"));
let delivered_text = delivered.text_content();
assert!(
delivered_text.contains("Peer message from test-mob/lead/l-multi"),
"peer message should carry typed comms source projection for {agent_identity}: {delivered_text:?}"
);
assert!(
delivered_text.contains(&format!("caption: fanout should reach {agent_identity}")),
"peer message text block should survive runtime delivery for {agent_identity}: {delivered_text:?}"
);
assert!(
delivered.has_images(),
"peer message image block should survive runtime delivery for {agent_identity}: {delivered:?}"
);
}
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after multi-recipient delivery assertion")
.expect("stop after multi-recipient delivery assertion");
}
#[tokio::test]
async fn test_running_peer_message_to_autonomous_member_queues_when_exact_boundary_is_unavailable()
{
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_artist = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-artist"),
None,
)
.await
.expect("spawn artist")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_helper = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-helper"),
None,
)
.await
.expect("spawn helper")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_worker = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-target"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-artist"),
AgentIdentity::from("w-target"),
)
.await
.expect("wire artist→worker");
handle
.wire(
AgentIdentity::from("l-helper"),
AgentIdentity::from("w-target"),
)
.await
.expect("wire helper→worker");
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle with immediate keep-alive completions");
handle
.wait_for_members_kickoff_complete(
&[
AgentIdentity::from("l-artist"),
AgentIdentity::from("l-helper"),
AgentIdentity::from("w-target"),
],
Some(Duration::from_secs(2)),
)
.await
.expect("kickoff should resolve before we switch into blocked-turn mode");
let baseline_prompts = service.applied_runtime_prompts(&sid_worker).await.len();
let baseline_system_appends = service
.applied_runtime_system_appends(&sid_worker)
.await
.len();
service.set_keep_alive_turns_complete_immediately(false);
let artist_comms = service.real_comms(&sid_artist).await.expect("artist comms");
let helper_comms = service.real_comms(&sid_helper).await.expect("helper comms");
CoreCommsRuntime::send(
&*artist_comms,
CommsCommand::PeerMessage {
objective_id: None,
content_taint: None,
to: test_peer_route(&*artist_comms, &test_comms_name("worker", "w-target")).await,
body: "body: first peer message".to_string(),
blocks: Some(vec![
meerkat_core::types::ContentBlock::Text {
text: "caption: first peer message".to_string(),
},
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "aGVsbG8=".into(),
},
]),
handling_mode: meerkat_core::types::HandlingMode::Steer,
},
)
.await
.expect("first peer message should succeed");
tokio::time::timeout(Duration::from_secs(5), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(baseline_prompts).any(|prompt| {
prompt.has_images()
|| prompt
.text_content()
.contains("caption: first peer message")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("worker should begin applying the first peer message");
CoreCommsRuntime::send(
&*helper_comms,
CommsCommand::PeerMessage {
objective_id: None,
content_taint: None,
to: test_peer_route(&*helper_comms, &test_comms_name("worker", "w-target")).await,
body: "body: second peer message while running".to_string(),
blocks: None,
handling_mode: meerkat_core::types::HandlingMode::Steer,
},
)
.await
.expect("second peer message should succeed");
let system_appends = service.applied_runtime_system_appends(&sid_worker).await;
assert!(
!system_appends
.iter()
.skip(baseline_system_appends)
.any(|append| append
.text()
.contains("body: second peer message while running")),
"the synthetic service must not claim an exact live append without boundary authority"
);
assert_eq!(
service.applied_runtime_prompts(&sid_worker).await.len(),
baseline_prompts + 1,
"queued fallback must wait for the active apply to release"
);
service.set_keep_alive_turns_complete_immediately(true);
service
.interrupt(&sid_worker)
.await
.expect("interrupt should release blocked worker apply");
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(baseline_prompts + 1).any(|prompt| {
prompt
.text_content()
.contains("body: second peer message while running")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("queued peer steer should run after the active apply releases");
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after running peer message assertion")
.expect("stop after running peer message assertion");
}
#[tokio::test]
async fn test_active_autonomous_direct_steer_falls_back_when_exact_boundary_is_unavailable() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_worker = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("w-direct-steer"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
handle
.wait_for_members_kickoff_complete(
&[AgentIdentity::from("w-direct-steer")],
Some(Duration::from_secs(2)),
)
.await
.expect("kickoff should resolve before blocked direct turn");
// Scope the block to this input. A global keep-alive flag can be observed
// late by an already-started ambient empty turn and block the wrong run.
let first_apply_entered = service
.arm_runtime_turn_content_barrier(&sid_worker, "first direct prompt holds active apply")
.await;
let worker = handle
.member(&AgentIdentity::from("w-direct-steer"))
.await
.expect("worker handle");
worker
.send(
"first direct prompt holds active apply",
HandlingMode::Queue,
)
.await
.expect("first queued direct turn should be admitted");
tokio::time::timeout(Duration::from_secs(5), first_apply_entered.notified())
.await
.expect("first queued direct turn should enter active apply");
let active_apply_count = service.applied_runtime_prompts(&sid_worker).await.len();
let active_append_count = service
.applied_runtime_system_appends(&sid_worker)
.await
.len();
service.set_append_system_context_delay_ms(250);
let steer_worker = handle
.member(&AgentIdentity::from("w-direct-steer"))
.await
.expect("worker handle for active steer");
let mut steer_task = tokio::spawn(async move {
steer_worker
.send(
"second direct steer uses the queued fallback",
HandlingMode::Steer,
)
.await
});
tokio::time::timeout(Duration::from_millis(150), &mut steer_task)
.await
.expect("typed boundary Unavailable must not wait behind the synthetic append delay")
.expect("active steer task should join")
.expect("active steer queued fallback should be admitted");
let appends = service.applied_runtime_system_appends(&sid_worker).await;
assert!(
!appends.iter().skip(active_append_count).any(|append| append
.text()
.contains("second direct steer uses the queued fallback")),
"a service without exact boundary preparation must not claim a live append; appends={appends:?}"
);
assert_eq!(
service.applied_runtime_prompts(&sid_worker).await.len(),
active_apply_count,
"queued fallback must wait for the blocked apply to release"
);
service
.clear_runtime_turn_content_barrier(&sid_worker)
.await;
service.release_one_runtime_turn();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(active_apply_count).any(|prompt| {
prompt
.text_content()
.contains("second direct steer uses the queued fallback")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("queued steer must run after the active turn releases");
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after queued steer fallback assertion")
.expect("stop after queued steer fallback assertion");
}
#[tokio::test]
async fn test_active_internal_submit_work_steer_falls_back_when_exact_boundary_is_unavailable() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_worker = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("w-internal-steer"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
handle
.wait_for_members_kickoff_complete(
&[AgentIdentity::from("w-internal-steer")],
Some(Duration::from_secs(2)),
)
.await
.expect("kickoff should resolve before blocked direct turn");
// Scope the block to this input. A global keep-alive flag can be observed
// late by an already-started ambient empty turn and block the wrong run.
let first_apply_entered = service
.arm_runtime_turn_content_barrier(&sid_worker, "first direct prompt holds active apply")
.await;
let worker = handle
.member(&AgentIdentity::from("w-internal-steer"))
.await
.expect("worker handle");
worker
.send(
"first direct prompt holds active apply",
HandlingMode::Queue,
)
.await
.expect("first queued direct turn should be admitted");
tokio::time::timeout(Duration::from_secs(5), first_apply_entered.notified())
.await
.expect("first queued direct turn should enter active apply");
let active_apply_count = service.applied_runtime_prompts(&sid_worker).await.len();
let active_append_count = service
.applied_runtime_system_appends(&sid_worker)
.await
.len();
let entry = handle
.get_member(&AgentIdentity::from("w-internal-steer"))
.await
.unwrap()
.expect("member entry");
handle
.submit_work_with_mode(
entry.agent_runtime_id,
entry.fence_token,
WorkRef::new(),
WorkSpec::new(
"internal identity bridge steer uses queued fallback".to_string(),
WorkOrigin::Internal,
),
HandlingMode::Steer,
)
.await
.expect("internal active steer should be admitted");
let appends = service.applied_runtime_system_appends(&sid_worker).await;
assert!(
!appends.iter().skip(active_append_count).any(|append| append
.text()
.contains("internal identity bridge steer uses queued fallback")),
"a service without exact boundary preparation must not claim a live append; appends={appends:?}"
);
assert_eq!(
service.applied_runtime_prompts(&sid_worker).await.len(),
active_apply_count,
"queued fallback must wait for the blocked apply to release"
);
service
.clear_runtime_turn_content_barrier(&sid_worker)
.await;
service.release_one_runtime_turn();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(active_apply_count).any(|prompt| {
prompt
.text_content()
.contains("internal identity bridge steer uses queued fallback")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("queued internal steer must run after the active turn releases");
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after internal queued fallback assertion")
.expect("stop after internal queued fallback assertion");
}
#[tokio::test]
async fn test_idle_internal_submit_work_steer_preserves_exact_content() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let member_id = AgentIdentity::from("w-idle-internal-steer");
let session_id = handle
.spawn(ProfileName::from("lead"), member_id.clone(), None)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
handle
.wait_for_members_kickoff_complete(
std::slice::from_ref(&member_id),
Some(Duration::from_secs(2)),
)
.await
.expect("kickoff should resolve before idle steer");
let prompt_baseline = service.applied_runtime_prompts(&session_id).await.len();
let entry = handle
.get_member(&member_id)
.await
.expect("read member")
.expect("member exists");
let marker = "idle identity bridge steer must preserve this exact content";
handle
.submit_work_with_mode(
entry.agent_runtime_id,
entry.fence_token,
WorkRef::new(),
WorkSpec::new(marker, WorkOrigin::Internal),
HandlingMode::Steer,
)
.await
.expect("idle internal steer should be admitted");
let delivery = tokio::time::timeout(Duration::from_secs(5), async {
loop {
let prompts = service.applied_runtime_prompts(&session_id).await;
if prompts
.iter()
.skip(prompt_baseline)
.any(|prompt| prompt.text_content().starts_with(marker))
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await;
assert!(
delivery.is_ok(),
"idle internal steer must reach the model with exact content; applied prompts: {:?}",
service.applied_runtime_prompts(&session_id).await
);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after idle internal steer assertion")
.expect("stop after idle internal steer assertion");
}
#[tokio::test]
async fn internal_submit_work_carries_stable_delivery_identity_to_runtime_admission() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let member_id = AgentIdentity::from("delivery-identity-worker");
let session_id = handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker")
.bridge_session_id()
.expect("runtime-backed member")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
let entry = handle
.get_member(&member_id)
.await
.expect("read member")
.expect("member exists");
let baseline = service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len();
let delivery_identity = crate::MobDeliveryIdentity::new(
"schedule:daily-report:occurrence:2026-08-03T14:00:00Z",
"019fdcda-836e-70b4-9d89-78e629a77c30",
)
.expect("valid delivery identity");
let expected_work_ref = WorkRef::for_delivery(
handle.mob_id(),
&member_id,
&delivery_identity.idempotency_key,
);
let receipt = handle
.submit_work_with_mode_and_delivery_identity(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkSpec::new("deliver scheduled work", WorkOrigin::Internal),
HandlingMode::Queue,
delivery_identity.clone(),
)
.await
.expect("stable internal delivery should be admitted");
assert_eq!(receipt.work_ref, expected_work_ref);
tokio::time::timeout(Duration::from_secs(2), async {
while service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len()
< baseline + 1
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("stable internal delivery should reach the runtime executor");
let stored = <meerkat_runtime::MeerkatMachine as meerkat_runtime::service_ext::SessionServiceRuntimeExt>::input_state_by_idempotency_key(
service.runtime_adapter.as_ref(),
&session_id,
&delivery_identity.idempotency_key,
)
.await
.expect("runtime identity lookup")
.expect("stable delivery must be indexed at admission");
assert_eq!(
stored
.state
.idempotency_key
.as_ref()
.map(ToString::to_string),
Some(delivery_identity.idempotency_key.clone())
);
let duplicate = handle
.submit_work_with_mode_and_delivery_identity(
entry.agent_runtime_id,
entry.fence_token,
WorkSpec::new("deliver scheduled work", WorkOrigin::Internal),
HandlingMode::Queue,
delivery_identity,
)
.await
.expect("crash-redelivered identity should deduplicate");
assert_eq!(duplicate.work_ref, receipt.work_ref);
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len(),
baseline + 1,
"redelivery under the same stable identity must not execute twice"
);
}
#[tokio::test]
async fn autonomous_internal_submit_work_carries_stable_delivery_identity_to_runtime_admission() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let member_id = AgentIdentity::from("autonomous-delivery-identity-worker");
let session_id = handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous worker")
.bridge_session_id()
.expect("runtime-backed member")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
handle
.wait_for_members_kickoff_complete(
std::slice::from_ref(&member_id),
Some(Duration::from_secs(2)),
)
.await
.expect("autonomous kickoff should settle before stable delivery");
let entry = handle
.get_member(&member_id)
.await
.expect("read member")
.expect("member exists");
let baseline = service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len();
let delivery_identity = crate::MobDeliveryIdentity::new(
"schedule:autonomous-report:occurrence:2026-08-03T14:00:00Z",
"019fdcda-836e-70b4-9d89-78e629a77c31",
)
.expect("valid delivery identity");
let expected_work_ref = WorkRef::for_delivery(
handle.mob_id(),
&member_id,
&delivery_identity.idempotency_key,
);
let receipt = handle
.submit_work_with_mode_and_delivery_identity(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkSpec::new("deliver autonomous scheduled work", WorkOrigin::Internal),
HandlingMode::Queue,
delivery_identity.clone(),
)
.await
.expect("stable autonomous delivery should be admitted");
assert_eq!(receipt.work_ref, expected_work_ref);
tokio::time::timeout(Duration::from_secs(2), async {
while service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len()
< baseline + 1
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("stable autonomous delivery should reach runtime input admission");
let stored = <meerkat_runtime::MeerkatMachine as meerkat_runtime::service_ext::SessionServiceRuntimeExt>::input_state_by_idempotency_key(
service.runtime_adapter.as_ref(),
&session_id,
&delivery_identity.idempotency_key,
)
.await
.expect("runtime identity lookup")
.expect("stable autonomous delivery must be indexed at admission");
assert_eq!(
stored
.state
.idempotency_key
.as_ref()
.map(ToString::to_string),
Some(delivery_identity.idempotency_key.clone())
);
let duplicate = handle
.submit_work_with_mode_and_delivery_identity(
entry.agent_runtime_id,
entry.fence_token,
WorkSpec::new("deliver autonomous scheduled work", WorkOrigin::Internal),
HandlingMode::Queue,
delivery_identity,
)
.await
.expect("autonomous crash redelivery should deduplicate");
assert_eq!(duplicate.work_ref, receipt.work_ref);
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
service
.applied_runtime_contributing_input_ids(&session_id)
.await
.len(),
baseline + 1,
"autonomous redelivery under the same stable identity must not execute twice"
);
}
#[tokio::test]
async fn test_turn_driven_submit_work_steer_queues_when_exact_boundary_is_unavailable() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let sid_worker = handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("w-turn-driven-active-steer"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
let prompt_baseline = service.applied_runtime_prompts(&sid_worker).await.len();
let context_baseline = service
.applied_runtime_system_appends(&sid_worker)
.await
.len();
let entry = handle
.get_member(&AgentIdentity::from("w-turn-driven-active-steer"))
.await
.unwrap()
.expect("member entry");
service.set_block_runtime_turns(true);
let first_handle = handle.clone();
let first_runtime_id = entry.agent_runtime_id.clone();
let first_fence = entry.fence_token;
let first_turn = tokio::spawn(async move {
first_handle
.submit_work(
first_runtime_id,
first_fence,
WorkRef::new(),
WorkSpec::new(
"first turn blocks inside runtime apply".to_string(),
WorkOrigin::Internal,
),
)
.await
});
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
)
.await
.expect("first runtime apply should start and block");
tokio::time::timeout(Duration::from_secs(2), first_turn)
.await
.expect("first ingress admission should not wait for blocked apply")
.expect("first submit task should join")
.expect("first submit should be admitted");
service.set_block_session_reads(true);
let steer_handle = handle.clone();
let steer_runtime_id = entry.agent_runtime_id.clone();
let steer_fence = entry.fence_token;
let steer = tokio::spawn(async move {
steer_handle
.submit_work_with_mode(
steer_runtime_id,
steer_fence,
WorkRef::new(),
WorkSpec::new(
"turn-driven active steer uses queued fallback".to_string(),
WorkOrigin::Internal,
),
HandlingMode::Steer,
)
.await
});
tokio::time::timeout(Duration::from_millis(200), steer)
.await
.expect("active steer admission must not wait for blocked runtime apply or session reads")
.expect("steer submit task should join")
.expect("active steer should be admitted");
let appends = service.applied_runtime_system_appends(&sid_worker).await;
assert!(
!appends.iter().skip(context_baseline).any(|append| append
.text()
.contains("turn-driven active steer uses queued fallback")),
"a service without exact boundary preparation must not claim a live append; appends={appends:?}"
);
assert_eq!(
service.applied_runtime_prompts(&sid_worker).await.len(),
prompt_baseline + 1,
"queued fallback must wait for the first runtime apply to release"
);
service.set_block_session_reads(false);
service.release_session_reads();
service.set_block_runtime_turns(false);
service.release_one_runtime_turn();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(prompt_baseline + 1).any(|prompt| {
prompt
.text_content()
.contains("turn-driven active steer uses queued fallback")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("queued turn-driven steer should run after the active apply releases");
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after active steer runtime-apply assertion")
.expect("stop after active steer runtime-apply assertion");
}
#[tokio::test]
async fn test_member_send_steer_queues_when_exact_boundary_is_unavailable() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let sid_worker = handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("w-member-send-active-steer"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("startup should settle");
let prompt_baseline = service.applied_runtime_prompts(&sid_worker).await.len();
let context_baseline = service
.applied_runtime_system_appends(&sid_worker)
.await
.len();
let worker = handle
.member(&AgentIdentity::from("w-member-send-active-steer"))
.await
.expect("worker handle");
service.set_block_runtime_turns(true);
let first_worker = worker.clone();
let first_turn = tokio::spawn(async move {
first_worker
.send(
"first member send blocks inside runtime apply",
HandlingMode::Queue,
)
.await
});
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
)
.await
.expect("first runtime apply should start and block");
tokio::time::timeout(Duration::from_secs(2), first_turn)
.await
.expect("first member send admission should not wait for blocked apply")
.expect("first member send task should join")
.expect("first member send should be admitted");
service.set_block_session_reads(true);
let steer_worker = worker.clone();
let steer = tokio::spawn(async move {
steer_worker
.send(
"member send active steer uses queued fallback",
HandlingMode::Steer,
)
.await
});
tokio::time::timeout(Duration::from_millis(200), steer)
.await
.expect("active member-send steer admission must not wait for blocked runtime apply or session reads")
.expect("active member-send steer task should join")
.expect("active member-send steer should be admitted");
let appends = service.applied_runtime_system_appends(&sid_worker).await;
assert!(
!appends.iter().skip(context_baseline).any(|append| append
.text()
.contains("member send active steer uses queued fallback")),
"a service without exact boundary preparation must not claim a live append; appends={appends:?}"
);
assert_eq!(
service.applied_runtime_prompts(&sid_worker).await.len(),
prompt_baseline + 1,
"queued member-send fallback must wait for the first runtime apply to release"
);
service.set_block_session_reads(false);
service.release_session_reads();
service.set_block_runtime_turns(false);
service.release_runtime_turns();
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_worker).await;
if prompts.iter().skip(prompt_baseline + 1).any(|prompt| {
prompt
.text_content()
.contains("member send active steer uses queued fallback")
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("queued member-send steer should run after the active apply releases");
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after active member-send steer assertion")
.expect("stop after active member-send steer assertion");
}
#[tokio::test]
async fn test_peer_response_reaches_requester_in_runtime_backed_real_comms() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_requester = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-requester"),
None,
)
.await
.expect("spawn requester")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_responder = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-responder"),
None,
)
.await
.expect("spawn responder")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-requester"),
AgentIdentity::from("w-responder"),
)
.await
.expect("wire requester↔responder");
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("wait_for_ready should resolve");
handle
.wait_for_members_kickoff_complete(
&[
AgentIdentity::from("l-requester"),
AgentIdentity::from("w-responder"),
],
Some(Duration::from_secs(2)),
)
.await
.expect("requester and responder kickoff turns should settle before idle peer request");
let requester_prompt_baseline = service.applied_runtime_prompts(&sid_requester).await.len();
let requester_context_baseline = service
.applied_runtime_system_appends(&sid_requester)
.await
.len();
let responder_baseline = service.applied_runtime_prompts(&sid_responder).await.len();
let requester_comms = service
.real_comms(&sid_requester)
.await
.expect("requester comms");
let responder_comms = service
.real_comms(&sid_responder)
.await
.expect("responder comms");
let responder_route_identity = responder_comms.public_key().to_peer_id().to_string();
let responder_display_identity = test_comms_name("worker", "w-responder");
let receipt = CoreCommsRuntime::send(
&*requester_comms,
CommsCommand::PeerRequest {
objective_id: None,
content_taint: None,
to: test_peer_route(&*requester_comms, &test_comms_name("worker", "w-responder")).await,
intent: "interpret_image".to_string(),
params: serde_json::json!({"description":"tower with a light"}),
blocks: None,
// This test covers request/response delivery, not active-turn
// steering. Both kickoff turns are explicitly settled above, so
// queue the request onto the idle responder instead of depending
// on a racy leftover active boundary.
handling_mode: meerkat_core::types::HandlingMode::Queue,
stream: meerkat_core::InputStreamMode::ReserveInteraction,
},
)
.await
.expect("peer request should succeed");
let request_id = match receipt {
SendReceipt::PeerRequestSent { interaction_id, .. } => interaction_id,
other => panic!("expected PeerRequestSent receipt, got {other:?}"),
};
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_responder).await;
if prompts
.iter()
.skip(responder_baseline)
.any(|prompt| prompt.text_content().contains("Peer request from"))
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("responder should receive the peer request");
CoreCommsRuntime::send(
&*responder_comms,
CommsCommand::PeerResponse {
objective_id: None,
content_taint: None,
to: test_peer_route(&*responder_comms, &test_comms_name("lead", "l-requester")).await,
in_reply_to: request_id,
status: meerkat_core::ResponseStatus::Completed,
result: serde_json::json!({"interpretation":"lighthouse"}),
blocks: None,
handling_mode: None,
},
)
.await
.expect("peer response should succeed");
let requester_response_request_id = request_id.to_string();
let requester_response_payload = serde_json::json!({"interpretation":"lighthouse"});
let requester_response_delivery = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let appends = service.applied_runtime_system_appends(&sid_requester).await;
if appends
.iter()
.skip(requester_context_baseline)
.any(|append| {
is_incoming_terminal_peer_response(
append,
&responder_route_identity,
&responder_display_identity,
&requester_response_request_id,
&requester_response_payload,
)
})
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await;
let requester_snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await
.expect("requester snapshot should exist");
if requester_response_delivery.is_ok() {
assert_peer_response_terminal_consumed(&requester_snapshot, HandlingMode::Queue);
}
if requester_response_delivery.is_err() {
let has_executor = service
.runtime_adapter
.session_has_executor(&sid_requester)
.await
.expect("session_has_executor should resolve");
let has_comms = service
.runtime_adapter
.session_has_comms(&sid_requester)
.await
.expect("session_has_comms should resolve");
panic!(
"requester should apply the terminal peer response as runtime-owned context; has_executor={has_executor} has_comms={has_comms} snapshot={requester_snapshot:?}"
);
}
let requester_snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await
.expect("requester snapshot should still exist after delivery");
assert!(
requester_snapshot.inputs.queue.is_empty(),
"terminal peer response should be consumed, not left queued: {requester_snapshot:?}"
);
let terminal_admission = requester_snapshot
.inputs
.admission_order
.iter()
.find(|input| {
input.content_shape.as_ref().is_some_and(|shape| {
shape.kind() == meerkat_runtime::identifiers::InputKind::PeerResponseTerminal
})
})
.expect("requester should admit a terminal peer response input");
assert_eq!(
terminal_admission.lifecycle,
Some(meerkat_runtime::InputLifecycleState::Consumed),
"terminal peer response should be applied and consumed: {requester_snapshot:?}"
);
// The fixture records typed appends before it records the provider-visible
// prompt for the same apply. Observing the append above therefore proves
// delivery, but is not yet a completion barrier for the reaction-turn
// probe. Wait for that second record rather than racing the two fixture
// locks.
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if service.applied_runtime_prompts(&sid_requester).await.len()
> requester_prompt_baseline
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("terminal peer response should start the requester reaction turn");
let requester_prompts_after_response = service.applied_runtime_prompts(&sid_requester).await;
assert_eq!(
requester_prompts_after_response.len(),
requester_prompt_baseline + 1,
"terminal peer response should append runtime system context and kick exactly one requester reaction turn: {requester_prompts_after_response:?}"
);
// The mandatory requester reaction turn carries the typed comms-notice
// projection as its model-visible content — never a fabricated empty
// prompt (providers reject empty user messages) and never raw user
// prose authored by the responder outside the typed notice rendering.
let reaction_prompt = requester_prompts_after_response
.last()
.expect("reaction turn prompt should exist")
.text_content();
assert!(
!reaction_prompt.trim().is_empty(),
"terminal peer response reaction turn must have model-visible content, \
not a fabricated empty prompt"
);
assert!(
reaction_prompt.contains(&request_id.to_string()),
"reaction turn content must be the typed comms-notice projection \
correlated to the request: {reaction_prompt:?}"
);
let duplicate_peer_id =
meerkat_core::comms::PeerId::parse(&responder_route_identity).expect("responder peer id");
let duplicate_display_name =
meerkat_core::comms::PeerName::new(responder_display_identity.clone())
.expect("responder display name");
let duplicate_request_id = meerkat_core::PeerCorrelationId::from_uuid(request_id.0);
let duplicate = meerkat_runtime::peer_response_terminal_input(
duplicate_peer_id,
Some(duplicate_display_name),
duplicate_request_id,
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
serde_json::json!({"interpretation":"lighthouse"}),
);
let (duplicate_outcome, _) = service
.runtime_adapter
.accept_input_with_completion(&sid_requester, duplicate)
.await
.expect("duplicate terminal response should be accepted for idempotent apply");
assert!(
duplicate_outcome.is_deduplicated(),
"the typed peer-route/request-correlation terminal key must reject replay before apply: \
{duplicate_outcome:?}"
);
let requester_contexts = service.applied_runtime_system_appends(&sid_requester).await;
let response_context_count = requester_contexts
.iter()
.filter(|append| {
is_incoming_terminal_peer_response(
append,
&responder_route_identity,
&responder_display_identity,
&requester_response_request_id,
&requester_response_payload,
)
})
.count();
assert_eq!(
response_context_count, 1,
"duplicate terminal response should not append its exact runtime system context twice: {requester_contexts:?}"
);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after peer response assertions")
.expect("stop after peer response assertions");
}
#[tokio::test]
async fn test_default_peer_response_inherits_request_steer_while_requester_running() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_requester = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-running-requester"),
None,
)
.await
.expect("spawn requester")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_responder = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-running-responder"),
None,
)
.await
.expect("spawn responder")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-running-requester"),
AgentIdentity::from("w-running-responder"),
)
.await
.expect("wire requester↔responder");
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("wait_for_ready should resolve");
handle
.wait_for_members_kickoff_complete(
&[
AgentIdentity::from("l-running-requester"),
AgentIdentity::from("w-running-responder"),
],
Some(Duration::from_secs(2)),
)
.await
.expect("kickoff should resolve before blocking turns");
let requester_prompt_baseline = service.applied_runtime_prompts(&sid_requester).await.len();
let requester_context_baseline = service
.applied_runtime_system_appends(&sid_requester)
.await
.len();
let responder_baseline = service.applied_runtime_prompts(&sid_responder).await.len();
let delivery_observation_timeout = Duration::from_secs(15);
let requester_comms = service
.real_comms(&sid_requester)
.await
.expect("requester comms");
let responder_comms = service
.real_comms(&sid_responder)
.await
.expect("responder comms");
let responder_route_identity = responder_comms.public_key().to_peer_id().to_string();
let responder_display_identity = test_comms_name("worker", "w-running-responder");
let requester_runtime_started_notify = service
.arm_runtime_turn_content_barrier(&sid_requester, "body: keep requester busy")
.await;
let requester_runtime_started = requester_runtime_started_notify.notified();
tokio::pin!(requester_runtime_started);
requester_runtime_started.as_mut().enable();
CoreCommsRuntime::send(
&*responder_comms,
CommsCommand::PeerMessage {
objective_id: None,
content_taint: None,
to: test_peer_route(
&*responder_comms,
&test_comms_name("lead", "l-running-requester"),
)
.await,
body: "body: keep requester busy".to_string(),
blocks: None,
handling_mode: meerkat_core::types::HandlingMode::Steer,
},
)
.await
.expect("busy message should send");
if tokio::time::timeout(delivery_observation_timeout, &mut requester_runtime_started)
.await
.is_err()
{
let prompts = service.applied_runtime_prompts(&sid_requester).await;
let snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await;
panic!(
"requester runtime apply should start and block before the pong lands; prompts={prompts:?} snapshot={snapshot:?}"
);
}
let requester_prompts = service.applied_runtime_prompts(&sid_requester).await;
assert!(
requester_prompts
.iter()
.skip(requester_prompt_baseline)
.any(|prompt| prompt.text_content().contains("body: keep requester busy")),
"runtime-start barrier fired without recording the busy requester prompt; prompts={requester_prompts:?}"
);
service.set_keep_alive_turns_complete_immediately(false);
// Install a fresh notifier before releasing the content barrier. The
// kickoff path may have left a stored Notify permit on the original test
// notifier, which would let this turn finish and make the response race a
// second apply instead of exercising the active-requester boundary.
service.reset_keep_alive_notifier(&sid_requester).await;
service
.clear_runtime_turn_content_barrier(&sid_requester)
.await;
service.release_one_runtime_turn();
let request_receipt = CoreCommsRuntime::send(
&*requester_comms,
CommsCommand::PeerRequest {
objective_id: None,
content_taint: None,
to: test_peer_route(
&*requester_comms,
&test_comms_name("worker", "w-running-responder"),
)
.await,
intent: "ping".to_string(),
params: serde_json::json!({"message":"ping"}),
blocks: None,
handling_mode: meerkat_core::types::HandlingMode::Steer,
stream: meerkat_core::InputStreamMode::ReserveInteraction,
},
)
.await
.expect("peer request should succeed");
let request_id = match request_receipt {
SendReceipt::PeerRequestSent { interaction_id, .. } => interaction_id,
other => panic!("expected PeerRequestSent receipt, got {other:?}"),
};
tokio::time::timeout(delivery_observation_timeout, async {
loop {
let prompts = service.applied_runtime_prompts(&sid_responder).await;
if prompts.iter().skip(responder_baseline).any(|prompt| {
let text = prompt.text_content();
text.contains("Peer request from")
&& text.contains("Intent: ping")
&& text.contains(&format!("Request ID: {request_id}"))
}) {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("responder should receive the steer ping request");
CoreCommsRuntime::send(
&*responder_comms,
CommsCommand::PeerResponse {
objective_id: None,
content_taint: None,
to: test_peer_route(
&*responder_comms,
&test_comms_name("lead", "l-running-requester"),
)
.await,
in_reply_to: request_id,
status: meerkat_core::ResponseStatus::Completed,
result: serde_json::json!({"message":"pong"}),
blocks: None,
handling_mode: None,
},
)
.await
.expect("default peer response should succeed");
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
service.applied_runtime_prompts(&sid_requester).await.len(),
requester_prompt_baseline + 1,
"default response to a steer request must not start another apply or cancel the active run"
);
service
.interrupt(&sid_requester)
.await
.expect("boundary release should let requester drain steered response");
let requester_response_request_id = request_id.to_string();
let requester_response_payload = serde_json::json!({"message":"pong"});
let requester_delivery = tokio::time::timeout(delivery_observation_timeout, async {
loop {
let appends = service.applied_runtime_system_appends(&sid_requester).await;
if appends
.iter()
.skip(requester_context_baseline)
.any(|append| {
is_incoming_terminal_peer_response(
append,
&responder_route_identity,
&responder_display_identity,
&requester_response_request_id,
&requester_response_payload,
)
})
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await;
if requester_delivery.is_err() {
let appends = service.applied_runtime_system_appends(&sid_requester).await;
let snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await
.expect("requester snapshot should exist");
panic!(
"default response to a steer request should drain after requester boundary; appends={appends:?} snapshot={snapshot:?}"
);
}
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after running requester response assertion")
.expect("stop after running requester response assertion");
}
#[tokio::test]
async fn test_mcp_send_request_response_terminal_steer_is_visible_to_requester() {
let _serial = lock_real_comms_tests();
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
service.set_keep_alive_turns_complete_immediately(true);
let sid_requester = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("l-mcp-requester"),
None,
)
.await
.expect("spawn requester")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_responder = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-mcp-responder"),
None,
)
.await
.expect("spawn responder")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(
AgentIdentity::from("l-mcp-requester"),
AgentIdentity::from("w-mcp-responder"),
)
.await
.expect("wire requester↔responder");
handle
.wait_for_ready(Some(Duration::from_secs(2)))
.await
.expect("wait_for_ready should resolve");
let responder_baseline = service.applied_runtime_prompts(&sid_responder).await.len();
let requester_context_baseline = service
.applied_runtime_system_appends(&sid_requester)
.await
.len();
let requester_comms = service
.real_comms(&sid_requester)
.await
.expect("requester comms");
let responder_comms = service
.real_comms(&sid_responder)
.await
.expect("responder comms");
let responder_peer_id = mcp_peer_id_for(
&requester_comms,
&test_comms_name("worker", "w-mcp-responder"),
)
.await;
let requester_peer_id = mcp_peer_id_for(
&responder_comms,
&test_comms_name("lead", "l-mcp-requester"),
)
.await;
let requester_ctx = mcp_tool_context_for(&requester_comms);
let request_result = meerkat_comms::handle_tools_call(
&requester_ctx,
"send_request",
&serde_json::json!({
"peer_id": responder_peer_id,
"intent": "checksum_token",
"params": {"subject": "alpha beta gamma"},
"handling_mode": "steer"
}),
)
.await
.expect("send_request tool should succeed");
assert_eq!(request_result["kind"], "peer_request");
assert_eq!(request_result["status"], "sent");
let responder_request_prompt = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let prompts = service.applied_runtime_prompts(&sid_responder).await;
if let Some(prompt) = prompts
.iter()
.skip(responder_baseline)
.find(|prompt| {
let text = prompt.text_content();
text.contains("Peer request from")
&& text.contains("Intent: checksum_token")
&& text.contains("Request ID: ")
})
.cloned()
{
break prompt;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await
.expect("responder should receive actionable MCP peer request");
let request_notice = responder_request_prompt.text_content();
let request_id = peer_request_notice_request_id(&request_notice);
let responder_ctx = mcp_tool_context_for(&responder_comms);
let response_result = meerkat_comms::handle_tools_call(
&responder_ctx,
"send_response",
&serde_json::json!({
"peer_id": requester_peer_id,
"in_reply_to": request_id.to_string(),
"status": "completed",
"result": {
"request_intent": "checksum_token",
"request_subject": "alpha beta gamma",
"token": "birch seventeen"
},
"handling_mode": "steer"
}),
)
.await
.expect("send_response tool should succeed");
assert_eq!(response_result["kind"], "peer_response");
assert_eq!(response_result["status"], "sent");
let responder_peer_id = responder_peer_id.to_string();
let responder_display_name = test_comms_name("worker", "w-mcp-responder");
let requester_response_request_id = request_id.to_string();
let requester_response_payload = serde_json::json!({
"request_intent": "checksum_token",
"request_subject": "alpha beta gamma",
"token": "birch seventeen"
});
let requester_terminal_context = tokio::time::timeout(Duration::from_secs(5), async {
loop {
let appends = service.applied_runtime_system_appends(&sid_requester).await;
if let Some(append) = appends
.iter()
.skip(requester_context_baseline)
.find(|append| {
is_incoming_terminal_peer_response(
append,
&responder_peer_id,
&responder_display_name,
&requester_response_request_id,
&requester_response_payload,
)
})
.cloned()
{
break append;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
})
.await;
let requester_terminal_context = match requester_terminal_context {
Ok(context) => context,
Err(_) => {
let appends = service.applied_runtime_system_appends(&sid_requester).await;
let snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await;
panic!(
"requester should apply terminal peer response context after sent tool result; appends={appends:?} snapshot={snapshot:?}"
);
}
};
assert_eq!(
requester_terminal_context.source, None,
"typed peer-response input authority, not a synthetic message key, owns deduplication"
);
assert_eq!(
requester_terminal_context.idempotency_key, None,
"ordinary ordered transcript messages must not recover the retired sidecar identity"
);
let requester_snapshot = service
.runtime_adapter
.meerkat_machine_spine_snapshot(&sid_requester)
.await
.expect("requester snapshot should exist");
assert_peer_response_terminal_consumed(&requester_snapshot, HandlingMode::Steer);
tokio::time::timeout(Duration::from_secs(2), handle.stop())
.await
.expect("stop timeout after MCP peer response assertion")
.expect("stop after MCP peer response assertion");
}
#[tokio::test]
async fn test_runtime_backed_turn_driven_send_preserves_render_metadata() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let member = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-rt"),
None,
)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let render_metadata = meerkat_core::types::RenderMetadata {
class: meerkat_core::types::RenderClass::ExternalEvent,
salience: meerkat_core::types::RenderSalience::Urgent,
};
handle
.member(&AgentIdentity::from("lead-rt"))
.await
.expect("member handle")
.send_with_render_metadata(
ContentInput::Text("hello".into()),
meerkat_core::types::HandlingMode::Queue,
Some(render_metadata.clone()),
)
.await
.expect("runtime-backed turn-driven send should succeed");
let deadline = Instant::now() + Duration::from_secs(2);
while service
.applied_runtime_render_metadata(&member)
.await
.is_empty()
{
assert!(
Instant::now() < deadline,
"runtime-backed admitted turn should eventually reach the exact executor"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
// render_metadata is runtime-owned and stripped at the provisioner
// level before reaching the session service (dogma: render_metadata
// must not leak into the session-service path).
assert_eq!(
service.applied_runtime_render_metadata(&member).await,
vec![None]
);
}
// WIP: turn-driven realtime-attached peer_response_terminal application
// lands the response in the `Steer` admission queue without flushing it
// onto the keep-alive runtime's prompt stream. The machine/DSL rebase on
// `codex/machine-dls-completion` is expected to own the final shape of
// this delivery seam; tracking this as ignored until then so CI and the
// audio smoke lane aren't blocked by the transitional gap.
#[tokio::test]
async fn test_member_status_keeps_idle_live_session_active() {
let inner = Arc::new(MockSessionService::new());
let _ = inner.enable_runtime_adapter();
let service = Arc::new(InactiveReadSessionService::new(inner));
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service)
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let snapshot = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status");
assert_eq!(
snapshot.status,
crate::runtime::handle::MobMemberStatus::Active,
"idle live sessions should remain Active rather than Completed"
);
assert!(!snapshot.is_final);
}
#[tokio::test]
async fn test_mob_session_service_exposes_event_injector_for_session() {
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let injector = service.event_injector(&sid).await;
assert!(
injector.is_some(),
"expected event injector for live comms-backed session"
);
}
#[tokio::test]
async fn test_autonomous_spawn_requires_interaction_event_injector() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
service.disable_interaction_event_injector();
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let result = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await;
let error = result.expect_err("autonomous spawn must fail before seating");
assert!(
matches!(
error,
MobError::MissingMemberCapability {
ref member_id,
capability: crate::error::MobMemberCapability::InteractionEventInjector,
context: "autonomous member dispatch",
} if member_id.as_str() == "l-1"
),
"unexpected error: {error}"
);
assert!(
handle.list_members().await.is_empty(),
"member must not be finalized/listed when console/RPC injection cannot work"
);
let events = handle.events().replay_all().await.expect("replay events");
assert!(
!events
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberSpawned(_))),
"failed autonomous spawn must not emit MemberSpawned"
);
assert_eq!(
service.inject_calls.load(Ordering::Relaxed),
0,
"failure should happen during capability validation, not at delivery time"
);
}
#[tokio::test]
async fn test_mob_session_service_subscribe_session_events_available() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let stream_result =
crate::runtime::session_service::MobSessionService::subscribe_session_events(
service.as_ref(),
&sid,
)
.await;
assert!(
stream_result.is_ok(),
"expected session-wide event stream contract from mob session service"
);
}
#[test]
fn test_runtime_owner_is_derived_from_machine_placement_before_session_shape() {
use crate::machines::mob_machine as mob_dsl;
let identity = AgentIdentity::from("placed-owner-predicate");
let dsl_identity = mob_dsl::AgentIdentity::from_domain(&identity);
let mut state = mob_dsl::MobMachineAuthority::new().state().clone();
// `Some(session_id)` alone is not local ownership: host-materialized refs
// intentionally project their remote resident session for fencing.
state.member_session_bindings.insert(
dsl_identity.clone(),
mob_dsl::SessionId("host-resident-session".to_string()),
);
assert!(
!super::member_runtime_is_host_owned(&state, &identity),
"a session binding without placement remains controller-owned"
);
state.member_placement.insert(
dsl_identity.clone(),
mob_dsl::HostId("member-host".to_string()),
);
assert!(
super::member_runtime_is_host_owned(&state, &identity),
"placement must override the Some(session_id) transport shape"
);
state.member_placement.remove(&dsl_identity);
assert!(
!super::member_runtime_is_host_owned(&state, &identity),
"removing placement returns ownership to the controller lane"
);
}
#[tokio::test]
async fn test_member_reads_bypass_saturated_actor_command_queue() {
use crate::machines::mob_machine as mob_dsl;
let (handle, _service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("w-observe");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(identity.as_str()),
None,
)
.await
.expect("spawn observable worker");
let session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("session-backed worker");
let (blocked_tx, _blocked_rx) = tokio::sync::mpsc::channel(1);
let (reply_tx, _reply_rx) = tokio::sync::oneshot::channel();
blocked_tx
.try_send(super::scope_gate::RoutedMobCommand::internal(
super::state::MobCommand::Retire {
agent_identity: AgentIdentity::from("queued-command"),
reply_tx,
},
))
.expect("test command channel should accept first command");
let blocked_handle = MobHandle {
command_tx: blocked_tx,
..handle.clone()
};
let machine_projected =
tokio::time::timeout(Duration::from_millis(50), handle.list_members()).await;
assert!(
machine_projected.is_ok(),
"control handle should remain healthy before swapping in blocked command queue"
);
let machine_projected =
tokio::time::timeout(Duration::from_millis(50), blocked_handle.list_members()).await;
let projected_members =
machine_projected.expect("machine-projected list_members must not wait for actor queue");
let projected_member = projected_members
.iter()
.find(|member| member.agent_identity == identity)
.expect("machine projection should include the active member");
assert_eq!(
projected_member.current_bridge_session_id.as_ref(),
Some(&session_id)
);
assert_eq!(
blocked_handle.status_observation_snapshot(),
MobState::Running,
"observation phase snapshot must not wait for the actor queue"
);
let observed = tokio::time::timeout(
Duration::from_millis(50),
blocked_handle.list_members_observation_snapshot(),
)
.await
.expect("observation snapshot must not wait for the actor queue");
let observed_member = observed
.iter()
.find(|member| member.agent_identity == identity)
.expect("observation snapshot should include the active member");
assert_eq!(
observed_member.current_bridge_session_id.as_ref(),
Some(&session_id)
);
let observed_session = tokio::time::timeout(
Duration::from_millis(50),
blocked_handle.resolve_bridge_session_id_observation(&identity),
)
.await
.expect("observation session lookup must not wait for the actor queue");
assert_eq!(observed_session.as_ref(), Some(&session_id));
let _stream = tokio::time::timeout(
Duration::from_millis(50),
blocked_handle.subscribe_agent_events_observation(&identity),
)
.await
.expect("observation event subscription must not wait for the actor queue")
.expect("session-backed member should expose an event stream");
let mut placed_state = blocked_handle.machine_state_watch_rx.borrow().clone();
placed_state.member_placement.insert(
mob_dsl::AgentIdentity::from_domain(&identity),
mob_dsl::HostId("observation-remote-host".to_string()),
);
let (_placed_state_tx, placed_state_rx) = tokio::sync::watch::channel(placed_state);
let placed_observation_handle = MobHandle {
machine_state_watch_rx: placed_state_rx,
..blocked_handle
};
let placed_result = tokio::time::timeout(
Duration::from_millis(50),
placed_observation_handle.subscribe_agent_events_observation(&identity),
)
.await
.expect("placed observation rejection must not wait for the actor queue");
assert!(
matches!(
placed_result,
Err(MobError::UnsupportedForMode { reason, .. })
if reason.contains("actor-owned external event pump")
),
"placed observation helper must not treat a host-resident session id as local"
);
}
#[tokio::test]
async fn test_queued_steer_during_running_turn_does_not_block_actor_commands() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(600_000);
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("lead-busy"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
handle
.member(&AgentIdentity::from("lead-busy"))
.await
.expect("lead member")
.send_with_render_metadata(
ContentInput::Text("hold the current turn open".into()),
HandlingMode::Queue,
None,
)
.await
.expect("first turn should be admitted");
tokio::time::timeout(Duration::from_secs(2), async {
while service.start_turn_call_count() == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("runtime should start the long-running turn");
let steer_member = handle
.member(&AgentIdentity::from("lead-busy"))
.await
.expect("lead member for steer");
let steer_task = tokio::spawn(async move {
steer_member
.send_with_render_metadata(
ContentInput::Text("queued steer while busy".into()),
HandlingMode::Steer,
None,
)
.await
});
tokio::time::sleep(Duration::from_millis(50)).await;
tokio::time::timeout(
Duration::from_millis(250),
handle.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("worker-after-steer"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
),
)
.await
.expect("queued steer admission must not park the mob actor")
.expect("unrelated actor command should still be processed");
steer_task.abort();
}
#[tokio::test]
async fn test_turn_completed_submission_does_not_block_actor_commands() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(600_000);
handle
.spawn_with_options(
ProfileName::from("lead"),
AgentIdentity::from("lead-completed-busy"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let turn_handle = handle.clone();
let turn_task = tokio::spawn(async move {
turn_handle
.internal_turn_for_member(
AgentIdentity::from("lead-completed-busy"),
ContentInput::Text("hold turn-completed submission open".into()),
)
.await
});
tokio::time::timeout(Duration::from_secs(2), async {
while service.start_turn_call_count() == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("runtime should start the long-running turn");
tokio::time::timeout(
Duration::from_millis(250),
handle.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from("worker-after-turn-completed"),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
),
)
.await
.expect("turn-completed submission must not park the mob actor")
.expect("unrelated actor command should still be processed");
turn_task.abort();
}
#[tokio::test]
async fn test_internal_queued_submit_work_drains_after_running_turn() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("lead-internal-queue-drain");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let entry = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("member exists");
service.set_start_turn_delay_ms(500);
handle
.submit_work_with_mode(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new(
"first internal queued turn".to_string(),
WorkOrigin::Internal,
),
HandlingMode::Queue,
)
.await
.expect("first internal queued turn should be admitted");
wait_for_start_turn_call_count(
&service,
1,
"first internal queued turn should start running",
)
.await;
handle
.submit_work_with_mode(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new(
"second internal queued turn".to_string(),
WorkOrigin::Internal,
),
HandlingMode::Queue,
)
.await
.expect("second internal queued turn should be admitted while first is running");
wait_for_start_turn_call_count(
&service,
2,
"second internal queued turn should drain after the running turn completes",
)
.await;
let prompts: Vec<String> = service
.recorded_start_turn_prompts()
.await
.into_iter()
.map(|(_, prompt)| prompt)
.collect();
assert!(
prompts
.iter()
.any(|prompt| prompt == "first internal queued turn"),
"first prompt should be delivered to runtime: {prompts:?}"
);
assert!(
prompts
.iter()
.any(|prompt| prompt == "second internal queued turn"),
"second prompt should drain to runtime after first completes: {prompts:?}"
);
}
/// Ask 1 (design item 7), turn-driven work lane: injected context on a
/// `WorkSpec` reaches the member's turn request as the user-channel shape
/// `[injected..., work content]`, in delivery order.
#[tokio::test]
async fn test_turn_driven_submit_work_delivers_injected_context_before_work_content() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("worker-injected-context");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("summarize the findings".to_string(), WorkOrigin::Internal)
.with_injected_context(vec![
meerkat_core::types::ContentInput::Text("ambient alpha".to_string()),
meerkat_core::types::ContentInput::Text("ambient beta".to_string()),
]),
)
.await
.expect("submit work with injected context");
wait_for_start_turn_call_count(
&service,
1,
"turn-driven work with injected context should start a turn",
)
.await;
let records = service.recorded_start_turn_user_channel().await;
let (_, user_channel) = records.last().expect("one turn request recorded");
assert_eq!(
user_channel,
&vec![
(true, "ambient alpha".to_string()),
(true, "ambient beta".to_string()),
(false, "summarize the findings".to_string()),
],
"injected context must reach the member turn request as typed \
injected-context entries before the work content, in order"
);
}
#[tokio::test]
async fn test_turn_driven_submit_work_allows_system_prompt_updates_between_turns() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("worker-system-prompt");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let baseline_start_turn_calls = service.start_turn_call_count();
let baseline_recorded_prompts = service.recorded_start_turn_system_prompts().await.len();
handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("first turn", WorkOrigin::Internal)
.with_system_prompt("first instructions"),
)
.await
.expect("submit first per-turn System message");
wait_for_start_turn_call_count(
&service,
baseline_start_turn_calls + 1,
"first System-bearing work should start",
)
.await;
handle
.submit_work(
entry.agent_runtime_id,
entry.fence_token,
WorkRef::new(),
WorkSpec::new("second turn", WorkOrigin::Internal)
.with_system_prompt("updated instructions"),
)
.await
.expect("submit updated per-turn System message");
wait_for_start_turn_call_count(
&service,
baseline_start_turn_calls + 2,
"updated System-bearing work should start",
)
.await;
let recorded = service.recorded_start_turn_system_prompts().await;
let prompts = recorded[baseline_recorded_prompts..]
.iter()
.map(|(_, prompts)| prompts.clone())
.collect::<Vec<_>>();
let recorded_turns = service.recorded_start_turn_prompts().await;
assert_eq!(
prompts,
[
vec!["first instructions".to_string()],
vec!["updated instructions".to_string()]
],
"System content is authored per turn and must not become immutable member configuration; observed turn prompts: {recorded_turns:?}"
);
}
/// Ask-15 addendum (turn-driven): a host-supplied `WorkSpec.interaction_id`
/// is stamped into the delivered turn's
/// `RuntimeTurnMetadata.transcript_identity`, so the runner persists it onto
/// the committed transcript messages and history backfill can join them with
/// the host's live interaction frames.
#[tokio::test]
async fn test_turn_driven_submit_work_interaction_id_stamps_transcript_identity() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("worker-interaction-id");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let interaction_id = InteractionId(uuid::Uuid::new_v4());
handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("summarize the findings".to_string(), WorkOrigin::Internal)
.with_interaction_id(interaction_id),
)
.await
.expect("submit work with interaction id");
wait_for_start_turn_call_count(
&service,
1,
"turn-driven work with an interaction id should start a turn",
)
.await;
let records = service.recorded_start_turn_metadata().await;
let (_, metadata) = records.last().expect("one turn request recorded");
let metadata = metadata
.as_ref()
.expect("submit-work delivery must carry runtime turn metadata");
assert_eq!(
metadata.transcript_identity.interaction_id,
Some(interaction_id),
"the host-supplied interaction id must ride the turn's transcript identity"
);
assert_eq!(
metadata.transcript_identity.run_id, None,
"the concrete run id is stamped later by the runner, not the work lane"
);
}
#[tokio::test]
async fn test_turn_driven_submit_work_objective_id_stamps_transcript_identity() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("worker-objective-id");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let objective_id = meerkat_core::interaction::ObjectiveId::new();
handle
.submit_work(
entry.agent_runtime_id,
entry.fence_token,
WorkRef::new(),
WorkSpec::new("delegated objective", WorkOrigin::Internal)
.with_objective_id(objective_id),
)
.await
.expect("submit objective-correlated work");
wait_for_start_turn_call_count(&service, 1, "objective work should start a turn").await;
let records = service.recorded_start_turn_metadata().await;
let metadata = records
.last()
.and_then(|(_, metadata)| metadata.as_ref())
.expect("objective delivery must carry turn metadata");
assert_eq!(
metadata.transcript_identity.objective_id,
Some(objective_id)
);
}
/// Ask-15 addendum (autonomous): a host-supplied `WorkSpec.interaction_id`
/// rides the injected inbox event, so the comms classification (and the
/// runtime transcript identity derived from it) carries the SAME id as the
/// host's live interaction frames instead of a fresh unrelated one.
#[tokio::test]
async fn test_autonomous_submit_work_interaction_id_reaches_inbox_injection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("lead-interaction-id");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous lead");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let interaction_id = InteractionId(uuid::Uuid::new_v4());
handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("do the thing".to_string(), WorkOrigin::External)
.with_interaction_id(interaction_id),
)
.await
.expect("submit keyed autonomous work");
assert_eq!(
service.recorded_injected_interaction_ids(),
vec![interaction_id],
"the autonomous inbox delivery must carry the host-supplied interaction id"
);
}
/// Ask 1 (design item 7), autonomous decision: the autonomous inbox path
/// flows through comms plain events (no user-channel work boundary), so
/// non-empty injected context is rejected fail-closed with a typed error —
/// never silently dropped.
#[tokio::test]
async fn test_autonomous_submit_work_with_injected_context_rejected() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("lead-injected-context");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous lead");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let baseline_injects = service.inject_call_count();
let err = handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("do the thing".to_string(), WorkOrigin::External).with_injected_context(
vec![meerkat_core::types::ContentInput::Text(
"ambient".to_string(),
)],
),
)
.await
.expect_err("autonomous inbox delivery cannot carry injected context");
assert!(
matches!(err, MobError::InjectedContextUndeliverable { .. }),
"expected typed InjectedContextUndeliverable, got {err:?}"
);
assert_eq!(
service.inject_call_count(),
baseline_injects,
"rejected work must not be partially injected"
);
}
#[tokio::test]
async fn test_autonomous_submit_work_with_system_prompt_rejected_before_injection() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("lead-system-prompt");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::AutonomousHost),
None,
)
.await
.expect("spawn autonomous lead");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let baseline_injects = service.inject_call_count();
let err = handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("do the thing", WorkOrigin::External)
.with_system_prompt("updated member instructions"),
)
.await
.expect_err("autonomous inbox delivery cannot carry ordinary System content");
assert!(
matches!(
&err,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::AutonomousHost,
..
}
),
"expected typed AutonomousHost rejection, got {err:?}"
);
assert!(
err.to_string()
.contains("no admitted turn boundary for ordinary System content"),
"rejection must name the unrepresentable System boundary: {err}"
);
assert_eq!(
service.inject_call_count(),
baseline_injects,
"rejected work must not be partially injected"
);
}
/// Ask 1 (design item 7), steer decision: steer dispatch realizes as live
/// system-context appends (no transcript boundary), so injected context on a
/// steer-mode work submission is rejected fail-closed with a typed error.
#[tokio::test]
async fn test_steer_submit_work_with_injected_context_rejected() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("worker-steer-injected");
handle
.spawn_with_options(
ProfileName::from("worker"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let entry = handle
.get_member(&member_id)
.await
.unwrap()
.expect("member exists");
let err = handle
.submit_work_with_mode(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("urgent update".to_string(), WorkOrigin::Internal).with_injected_context(
vec![meerkat_core::types::ContentInput::Text(
"ambient".to_string(),
)],
),
HandlingMode::Steer,
)
.await
.expect_err("steer dispatch cannot carry injected context");
assert!(
matches!(err, MobError::InjectedContextUndeliverable { .. }),
"expected typed InjectedContextUndeliverable, got {err:?}"
);
}
static REAL_COMMS_TEST_LOCK: Mutex<()> = Mutex::new(());
fn lock_real_comms_tests() -> std::sync::MutexGuard<'static, ()> {
// This mutex only serializes access to the process-global inproc registry.
// A prior test panic does not corrupt registry state owned by this guard,
// so retain serialization without masking every later assertion.
REAL_COMMS_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[tokio::test]
async fn test_wire_enables_peer_request_delivery() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire should succeed");
let comms_a = service.real_comms(&sid_a).await.expect("comms for l-1");
let comms_b = service.real_comms(&sid_b).await.expect("comms for w-1");
let entry_a = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("l-1 should be in roster");
let entry_b = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("w-1 should be in roster");
let comms_name_a = format!(
"{}/{}/{}",
handle.mob_id(),
entry_a.role,
entry_a.agent_identity
);
let comms_name_b = format!(
"{}/{}/{}",
handle.mob_id(),
entry_b.role,
entry_b.agent_identity
);
let peers_a = CoreCommsRuntime::peers(&*comms_a).await;
let peers_b = CoreCommsRuntime::peers(&*comms_b).await;
assert!(
peers_a
.iter()
.any(|entry| entry.name.as_str() == comms_name_b),
"wire should expose worker in lead peers()"
);
assert!(
peers_b
.iter()
.any(|entry| entry.name.as_str() == comms_name_a),
"wire should expose lead in worker peers()"
);
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_a).await;
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_b).await;
let cmd = meerkat_core::comms::CommsCommand::PeerRequest {
objective_id: None,
content_taint: None,
to: test_peer_route(&*comms_a, &comms_name_b).await,
intent: "mob.test_ping".to_string(),
params: serde_json::json!({"test": true}),
blocks: None,
handling_mode: meerkat_core::types::HandlingMode::Queue,
stream: meerkat_core::comms::InputStreamMode::None,
};
let receipt = CoreCommsRuntime::send(&*comms_a, cmd)
.await
.expect("PeerRequest from l-1 to w-1 should succeed after wire()");
assert!(
matches!(
receipt,
meerkat_core::comms::SendReceipt::PeerRequestSent { .. }
),
"expected PeerRequestSent, got: {receipt:?}"
);
let interactions = CoreCommsRuntime::drain_inbox_interactions(&*comms_b).await;
assert_eq!(
interactions.len(),
1,
"w-1 should have received exactly one interaction after wire"
);
match &interactions[0].content {
meerkat_core::InteractionContent::Request { intent, params, .. } => {
assert_eq!(intent, "mob.test_ping");
assert_eq!(params["test"], true);
}
other => panic!("expected Request interaction, got: {other:?}"),
}
}
#[tokio::test]
async fn test_unwire_updates_peers_and_sends_retired_notifications() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let comms_a = service.real_comms(&sid_a).await.expect("comms for l-1");
let comms_b = service.real_comms(&sid_b).await.expect("comms for w-1");
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_a).await;
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_b).await;
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("unwire");
let peers_a = CoreCommsRuntime::peers(&*comms_a).await;
let peers_b = CoreCommsRuntime::peers(&*comms_b).await;
assert!(
!peers_a
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-1")),
"unwire should remove worker from lead peers()"
);
assert!(
!peers_b
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("lead", "l-1")),
"unwire should remove lead from worker peers()"
);
}
#[tokio::test]
async fn test_stale_member_trust_obligation_cannot_readd_local_trust_when_machine_edge_absent() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let stale_machine_state = handle
.query_machine_state()
.await
.expect("query stale machine state after wire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("initial unwire");
let comms_a = service.real_comms(&sid_a).await.expect("comms for l-1");
let comms_b = service.real_comms(&sid_b).await.expect("comms for w-1");
let name_a = test_comms_name("lead", "l-1");
let name_b = test_comms_name("worker", "w-1");
let key_b = comms_b.public_key();
let peer_id_b = key_b.to_peer_id().to_string();
let stale_edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity("l-1".to_string()),
crate::machines::mob_machine::AgentIdentity("w-1".to_string()),
);
let mut stale_authority =
crate::machines::mob_machine::MobMachineAuthority::recover_from_state(stale_machine_state)
.expect("recover stale wired machine state");
let stale_transition = crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::AuthorizeMemberTrustWiring {
edge: stale_edge.clone(),
a_identity: stale_edge.a.clone(),
b_identity: stale_edge.b.clone(),
},
)
.expect("authorize stale member trust wiring");
let raw_stale_obligation =
crate::generated::protocol_mob_member_trust_wiring::extract_obligations(&stale_transition)
.pop()
.expect("generated member wiring obligation without freshness");
let raw_error =
crate::generated::protocol_mob_member_trust_wiring::wiring_authority_for_identity(
&raw_stale_obligation,
"w-1",
&peer_id_b,
)
.expect_err("raw stale member wiring obligation must fail closed");
assert!(
raw_error.contains("requires live authority validation"),
"unexpected raw stale member wiring error: {raw_error}"
);
let current_machine_state = handle
.query_machine_state()
.await
.expect("query current machine state after unwire");
let current_authority = crate::machines::mob_machine::MobMachineAuthority::recover_from_state(
current_machine_state,
)
.expect("recover current unwired machine state");
let current_topology_epoch = Arc::new(AtomicU64::new(current_authority.state().topology_epoch));
let stale_obligation =
crate::generated::protocol_mob_member_trust_wiring::extract_obligations_with_freshness(
&stale_transition,
crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(
current_topology_epoch,
current_authority.generated_authority_owner_token(),
),
)
.pop()
.expect("generated stale member wiring obligation");
let stale_error =
crate::generated::protocol_mob_member_trust_wiring::wiring_authority_for_identity_with_live_authority(
&stale_obligation,
"w-1",
&peer_id_b,
¤t_authority,
)
.expect_err("stale member wiring obligation must fail closed");
assert!(
stale_error.contains("stale generated MobMachine trust obligation"),
"unexpected stale member wiring error: {stale_error}"
);
let peers_a = CoreCommsRuntime::peers(&*comms_a).await;
let peers_b = CoreCommsRuntime::peers(&*comms_b).await;
assert!(
!peers_a.iter().any(|entry| entry.name.as_str() == name_b),
"stale member obligation must not re-add worker trust on lead"
);
assert!(
!peers_b.iter().any(|entry| entry.name.as_str() == name_a),
"stale member obligation must not re-add lead trust on worker"
);
}
#[tokio::test]
async fn test_member_trust_mutation_rejects_foreign_mob_owner() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let comms_a = service.real_comms(&sid_a).await.expect("comms for l-1");
let comms_b = service.real_comms(&sid_b).await.expect("comms for w-1");
let foreign_state = handle
.query_machine_state()
.await
.expect("query wired machine state");
let mut foreign_authority =
crate::machines::mob_machine::MobMachineAuthority::recover_from_state(foreign_state)
.expect("recover foreign authority over equivalent state");
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity("l-1".to_string()),
crate::machines::mob_machine::AgentIdentity("w-1".to_string()),
);
let transition = crate::machines::mob_machine::MobMachineMutator::apply(
&mut foreign_authority,
crate::machines::mob_machine::MobMachineInput::AuthorizeMemberTrustWiring {
edge: edge.clone(),
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
},
)
.expect("foreign authority can produce a generated handoff for its own owner");
let topology_epoch = Arc::new(AtomicU64::new(foreign_authority.state().topology_epoch));
let obligation =
crate::generated::protocol_mob_member_trust_wiring::extract_obligations_with_freshness(
&transition,
crate::generated::protocol_mob_member_trust_wiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(
topology_epoch,
foreign_authority.generated_authority_owner_token(),
),
)
.pop()
.expect("foreign generated member wiring obligation");
let peer = TrustedPeerDescriptor::unsigned_with_pubkey(
test_comms_name("worker", "w-1"),
comms_b.peer_id().expect("worker peer id").to_string(),
comms_b.public_key_bytes().expect("worker pubkey bytes"),
CoreCommsRuntime::advertised_address(&*comms_b).expect("worker advertised address"),
)
.expect("valid worker descriptor");
let peer_id = peer.peer_id.to_string();
let authority =
crate::generated::protocol_mob_member_trust_wiring::wiring_authority_for_identity_with_live_authority(
&obligation,
"w-1",
&peer_id,
&foreign_authority,
)
.expect("foreign generated authority");
let error = CoreCommsRuntime::apply_trust_mutation(
&*comms_a,
CommsTrustMutation::AddTrustedPeer { peer, authority },
)
.await
.expect_err("foreign MobMachine owner must not mutate target trust");
assert!(
matches!(error, SendError::Validation(ref message) if message.contains("different generated owner")),
"unexpected foreign-owner rejection: {error:?}"
);
}
#[tokio::test]
async fn test_stale_external_peer_trust_obligation_cannot_readd_trust_when_machine_edge_absent() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let external_keypair = meerkat_comms::Keypair::generate();
let external_pubkey = external_keypair.public_key();
let spec = TrustedPeerDescriptor::unsigned_with_pubkey(
"remote-mob/worker/agent-x",
external_pubkey.to_peer_id().to_string(),
*external_pubkey.as_bytes(),
"inproc://remote-mob/worker/agent-x",
)
.expect("valid external peer");
handle
.wire(
AgentIdentity::from("l-1"),
PeerTarget::External(spec.clone()),
)
.await
.expect("wire external");
handle
.unwire(
AgentIdentity::from("l-1"),
PeerTarget::External(spec.clone()),
)
.await
.expect("initial external unwire");
let comms = service.real_comms(&sid).await.expect("comms for l-1");
let peer_id = spec.peer_id.to_string();
let stale_edge = crate::machines::mob_machine::ExternalPeerEdge::new(
crate::machines::mob_machine::AgentIdentity("l-1".to_string()),
crate::machines::mob_machine::ExternalPeerEndpoint::from(&spec),
);
let stale_key = crate::machines::mob_machine::ExternalPeerKey::new(
stale_edge.local.clone(),
stale_edge.endpoint.name.clone(),
);
let mut stale_authority = crate::machines::mob_machine::MobMachineAuthority::new();
let local_peer_id = comms.peer_id().expect("local comms peer id");
let local_pubkey = comms.public_key_bytes().expect("local comms pubkey bytes");
let local_name = test_comms_name("lead", "l-1");
let local_descriptor = TrustedPeerDescriptor::unsigned_with_pubkey(
local_name.clone(),
local_peer_id.to_string(),
local_pubkey,
comms.advertised_address(),
)
.expect("valid local trusted spec");
crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: stale_edge.local.clone(),
profile_name: "lead".to_string(),
model: "test-model".to_string(),
profile_material_digest: "stale-profile-digest".to_string(),
tool_config_digest: "stale-tool-config-digest".to_string(),
skills_digest: "stale-skills-digest".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: true,
resolved_spec_digest: None,
},
)
.expect("authorize stale local member");
crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::BeginSpawnExec {
agent_identity: stale_edge.local.clone(),
agent_runtime_id: crate::machines::mob_machine::AgentRuntimeId(
"stale-l-1-runtime".to_string(),
),
fence_token: crate::machines::mob_machine::FenceToken(1),
generation: crate::machines::mob_machine::Generation(0),
profile_material_digest: "stale-profile-digest".to_string(),
external_addressable: true,
runtime_mode: crate::machines::mob_machine::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(crate::machines::mob_machine::SessionId(
"stale-l-1-session".to_string(),
)),
replacing: None,
placement: None,
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: false,
default_llm_client_override_present: false,
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
},
)
.expect("begin spawn exec for stale local member");
crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::CommitSpawnMembership {
agent_identity: stale_edge.local.clone(),
agent_runtime_id: crate::machines::mob_machine::AgentRuntimeId(
"stale-l-1-runtime".to_string(),
),
fence_token: crate::machines::mob_machine::FenceToken(1),
generation: crate::machines::mob_machine::Generation(0),
profile_material_digest: "stale-profile-digest".to_string(),
external_addressable: true,
runtime_mode: crate::machines::mob_machine::SpawnPolicyRuntimeMode::AutonomousHost,
bridge_session_id: Some(crate::machines::mob_machine::SessionId(
"stale-l-1-session".to_string(),
)),
replacing: None,
member_peer_endpoint: None,
spec_digest_echo: None,
ack_engine_version: None,
placed_spawn_id: None,
provision_operation_id: None,
},
)
.expect("spawn stale local member");
crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::RegisterMemberPeer {
agent_identity: stale_edge.local.clone(),
agent_runtime_id: crate::machines::mob_machine::AgentRuntimeId(
"stale-l-1-runtime".to_string(),
),
generation: crate::machines::mob_machine::Generation(0),
fence_token: crate::machines::mob_machine::FenceToken(1),
peer_endpoint: crate::machines::mob_machine::MemberPeerEndpoint::from(
&local_descriptor,
),
},
)
.expect("register stale local member peer");
let stale_transition = crate::machines::mob_machine::MobMachineMutator::apply(
&mut stale_authority,
crate::machines::mob_machine::MobMachineInput::WireExternalPeer {
key: stale_key,
edge: stale_edge.clone(),
},
)
.expect("wire stale external peer");
let raw_stale_obligation =
crate::generated::protocol_mob_external_peer_trust_wiring::extract_obligations(
&stale_transition,
)
.pop()
.expect("generated external wiring obligation without freshness");
let raw_error =
crate::generated::protocol_mob_external_peer_trust_wiring::wiring_authority_for_peer(
&raw_stale_obligation,
&peer_id,
)
.expect_err("raw stale external wiring obligation must fail closed");
assert!(
raw_error.contains("topology freshness authority is absent"),
"unexpected raw stale external wiring error: {raw_error}"
);
let current_machine_state = handle
.query_machine_state()
.await
.expect("query current machine state after external unwire");
let current_authority = crate::machines::mob_machine::MobMachineAuthority::recover_from_state(
current_machine_state,
)
.expect("recover current external-unwired machine state");
let current_topology_epoch = Arc::new(AtomicU64::new(current_authority.state().topology_epoch));
let stale_obligation =
crate::generated::protocol_mob_external_peer_trust_wiring::extract_obligations_with_freshness(
&stale_transition,
crate::generated::protocol_mob_external_peer_trust_wiring::MobTopologyFreshnessAuthority::from_live_topology_epoch(
current_topology_epoch,
current_authority.generated_authority_owner_token(),
),
)
.pop()
.expect("generated stale external wiring obligation");
let stale_error =
crate::generated::protocol_mob_external_peer_trust_wiring::wiring_authority_for_peer(
&stale_obligation,
&peer_id,
)
.expect_err("stale external wiring obligation must fail closed");
assert!(
stale_error.contains("stale generated MobMachine trust obligation"),
"unexpected stale external wiring error: {stale_error}"
);
let peers = CoreCommsRuntime::peers(&*comms).await;
assert!(
!peers
.iter()
.any(|entry| entry.name.as_str() == spec.name.as_str()),
"stale external obligation must not re-add external trust"
);
}
#[tokio::test]
async fn test_retire_updates_peers_and_sends_retired_notifications() {
let _serial = lock_real_comms_tests();
let (handle, service) = create_test_mob_with_real_comms(sample_definition()).await;
let sid_a = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_b = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
let comms_a = service.real_comms(&sid_a).await.expect("comms for w-1");
let comms_b = service.real_comms(&sid_b).await.expect("comms for w-2");
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_a).await;
let _ = CoreCommsRuntime::drain_inbox_interactions(&*comms_b).await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"retired meerkat should leave roster"
);
let entry_w2 = handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.expect("w-2 remains active");
assert!(
!entry_w2.wired_to.contains(&AgentIdentity::from("w-1")),
"retire should remove wiring from remaining peer"
);
let peers_w2 = CoreCommsRuntime::peers(&*comms_b).await;
assert!(
!peers_w2
.iter()
.any(|entry| entry.name.as_str() == test_comms_name("worker", "w-1")),
"retire should remove retired peer from peers()"
);
}
#[tokio::test]
async fn test_unwire_sends_required_peer_unwired_notifications() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_l = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
handle
.unwire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("unwire");
let intents_lead = service.sent_intents(&sid_l).await;
let intents_worker = service.sent_intents(&sid_w).await;
assert!(
intents_lead
.iter()
.any(|intent| intent == "mob.peer_unwired"),
"lead should send mob.peer_unwired during unwire"
);
assert!(
intents_worker
.iter()
.any(|intent| intent == "mob.peer_unwired"),
"worker should send mob.peer_unwired during unwire"
);
}
#[tokio::test]
async fn test_retire_sends_required_peer_retired_notifications() {
let (handle, service) = create_test_mob(sample_definition()).await;
let sid_w1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1")
.bridge_session_id()
.expect("session-backed")
.clone();
let sid_w2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2")
.bridge_session_id()
.expect("session-backed")
.clone();
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire");
let intents_w1 = service.sent_intents(&sid_w1).await;
let intents_w2 = service.sent_intents(&sid_w2).await;
assert!(
intents_w1.iter().any(|intent| intent == "mob.peer_retired"),
"retiring peer should send mob.peer_retired notifications"
);
assert!(
!intents_w2.iter().any(|intent| intent == "mob.peer_retired"),
"non-retiring peer should not be the sender for retire notifications"
);
}
// -----------------------------------------------------------------------
// Disposal pipeline integration tests
// -----------------------------------------------------------------------
/// Verifies that critical archive failure is surfaced while the non-routable
/// retry anchor is retained even when best-effort peer cleanup also fails.
#[tokio::test]
async fn test_dispose_member_retains_roster_when_archive_fails() {
let (handle, service) = create_test_mob(sample_definition()).await;
// Configure w-1's comms to fail both send and trust removal.
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_send_peer_retired: true,
fail_remove_trust: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
// Archive also fails.
let sid_w1 = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.unwrap()
.bridge_session_id()
.cloned()
.unwrap();
service.set_archive_failure(&sid_w1).await;
// Archive failure is critical — retire must surface it even when comms
// steps also failed (comms failures are still best-effort).
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
result.is_err(),
"retire must return Err when ArchiveSession fails, regardless of other step failures"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"critical archive failure must retain the retirement retry anchor"
);
}
/// Verifies that retiring two identical-topology members produces the same
/// disposal step sequence (deterministic ordering).
#[tokio::test]
async fn test_disposal_report_ordering_is_deterministic() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
// Retire both in sequence — same topology, same cleanup steps.
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire w-1");
handle
.retire(AgentIdentity::from("w-2"))
.await
.expect("retire w-2");
// Both members removed — disposal executed identically.
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none()
);
assert!(
handle
.get_member(&AgentIdentity::from("w-2"))
.await
.unwrap()
.is_none()
);
// Verify events show two retirements in order.
let events = handle.events().replay_all().await.expect("replay");
let retire_events: Vec<_> = events
.iter()
.filter(|e| matches!(e.kind, MobEventKind::MemberRetired { .. }))
.collect();
assert_eq!(
retire_events.len(),
2,
"both members should have retire events"
);
}
// -----------------------------------------------------------------------
// MobHandle::reset()
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_reset_clears_roster_events_and_returns_to_running() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle.reset().await.expect("reset");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
assert!(
handle.list_members().await.is_empty(),
"reset should retire all members"
);
let events = handle.events().replay_all().await.expect("replay");
// Append-only: old epoch events remain, new epoch ends with MobReset.
let len = events.len();
assert!(len >= 1, "should have at least MobReset");
assert!(
matches!(events[len - 1].kind, MobEventKind::MobReset),
"last event should be MobReset"
);
// Roster projection after full replay should be empty (MobReset clears it).
let roster = crate::roster::Roster::project(&events);
assert!(
roster.list().next().is_none(),
"roster projection should be empty after reset epoch"
);
}
#[tokio::test]
async fn test_reset_allows_spawn_after_reset() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle.reset().await.expect("reset");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn after reset should succeed");
assert_eq!(handle.list_members().await.len(), 1);
}
#[tokio::test]
async fn test_reset_from_stopped_state() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle.stop().await.expect("stop");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
handle.reset().await.expect("reset from stopped");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
}
#[tokio::test]
async fn test_reset_from_completed_state() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle.complete().await.expect("complete");
assert_eq!(handle.status().await.unwrap(), MobState::Completed);
handle.reset().await.expect("reset from completed");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
}
#[tokio::test]
async fn test_reset_rejects_from_destroyed() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle.destroy().await.expect("destroy");
assert_eq!(handle.status().await.unwrap(), MobState::Destroyed);
let err = handle
.reset()
.await
.expect_err("reset after destroy must fail");
assert!(
matches!(err, MobError::ActorCommandChannelClosed),
"reset after terminal destroy should fail because the actor is gone: {err:?}"
);
}
#[tokio::test]
async fn test_structural_roster_reads_round_trip_through_machine_command_surface() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let roster = handle.roster().await;
assert!(
roster.get(&AgentIdentity::from("w-1")).is_some(),
"machine-routed roster snapshot should include spawned member"
);
let all_members = handle.list_all_members().await;
assert_eq!(all_members.len(), 1);
assert_eq!(all_members[0].agent_identity, AgentIdentity::from("w-1"));
let active_members = handle.list_members().await;
assert_eq!(active_members.len(), 1);
assert_eq!(active_members[0].agent_identity, AgentIdentity::from("w-1"));
let including_retiring = handle.list_members_including_retiring().await;
assert_eq!(including_retiring.len(), 1);
assert_eq!(
including_retiring[0].agent_identity,
AgentIdentity::from("w-1")
);
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
assert_eq!(entry.agent_identity, AgentIdentity::from("w-1"));
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect("retire worker");
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"machine-routed get_member should reflect retirement"
);
}
#[tokio::test]
async fn test_member_status_round_trips_through_machine_command_surface() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let snapshot = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status");
assert_eq!(snapshot.status, crate::runtime::MobMemberStatus::Active);
assert_eq!(
snapshot.current_bridge_session_id.as_ref(),
Some(receipt.bridge_session_id().expect("session-backed")),
"machine-routed member_status should preserve the active session binding"
);
}
#[tokio::test]
async fn test_busy_member_execution_snapshot_cannot_block_mob_lifecycle_commands() {
let (handle, service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-busy-status"),
None,
)
.await
.expect("spawn worker");
let objective_id = handle
.member_status(&AgentIdentity::from("w-busy-status"))
.await
.expect("initial member status")
.kickoff
.and_then(|kickoff| kickoff.objective_id)
.expect("spawned worker should have a kickoff objective");
service.set_execution_snapshot_delay_ms(5_000);
let status_handle = handle.clone();
let status_started_at = Instant::now();
let status_task = tokio::spawn(async move {
status_handle
.member_status(&AgentIdentity::from("w-busy-status"))
.await
});
tokio::time::timeout(
Duration::from_secs(1),
service.wait_for_execution_snapshot(),
)
.await
.expect("member status should begin the execution observation");
tokio::time::timeout(
Duration::from_secs(1),
handle.conclude_objective(
&AgentIdentity::from("w-busy-status"),
objective_id,
"worker completed while its status observation was busy",
),
)
.await
.expect("a busy status observation must not hold the mob command lane")
.expect("objective conclusion should succeed after the bounded observation");
let snapshot = tokio::time::timeout(Duration::from_secs(1), status_task)
.await
.expect("member status should return promptly")
.expect("member status task should not panic")
.expect("member status should return an observation");
assert!(
status_started_at.elapsed() < Duration::from_secs(1),
"member status must degrade to unknown instead of awaiting the busy session turn"
);
assert_eq!(
snapshot.progress.map(|progress| progress.run_state),
Some(crate::runtime::handle::MemberRunState::Unknown),
"a timed-out execution observation must be represented truthfully as unknown"
);
}
#[tokio::test]
async fn test_member_status_marks_current_missing_bridge_session_broken() {
let (handle, service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
service
.archive(&bridge_session_id)
.await
.expect("test should remove the live bridge session");
let snapshot = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status should still return a diagnostic snapshot");
assert_eq!(snapshot.status, crate::runtime::MobMemberStatus::Broken);
assert!(snapshot.is_final);
assert_eq!(snapshot.current_session_id, Some(bridge_session_id.clone()));
assert!(
matches!(
snapshot.peer_connectivity,
Some(meerkat_contracts::WirePeerConnectivity::NotApplicable)
),
"broken member without a live bridge-session comms runtime should report NotApplicable connectivity, got {:?}",
snapshot.peer_connectivity
);
assert!(
snapshot
.error
.as_deref()
.is_some_and(|message| message.contains("missing bridge session")),
"broken member should surface the missing-session reason: {snapshot:?}"
);
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
assert_eq!(
machine_state.member_lifecycle_for_identity(&machine_identity),
crate::machines::mob_machine::MobMemberLifecycleMaterial {
status: crate::machines::mob_machine::MobMemberLifecycleStatus::Broken,
terminal_class: crate::machines::mob_machine::MobMemberTerminalClass::TerminalFailure,
error: Some(format!(
"missing bridge session snapshot for '{bridge_session_id}'"
)),
},
"missing bridge-session classification must be owned by MobMachine state"
);
}
#[tokio::test]
async fn test_missing_bridge_session_stops_member_from_looking_runnable_after_status_probe() {
let (handle, service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
service
.archive(&bridge_session_id)
.await
.expect("test should remove the live bridge session");
let _ = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("status probe should record the missing session");
let listed = handle.list_members().await;
let entry = listed
.iter()
.find(|entry| entry.agent_identity == "w-1")
.expect("member remains visible for diagnostics");
assert_eq!(entry.status, crate::runtime::MobMemberStatus::Broken);
assert!(entry.is_final);
assert_eq!(entry.current_session_id, Some(bridge_session_id.clone()));
let error = match handle.member(&AgentIdentity::from("w-1")).await {
Err(error) => error,
Ok(member) => member
.internal_turn(ContentInput::from("should fail fast".to_string()))
.await
.expect_err("broken member must reject queued work"),
};
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
reason,
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(bridge_session_id));
assert!(
reason.contains("missing bridge session"),
"unexpected restore failure reason: {reason}"
);
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
}
#[tokio::test]
async fn test_submit_work_marks_missing_bridge_session_broken_without_prior_status_probe() {
let (handle, service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
let active = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("initial member status");
let (runtime_id, fence_token) = active
.runtime_identity_fields()
.expect("active member should have runtime identity");
let runtime_id = runtime_id.clone();
service
.archive(&bridge_session_id)
.await
.expect("test should remove the live bridge session");
let error = handle
.submit_work(
runtime_id,
fence_token,
WorkRef::new(),
WorkSpec::new(
ContentInput::from("dispatch should fail fast".to_string()),
WorkOrigin::Internal,
),
)
.await
.expect_err("submit_work should diagnose the missing bridge session");
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
reason,
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(bridge_session_id.clone()));
assert!(
reason.contains("missing bridge session"),
"unexpected restore failure reason: {reason}"
);
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
let after = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("post-failure member status");
assert_eq!(after.status, crate::runtime::MobMemberStatus::Broken);
assert_eq!(after.current_session_id, Some(bridge_session_id));
}
/// Task #37 regression: a fail-closed live-session discard (live runtime +
/// comms gone, durable snapshot intact, member still Active in MobMachine)
/// must not leave the member a zombie for direct dispatch. The next dispatch
/// feeds the typed live-materialization observation to MobMachine, which
/// authorizes a revival that rebuilds the live session through the existing
/// resume materialization path; the admitted turn then succeeds.
#[tokio::test]
async fn test_discarded_live_session_revived_by_machine_authorized_dispatch() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
// Production-shaped prompt history: automatic rematerialization must
// preserve every persisted System row, including byte-identical repeats.
// These are transcript events, not configuration entries to normalize.
let expected_before_revival = {
let mut persisted_sessions = service.persisted_sessions.write().await;
let persisted = persisted_sessions
.get_mut(&bridge_session_id)
.expect("durable bridge session");
for _ in 0..12 {
persisted.append_system_message("repeated runtime instruction".to_string());
}
let expected = persisted.messages().to_vec();
service
.live_session_data
.write()
.await
.insert(bridge_session_id.clone(), persisted.clone());
expected
};
// Model the #34 fail-closed discard: the live session (and with it the
// comms runtime) is dropped out from under MobMachine while the durable
// snapshot stays loadable and the member stays Active.
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"discard must remove the live materialization"
);
handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("come back online".to_string()))
.await
.expect("machine-authorized revival must rebuild the live session and dispatch the turn");
// The live session was re-materialized under the SAME machine-owned
// binding and the admitted turn actually ran on it.
assert!(
service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"revival must rebuild the live session under the unchanged binding"
);
let revived = service
.live_session_clone(&bridge_session_id)
.await
.expect("revived bridge session");
assert_eq!(
revived.messages(),
expected_before_revival,
"machine-authorized revival must preserve repeated System rows byte-for-byte and in order",
);
let turn_prompts = service.start_turn_prompts.read().await.clone();
assert!(
turn_prompts
.iter()
.any(|(session_id, prompt)| session_id == &bridge_session_id
&& prompt.contains("come back online")),
"the dispatched turn must run on the revived bridge session: {turn_prompts:?}"
);
// MobMachine state: member stays Active, no restore failure, and the
// revival obligation is resolved (not dangling).
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
let lifecycle = machine_state.member_lifecycle_for_identity(&machine_identity);
assert_eq!(
lifecycle.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Active,
"revived member must remain machine-owned Active: {lifecycle:?}"
);
assert!(
!machine_state
.member_revival_pending
.contains(&machine_identity),
"successful revival must resolve the machine-owned revival obligation"
);
let snapshot = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status");
assert_eq!(snapshot.status, crate::runtime::MobMemberStatus::Active);
}
/// Delivery-time MissingLive regression: the member and its durable session
/// snapshot remain Active, but a completed runtime stop left the registered
/// session with no executor. Delivery-time binding preparation readmits that
/// ownerless runtime and a stale binding tuple advances it to Attached. The
/// next turn must authorize the narrow MissingLiveMaterialization redrive,
/// retire the completed stop receipt, rebuild the exact bridge session, and
/// execute without classifying the member Broken.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_ownerless_stale_runtime_binding_is_redriven_by_missing_live_dispatch() {
use meerkat_runtime::SessionServiceRuntimeExt as _;
use meerkat_runtime::composition::{ConsumerSurface as _, OwnedFieldValue};
use meerkat_runtime::generated::meerkat_mob_seam as seam_facts;
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.expect("inline worker profile")
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let adapter = service.enable_runtime_adapter();
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-missing-live-redrive"),
None,
)
.await
.expect("spawn turn-driven worker");
let bridge_session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member");
let initial_status = handle
.member_status(&AgentIdentity::from("w-missing-live-redrive"))
.await
.expect("initial member status");
let (agent_runtime_id, fence_token) = initial_status
.runtime_identity_fields()
.expect("active member has runtime identity");
let agent_runtime_id = agent_runtime_id.clone();
let baseline_create_requests = service.recorded_create_requests().await.len();
let baseline_turn_calls = service.start_turn_call_count();
assert!(
adapter
.session_has_executor(&bridge_session_id)
.await
.expect("inspect initial executor"),
"spawn must publish the original runtime executor"
);
adapter
.stop_runtime_executor(&bridge_session_id, "completed limbo shutdown")
.await
.expect("canonically stop the original runtime executor");
assert!(
adapter.contains_session(&bridge_session_id).await,
"completed stop must retain the registered runtime entry"
);
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("inspect live materialization after stop"),
"executor-owned cleanup must remove only the live materialization"
);
assert!(
service
.persisted_session_clone(&bridge_session_id)
.await
.is_some(),
"canonical stop must leave the durable mock session snapshot revivable"
);
let machine_after_stop = handle
.query_machine_state()
.await
.expect("query MobMachine after runtime stop");
let machine_identity_after_stop = crate::machines::mob_machine::AgentIdentity::from_domain(
&AgentIdentity::from("w-missing-live-redrive"),
);
let lifecycle_after_stop =
machine_after_stop.member_lifecycle_for_identity(&machine_identity_after_stop);
assert_eq!(
lifecycle_after_stop.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Active,
"runtime loss must not terminalize the actor-owned member: {lifecycle_after_stop:?}"
);
assert!(lifecycle_after_stop.error.is_none());
adapter
.prepare_local_session_bindings(bridge_session_id.clone())
.await
.expect("delivery-time preparation must readmit the stopped runtime");
assert!(
!adapter
.session_has_executor(&bridge_session_id)
.await
.expect("inspect readmitted ownerless runtime"),
"delivery-time binding preparation must not fabricate an executor"
);
meerkat_runtime::meerkat_machine::composition::MeerkatConsumerSurface::pinned(
Arc::clone(&adapter),
bridge_session_id.clone(),
)
.apply_routed_input(
seam_facts::inputs::prepare_bindings(),
vec![
(
seam_facts::fields::agent_runtime_id(),
OwnedFieldValue::Str(agent_runtime_id.to_string()),
),
(
seam_facts::fields::fence_token(),
OwnedFieldValue::U64(fence_token.get()),
),
(
seam_facts::fields::generation(),
OwnedFieldValue::U64(agent_runtime_id.generation.get()),
),
(
seam_facts::fields::session_id(),
OwnedFieldValue::Str(bridge_session_id.to_string()),
),
],
)
.await
.expect("public composition route must manufacture the stale binding tuple");
assert_eq!(
adapter
.runtime_state(&bridge_session_id)
.await
.expect("inspect stale ownerless runtime"),
meerkat_runtime::RuntimeState::Attached,
"PrepareBindings on the ownerless registration exposes the supported Attached shape"
);
assert!(
!adapter
.session_has_executor(&bridge_session_id)
.await
.expect("inspect stale ownerless executor"),
"binding projection must not fabricate a live executor"
);
tokio::time::timeout(
Duration::from_secs(3),
handle
.member(&AgentIdentity::from("w-missing-live-redrive"))
.await
.expect("member handle")
.internal_turn(ContentInput::from(
"execute after missing-live redrive".to_string(),
)),
)
.await
.expect("missing-live materialization must not wedge")
.expect("machine-authorized MissingLive revival and turn must succeed");
assert!(
service
.has_live_session(&bridge_session_id)
.await
.expect("inspect revived materialization"),
"MissingLive revival must rebuild the same bridge session"
);
assert_eq!(
service.recorded_create_requests().await.len(),
baseline_create_requests + 1,
"the missing live materialization must be rebuilt exactly once"
);
assert_eq!(
service.start_turn_call_count(),
baseline_turn_calls + 1,
"the admitted turn must execute exactly once on the replacement runtime"
);
let turn_prompts = service.start_turn_prompts.read().await.clone();
assert!(
turn_prompts.iter().any(|(session_id, prompt)| {
session_id == &bridge_session_id
&& prompt.contains("execute after missing-live redrive")
}),
"replacement executor must run the dispatched turn: {turn_prompts:?}"
);
assert!(
adapter
.session_has_executor(&bridge_session_id)
.await
.expect("inspect replacement executor"),
"successful materialization must publish a replacement executor"
);
let final_status = handle
.member_status(&AgentIdentity::from("w-missing-live-redrive"))
.await
.expect("final member status");
assert_eq!(final_status.status, crate::runtime::MobMemberStatus::Active);
assert!(!final_status.is_final);
let (final_runtime_id, final_fence_token) = final_status
.runtime_identity_fields()
.expect("revived member retains runtime identity");
assert_eq!(final_runtime_id, &agent_runtime_id);
assert_eq!(final_fence_token, fence_token);
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine after missing-live revival");
let machine_identity = crate::machines::mob_machine::AgentIdentity::from_domain(
&AgentIdentity::from("w-missing-live-redrive"),
);
let lifecycle = machine_state.member_lifecycle_for_identity(&machine_identity);
assert_eq!(
lifecycle.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Active,
"successful missing-live redrive must never classify the member Broken: {lifecycle:?}"
);
assert!(lifecycle.error.is_none());
assert!(
!machine_state
.member_revival_pending
.contains(&machine_identity),
"successful materialization must resolve the machine-owned revival obligation"
);
}
#[tokio::test]
async fn test_model_override_is_field_scoped_and_survives_revival_without_profile_snapshot() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.expect("inline worker profile")
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let mut spec = SpawnMemberSpec::new("worker", "w-model-only");
spec.model_override = Some("override-model".to_string());
handle.spawn_spec(spec).await.expect("spawn model override");
let entry = handle
.get_member(&AgentIdentity::from("w-model-only"))
.await
.expect("get member")
.expect("member exists");
assert_eq!(
entry.effective_model_override.as_deref(),
Some("override-model")
);
assert!(
entry.effective_profile_override.is_none(),
"model-only override must not freeze the rest of the profile"
);
let bridge_session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-model-only"))
.await
.expect("session-backed member");
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
handle
.member(&AgentIdentity::from("w-model-only"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("revive with model patch"))
.await
.expect("revive model-only member");
let requests = service.recorded_create_requests().await;
assert!(
requests.len() >= 2,
"spawn and revival must both materialize"
);
assert_eq!(
requests.last().map(|request| request.model.as_str()),
Some("override-model"),
"revival must reapply the model patch over the resolved profile"
);
}
#[tokio::test]
async fn test_combined_profile_and_model_override_store_final_composed_profile() {
let definition = sample_definition();
let mut override_profile = definition
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.expect("inline worker profile")
.clone();
override_profile.model = "pre-model-full-override".to_string();
override_profile.peer_description = "combined override marker".to_string();
let (handle, _service) = create_test_mob(definition).await;
let mut spec = SpawnMemberSpec::new("worker", "w-combined-override");
spec.override_profile = Some(override_profile);
spec.model_override = Some("final-composed-model".to_string());
handle
.spawn_spec(spec)
.await
.expect("spawn combined override");
let entry = handle
.get_member(&AgentIdentity::from("w-combined-override"))
.await
.expect("get combined override member")
.expect("combined override member exists");
let effective_profile = entry
.effective_profile_override
.expect("full override presence is retained");
assert_eq!(effective_profile.model, "final-composed-model");
assert_eq!(
effective_profile.peer_description, "combined override marker",
"model composition must not freeze or replace unrelated full-profile fields"
);
assert_eq!(
entry.effective_model_override.as_deref(),
Some("final-composed-model")
);
}
/// Task #37 race regression: if the dispatch-triggered revival observes the
/// live session as missing but the materialization path then hits the
/// session-scoped comms identity's "already active" guard, the mob shell must
/// treat that as idempotent success only after re-checking that the bridge
/// session is live. The machine still owns the revival obligation and receives
/// `ResolveMemberRevivalSucceeded`; the admitted turn then delivers normally.
#[tokio::test]
async fn test_revival_already_active_identity_is_idempotent_success() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"discard must remove the live materialization"
);
service
.set_create_session_identity_already_active(&bridge_session_id)
.await;
handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from(
"deliver despite already-active race".to_string(),
))
.await
.expect("already-active comms identity must be treated as live revival success");
assert!(
service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"idempotent revival success must leave the bridge session live"
);
let turn_prompts = service.start_turn_prompts.read().await.clone();
assert!(
turn_prompts
.iter()
.any(|(session_id, prompt)| session_id == &bridge_session_id
&& prompt.contains("deliver despite already-active race")),
"the admitted turn must run after the idempotent revival success: {turn_prompts:?}"
);
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
let lifecycle = machine_state.member_lifecycle_for_identity(&machine_identity);
assert_eq!(
lifecycle.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Active,
"already-live revival must leave the member Active: {lifecycle:?}"
);
assert!(
!machine_state
.member_revival_pending
.contains(&machine_identity),
"already-live revival success must resolve the machine-owned obligation"
);
}
/// Task #37 regression (failure path): when the machine-authorized revival
/// itself fails terminally, the outcome is the typed `MemberRestoreFailed`
/// and the machine-owned Broken classification — and the machine refuses any
/// further revival authorization (no retry loop), even if a later attempt
/// would have succeeded.
#[tokio::test]
async fn test_revival_failure_is_typed_terminal_without_retry_loop() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
// The durable snapshot is present (revival is authorized), but the
// re-materialization fails.
service.set_create_session_failure(&bridge_session_id).await;
let error = handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("come back online".to_string()))
.await
.expect_err("failed revival must surface the typed terminal restore failure");
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
reason,
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(bridge_session_id.clone()));
assert!(
reason.contains("machine-authorized revival"),
"failure reason must carry the typed revival outcome: {reason}"
);
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
// Machine state: terminal Broken, obligation resolved.
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
let lifecycle = machine_state.member_lifecycle_for_identity(&machine_identity);
assert_eq!(
lifecycle.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Broken,
"failed revival must resolve into the machine-owned Broken classification: {lifecycle:?}"
);
assert!(
!machine_state
.member_revival_pending
.contains(&machine_identity),
"failed revival must resolve the machine-owned revival obligation"
);
// No retry loop: even though a fresh materialization attempt would now
// succeed, the Broken classification refuses re-authorization and the
// next dispatch fails typed without touching the session service.
service
.clear_create_session_failure(&bridge_session_id)
.await;
// The typed terminal classification surfaces at handle acquisition:
// `member()` refuses to mint a capability handle for a Broken member.
let error = match handle.member(&AgentIdentity::from("w-1")).await {
Err(error) => error,
Ok(_) => panic!(
"broken member must stay terminal; no machine re-authorization after failed revival"
),
};
assert!(
matches!(error, MobError::MemberRestoreFailed { .. }),
"second dispatch must fail typed: {error:?}"
);
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"no shell retry may re-materialize a broken member's session"
);
}
/// Direct regression for the user-reported mob bridge failure
/// ("...machine-authorized revival of bridge session '...' failed: ...Comms
/// runtime failed: Failed to create inproc comms runtime: Session identity
/// already active: ..."): on revival, the re-provision fails with the
/// "Session identity already active" comms-claim error AND there is no live
/// session (the discard already removed it), because a not-yet-dropped prior
/// runtime still holds the process-scoped identity claim. The mob shell must
/// surface the typed terminal `MemberRestoreFailed` (member Broken) — NOT
/// silently treat it as idempotent success, which is what the
/// `has_live_session` re-check guards.
///
/// This is the exact (already-active ∧ has_live_session=false) branch that no
/// prior test could reach: the idempotent-success test leaves the session live
/// (so `has_live_session` is true), and the generic terminal test injects a
/// `Store` error that never matches the already-active classifier. Closing that
/// fidelity gap is why the original failure shipped uncaught. The runtime-layer
/// drain-task join (`detach_drops_drain_task_runtime_clone_so_identity_claim_is_released`)
/// removes the race at its source; this pins the mob shell's recovery decision
/// boundary as the fail-closed backstop.
#[tokio::test]
async fn test_revival_already_active_without_live_session_is_typed_terminal() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let bridge_session_id = receipt
.bridge_session_id()
.expect("session-backed member")
.clone();
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
assert!(
!service
.has_live_session(&bridge_session_id)
.await
.expect("has_live_session"),
"discard must remove the live materialization"
);
// Re-provision fails with the production-faithful already-active error and
// leaves NO live session — the leaked-claim-without-live-session shape.
service
.set_create_session_identity_already_active_without_live_session(&bridge_session_id)
.await;
let error = handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("come back online".to_string()))
.await
.expect_err(
"already-active with no live session must surface the typed terminal restore failure, \
not idempotent success",
);
match error {
MobError::MemberRestoreFailed {
member_id,
session_id,
reason,
} => {
assert_eq!(member_id, AgentIdentity::from("w-1"));
assert_eq!(session_id, Some(bridge_session_id.clone()));
assert!(
reason.contains("machine-authorized revival"),
"failure reason must carry the typed revival outcome: {reason}"
);
assert!(
reason.contains("Session identity already active"),
"failure reason must carry the underlying comms-claim cause: {reason}"
);
}
other => panic!("expected MemberRestoreFailed, got {other:?}"),
}
// Machine state: terminal Broken, revival obligation resolved (no dangling
// pending revival, no retry loop).
let machine_state = handle
.query_machine_state()
.await
.expect("query MobMachine state");
let machine_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&AgentIdentity::from("w-1"));
let lifecycle = machine_state.member_lifecycle_for_identity(&machine_identity);
assert_eq!(
lifecycle.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Broken,
"already-active-with-no-live-session revival must resolve into Broken: {lifecycle:?}"
);
assert!(
!machine_state
.member_revival_pending
.contains(&machine_identity),
"failed revival must resolve the machine-owned revival obligation"
);
}
#[tokio::test]
async fn test_member_status_surface_exposes_current_session_id_for_diagnostics_only() {
// `current_session_id` remains observable for status/continuity
// diagnostics, but it is not a realtime routing contract. Public
// realtime callers use `RealtimeChannelTarget::MobMember` so the server
// resolves the machine-owned bridge binding at open/reconnect time.
let (handle, _service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let snapshot = handle
.member_status(&AgentIdentity::from("w-1"))
.await
.expect("member status");
let expected_session_id = receipt
.bridge_session_id()
.expect("session-backed receipt")
.clone();
let json_value = serde_json::to_value(&snapshot).expect("serialize snapshot");
assert_eq!(
json_value
.get("current_session_id")
.and_then(|v| v.as_str()),
Some(expected_session_id.to_string().as_str()),
"mob/member_status response should surface current_session_id for diagnostics: {json_value}"
);
// The bridge-internal alias stays hidden — it is not part of the
// public identity contract.
assert!(
json_value.get("current_bridge_session_id").is_none(),
"current_bridge_session_id is bridge-internal and must not leak over the wire: {json_value}"
);
}
#[tokio::test]
async fn test_member_handle_bridge_session_helper_round_trips_through_machine_projection_surface() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let member = handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle");
let current_bridge_session_id = member
.current_bridge_session_id()
.await
.expect("current session id");
assert_eq!(
current_bridge_session_id.as_ref(),
Some(receipt.bridge_session_id().expect("session-backed")),
"member handle should expose the machine-routed current session binding"
);
}
#[tokio::test]
async fn test_agent_event_subscriptions_resolve_through_machine_routed_member_reads() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let _stream = handle
.subscribe_agent_events(&AgentIdentity::from("l-1"))
.await
.expect("member stream");
let streams = handle
.subscribe_all_agent_events()
.await
.expect("all-member streams");
assert_eq!(streams.len(), 1);
assert_eq!(streams[0].0, AgentIdentity::from("l-1"));
}
#[tokio::test]
async fn test_mob_events_view_round_trips_through_machine_command_surface() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let replay = handle.events().replay_all().await.expect("replay all");
assert!(
replay
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberSpawned(..))),
"replayed event view should include spawn events through the machine command surface"
);
let polled = handle.events().poll(0, 32).await.expect("poll events");
assert!(
!polled.is_empty(),
"polled event view should surface stored events through the machine command surface"
);
let direct_polled = handle.poll_events(0, 32).await.expect("direct poll events");
assert!(
!direct_polled.is_empty(),
"direct handle poll_events should also round-trip through the machine command surface"
);
}
#[tokio::test]
async fn test_mob_events_view_latest_cursor_uses_store_cursor_without_replay() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let replay_calls_before = events.replay_calls();
let cursor = handle
.events()
.latest_cursor()
.await
.expect("latest cursor");
assert!(cursor > 0, "latest cursor should reflect stored events");
assert_eq!(
events.replay_calls(),
replay_calls_before,
"latest_cursor should not replay the full event log"
);
}
#[tokio::test]
async fn test_mob_events_view_subscribe_streams_structural_events() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
let poll_calls_before = events.poll_calls();
let mut subscription = handle
.events()
.subscribe_with_config(MobEventsSubscriptionConfig {
batch_limit: 16,
channel_capacity: 16,
..MobEventsSubscriptionConfig::default()
})
.await
.expect("subscribe to structural mob events");
assert_eq!(
events.poll_calls(),
poll_calls_before,
"live structural subscription should not poll to start at the latest cursor"
);
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let observed = tokio::time::timeout(Duration::from_secs(1), async {
while let Some(event) = subscription.event_rx.recv().await {
if matches!(event.kind, MobEventKind::MemberSpawned(..)) {
return Some(event);
}
}
None
})
.await
.expect("subscription should receive a structural mob event")
.expect("subscription stream should remain open until the spawn event arrives");
assert!(observed.cursor > 0);
assert_eq!(
events.poll_calls(),
poll_calls_before,
"live structural subscription should receive append notifications without polling"
);
}
#[tokio::test]
async fn test_mob_events_view_subscribe_after_rejects_future_cursor() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events).await;
let latest_cursor = handle
.events()
.latest_cursor()
.await
.expect("latest cursor");
let after_cursor = latest_cursor.saturating_add(10);
match handle.events().subscribe_after(after_cursor).await {
Err(MobError::StaleEventCursor {
after_cursor: observed_after_cursor,
latest_cursor: observed_latest_cursor,
}) => {
assert_eq!(observed_after_cursor, after_cursor);
assert_eq!(observed_latest_cursor, latest_cursor);
}
Err(error) => panic!("expected stale cursor error, got {error:?}"),
Ok(_) => panic!("future cursor subscription should fail"),
}
}
#[tokio::test]
async fn test_mob_events_view_poll_strict_rejects_future_cursor() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
let latest_cursor = handle
.events()
.latest_cursor()
.await
.expect("latest cursor");
let after_cursor = latest_cursor.saturating_add(10);
let poll_calls_before = events.poll_calls();
match handle.events().poll_strict(after_cursor, 16).await {
Err(MobError::StaleEventCursor {
after_cursor: observed_after_cursor,
latest_cursor: observed_latest_cursor,
}) => {
assert_eq!(observed_after_cursor, after_cursor);
assert_eq!(observed_latest_cursor, latest_cursor);
}
Err(error) => panic!("expected stale cursor error, got {error:?}"),
Ok(_) => panic!("future cursor strict poll should fail"),
}
assert_eq!(
events.poll_calls(),
poll_calls_before,
"strict poll should not read events after MobMachine rejects the cursor"
);
}
#[tokio::test]
async fn test_mob_handle_list_runs_reads_public_run_listing() {
let store = Arc::new(InMemoryMobRunStore::new());
let definition = sample_definition();
let mob_id = definition.id.clone();
let alpha_flow = FlowId::from("alpha");
let beta_flow = FlowId::from("beta");
let alpha_run = MobRun::authority_backed_for_steps(
crate::ids::RunId::new(),
mob_id.clone(),
alpha_flow.clone(),
std::iter::empty::<crate::ids::StepId>(),
MobRunStatus::Pending,
serde_json::json!({"flow": "alpha"}),
)
.expect("authority-backed alpha run");
let alpha_run_id = alpha_run.run_id.clone();
let beta_run = MobRun::authority_backed_for_steps(
crate::ids::RunId::new(),
mob_id,
beta_flow,
std::iter::empty::<crate::ids::StepId>(),
MobRunStatus::Pending,
serde_json::json!({"flow": "beta"}),
)
.expect("authority-backed beta run");
store.create_run(alpha_run).await.expect("create alpha run");
store.create_run(beta_run).await.expect("create beta run");
let (handle, _service) = create_test_mob_with_run_store(definition, store).await;
let all_runs = handle.list_runs(None).await.expect("list all runs");
assert_eq!(all_runs.len(), 2);
let filtered = handle
.list_runs(Some(&alpha_flow))
.await
.expect("list runs by flow");
assert_eq!(filtered.len(), 1);
assert_eq!(filtered[0].run_id, alpha_run_id);
}
#[tokio::test]
async fn test_record_operator_action_provenance_round_trips_through_machine_command_surface() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let caller_provenance = meerkat_core::service::MobToolCallerProvenance::new()
.with_session_id(SessionId::from_uuid(Uuid::nil()))
.with_mob_id("caller-mob")
.with_member_id("caller-lead");
let authority = meerkat_runtime::mob_operator_authority::with_caller_provenance(
&meerkat_runtime::mob_operator_authority::create_only_mob_operator_authority()
.expect("generated create-only mob authority should be accepted"),
caller_provenance.clone(),
)
.expect("generated mob authority should attach caller provenance");
let authority = meerkat_runtime::mob_operator_authority::with_audit_invocation_id(
&authority,
"audit-machine-surface",
)
.expect("generated mob authority should attach audit invocation id");
let expected_principal = authority.principal_token().clone();
handle
.record_operator_action_provenance("spawn_member", &authority)
.await
.expect("record operator action");
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| matches!(
&event.kind,
MobEventKind::OperatorActionRecorded {
tool_name,
principal_token,
caller_provenance: recorded_caller_provenance,
audit_invocation_id,
..
} if tool_name == "spawn_member"
&& principal_token == &expected_principal
&& recorded_caller_provenance.as_ref() == Some(&caller_provenance)
&& audit_invocation_id.as_deref() == Some("audit-machine-surface")
)),
"operator action provenance should be appended through generated machine authority"
);
}
#[tokio::test]
async fn test_mob_event_router_stays_alive_across_machine_routed_spawn_tracking() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let mut router = handle
.subscribe_mob_events_with_config(MobEventRouterConfig {
poll_interval: Duration::from_millis(10),
channel_capacity: 32,
})
.await
.expect("subscribe mob events");
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
matches!(
router.event_rx.try_recv(),
Err(tokio::sync::mpsc::error::TryRecvError::Empty)
),
"router should remain live after tracking a spawned member through machine-routed bootstrap"
);
router.cancel();
}
#[tokio::test]
async fn test_reset_append_failure_transitions_to_stopped() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
assert_eq!(handle.status().await.unwrap(), MobState::Running);
// Fail the MobReset append. After destructive steps (retire/stop),
// fail-closed to Stopped.
events.fail_appends_for("MobReset").await;
let result = handle.reset().await;
assert!(
matches!(result, Err(MobError::StorageError(_))),
"reset should surface append failure"
);
assert_eq!(
handle.status().await.unwrap(),
MobState::Stopped,
"failed reset after destructive steps must transition to Stopped"
);
// The original creation event should still remain available for resume.
let all_events = events.replay_all().await.expect("replay");
assert!(
all_events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MobCreated { .. })),
"original MobCreated must survive a failed reset"
);
}
#[tokio::test]
async fn test_reset_failure_from_stopped_stays_stopped() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let (handle, _service) = create_test_mob_with_events(sample_definition(), events.clone()).await;
handle.stop().await.expect("stop");
assert_eq!(handle.status().await.unwrap(), MobState::Stopped);
// Fail the MobReset append to trigger reset failure from Stopped.
events.fail_appends_for("MobReset").await;
let result = handle.reset().await;
assert!(result.is_err(), "reset should fail");
assert_eq!(
handle.status().await.unwrap(),
MobState::Stopped,
"failed reset from Stopped must remain Stopped"
);
}
#[tokio::test]
async fn test_shutdown_does_not_stall_on_stuck_lifecycle_notification() {
// Create a definition with a TurnDriven orchestrator so lifecycle
// notifications go through start_turn (which we can delay).
let mut def = sample_definition();
if let Some(profile) = def
.profiles
.get_mut(&ProfileName::from("lead"))
.and_then(|b| b.as_inline_mut())
{
profile.runtime_mode = crate::MobRuntimeMode::TurnDriven;
}
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(def, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
// Spawn the orchestrator so the notification path is active.
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
// Make start_turn hang for 10 minutes — simulates a stuck backend.
service.set_start_turn_delay_ms(600_000);
// Stop the mob. This fires a lifecycle notification that will hang
// in start_turn due to the delay. The notification is spawned onto
// the JoinSet, so stop itself returns immediately.
handle.stop().await.expect("stop");
// Shutdown must complete quickly despite the stuck notification task.
// abort_all cancels in-flight tasks instead of awaiting them.
let shutdown_result = tokio::time::timeout(Duration::from_secs(5), handle.shutdown()).await;
assert!(
shutdown_result.is_ok(),
"shutdown must not stall on stuck lifecycle notification tasks"
);
shutdown_result.unwrap().expect("shutdown should succeed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_lifecycle_notification_burst_backpressures_without_dropping() {
const NOTIFICATIONS: usize = 48;
const INJECT_DELAY_MS: u64 = 20;
let def = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(def, MobStorage::in_memory())
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
)
.await
.expect("spawn autonomous orchestrator");
service.set_inject_delay_ms(INJECT_DELAY_MS);
let baseline = service.inject_call_count();
let started = Instant::now();
handle
.debug_lifecycle_notification_burst(NOTIFICATIONS, "burst lifecycle")
.await
.expect("lifecycle burst");
let command_elapsed = started.elapsed();
let deadline = Instant::now() + Duration::from_secs(2);
while service.inject_call_count() < baseline + NOTIFICATIONS as u64 {
assert!(
Instant::now() < deadline,
"lifecycle burst must inject every notification instead of dropping at the task cap"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
let total_elapsed = started.elapsed();
assert_eq!(
service.inject_call_count(),
baseline + NOTIFICATIONS as u64,
"lifecycle burst must inject every notification instead of dropping at the task cap"
);
assert!(
command_elapsed >= Duration::from_millis(INJECT_DELAY_MS),
"burst should apply backpressure once the lifecycle task set saturates"
);
assert!(
total_elapsed < Duration::from_secs(3),
"backpressured lifecycle burst should complete in bounded waves, not stall indefinitely (elapsed={total_elapsed:?})"
);
eprintln!(
"lifecycle notification stress: notifications={NOTIFICATIONS}, inject_delay_ms={INJECT_DELAY_MS}, command={command_elapsed:?}, total={total_elapsed:?}"
);
}
// ---------------------------------------------------------------------------
// P1-5: Disposal policy — archive failure must propagate as Err
// ---------------------------------------------------------------------------
/// Archive failure during retirement must return Err to the caller.
///
/// Dogma §15: "success means truthful" — returning Ok(()) when ArchiveSession
/// failed creates an orphan session that the caller believes was cleaned up.
/// The non-routable retiring roster anchor is retained so the caller can retry
/// the exact session/runtime tuple after the transient fault clears.
#[tokio::test]
async fn test_retire_returns_err_when_archive_fails() {
let (handle, service) = create_test_mob(sample_definition()).await;
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
service.set_archive_failure(&session_id).await;
// Archive failure is critical — retire must surface it.
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
result.is_err(),
"retire must return Err when ArchiveSession fails — got Ok"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_some(),
"archive failure must retain the non-routable retry anchor"
);
// Only the start carrier was persisted before disposal; final publication
// is archive-confirmed.
let events = handle.events().replay_all().await.expect("replay");
assert!(
events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetirementStarted { .. })),
"retirement start must persist before archive"
);
assert!(
!events
.iter()
.any(|e| matches!(e.kind, MobEventKind::MemberRetired { .. })),
"archive failure must not publish a false final"
);
}
/// The routed runtime effect may commit before the final retirement event can
/// be published (here the intervening archive fails). A process restart must
/// replay the durable start marker, retain the roster entry, and retry the
/// already-Retired runtime idempotently before publishing completion.
#[tokio::test]
async fn test_retire_effect_before_final_event_survives_cold_restart() {
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
service.set_archive_failure(&session_id).await;
handle
.retire(AgentIdentity::from("w-1"))
.await
.expect_err("archive failure must interrupt retirement completion");
assert_eq!(
meerkat_runtime::RuntimeControlPlane::runtime_state(
adapter.as_ref(),
&meerkat_runtime::LogicalRuntimeId::for_session(&session_id),
)
.await
.expect("runtime must remain observable after failed archive"),
meerkat_runtime::RuntimeState::Retired,
"the routed runtime-retire effect must have committed before archive failed"
);
let interrupted = events
.replay_all()
.await
.expect("replay interrupted events");
assert!(
interrupted
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberRetirementStarted { .. }))
);
assert!(
interrupted
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
handle
.shutdown()
.await
.expect("shutdown actor at effect-before-final-event boundary");
service.clear_archive_failure(&session_id).await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume in-flight retirement");
assert!(
resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read recovered roster anchor")
.is_some(),
"start-event replay must retain the member until cleanup completes"
);
resumed
.retire(AgentIdentity::from("w-1"))
.await
.expect("cold retry must accept an already-Retired runtime and finish archive");
assert!(
resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read completed roster")
.is_none()
);
let completed = events.replay_all().await.expect("replay completed events");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"effect retry must close with exactly one final event"
);
}
/// If archive succeeds but final publication fails, retry on the same actor
/// must recognize archive authority before touching the consumed provisioner
/// operation binding, then publish completion exactly once.
#[tokio::test]
async fn test_retire_final_event_append_failure_retries_on_same_actor() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), MobStorage::with_events(events.clone()))
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("w-same-actor-archive-retry");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle
.spawn_spec(spec)
.await
.expect("spawn turn-driven worker");
let session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("session-backed worker");
events.fail_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect_err("final retirement event append should be fault-injected");
assert_eq!(service.active_session_count().await, 0);
assert!(!adapter.contains_session(&session_id).await);
events.allow_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect("same-actor retry must accept existing archive authority");
assert!(
handle
.get_member(&identity)
.await
.expect("read completed roster")
.is_none()
);
let completed = events.replay_all().await.expect("replay completed events");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"same-actor retry must publish exactly one final event"
);
}
#[tokio::test]
async fn test_destroy_takes_over_released_ordinary_retirement() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), MobStorage::with_events(events.clone()))
.with_session_service(service)
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("w-destroy-takeover");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle
.spawn_spec(spec)
.await
.expect("spawn turn-driven worker");
events.fail_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect_err("ordinary final retirement append should be fault-injected");
events.allow_appends_for("MemberRetired").await;
events.fail_clear();
let retry_error = handle
.destroy()
.await
.expect_err("final event clear remains fault-injected after destroy takeover");
let report = match retry_error {
crate::runtime::handle::MobDestroyError::Incomplete { report } => report,
other => panic!("expected incomplete destroy takeover, got {other:?}"),
};
assert!(
report.errors.iter().all(|error| {
!error.contains("destroy retire admission failed")
&& !error.contains("missing generated owner operation binding")
}),
"destroy must adopt the released ordinary retirement correlation: {report:?}"
);
assert!(
handle.list_members_observation_snapshot().await.is_empty(),
"destroy takeover must remove the retired roster anchor"
);
let completed = events.replay_all().await.expect("replay destroy takeover");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"destroy takeover must publish exactly one terminal member event"
);
events.allow_clear();
handle
.destroy()
.await
.expect("destroy takeover must finish after event clear recovers");
}
/// If archive succeeds but the final journal append fails, cold replay must
/// not route retirement into the now-unregistered runtime and manufacture a
/// new refusal. The archive authority proves the effect is already discharged;
/// the retained start marker then authorizes publishing completion exactly once.
#[tokio::test]
async fn test_retire_final_event_append_failure_survives_cold_restart() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let storage = MobStorage::with_events(events.clone());
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
events.fail_appends_for("MemberRetired").await;
let error = handle
.retire(AgentIdentity::from("w-1"))
.await
.expect_err("final retirement event append should be fault-injected");
assert!(
matches!(error, MobError::StorageError(_)),
"final append failure must surface as storage error: {error}"
);
assert_eq!(
service.active_session_count().await,
0,
"archive must have completed before the final append failed"
);
assert!(
!adapter.contains_session(&session_id).await,
"successful archive must unregister the runtime before final publication"
);
let interrupted = events
.replay_all()
.await
.expect("replay interrupted events");
assert!(
interrupted
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberRetirementStarted { .. }))
);
assert!(
interrupted
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
handle
.shutdown()
.await
.expect("shutdown after final-event failure");
events.allow_appends_for("MemberRetired").await;
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume from durable retirement-start marker");
resumed
.retire(AgentIdentity::from("w-1"))
.await
.expect("archive-complete retry should publish final retirement without rerouting");
assert!(
resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read roster after retry")
.is_none()
);
let completed = events.replay_all().await.expect("replay completed events");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"cold retry must publish exactly one final event"
);
}
/// A terminal journal failure retains a Retiring roster row after the session
/// archive and exact runtime unregister have already completed. For an
/// autonomous member that row is a publication retry anchor, not a live host;
/// the MobMachine shutdown disposition must not interrupt the absent runtime.
#[tokio::test]
async fn test_shutdown_skips_interrupt_for_archived_autonomous_retirement_anchor() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), MobStorage::with_events(events.clone()))
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("w-autonomous-retire-anchor");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::AutonomousHost);
handle
.spawn_spec(spec)
.await
.expect("spawn autonomous worker");
let session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("session-backed autonomous worker");
events.fail_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect_err("final retirement event append should be fault-injected");
assert!(
handle
.get_member(&identity)
.await
.expect("read retained retirement anchor")
.is_some(),
"failed terminal publication must retain the exact roster retry anchor"
);
assert!(
!adapter.contains_session(&session_id).await,
"archive completion must unregister the runtime before terminal publication"
);
assert_eq!(
service.active_session_count().await,
0,
"archive completion must remove the live session before terminal publication"
);
handle
.shutdown()
.await
.expect("shutdown must treat an archived Retiring row as a journal retry anchor");
}
/// Host-owned session disposal unregisters the Mob-attached runtime without
/// archiving or deleting the host's durable session document. If terminal
/// `MemberRetired` publication then fails, shutdown must recognize the
/// retained Retiring row as a publication retry anchor instead of trying to
/// interrupt the already-unregistered runtime.
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn test_shutdown_skips_interrupt_for_host_owned_runtime_release_anchor() {
let events = Arc::new(FaultInjectedMobEventStore::new());
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), MobStorage::with_events(events.clone()))
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let identity = AgentIdentity::from("w-host-owned-retire-anchor");
let mut spec = SpawnMemberSpec::new("worker", identity.as_str());
spec.runtime_mode = Some(crate::MobRuntimeMode::AutonomousHost);
handle
.spawn_spec(spec)
.await
.expect("spawn autonomous worker");
let session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("session-backed autonomous worker");
service.set_archive_authority_unknown(&session_id).await;
service.set_archive_not_found(&session_id).await;
events.fail_appends_for("MemberRetired").await;
handle
.retire(identity.clone())
.await
.expect_err("final retirement event append should be fault-injected");
assert!(
handle
.get_member(&identity)
.await
.expect("read retained host-owned retirement anchor")
.is_some(),
"failed terminal publication must retain the exact roster retry anchor"
);
assert!(
!adapter.contains_session(&session_id).await,
"host-owned disposal must unregister the Mob-attached runtime before terminal publication"
);
assert!(
service
.load_persisted_session(&session_id)
.await
.expect("load host-owned durable session")
.is_some(),
"host-owned disposal must retain the host's durable session projection"
);
assert!(
!service
.live_session_actor_registered(&session_id)
.await
.expect("observe actor registry after host-owned disposal"),
"host-owned runtime release must remove the Mob-attached live actor"
);
let baseline_interrupts = service.interrupt_call_count();
handle
.shutdown()
.await
.expect("shutdown must treat a host-owned released Retiring row as a journal retry anchor");
assert_eq!(
service.interrupt_call_count(),
baseline_interrupts,
"shutdown must not interrupt an already-released host-owned runtime"
);
}
#[tokio::test]
async fn test_retire_consumer_refusal_survives_cold_restart_and_retries() {
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
handle.spawn_spec(spec).await.expect("spawn w-1");
let session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("session-backed");
unregister_runtime_session_until_complete(adapter.as_ref(), &session_id)
.await
.expect("bridge session should unregister cleanly");
let refusal = handle
.retire(AgentIdentity::from("w-1"))
.await
.expect_err("unregistered runtime consumer must refuse the routed retire");
assert!(
refusal.is_closed_runtime_effect_refusal(),
"retire refusal must close through generated MobMachine feedback: {refusal}"
);
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read retained member")
.is_some(),
"consumer refusal must retain the roster cleanup anchor"
);
let after_refusal = events.replay_all().await.expect("replay refusal events");
assert!(
after_refusal
.iter()
.any(|event| matches!(event.kind, MobEventKind::MemberRetirementStarted { .. }))
);
assert!(
after_refusal
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberRetired { .. }))
);
handle
.shutdown()
.await
.expect("shutdown refused actor before cold restart");
// Repair the external runtime reachability that caused the first refusal.
// Resume must derive RETIRE retry authority solely from the durable start
// event; the in-memory refusal maps and actor state are gone.
adapter
.register_session(session_id.clone())
.await
.expect("restore runtime reachability before retry");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events.clone(),
runtime_metadata,
))
.with_session_service(service)
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume retiring mob from durable event journal");
let recovered = resumed
.query_machine_state()
.await
.expect("query recovered machine state");
assert!(
recovered
.runtime_retire_pending_sessions
.values()
.any(|pending| pending.0 == session_id.to_string()),
"cold replay must restore the exact pending runtime-retire correlation"
);
resumed
.retire(AgentIdentity::from("w-1"))
.await
.expect("cold-restarted pending retirement should retry and complete");
assert!(
resumed
.get_member(&AgentIdentity::from("w-1"))
.await
.expect("read member after completed retry")
.is_none(),
"successful retry must remove the completed roster entry"
);
let completed = events.replay_all().await.expect("replay completed events");
assert_eq!(
completed
.iter()
.filter(|event| matches!(event.kind, MobEventKind::MemberRetired { .. }))
.count(),
1,
"retry must publish exactly one final MemberRetired event"
);
assert_eq!(
resumed.status().await.expect("query recovered lifecycle"),
MobState::Stopped,
"graceful pre-restart shutdown is durable; retirement retry must not implicitly resume the mob"
);
}
/// Lifecycle-notification failures during retirement remain best-effort.
///
/// The notification may legitimately fail during concurrent teardown, but
/// reciprocal trust cleanup remains a separate critical convergence boundary.
#[tokio::test]
async fn test_retire_comms_failures_remain_best_effort() {
let (handle, service) = create_test_mob(sample_definition()).await;
// Fail only the retiring member's lifecycle notification send. Trust
// cleanup runs against the surviving peer and must still complete.
service
.set_comms_behavior(
&test_comms_name("worker", "w-1"),
MockCommsBehavior {
fail_send_peer_retired: true,
..MockCommsBehavior::default()
},
)
.await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.wire(AgentIdentity::from("w-1"), AgentIdentity::from("w-2"))
.await
.expect("wire");
// Lifecycle notification failure is best-effort — retire still succeeds
// after reciprocal trust is removed.
let result = handle.retire(AgentIdentity::from("w-1")).await;
assert!(
result.is_ok(),
"retire must succeed when only lifecycle notification fails — got {:?}",
result.unwrap_err()
);
// Roster removal still happens.
assert!(
handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.is_none(),
"roster entry must be removed after best-effort comms failures"
);
}
// ---------------------------------------------------------------------------
// NameFilteredDispatcher tests
// ---------------------------------------------------------------------------
/// A configurable dispatcher for testing name filtering and external tools.
struct MultiToolDispatcher {
defs: Arc<[Arc<ToolDef>]>,
}
impl MultiToolDispatcher {
fn new(names: &[&str]) -> Self {
let defs: Vec<Arc<ToolDef>> = names
.iter()
.map(|name| {
Arc::new(ToolDef {
name: (*name).into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: None,
})
})
.collect();
Self { defs: defs.into() }
}
}
#[async_trait]
impl AgentToolDispatcher for MultiToolDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.defs)
}
async fn dispatch(&self, call: ToolCallView<'_>) -> Result<ToolDispatchOutcome, ToolError> {
if self.defs.iter().any(|d| d.name == call.name) {
Ok(ToolResult::new(
call.id.to_string(),
format!("{{\"tool\":\"{}\"}}", call.name),
false,
)
.into())
} else {
Err(ToolError::not_found(call.name))
}
}
fn capabilities(&self) -> meerkat_core::agent::DispatcherCapabilities {
meerkat_core::agent::DispatcherCapabilities::default()
}
}
struct ExactHybridExecutionDispatcher {
catalog: Arc<[meerkat_core::ToolCatalogEntry]>,
}
impl ExactHybridExecutionDispatcher {
fn new(defs: Vec<Arc<ToolDef>>) -> Self {
let detached = meerkat_core::DetachedToolExecutionPolicy::new(
meerkat_core::RunnerIdentity::new("test.hybrid", "v1").expect("valid runner identity"),
meerkat_core::RestartClass::NonResumable,
meerkat_core::IdempotencyScope::InteractionAndArguments,
Duration::from_secs(10),
)
.expect("valid detached policy");
let execution = meerkat_core::ToolExecutionContract::new(
BTreeSet::from([
meerkat_core::ToolExecutionMode::Fast,
meerkat_core::ToolExecutionMode::Detached,
]),
meerkat_core::ToolExecutionMode::Fast,
None,
Some(detached),
)
.expect("valid hybrid execution contract");
let catalog = defs
.into_iter()
.map(|tool| {
meerkat_core::ToolCatalogEntry::session_inline(tool, true)
.with_execution_contract(execution.clone())
})
.collect::<Vec<_>>()
.into();
Self { catalog }
}
}
#[async_trait]
impl AgentToolDispatcher for ExactHybridExecutionDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
self.catalog
.iter()
.map(|entry| Arc::clone(&entry.tool))
.collect::<Vec<_>>()
.into()
}
fn tool_catalog_capabilities(&self) -> meerkat_core::ToolCatalogCapabilities {
meerkat_core::ToolCatalogCapabilities {
exact_catalog: true,
may_require_catalog_control_plane: false,
}
}
fn tool_catalog(&self) -> Arc<[meerkat_core::ToolCatalogEntry]> {
Arc::clone(&self.catalog)
}
fn resolve_execution_plan(
&self,
call: ToolCallView<'_>,
_dispatch_context: &meerkat_core::ToolDispatchContext,
resolution_context: &meerkat_core::ToolExecutionResolutionContext,
) -> Result<meerkat_core::ResolvedToolExecutionPlan, meerkat_core::ToolExecutionResolutionError>
{
let entry = self
.catalog
.iter()
.find(|entry| entry.tool.name == call.name)
.ok_or_else(|| meerkat_core::ToolExecutionResolutionError::NotFound {
tool_name: call.name.to_string(),
})?;
let arguments: serde_json::Value =
serde_json::from_str(call.args.get()).map_err(|error| {
meerkat_core::ToolExecutionResolutionError::InvalidArguments {
tool_name: call.name.to_string(),
reason: error.to_string(),
}
})?;
let mode = if arguments["detach"].as_bool() == Some(true) {
meerkat_core::ToolExecutionMode::Detached
} else {
meerkat_core::ToolExecutionMode::Fast
};
entry
.execution
.resolve(mode, resolution_context.deadlines().clone())
.map_err(Into::into)
}
async fn dispatch(&self, call: ToolCallView<'_>) -> Result<ToolDispatchOutcome, ToolError> {
if self
.catalog
.iter()
.any(|entry| entry.tool.name == call.name)
{
Ok(ToolResult::new(call.id.to_string(), "{}".to_string(), false).into())
} else {
Err(ToolError::not_found(call.name))
}
}
async fn dispatch_resolved_with_context(
&self,
call: ToolCallView<'_>,
_context: &meerkat_core::ToolDispatchContext,
plan: &meerkat_core::ResolvedToolExecutionPlan,
) -> Result<ToolDispatchOutcome, ToolError> {
if plan.mode() != meerkat_core::ToolExecutionMode::Detached {
return Err(ToolError::execution_failed(
"test detached owner received the wrong plan",
));
}
self.dispatch(call).await
}
}
fn hybrid_execution_resolution_context() -> meerkat_core::ToolExecutionResolutionContext {
meerkat_core::ToolExecutionResolutionContext::new(
meerkat_core::ToolDeadlineChain::new(vec![meerkat_core::ToolDeadlineContributor::finite(
meerkat_core::ToolDeadlineOwner::CoreToolDispatch,
Duration::from_secs(600),
)])
.expect("valid test deadline chain"),
)
}
struct ContextRecordingDispatcher {
defs: Arc<[Arc<ToolDef>]>,
saw_context_image: Arc<AtomicBool>,
}
impl ContextRecordingDispatcher {
fn new(defs: Vec<Arc<ToolDef>>, saw_context_image: Arc<AtomicBool>) -> Self {
Self {
defs: Arc::from(defs),
saw_context_image,
}
}
}
#[async_trait]
impl AgentToolDispatcher for ContextRecordingDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.defs)
}
async fn dispatch(&self, call: ToolCallView<'_>) -> Result<ToolDispatchOutcome, ToolError> {
if self.defs.iter().any(|d| d.name.as_str() == call.name) {
Ok(ToolResult::new(call.id.to_string(), "{}".to_string(), false).into())
} else {
Err(ToolError::not_found(call.name))
}
}
async fn dispatch_with_context(
&self,
call: ToolCallView<'_>,
context: &meerkat_core::ToolDispatchContext,
) -> Result<ToolDispatchOutcome, ToolError> {
let saw_context_image = context
.current_turn()
.and_then(|turn| turn.image_ref(0))
.and_then(|image_ref| context.current_turn_image(image_ref))
.is_some();
self.saw_context_image
.store(saw_context_image, Ordering::SeqCst);
self.dispatch(call).await
}
}
fn context_recording_tool(name: &str, provenance: Option<ToolProvenance>) -> Arc<ToolDef> {
Arc::new(ToolDef {
name: name.into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance,
})
}
fn image_dispatch_context() -> meerkat_core::ToolDispatchContext {
meerkat_core::ToolDispatchContext::from_current_turn_input(
&meerkat_core::types::ContentInput::Blocks(vec![
meerkat_core::types::ContentBlock::Image {
media_type: "image/png".to_string(),
data: "abc123".to_string().into(),
},
]),
)
}
#[tokio::test]
async fn test_name_filtered_dispatcher() {
use super::tools::NameFilteredDispatcher;
let inner = Arc::new(MultiToolDispatcher::new(&["a", "b", "c"]));
let excluded: HashSet<String> = ["b".to_string()].into_iter().collect();
let filtered = NameFilteredDispatcher::new(inner, excluded);
// tools() should return only ["a", "c"]
let tool_names: Vec<String> = filtered
.tools()
.iter()
.map(|t| t.name.to_string())
.collect();
assert_eq!(tool_names.len(), 2);
assert!(tool_names.contains(&"a".to_string()));
assert!(tool_names.contains(&"c".to_string()));
assert!(!tool_names.contains(&"b".to_string()));
// dispatch("b") should error
let raw_args = RawValue::from_string("{}".to_string()).unwrap();
let result = filtered
.dispatch(ToolCallView {
id: "call-1",
name: "b",
args: &raw_args,
})
.await;
assert!(result.is_err(), "excluded tool should return not_found");
// dispatch("a") should succeed
let result = filtered
.dispatch(ToolCallView {
id: "call-2",
name: "a",
args: &raw_args,
})
.await;
assert!(result.is_ok(), "non-excluded tool should delegate to inner");
}
#[tokio::test]
async fn name_filtered_dispatcher_preserves_dispatch_context() {
use super::tools::NameFilteredDispatcher;
let saw_context_image = Arc::new(AtomicBool::new(false));
let inner = Arc::new(ContextRecordingDispatcher::new(
vec![context_recording_tool("visible", None)],
Arc::clone(&saw_context_image),
));
let excluded: HashSet<String> = ["hidden".to_string()].into_iter().collect();
let filtered = NameFilteredDispatcher::new(inner, excluded);
let raw_args = RawValue::from_string("{}".to_string()).unwrap();
let context = image_dispatch_context();
filtered
.dispatch_with_context(
ToolCallView {
id: "call-visible",
name: "visible",
args: &raw_args,
},
&context,
)
.await
.expect("visible tool should dispatch");
assert!(
saw_context_image.load(Ordering::SeqCst),
"name filter must forward ToolDispatchContext to inner dispatcher"
);
}
#[tokio::test]
async fn name_filtered_dispatcher_preserves_exact_catalog_and_hybrid_resolution() {
use super::tools::NameFilteredDispatcher;
let inner = Arc::new(ExactHybridExecutionDispatcher::new(vec![
context_recording_tool("visible", None),
context_recording_tool("hidden", None),
]));
let excluded: HashSet<String> = ["hidden".to_string()].into_iter().collect();
let filtered = NameFilteredDispatcher::new(inner, excluded);
assert!(
filtered.tool_catalog_capabilities().exact_catalog,
"an exact inner catalog remains exact after deterministic name filtering"
);
let catalog = filtered.tool_catalog();
assert_eq!(catalog.len(), 1);
assert_eq!(catalog[0].tool.name, "visible");
assert!(
catalog[0]
.execution
.supported_modes()
.contains(&meerkat_core::ToolExecutionMode::Detached),
"filter must preserve the exact inner execution contract"
);
let args = RawValue::from_string(r#"{"detach":true}"#.to_string()).unwrap();
let visible = filtered
.resolve_execution_plan(
ToolCallView {
id: "resolve-visible",
name: "visible",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&hybrid_execution_resolution_context(),
)
.expect("visible execution resolution reaches the hybrid inner resolver");
assert_eq!(visible.mode(), meerkat_core::ToolExecutionMode::Detached);
filtered
.dispatch_resolved_with_context(
ToolCallView {
id: "resolve-visible",
name: "visible",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&visible,
)
.await
.expect("name filter forwards the resolved plan");
let hidden = filtered
.resolve_execution_plan(
ToolCallView {
id: "resolve-hidden",
name: "hidden",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&hybrid_execution_resolution_context(),
)
.expect_err("excluded names must not resolve an execution plan");
assert_eq!(
hidden,
meerkat_core::ToolExecutionResolutionError::NotFound {
tool_name: "hidden".to_string(),
}
);
}
#[tokio::test]
async fn mcp_provenance_filter_scopes_only_mcp_tools_to_allowlist() {
use super::tools::McpProvenanceFilter;
use meerkat_core::types::{ToolProvenance, ToolSourceId, ToolSourceKind};
fn mcp_tool(name: &str, source: &str) -> Arc<ToolDef> {
Arc::new(ToolDef {
name: name.into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new(source),
}),
})
}
fn callback_tool(name: &str) -> Arc<ToolDef> {
Arc::new(ToolDef {
name: name.into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: Some(ToolProvenance {
kind: ToolSourceKind::Callback,
source_id: ToolSourceId::new("inline"),
}),
})
}
struct StaticDefsDispatcher {
defs: Arc<[Arc<ToolDef>]>,
}
#[async_trait]
impl AgentToolDispatcher for StaticDefsDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.defs)
}
async fn dispatch(&self, call: ToolCallView<'_>) -> Result<ToolDispatchOutcome, ToolError> {
if self.defs.iter().any(|d| d.name.as_str() == call.name) {
Ok(ToolResult::new(call.id.to_string(), "{}".to_string(), false).into())
} else {
Err(ToolError::not_found(call.name))
}
}
}
let defs: Vec<Arc<ToolDef>> = vec![
mcp_tool("search", "linear"),
mcp_tool("create_issue", "linear"),
mcp_tool("ping", "github"),
callback_tool("workpad_save"),
];
let inner = Arc::new(StaticDefsDispatcher {
defs: Arc::from(defs),
}) as Arc<dyn AgentToolDispatcher>;
let allowlist: HashSet<String> = ["linear".to_string()].into_iter().collect();
let filter = McpProvenanceFilter::new(inner, allowlist);
let visible: std::collections::BTreeSet<String> =
filter.tools().iter().map(|t| t.name.to_string()).collect();
assert!(visible.contains("search"));
assert!(visible.contains("create_issue"));
assert!(
!visible.contains("ping"),
"github MCP tool must be hidden when only linear is allowed"
);
assert!(
visible.contains("workpad_save"),
"non-MCP tools (callback, builtin, etc.) must pass through unchanged"
);
// Dispatch on a hidden MCP tool returns NotFound.
let raw_args = RawValue::from_string("{}".to_string()).unwrap();
let err = filter
.dispatch(ToolCallView {
id: "call-ping",
name: "ping",
args: &raw_args,
})
.await
.expect_err("hidden MCP tool must report NotFound");
assert!(matches!(err, ToolError::NotFound { .. }));
// Dispatch on a visible MCP tool reaches the inner dispatcher.
let ok = filter
.dispatch(ToolCallView {
id: "call-search",
name: "search",
args: &raw_args,
})
.await;
assert!(ok.is_ok(), "visible MCP tool must dispatch to inner");
}
#[tokio::test]
async fn mcp_provenance_filter_preserves_dispatch_context() {
use super::tools::McpProvenanceFilter;
use meerkat_core::types::{ToolProvenance, ToolSourceId, ToolSourceKind};
let saw_context_image = Arc::new(AtomicBool::new(false));
let inner = Arc::new(ContextRecordingDispatcher::new(
vec![
context_recording_tool(
"search",
Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new("linear"),
}),
),
context_recording_tool(
"hidden",
Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new("github"),
}),
),
],
Arc::clone(&saw_context_image),
)) as Arc<dyn AgentToolDispatcher>;
let allowlist: HashSet<String> = ["linear".to_string()].into_iter().collect();
let filter = McpProvenanceFilter::new(inner, allowlist);
let raw_args = RawValue::from_string("{}".to_string()).unwrap();
let context = image_dispatch_context();
let hidden = filter
.dispatch_with_context(
ToolCallView {
id: "call-hidden",
name: "hidden",
args: &raw_args,
},
&context,
)
.await;
assert!(matches!(hidden, Err(ToolError::NotFound { .. })));
assert!(
!saw_context_image.load(Ordering::SeqCst),
"hidden MCP tools must not dispatch to inner"
);
filter
.dispatch_with_context(
ToolCallView {
id: "call-search",
name: "search",
args: &raw_args,
},
&context,
)
.await
.expect("visible MCP tool should dispatch");
assert!(
saw_context_image.load(Ordering::SeqCst),
"MCP provenance filter must forward ToolDispatchContext to inner dispatcher"
);
}
#[tokio::test]
async fn mcp_provenance_filter_preserves_exact_catalog_and_gates_hybrid_resolution() {
use super::tools::McpProvenanceFilter;
use meerkat_core::types::{ToolSourceId, ToolSourceKind};
let inner = Arc::new(ExactHybridExecutionDispatcher::new(vec![
context_recording_tool(
"allowed",
Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new("linear"),
}),
),
context_recording_tool(
"blocked",
Some(ToolProvenance {
kind: ToolSourceKind::Mcp,
source_id: ToolSourceId::new("github"),
}),
),
])) as Arc<dyn AgentToolDispatcher>;
let allowlist: HashSet<String> = ["linear".to_string()].into_iter().collect();
let filter = McpProvenanceFilter::new(inner, allowlist);
assert!(
filter.tool_catalog_capabilities().exact_catalog,
"an exact inner catalog remains exact after provenance filtering"
);
let catalog = filter.tool_catalog();
assert_eq!(catalog.len(), 1);
assert_eq!(catalog[0].tool.name, "allowed");
assert!(
catalog[0]
.execution
.supported_modes()
.contains(&meerkat_core::ToolExecutionMode::Detached),
"filter must preserve the exact inner execution contract"
);
let args = RawValue::from_string(r#"{"detach":true}"#.to_string()).unwrap();
let allowed = filter
.resolve_execution_plan(
ToolCallView {
id: "resolve-allowed",
name: "allowed",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&hybrid_execution_resolution_context(),
)
.expect("allowed MCP tool reaches the hybrid inner resolver");
assert_eq!(allowed.mode(), meerkat_core::ToolExecutionMode::Detached);
filter
.dispatch_resolved_with_context(
ToolCallView {
id: "resolve-allowed",
name: "allowed",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&allowed,
)
.await
.expect("provenance filter forwards the resolved plan");
let blocked = filter
.resolve_execution_plan(
ToolCallView {
id: "resolve-blocked",
name: "blocked",
args: &args,
},
&meerkat_core::ToolDispatchContext::default(),
&hybrid_execution_resolution_context(),
)
.expect_err("blocked MCP provenance must not resolve an execution plan");
assert_eq!(
blocked,
meerkat_core::ToolExecutionResolutionError::NotFound {
tool_name: "blocked".to_string(),
}
);
}
// ── RuntimeBinding TDD tests ────────────────────────────────────────────
//
// These tests define the expected behavior for RuntimeBinding — the first
// step toward identity-first mobs. They verify that external members carry
// real process identity, not phantom placeholder comms keys.
#[tokio::test]
async fn test_external_spawn_with_binding_uses_real_identity() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-spawn-real-identity",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-real")).await;
let crate::RuntimeBinding::External {
peer_id: expected_peer_id,
address: expected_address,
bootstrap_token: _expected_bootstrap_token,
pubkey: _expected_pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let mut spec = SpawnMemberSpec::new("worker", "w-real");
spec.binding = Some(external.binding());
let spawn_result = spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn with binding");
// Verify identity was created
assert!(!spawn_result.agent_identity.as_str().is_empty());
// Verify backend binding through roster
let entry = handle
.roster()
.await
.get_by_identity(&spawn_result.agent_identity)
.cloned()
.expect("roster entry");
match &entry.member_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(
peer_id, &expected_peer_id,
"BackendPeer.peer_id must be the committed external process key"
);
assert_eq!(
address,
canonical_external_address(&expected_address),
"BackendPeer.address must be the committed external process transport address"
);
assert!(
session_id.is_none(),
"peer-only external members should not keep a hidden bridge session"
);
}
other => panic!("expected BackendPeer member ref, got {other:?}"),
}
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_external_tcp_bind_and_peer_turn_use_routable_supervisor_bridge() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-tcp-bind-route",
);
let mob_id = definition.id.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let owner_bridge_session_id = SessionId::new();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_owner_bridge_session_create_authority(owner_bridge_session_id.clone(), false, false)
.create()
.await
.expect("create mob");
let external_name = test_comms_name_for(&mob_id, "lead", "l-tcp");
let external = spawn_live_external_tcp_peer(&external_name).await;
let crate::RuntimeBinding::External {
peer_id: expected_peer_id,
address: expected_address,
..
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
assert!(
expected_address.starts_with("tcp://127.0.0.1:"),
"test harness must exercise TCP routing, got {expected_address}"
);
let mut spec = SpawnMemberSpec::new("lead", "l-tcp");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
spec.binding = Some(external.binding());
handle
.spawn_spec_with_generated_owner_context(spec, owner_bridge_session_id)
.await
.expect("spawn TCP external member");
handle
.member(&AgentIdentity::from("l-tcp"))
.await
.expect("member handle")
.send("tcp external turn", HandlingMode::Queue)
.await
.expect("peer turn should route to TCP external runtime");
assert_eq!(
service.start_turn_call_count(),
0,
"peer-only TCP external dispatch must not fall back to local session start_turn"
);
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"remote runtime must receive the post-bind peer turn"
);
handle.stop().await.expect("stop before peer-only resume");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.notify_orchestrator_on_resume(false)
.resume()
.await
.expect("resume peer-only TCP member from durable endpoint");
let resumed_entry = resumed
.get_member(&AgentIdentity::from("l-tcp"))
.await
.expect("read resumed peer-only TCP member")
.expect("resumed peer-only TCP member");
match resumed_entry.member_ref {
MemberRef::BackendPeer {
peer_id,
address,
session_id,
..
} => {
assert_eq!(peer_id, expected_peer_id);
assert_eq!(address, canonical_external_address(&expected_address));
assert!(session_id.is_none());
}
other => panic!("expected peer-only TCP backend member, got {other:?}"),
}
resumed
.resume()
.await
.expect("explicitly resume the stopped mob before delivering another peer turn");
let delivered_before_resumed_turn = external.delivered_input_ids().await.len();
resumed
.member(&AgentIdentity::from("l-tcp"))
.await
.expect("resumed peer-only TCP handle")
.send("tcp external turn after resume", HandlingMode::Queue)
.await
.expect("resumed peer turn should route to TCP external runtime");
assert!(
external.delivered_input_ids().await.len() > delivered_before_resumed_turn,
"remote runtime must receive the post-resume peer turn"
);
}
#[cfg(not(target_arch = "wasm32"))]
fn unused_loopback_port() -> u16 {
std::net::TcpListener::bind(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
.expect("reserve loopback port")
.local_addr()
.expect("reserved listener local addr")
.port()
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_external_tcp_bind_uses_configured_supervisor_advertised_address() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-tcp-advertised-supervisor",
);
let mob_id = definition.id.clone();
let port = unused_loopback_port();
let advertised_address = format!("tcp://127.0.0.1:{port}");
definition
.backend
.external
.as_mut()
.expect("external backend")
.supervisor_bridge = Some(crate::definition::SupervisorBridgeEndpointConfig {
bind_address: Some(format!("0.0.0.0:{port}")),
advertised_address: Some(advertised_address.clone()),
});
let (handle, service) = create_test_mob(definition).await;
let external_name = test_comms_name_for(&mob_id, "lead", "l-tcp-advertised");
let external = spawn_live_external_tcp_peer(&external_name).await;
let mut spec = SpawnMemberSpec::new("lead", "l-tcp-advertised");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
spec.binding = Some(external.binding());
handle
.spawn_spec_with_generated_owner_context(spec, SessionId::new())
.await
.expect("spawn TCP external member");
assert_eq!(
external.supervisor_addresses().await,
vec![advertised_address.clone()],
"external runtime must be told to trust the configured supervisor bridge address"
);
handle
.member(&AgentIdentity::from("l-tcp-advertised"))
.await
.expect("member handle")
.send(
"tcp external turn through advertised supervisor",
HandlingMode::Queue,
)
.await
.expect("peer turn should route through configured advertised supervisor");
assert_eq!(
service.start_turn_call_count(),
0,
"configured peer-only TCP external dispatch must not fall back to local session start_turn"
);
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"remote runtime must receive the post-bind peer turn"
);
handle
.rotate_supervisor()
.await
.expect("rotate configured supervisor bridge");
let supervisor_addresses = external.supervisor_addresses().await;
assert!(
supervisor_addresses.len() >= 2
&& supervisor_addresses
.iter()
.all(|address| address == &advertised_address),
"rotation must preserve the configured routable supervisor address, got {supervisor_addresses:?}"
);
handle
.member(&AgentIdentity::from("l-tcp-advertised"))
.await
.expect("member handle")
.send(
"tcp external turn after advertised supervisor rotation",
HandlingMode::Queue,
)
.await
.expect("peer turn after rotation should route through configured advertised supervisor");
assert_eq!(
service.start_turn_call_count(),
0,
"configured peer-only TCP external dispatch after rotation must not fall back to local session start_turn"
);
assert_eq!(
external.delivered_input_ids().await.len(),
2,
"remote runtime must receive the post-rotation peer turn"
);
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_external_tcp_fixed_supervisor_bridge_pending_rotation_retry_reuses_active_listener() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-tcp-fixed-supervisor-pending-retry",
);
let mob_id = definition.id.clone();
let port = unused_loopback_port();
let advertised_address = format!("tcp://127.0.0.1:{port}");
definition
.backend
.external
.as_mut()
.expect("external backend")
.supervisor_bridge = Some(crate::definition::SupervisorBridgeEndpointConfig {
bind_address: Some(format!("0.0.0.0:{port}")),
advertised_address: Some(advertised_address.clone()),
});
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(ProfileBinding::as_inline_mut)
.expect("worker profile exists")
.external_addressable = true;
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
// External peer-only members are owned by the MobMachine owner bridge
// session; resume restore (restore_generated_member_operation_bindings)
// re-establishes their ops-owner binding from this canonical authority.
.with_owner_bridge_session_create_authority(SessionId::new(), false, false)
.create()
.await
.expect("create mob");
let external_a =
spawn_live_external_tcp_peer(&test_comms_name_for(&mob_id, "worker", "w-a")).await;
let external_b =
spawn_live_external_tcp_peer(&test_comms_name_for(&mob_id, "worker", "w-b")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-a"),
None,
external_a.binding(),
)
.await
.expect("spawn first TCP external worker");
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-b"),
None,
external_b.binding(),
)
.await
.expect("spawn second TCP external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
external_b.fail_next_authorize();
let error = handle
.rotate_supervisor()
.await
.expect_err("partial rotation should leave pending supervisor authority");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
attempted_public_peer_id,
rotated_peer_count,
pending_authority_recorded,
..
} => {
assert_eq!(previous_epoch, original.epoch);
assert_eq!(attempted_epoch, original.epoch + 1);
assert_eq!(rotated_peer_count, 1);
assert!(
pending_authority_recorded,
"partial TCP rotation should persist pending authority for retry"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str()),
"first remote should accept the attempted authority before retry"
);
handle
.shutdown()
.await
.expect("shutdown original actor before retry");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume mob with pending supervisor rotation");
let retry_report = resumed
.rotate_supervisor()
.await
.expect("fixed-port retry should reuse the active supervisor listener");
assert_eq!(retry_report.previous_epoch, original.epoch);
assert_eq!(retry_report.current_epoch, original.epoch + 1);
let retried = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after retry")
.expect("authority after retry");
assert_eq!(retried.public_peer_id, retry_report.public_peer_id);
assert!(
retried.pending_rotation.is_none(),
"successful fixed-port retry must clear pending rotation metadata"
);
assert_eq!(
external_a.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should reconcile the already-accepted TCP remote"
);
assert_eq!(
external_b.authorized_supervisor_peer_id().await.as_deref(),
Some(retried.public_peer_id.as_str()),
"retry should rotate the TCP remote that rejected the first attempt"
);
let supervisor_addresses = external_a.supervisor_addresses().await;
assert!(
supervisor_addresses
.iter()
.all(|address| address == &advertised_address),
"retry must keep publishing the configured routable supervisor address, got {supervisor_addresses:?}"
);
resumed
.member(&AgentIdentity::from("w-a"))
.await
.expect("member handle")
.send("tcp fixed-port retry peer turn", HandlingMode::Queue)
.await
.expect("peer turn should route after fixed-port retry");
assert_eq!(
service.start_turn_call_count(),
0,
"peer-only TCP external dispatch after retry must not fall back to local session start_turn"
);
assert_eq!(
external_a.delivered_input_ids().await.len(),
1,
"remote runtime must receive the post-retry peer turn"
);
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_fixed_port_activation_rebuild_failure_retries_same_durable_operation() {
let _serial = lock_real_comms_tests();
let mut definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"fixed-port-activation-rebuild-retry-anchor",
);
let mob_id = definition.id.clone();
let port = unused_loopback_port();
let advertised_address = format!("tcp://127.0.0.1:{port}");
definition
.backend
.external
.as_mut()
.expect("external backend")
.supervisor_bridge = Some(crate::definition::SupervisorBridgeEndpointConfig {
bind_address: Some(format!("0.0.0.0:{port}")),
advertised_address: Some(advertised_address),
});
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create fixed-port mob");
let external =
spawn_live_external_tcp_peer(&test_comms_name_for(&mob_id, "worker", "w-fixed")).await;
handle
.spawn_with_binding(
ProfileName::from("worker"),
AgentIdentity::from("w-fixed"),
None,
external.binding(),
)
.await
.expect("spawn TCP external worker");
let original = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original authority")
.expect("original authority record");
super::supervisor_bridge::fail_next_fixed_port_rotation_rebuild_for_test();
let error = handle
.rotate_supervisor()
.await
.expect_err("fixed-port rebuild failure must leave local activation pending");
let attempted_public_peer_id = match error {
MobError::SupervisorRotationIncomplete {
attempted_public_peer_id,
pending_authority_recorded,
reason,
..
} => {
assert!(pending_authority_recorded);
assert!(
reason.contains("local authority activation failed")
&& reason.contains("fixed-port supervisor bridge rebuild failure"),
"rebuild failure should remain visible with its durable anchor: {reason}"
);
attempted_public_peer_id
}
other => panic!("expected typed incomplete supervisor rotation, got: {other:?}"),
};
let after_failure = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load authority after rebuild failure")
.expect("authority after rebuild failure");
assert_eq!(after_failure.epoch, original.epoch);
assert_eq!(after_failure.public_peer_id, original.public_peer_id);
let pending = after_failure
.pending_rotation
.expect("remote-complete/local-activation-pending anchor");
let operation_id = pending.operation_id.expect("stable operation id");
assert_eq!(pending.public_peer_id, attempted_public_peer_id);
assert_eq!(pending.accepted_peer_ids.len(), 1);
assert_eq!(pending.member_targets.len(), 1);
assert_eq!(external.rotation_operation_ids().await, vec![operation_id]);
assert_eq!(
external.authorized_supervisor_peer_id().await.as_deref(),
Some(attempted_public_peer_id.as_str()),
"remote completion must remain fenced onto the attempted authority"
);
let report = handle
.rotate_supervisor()
.await
.expect("retry should rebuild locally and finalize the same operation");
assert_eq!(report.previous_epoch, original.epoch);
assert_eq!(report.current_epoch, original.epoch + 1);
assert_eq!(report.public_peer_id, attempted_public_peer_id);
assert_eq!(
external.rotation_operation_ids().await,
vec![operation_id],
"accepted remote receipt must be observed, not resubmitted as a new operation"
);
let committed = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load committed authority")
.expect("committed authority");
assert_eq!(committed.public_peer_id, attempted_public_peer_id);
assert!(committed.pending_rotation.is_none());
}
#[cfg(not(target_arch = "wasm32"))]
#[tokio::test]
async fn test_private_ephemeral_tcp_supervisor_rotation_rejected_before_mutation() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-tcp-ephemeral-rotate-rejected",
);
let mob_id = definition.id.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let owner_bridge_session_id = SessionId::new();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_owner_bridge_session_create_authority(owner_bridge_session_id.clone(), false, false)
.create()
.await
.expect("create mob");
let external_name = test_comms_name_for(&mob_id, "lead", "l-tcp-rotate");
let external = spawn_live_external_tcp_peer(&external_name).await;
let mut spec = SpawnMemberSpec::new("lead", "l-tcp-rotate");
spec.runtime_mode = Some(crate::MobRuntimeMode::TurnDriven);
spec.binding = Some(external.binding());
handle
.spawn_spec_with_generated_owner_context(spec, owner_bridge_session_id)
.await
.expect("spawn TCP external member");
let original_authority = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load original supervisor authority")
.expect("original supervisor authority");
let original_remote_authority = external.authorized_supervisor_peer_id().await;
let original_rotation_operations = external.rotation_operation_ids().await;
let error = handle
.rotate_supervisor()
.await
.expect_err("private ephemeral TCP rotation must fail before mutation");
assert!(
matches!(&error, MobError::WiringError(message) if message.contains("requires a stable callback endpoint")),
"unexpected ephemeral TCP rotation failure: {error:?}"
);
let retained_authority = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor authority after rejected rotation")
.expect("retained supervisor authority");
assert_eq!(retained_authority, original_authority);
assert!(retained_authority.pending_rotation.is_none());
assert_eq!(
external.authorized_supervisor_peer_id().await,
original_remote_authority,
"rejected rotation must not change remote supervisor authority"
);
assert_eq!(
external.rotation_operation_ids().await,
original_rotation_operations,
"rejected rotation must not submit a remote operation"
);
handle
.member(&AgentIdentity::from("l-tcp-rotate"))
.await
.expect("member handle")
.send(
"tcp external turn after rejected rotation",
HandlingMode::Queue,
)
.await
.expect("retained TCP route should remain usable");
assert_eq!(
service.start_turn_call_count(),
0,
"retained peer-only TCP dispatch must not fall back to local session start_turn"
);
assert_eq!(
external.delivered_input_ids().await.len(),
1,
"remote runtime must receive the turn after rejected rotation"
);
}
#[tokio::test]
async fn test_force_cancel_peer_only_external_member_uses_cooperative_bridge_interrupt() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"external-force-cancel",
);
let mob_id = definition.id.clone();
let (handle, _service) = create_test_mob(definition).await;
let external =
spawn_live_external_peer(&test_comms_name_for(&mob_id, "worker", "w-cancel")).await;
let mut spec = SpawnMemberSpec::new("worker", "w-cancel");
spec.binding = Some(external.binding());
let spawn = spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn peer-only external member");
handle
.force_cancel_member(spawn.agent_identity.clone())
.await
.expect("force-cancel peer-only external member");
assert_eq!(
external.interrupt_count(),
1,
"peer-only force-cancel must use the cooperative interrupt bridge command"
);
assert_eq!(
external.hard_cancel_reasons().await,
Vec::<String>::new(),
"peer-only force-cancel must not use the hard-cancel bridge command"
);
}
#[tokio::test]
async fn test_external_spawn_without_binding_errors() {
let (handle, _service) = create_test_mob(sample_definition_with_external_backend()).await;
// Bare MobBackendKind::External without RuntimeBinding must fail — you
// can't spawn an external member without saying who the real process is.
let mut spec = SpawnMemberSpec::new("worker", "w-bare");
spec.backend = Some(MobBackendKind::External);
// No binding set.
let result = handle.spawn_spec(spec).await;
assert!(
result.is_err(),
"external backend without RuntimeBinding must be rejected: {result:?}"
);
}
#[tokio::test]
async fn test_session_spawn_default_binding() {
// Regression guard: session backend spawns continue to work with no binding.
let (handle, _service) = create_test_mob(sample_definition_with_external_backend()).await;
let member_ref = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-session"),
None,
)
.await
.expect("spawn session member");
assert!(
matches!(member_ref, MemberRef::Session { .. }),
"default spawn should produce session member ref"
);
}
#[tokio::test]
async fn test_trusted_peer_spec_uses_real_external_identity_for_peer_only_members() {
// Peer-only external members should wire using the real external transport
// identity rather than a hidden local bridge session.
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"trusted-peer-spec",
);
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob_with_real_comms(definition).await;
let external_name = test_comms_name_for(&mob_id, "worker", "w-bridge");
let external = spawn_live_external_peer(&external_name).await;
let crate::RuntimeBinding::External {
peer_id,
address,
bootstrap_token,
pubkey,
} = external.binding()
else {
panic!("live external peer must produce external binding");
};
let mut spec = SpawnMemberSpec::new("worker", "w-bridge");
spec.binding = Some(crate::RuntimeBinding::External {
peer_id: peer_id.clone(),
address: address.clone(),
bootstrap_token: bootstrap_token.clone(),
pubkey,
});
let _member_ref = spawn_spec_with_test_ops_owner(&handle, spec)
.await
.expect("spawn with real binding");
// Wire to orchestrator to trigger trust spec resolution
let lead_ref = handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
)
.await
.expect("spawn lead");
handle
.wire(
AgentIdentity::from("lead-1"),
AgentIdentity::from("w-bridge"),
)
.await
.expect("wire lead to external member");
// Check what trust was added to the lead's comms runtime.
let lead_sid = lead_ref
.bridge_session_id()
.cloned()
.expect("lead session id");
let trusted_peers = service
.real_comms(&lead_sid)
.await
.expect("lead comms runtime")
.peers()
.await;
let trusted_peer = trusted_peers
.iter()
.find(|entry| entry.name.as_str() == external_name)
.expect("lead should trust the real external peer");
assert!(
trusted_peer.address.to_string() == canonical_external_address(&address),
"peer-only trust specs should use the real external address. Got: {trusted_peers:?}",
);
assert!(
trusted_peer.peer_id.to_string() == peer_id,
"peer-only trust should retain the real external peer identity. Got: {trusted_peers:?}",
);
}
#[tokio::test]
async fn test_supervisor_trust_does_not_leak_into_member_peer_directory() {
// Fix 2 regression (negative invariant): `finalize_spawn_from_pending`
// bootstraps supervisor trust on every session-backed member so
// lifecycle notifications land at the classified inbox. That bootstrap
// must NOT register the supervisor as a publicly-listed trusted peer —
// it's a control-plane edge and would otherwise surface as an ordinary
// sendable peer in the member's `comms.peers` output (and downstream
// REST/RPC/MCP). The private-trust seam uses the router's
// private peer-id directory-filter to keep the entry invisible.
//
// The *positive* complement — "the supervisor is still trusted enough
// for the member to send back to it" — is enforced by
// `test_supervisor_private_trust_preserves_send_resolution` below.
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(sample_definition(), "no-supervisor-leak");
let (handle, service) = create_test_mob_with_real_comms(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-private"),
None,
)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let member_comms = service
.real_comms(&session_id)
.await
.expect("member comms runtime");
let directory = member_comms.peers().await;
let supervisor_entry = directory
.iter()
.find(|entry| entry.name.as_str().contains("__mob_supervisor__"));
assert!(
supervisor_entry.is_none(),
"supervisor must NOT appear in the session-backed member's public \
peer directory; the private-trust seam is the whole point. Got: {directory:?}"
);
}
#[tokio::test]
async fn test_supervisor_private_trust_failure_does_not_commit_spawn() {
let definition = with_unique_mob_id(sample_definition(), "supervisor-trust-failure");
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob(definition).await;
service
.set_comms_behavior(
&format!("{mob_id}/worker/w-trust-fail"),
MockCommsBehavior {
fail_add_trust: true,
..MockCommsBehavior::default()
},
)
.await;
let error = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-trust-fail"),
None,
)
.await
.expect_err("supervisor private trust failure should fail spawn");
assert!(
error
.to_string()
.contains("supervisor private trust publication failed"),
"spawn should surface the authority-gated trust failure, got: {error}"
);
assert!(
handle.list_members().await.is_empty(),
"failed supervisor trust publication must not commit roster membership"
);
assert_eq!(
service.active_session_count().await,
0,
"failed supervisor trust publication must roll back the provisioned session"
);
let events = handle.events().replay_all().await.expect("replay events");
assert!(
events
.iter()
.all(|event| !matches!(event.kind, MobEventKind::MemberSpawned(_))),
"failed supervisor trust publication must not append MemberSpawned"
);
}
#[tokio::test]
async fn test_supervisor_private_trust_preserves_send_resolution() {
// Fix 2 positive invariant: the private-trust seam must still keep
// the supervisor fully send-resolvable from the member. When splitting
// `add_trusted_peer` into "admission + directory + send-resolution" vs
// "admission + send-resolution (no directory)", the positive side was
// the one that broke first — e2e-smoke caught pictionary kickoff
// timing out because the admission-only path left the member's
// router unable to resolve the supervisor's `PeerName` to a `PeerAddr`,
// so reply-sends silently failed. This test pins the required
// invariants: the supervisor entry is in the router's `TrustStore`
// (so `send_peer_command` can resolve), is marked private (so the
// directory hides it), and the member's classified inbox knows the
// supervisor pubkey (so admission accepts lifecycle notifications).
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(sample_definition(), "supervisor-send-resolution");
let (handle, service) = create_test_mob_with_real_comms(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-send"),
None,
)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let member_comms = service
.real_comms(&session_id)
.await
.expect("member comms runtime");
// Look up the supervisor pubkey via the member's trust list — this is
// the same list the router uses to resolve `PeerName → PeerAddr`.
let supervisor_pubkey = {
let trusted = member_comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
let entry = trusted_peers
.iter()
.find(|p| p.name.as_str().contains("__mob_supervisor__"))
.expect(
"send-resolution invariant: supervisor must still appear in the \
member's internal TrustStore — that's what the router consults \
to resolve the supervisor's PeerName to a PeerAddr on reply-sends",
);
entry.pubkey
};
// The directory-filter side-channel must mark this pubkey private.
assert!(
member_comms.router().is_private(&supervisor_pubkey),
"directory-filter invariant: supervisor pubkey must be marked \
private so `resolve_peer_directory()` hides it"
);
// And the directory must in fact hide it. This is the same fact the
// negative-invariant test asserts at the `peers()` API, pinned here
// as the end-to-end consequence of the two internal invariants above.
let directory = member_comms.peers().await;
assert!(
directory
.iter()
.all(|entry| !entry.name.as_str().contains("__mob_supervisor__")),
"directory output must not list the supervisor; got: {directory:?}"
);
}
#[tokio::test]
async fn test_rotate_supervisor_reinstalls_private_trust_on_session_backed_member() {
// W2-F integration test: after a supervisor rotation, session-backed
// members must privately trust the NEW supervisor peer id and must no
// longer trust the previous one. Without the re-install step, the old
// peer id sticks in the member's private-trust set and new-supervisor
// lifecycle notifications (`mob.peer_added`, `mob.peer_retired`, …) are
// dropped at the classified inbox's admission gate.
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(sample_definition(), "rotate-private-trust");
let mob_id = definition.id.clone();
let (handle, service) = create_test_mob_with_real_comms(definition).await;
let receipt = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-rotate"),
None,
)
.await
.expect("spawn session-backed member");
let session_id = receipt
.bridge_session_id()
.cloned()
.expect("session-backed member has bridge session id");
let member_comms = service
.real_comms(&session_id)
.await
.expect("member comms runtime");
// Capture the pre-rotation supervisor pubkey. This is what the member's
// private trust set must no longer trust after the rotation.
let previous_supervisor_pubkey = {
let trusted = member_comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
let entry = trusted_peers
.iter()
.find(|p| p.name.as_str().contains("__mob_supervisor__"))
.expect("spawn bootstraps supervisor private trust");
entry.pubkey
};
assert!(
member_comms
.router()
.is_private(&previous_supervisor_pubkey),
"pre-rotation supervisor must start as a private-trust entry"
);
let previous_name = {
let trusted = member_comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
trusted_peers
.iter()
.find(|p| p.name.as_str().contains("__mob_supervisor__"))
.expect("supervisor trust entry")
.name
.clone()
};
let report = handle.rotate_supervisor().await.expect("rotate supervisor");
assert_eq!(
report.previous_epoch + 1,
report.current_epoch,
"rotation must bump epoch by exactly one"
);
assert_ne!(
report.public_peer_id,
previous_supervisor_pubkey.to_peer_id().to_string(),
"rotation must replace the supervisor keypair"
);
// Post-rotation: the old supervisor pubkey must no longer be trusted,
// and the rotated supervisor pubkey must be present and marked private
// so the directory still hides it while the admission gate still
// accepts lifecycle notifications.
let (new_supervisor_pubkey_opt, old_still_present) = {
let trusted = member_comms.trusted_peers_shared();
let trusted_peers = trusted.entries();
let new_entry = trusted_peers
.iter()
.find(|p| p.name.as_str().contains("__mob_supervisor__"));
let old_still_present = trusted_peers
.iter()
.any(|p| p.pubkey == previous_supervisor_pubkey);
(new_entry.map(|entry| entry.pubkey), old_still_present)
};
assert!(
!old_still_present,
"previous supervisor private trust must be revoked by the rotation \
trust re-install; leaving it in would accept lifecycle notifications \
from the superseded authority"
);
let new_supervisor_pubkey = new_supervisor_pubkey_opt
.expect("rotated supervisor private trust must be installed on the session-backed member");
assert_ne!(
new_supervisor_pubkey, previous_supervisor_pubkey,
"rotation must change the trusted supervisor pubkey"
);
assert!(
member_comms.router().is_private(&new_supervisor_pubkey),
"rotated supervisor must stay on the private-trust edge — it is a \
control-plane peer and must not leak into `comms.peers`"
);
assert_eq!(
new_supervisor_pubkey.to_peer_id().to_string(),
report.public_peer_id,
"member-side private trust must agree with the rotation report"
);
// And the directory must still hide the (rotated) supervisor, matching
// the invariant from `test_supervisor_trust_does_not_leak_into_member_peer_directory`.
let directory = member_comms.peers().await;
assert!(
directory
.iter()
.all(|entry| !entry.name.as_str().contains("__mob_supervisor__")),
"rotated supervisor must remain absent from the public peer directory; got: {directory:?}"
);
// Sanity check: we really did spawn a session-backed member of the
// expected mob (the trust-entry name embeds the mob id).
assert!(
previous_name.as_str().starts_with(mob_id.as_str()),
"private-trust entry name must be scoped to this mob; got: {previous_name}"
);
}
#[tokio::test]
async fn test_external_tools_provider_called_per_spawn() {
let counter = Arc::new(std::sync::atomic::AtomicU32::new(0));
let counter_clone = counter.clone();
let provider: crate::ExternalToolsProvider = Arc::new(move || {
counter_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Some(Arc::new(MultiToolDispatcher::new(&["ext_tool"])) as Arc<dyn AgentToolDispatcher>)
});
let definition = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_default_external_tools_provider(Some(provider))
.create()
.await
.expect("create mob");
// Spawn 3 workers
for i in 1..=3 {
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from(format!("w-{i}")),
None,
)
.await
.expect("spawn");
}
assert_eq!(
counter.load(std::sync::atomic::Ordering::Relaxed),
3,
"provider should be called once per spawn"
);
// Verify external tools are visible on spawned sessions
let flags = service.recorded_external_tools_flags().await;
assert!(
flags.iter().all(|&f| f),
"all spawned sessions should have external tools"
);
}
#[tokio::test]
async fn test_per_spawn_external_tools_compose_with_profile_and_default_tools() {
let provider: crate::ExternalToolsProvider = Arc::new(|| {
Some(
Arc::new(MultiToolDispatcher::new(&["default_only", "shared_tool"]))
as Arc<dyn AgentToolDispatcher>,
)
});
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|binding| binding.as_inline_mut())
.expect("worker profile")
.tools
.rust_bundles = vec!["profile-bundle".to_string()];
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service.clone())
.register_tool_bundle(
"profile-bundle",
Arc::new(MultiToolDispatcher::new(&["profile_only", "shared_tool"])),
)
.with_default_external_tools_provider(Some(provider))
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-per-spawn");
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&[
"spawn_only",
"shared_tool",
])));
let spawned = handle.spawn_spec(spec).await.expect("spawn");
let session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session id");
let tool_names = service.external_tool_names(&session_id).await;
assert!(tool_names.contains(&"profile_only".to_string()));
assert!(tool_names.contains(&"spawn_only".to_string()));
assert!(tool_names.contains(&"default_only".to_string()));
let shared_count = tool_names
.iter()
.filter(|name| name.as_str() == "shared_tool")
.count();
assert_eq!(
shared_count, 1,
"profile > per-spawn > default collision order must expose one deterministic shared_tool"
);
}
#[tokio::test]
async fn test_per_spawn_external_tools_are_not_restored_from_storage() {
let definition = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-live-dispatcher");
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["live_only"])));
let spawned = handle.spawn_spec(spec).await.expect("spawn");
let old_session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("old bridge session");
assert!(
service
.external_tool_names(&old_session_id)
.await
.contains(&"live_only".to_string()),
"live per-spawn dispatcher should be installed on the original create"
);
handle.stop().await.expect("stop");
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal for durable snapshots).
MobSessionService::discard_live_session(service.as_ref(), &old_session_id)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
resumed
.resume()
.await
.expect("explicit resume should rebuild stopped member sessions");
let flags = service.recorded_external_tools_flags().await;
assert_eq!(
flags.last().copied(),
Some(false),
"per-spawn trait-object dispatchers must be supplied again by the upper layer on restore"
);
}
fn sample_definition_with_turn_driven_worker() -> MobDefinition {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline_mut()
.expect("worker profile should be inline")
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
definition
}
/// M2 Wave-1 regression (D1): the per-spawn external-tools overlay must be
/// retained by the actor and recomposed when machine-authorized revival
/// rebuilds a discarded live session. Before the retention map,
/// `materialize_revived_member_session` passed `None` to
/// `external_tools_for_profile` and revived members silently lost their
/// per-spawn tools.
#[tokio::test]
async fn test_revival_recomposes_per_spawn_external_tools_overlay() {
let (handle, service) = create_test_mob(sample_definition_with_turn_driven_worker()).await;
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["per_spawn_probe"])));
let spawned = handle.spawn_spec(spec).await.expect("spawn worker");
let bridge_session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session id");
assert!(
service
.external_tool_names(&bridge_session_id)
.await
.contains(&"per_spawn_probe".to_string()),
"sanity: the original create must expose the per-spawn overlay"
);
MobSessionService::discard_live_session(service.as_ref(), &bridge_session_id)
.await
.expect("discard live session");
handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("come back online".to_string()))
.await
.expect("machine-authorized revival must rebuild the live session");
let revived_tool_names = service.external_tool_names(&bridge_session_id).await;
assert!(
revived_tool_names.contains(&"per_spawn_probe".to_string()),
"revival must recompose the retained per-spawn overlay: {revived_tool_names:?}"
);
}
/// M2 Wave-2 regression (D2): per-spawn declarative tooling carried on
/// `SpawnMemberSpec.override_profile` must survive a cold process restart
/// WITHOUT a `SpawnMemberCustomizer`. Before `MemberSpawnedEvent` carried
/// `effective_profile_override`, roster replay projected `None` and the
/// restore rebuilt the member from the definition profile, silently dropping
/// per-spawn MCP servers.
#[tokio::test]
async fn test_cold_restart_restores_per_spawn_profile_override_without_customizer() {
let definition = sample_definition();
let mut override_profile = definition
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.expect("worker profile should be inline")
.clone();
override_profile.tools.mcp_servers = vec![meerkat_core::mcp_config::McpServerConfig::stdio(
"planner",
"/bin/echo",
vec![],
HashMap::new(),
)];
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-override");
spec.override_profile = Some(override_profile);
let spawned = handle
.spawn_spec(spec)
.await
.expect("spawn override worker");
let old_session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session id");
handle.stop().await.expect("stop");
MobSessionService::discard_live_session(service.as_ref(), &old_session_id)
.await
.expect("discard live session");
// Cold restart: rebuild the mob from durable storage with NO customizer.
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
resumed
.resume()
.await
.expect("explicit resume should rebuild stopped member sessions");
let entry = resumed
.get_member(&AgentIdentity::from("w-override"))
.await
.expect("get member")
.expect("restored member");
let restored_override = entry
.effective_profile_override
.expect("replayed MemberSpawned must repopulate effective_profile_override");
assert_eq!(restored_override.tools.mcp_servers.len(), 1);
assert_eq!(restored_override.tools.mcp_servers[0].name, "planner");
let mcp_creates = service.recorded_mcp_server_names().await;
let (_, last_names) = mcp_creates
.last()
.expect("restore must re-create the member session");
assert!(
last_names.contains(&"planner".to_string()),
"restored member build must carry the per-spawn declarative MCP servers: {mcp_creates:?}"
);
}
#[tokio::test]
async fn test_cold_restart_does_not_replay_persisted_system_prompt_configuration() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let prompt = "configured once, retained across rematerialization";
let mut spec = SpawnMemberSpec::new("worker", "w-prompt-restore");
spec.system_prompt_override = Some(SpawnSystemPromptOverride::Replace(prompt.to_string()));
let spawned = handle.spawn_spec(spec).await.expect("spawn worker");
let session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session id");
handle.stop().await.expect("stop");
MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.resume()
.await
.expect("resume");
resumed
.resume()
.await
.expect("rebuild stopped member session");
let restored = service
.live_session_clone(&session_id)
.await
.expect("restored live session");
assert_eq!(
restored
.messages()
.iter()
.filter(
|message| matches!(message, Message::System(system) if system.content == prompt)
)
.count(),
1,
"automatic rematerialization must not append persisted prompt configuration",
);
}
struct RespawnOverlayCustomizer;
impl SpawnMemberCustomizer for RespawnOverlayCustomizer {
fn customize_spawn(
&self,
ctx: &SpawnCustomizationContext,
spec: &mut SpawnMemberSpec,
) -> Result<(), MobError> {
if ctx.spawn_source == SpawnSource::Respawn {
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["respawn_tool"])));
}
Ok(())
}
}
/// Respawn replacement semantics for the retention map: the replacement
/// spawn's overlay replaces the retired incarnation's entry, so a later
/// machine-authorized revival composes the replacement tools, not the
/// original spawn's.
#[tokio::test]
async fn test_respawn_replaces_retained_per_spawn_external_tools_for_revival() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(
sample_definition_with_turn_driven_worker(),
MobStorage::in_memory(),
)
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(RespawnOverlayCustomizer))
.create()
.await
.expect("create mob");
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["tool_a"])));
handle.spawn_spec(spec).await.expect("spawn worker");
handle
.respawn(AgentIdentity::from("w-1"), None)
.await
.expect("respawn worker");
let respawn_session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("respawn bridge session");
MobSessionService::discard_live_session(service.as_ref(), &respawn_session_id)
.await
.expect("discard live session");
handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("revive replacement".to_string()))
.await
.expect("revival after respawn");
let revived_tool_names = service.external_tool_names(&respawn_session_id).await;
assert!(
revived_tool_names.contains(&"respawn_tool".to_string()),
"revival after respawn must compose the replacement overlay: {revived_tool_names:?}"
);
assert!(
!revived_tool_names.contains(&"tool_a".to_string()),
"the retired incarnation's overlay must not leak into the replacement's revival: {revived_tool_names:?}"
);
}
/// Retention cleanup / no leak: the old incarnation's disposal
/// (`dispose_remove_from_roster`) drops the retained overlay, and a
/// replacement spawned WITHOUT external_tools keeps it absent (`None` =
/// replacement semantics), so the new incarnation revives without the old
/// tool. A later incarnation reuses the identity through the same
/// machine-owned generation high-water path, so this respawn test guards the
/// replacement-overlay contract without relying on a current-binding tombstone.
#[tokio::test]
async fn test_respawn_without_replacement_overlay_clears_retained_per_spawn_tools() {
let (handle, service) = create_test_mob(sample_definition_with_turn_driven_worker()).await;
let mut spec = SpawnMemberSpec::new("worker", "w-1");
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["stale_tool"])));
handle.spawn_spec(spec).await.expect("spawn worker");
handle
.respawn(AgentIdentity::from("w-1"), None)
.await
.expect("respawn worker without a replacement overlay");
let respawn_session_id = handle
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("respawn bridge session");
MobSessionService::discard_live_session(service.as_ref(), &respawn_session_id)
.await
.expect("discard live session");
handle
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("revive fresh incarnation".to_string()))
.await
.expect("revival of fresh incarnation");
let revived_tool_names = service.external_tool_names(&respawn_session_id).await;
assert!(
!revived_tool_names.contains(&"stale_tool".to_string()),
"disposal must drop the retained overlay; the fresh incarnation revives clean: {revived_tool_names:?}"
);
}
struct ResumeOverlayCustomizer;
impl SpawnMemberCustomizer for ResumeOverlayCustomizer {
fn customize_spawn(
&self,
ctx: &SpawnCustomizationContext,
spec: &mut SpawnMemberSpec,
) -> Result<(), MobError> {
if ctx.spawn_source == SpawnSource::Resume {
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&[
"customizer_resume_probe",
])));
}
Ok(())
}
}
/// Restore-seeded retention: tools attached by a `SpawnMemberCustomizer` at
/// `SpawnSource::Resume` seed the actor's retention map, so a later
/// machine-authorized revival in the restored incarnation recomposes them
/// (guards the reconcile_resume -> MobActor construction seeding).
#[tokio::test]
async fn test_restore_seeded_per_spawn_tools_survive_machine_authorized_revival() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(sample_definition_with_turn_driven_worker(), storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let spawned = handle
.spawn_spec(SpawnMemberSpec::new("worker", "w-1"))
.await
.expect("spawn worker");
let old_session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session id");
handle.stop().await.expect("stop");
MobSessionService::discard_live_session(service.as_ref(), &old_session_id)
.await
.expect("discard live session");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(ResumeOverlayCustomizer))
.resume()
.await
.expect("resume");
resumed
.resume()
.await
.expect("explicit resume should rebuild stopped member sessions");
let restored_session_id = resumed
.resolve_bridge_session_id(&AgentIdentity::from("w-1"))
.await
.expect("restored bridge session");
assert!(
service
.external_tool_names(&restored_session_id)
.await
.contains(&"customizer_resume_probe".to_string()),
"sanity: the restore build must compose the customizer overlay"
);
MobSessionService::discard_live_session(service.as_ref(), &restored_session_id)
.await
.expect("discard restored live session");
resumed
.member(&AgentIdentity::from("w-1"))
.await
.expect("member handle")
.internal_turn(ContentInput::from("revive restored member".to_string()))
.await
.expect("revival after restore");
let revived_tool_names = service.external_tool_names(&restored_session_id).await;
assert!(
revived_tool_names.contains(&"customizer_resume_probe".to_string()),
"customizer-restored tools must survive machine-authorized revival: {revived_tool_names:?}"
);
}
#[tokio::test]
async fn test_external_tools_name_collision_profile_wins() {
// Provider returns a dispatcher with tool "spawn_member". When
// profile.tools.mob = true, the canonical resolver mounts the mob-owned
// spawn_member dispatcher; on name collision, the profile-declared tool
// wins and shadows the callback's spawn_member. The callback's unique
// tool stays visible. Dispatching the colliding name reaches the
// mob-owned dispatcher, which denies on scope (default create-only
// authority has no managed_mob_scope for this mob).
let provider: crate::ExternalToolsProvider = Arc::new(|| {
Some(
Arc::new(MultiToolDispatcher::new(&["spawn_member", "my_callback"]))
as Arc<dyn AgentToolDispatcher>,
)
});
let definition = sample_definition_with_mob_tools();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_default_external_tools_provider(Some(provider))
.create()
.await
.expect("create mob");
let member_ref = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn should succeed despite name collision");
let session_id = member_ref
.bridge_session_id()
.expect("session-backed")
.clone();
let tool_names = service.external_tool_names(&session_id).await;
assert!(
tool_names.contains(&"spawn_member".to_string()),
"spawn_member should be visible (the profile-declared mob-owned tool wins)"
);
assert!(
tool_names.contains(&"my_callback".to_string()),
"non-colliding callback tool should be present"
);
// Dispatch spawn_member — reaches the mob-owned dispatcher, not the
// colliding callback tool.
let raw_args = serde_json::json!({"profile": "worker", "member_id": "w-test"});
service
.dispatch_external_tool_outcome(&session_id, "spawn_member", raw_args)
.await
.expect("mob-owned spawn_member should win and use generated definition-profile scope");
}
#[tokio::test]
async fn test_spawn_member_with_initial_turn_returns_before_child_turn_completion() {
let mut definition = sample_definition_with_mob_tools();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(|binding| binding.as_inline_mut())
.expect("worker profile")
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let member_ref = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("spawner"),
None,
)
.await
.expect("spawn should expose spawn_member");
let session_id = member_ref
.bridge_session_id()
.expect("session-backed spawner")
.clone();
service.set_start_turn_delay_ms(600_000);
let raw_args = serde_json::json!({
"profile": "worker",
"member_id": "child-with-slow-initial-turn",
"initial_message": "hold child turn open"
});
let outcome = tokio::time::timeout(
Duration::from_millis(250),
service.dispatch_external_tool_outcome(&session_id, "spawn_member", raw_args),
)
.await
.expect("spawn_member tool must return after child turn ingress admission")
.expect("spawn_member should dispatch");
let payload: serde_json::Value =
serde_json::from_str(&outcome.result.text_content()).expect("spawn result json");
assert_eq!(
payload["agent_identity"],
serde_json::json!("child-with-slow-initial-turn")
);
tokio::time::timeout(Duration::from_secs(2), async {
while service.start_turn_call_count() == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("detached child initial turn should start after spawn_member returns");
let child = tokio::time::timeout(
Duration::from_millis(250),
handle.get_member(&AgentIdentity::from("child-with-slow-initial-turn")),
)
.await
.expect("actor must stay responsive while child initial turn is still running")
.expect("get_member command")
.expect("child should be visible in roster");
assert_eq!(
child.agent_identity,
AgentIdentity::from("child-with-slow-initial-turn")
);
}
#[tokio::test]
async fn test_default_helper_spawns_remain_ephemeral() {
let definition = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.create()
.await
.expect("create mob");
handle
.spawn_helper(
AgentIdentity::from("helper-ephemeral"),
"short helper task",
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
..HelperOptions::default()
},
)
.await
.expect("helper spawn");
let events = handle.events().replay_all().await.expect("events");
let helper_spawn = events
.iter()
.filter_map(|event| event.kind.member_spawned())
.find(|spawn| spawn.agent_identity.as_str() == "helper-ephemeral")
.expect("helper spawn event");
assert_eq!(
helper_spawn.continuity_intent,
SpawnContinuityIntent::Ephemeral,
"helpers must not become durable unless a customizer explicitly promotes them"
);
}
#[derive(Clone)]
struct RecordingSpawnCustomizer {
calls: Arc<Mutex<Vec<SpawnCustomizationContext>>>,
}
impl RecordingSpawnCustomizer {
fn new() -> Self {
Self {
calls: Arc::new(Mutex::new(Vec::new())),
}
}
fn calls(&self) -> Vec<SpawnCustomizationContext> {
self.calls.lock().expect("customizer calls").clone()
}
}
impl SpawnMemberCustomizer for RecordingSpawnCustomizer {
fn customize_spawn(
&self,
ctx: &SpawnCustomizationContext,
spec: &mut SpawnMemberSpec,
) -> Result<(), MobError> {
self.calls
.lock()
.expect("customizer calls")
.push(ctx.clone());
spec.labels.get_or_insert_with(BTreeMap::new).insert(
"customized".to_string(),
ctx.spawn_source.as_str().to_string(),
);
let tool_name = format!("customizer_{}", ctx.spawn_source.as_str());
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&[tool_name.as_str()])));
spec.system_prompt_override = Some(SpawnSystemPromptOverride::Replace(format!(
"custom prompt for {}",
ctx.spawn_source.as_str()
)));
if ctx.spawn_source == SpawnSource::HelperSpawn {
spec.continuity_intent = SpawnContinuityIntent::DurableIdentity {
continuity_key: format!("helper/{}", spec.identity),
};
}
Ok(())
}
}
#[derive(Clone)]
struct ResumeAddressabilityCustomizer {
calls: Arc<Mutex<Vec<SpawnCustomizationContext>>>,
override_profile: Profile,
}
impl ResumeAddressabilityCustomizer {
fn new(override_profile: Profile) -> Self {
Self {
calls: Arc::new(Mutex::new(Vec::new())),
override_profile,
}
}
fn calls(&self) -> Vec<SpawnCustomizationContext> {
self.calls.lock().expect("resume customizer calls").clone()
}
}
impl SpawnMemberCustomizer for ResumeAddressabilityCustomizer {
fn customize_spawn(
&self,
ctx: &SpawnCustomizationContext,
spec: &mut SpawnMemberSpec,
) -> Result<(), MobError> {
self.calls
.lock()
.expect("resume customizer calls")
.push(ctx.clone());
if ctx.spawn_source == SpawnSource::Resume {
spec.override_profile = Some(self.override_profile.clone());
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&["customizer_resume"])));
}
Ok(())
}
}
#[tokio::test]
async fn test_spawn_member_customizer_fires_with_source_and_spawner_provenance() {
let customizer = RecordingSpawnCustomizer::new();
let definition = sample_definition_with_mob_tools();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(customizer.clone()))
.create()
.await
.expect("create mob");
let lead_ref = handle
.spawn_spec(SpawnMemberSpec::new("lead", "lead-1"))
.await
.expect("spawn root");
let lead_session_id = handle
.resolve_bridge_session_id(&lead_ref.agent_identity)
.await
.expect("lead bridge session");
let worker_spec = SpawnMemberSpec::new("worker", "worker-agent-tool");
let owner_context = handle
.generated_ops_owner_context_for_test(lead_session_id.clone())
.await
.expect("generated owner context for lead");
handle
.spawn_spec_receipt_with_owner_context(worker_spec, owner_context)
.await
.expect("agent spawn_member");
let mut batch = handle
.spawn_many(vec![
SpawnMemberSpec::new("worker", "batch-a"),
SpawnMemberSpec::new("worker", "batch-b"),
])
.await
.expect("spawn_many batch customizer authority");
assert!(batch.drain(..).all(|result| result.is_ok()));
handle
.spawn_helper(
AgentIdentity::from("helper-promoted"),
"do helper work",
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
..HelperOptions::default()
},
)
.await
.expect("helper spawn");
handle
.fork_helper(
&AgentIdentity::from("lead-1"),
AgentIdentity::from("fork-helper"),
"do forked helper work",
crate::launch::ForkContext::LastMessages { count: 1 },
HelperOptions {
role_name: Some(ProfileName::from("worker")),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
..HelperOptions::default()
},
)
.await
.expect("fork helper spawn");
let calls = customizer.calls();
assert!(
calls
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::Consumer
&& ctx.spawner_identity.is_none()
&& ctx.requested_profile.as_str() == "lead"),
"root spawn must be customized as Consumer without spawner provenance"
);
assert!(
calls
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::AgentSpawnMember
&& ctx.spawner_identity.as_ref() == Some(&AgentIdentity::from("lead-1"))
&& ctx
.spawner_runtime_id
.as_ref()
.is_some_and(|id| id.identity.as_str() == "lead-1")),
"agent spawn_member must carry typed spawning-agent identity/runtime provenance"
);
assert!(
calls
.iter()
.filter(|ctx| ctx.spawn_source == SpawnSource::BatchItem)
.count()
>= 2,
"spawn_many items must be customized as BatchItem"
);
assert!(
calls
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::HelperSpawn),
"spawn_helper must be customized as HelperSpawn"
);
assert!(
calls
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::Fork),
"fork launch-mode spawns must be customized as Fork"
);
let root_labels = handle
.get_member(&AgentIdentity::from("lead-1"))
.await
.unwrap()
.expect("lead roster entry")
.labels;
assert_eq!(
root_labels.get("customized").map(String::as_str),
Some("consumer"),
"customizer mutations must land in the created member"
);
assert!(
service
.external_tool_names(&lead_session_id)
.await
.contains(&"customizer_consumer".to_string()),
"customizer-attached external tools must reach the created session"
);
let lead_session = service
.live_session_clone(&lead_session_id)
.await
.expect("lead live session");
assert!(
matches!(
lead_session.messages().first(),
Some(Message::System(system)) if system.content == "custom prompt for consumer"
),
"customizer system_prompt_override must reach the created session"
);
let events = handle.events().replay_all().await.expect("events");
let helper_spawn = events
.iter()
.filter_map(|event| event.kind.member_spawned())
.find(|spawn| spawn.agent_identity.as_str() == "helper-promoted")
.expect("helper spawn event");
assert_eq!(
helper_spawn.continuity_intent,
SpawnContinuityIntent::DurableIdentity {
continuity_key: "helper/helper-promoted".to_string(),
},
"promoted helper continuity intent must survive spawn finalization"
);
}
#[tokio::test]
async fn test_spawn_member_customizer_spawner_provenance_ignores_roster_state_mirror() {
let customizer = RecordingSpawnCustomizer::new();
let definition = sample_definition_with_mob_tools();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.with_spawn_member_customizer(Arc::new(customizer.clone()))
.create()
.await
.expect("create mob");
let lead = AgentIdentity::from("lead-roster-mirror");
let lead_ref = handle
.spawn_spec(SpawnMemberSpec::new("lead", lead.as_str()))
.await
.expect("spawn lead");
let lead_session_id = handle
.resolve_bridge_session_id(&lead_ref.agent_identity)
.await
.expect("lead bridge session");
{
let mut snapshot = handle.roster.read().await.snapshot();
let entry = snapshot.get_mut(&lead).expect("lead roster entry");
entry.agent_runtime_id = AgentRuntimeId::new(lead.clone(), crate::ids::Generation::new(99));
entry.member_ref = crate::event::MemberRef::from_bridge_session_id(SessionId::new());
*handle.roster.write().await =
super::roster_authority::RosterAuthority::from_roster(snapshot);
}
let owner_context = handle
.generated_ops_owner_context_for_test(lead_session_id)
.await
.expect("generated owner context for lead");
handle
.spawn_spec_receipt_with_owner_context(
SpawnMemberSpec::new("worker", "worker-roster-mirror"),
owner_context,
)
.await
.expect("agent spawn_member");
assert!(
customizer
.calls()
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::AgentSpawnMember
&& ctx.spawner_identity.as_ref() == Some(&lead)
&& ctx
.spawner_runtime_id
.as_ref()
.is_some_and(|id| id.identity.as_str() == lead.as_str())),
"spawn provenance must be gated by MobMachine lifecycle projection, not the roster compatibility state mirror"
);
}
#[tokio::test]
async fn test_resume_reconciliation_runs_spawn_customizer_for_missing_session_rebuild() {
let definition = sample_definition();
let mut override_profile = definition
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.expect("worker profile should be inline")
.clone();
override_profile.external_addressable = true;
let customizer = ResumeAddressabilityCustomizer::new(override_profile);
let service = Arc::new(MockSessionService::new());
let adapter = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let spawned = handle
.spawn_spec(SpawnMemberSpec::new("worker", "resume-reconcile-member"))
.await
.expect("spawn");
let session_id = handle
.resolve_bridge_session_id(&spawned.agent_identity)
.await
.expect("bridge session");
// Mirror real teardown ordering: unregister while the live session still
// owns the interrupt capability needed to unwind an autonomous turn. Only
// then drop the live material while keeping the durable snapshot loadable
// (archive would be terminal for that snapshot). Reversing this order
// strands the exact executor in a non-interruptible mock turn and violates
// the production archive owner contract.
unregister_runtime_session_until_complete(adapter.as_ref(), &session_id)
.await
.expect("bridge session should unregister cleanly");
MobSessionService::discard_live_session(service.as_ref(), &session_id)
.await
.expect("discard live session before resume reconciliation");
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(customizer.clone()))
.resume()
.await
.expect("resume");
assert!(
customizer
.calls()
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::Resume
&& ctx.requested_profile.as_str() == "worker"),
"resume reconciliation must call the supplied spawn customizer before rebuilding a missing session"
);
assert!(
service
.external_tool_names(&session_id)
.await
.contains(&"customizer_resume".to_string()),
"resume reconciliation must use customizer-supplied external tools on the rebuilt session"
);
assert!(
service
.recorded_create_requests()
.await
.last()
.is_some_and(|request| request.runtime_build_mode_session_owned),
"resume reconciliation must rebuild through the mob provisioner so runtime bindings are SessionOwned"
);
let member = resumed
.member(&AgentIdentity::from("resume-reconcile-member"))
.await
.expect("member handle after resume reconciliation");
member
.send(
"external resume hello".to_string(),
meerkat_core::types::HandlingMode::Queue,
)
.await
.expect("resume customizer effective profile must seed machine addressability");
}
struct StaticLeadSpawnPolicy;
#[async_trait]
impl super::spawn_policy::SpawnPolicy for StaticLeadSpawnPolicy {
async fn resolve(&self, _target: &AgentIdentity) -> Option<super::spawn_policy::SpawnSpec> {
Some(super::spawn_policy::SpawnSpec {
profile: ProfileName::from("lead"),
runtime_mode: Some(crate::MobRuntimeMode::TurnDriven),
})
}
}
fn sample_definition_with_static_spawn_policy(target: &str, profile: &str) -> MobDefinition {
let mut definition = sample_definition();
definition.spawn_policy = Some(SpawnPolicyConfig::Auto {
profile_map: BTreeMap::from([(target.to_string(), ProfileName::from(profile))]),
});
definition
}
#[tokio::test]
async fn test_definition_spawn_policy_lowers_to_machine_authority_on_create() {
let target = AgentIdentity::from("definition-policy-create");
let (handle, _service) = create_test_mob(sample_definition_with_static_spawn_policy(
target.as_str(),
"lead",
))
.await;
let machine_state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(
machine_state.spawn_policy_enabled,
"definition spawn policy must seed MobMachine policy authority"
);
assert_eq!(
machine_state.spawn_policy_revision, 1,
"definition spawn policy must enter through SetSpawnPolicy"
);
handle
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new(
"queued for definition policy spawn".to_string(),
WorkOrigin::External,
),
)
.await
.expect("definition policy auto-spawn should admit through machine authority");
assert!(
handle.get_member(&target).await.unwrap().is_some(),
"definition policy should seed the runtime observation source after authority"
);
}
#[tokio::test]
async fn test_definition_spawn_policy_lowers_to_machine_authority_on_resume() {
let target = AgentIdentity::from("definition-policy-resume");
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
let created = MobBuilder::new(
sample_definition_with_static_spawn_policy(target.as_str(), "lead"),
storage,
)
.with_session_service(service.clone())
.create()
.await
.expect("create mob with definition policy");
let created_state = created
.query_machine_state()
.await
.expect("query created machine state");
assert!(created_state.spawn_policy_enabled);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service)
.resume()
.await
.expect("resume mob with definition policy");
let machine_state = resumed
.query_machine_state()
.await
.expect("query resumed machine state");
assert!(
machine_state.spawn_policy_enabled,
"recovered definition spawn policy must seed MobMachine policy authority"
);
assert_eq!(
machine_state.spawn_policy_revision, 1,
"recovered definition policy must enter through SetSpawnPolicy"
);
resumed
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new(
"queued for resumed definition policy spawn".to_string(),
WorkOrigin::External,
),
)
.await
.expect("resumed definition policy auto-spawn should use machine authority");
assert!(resumed.get_member(&target).await.unwrap().is_some());
}
#[derive(Clone)]
struct PolicyAddressabilityCustomizer {
override_profile: Profile,
}
impl SpawnMemberCustomizer for PolicyAddressabilityCustomizer {
fn customize_spawn(
&self,
ctx: &SpawnCustomizationContext,
spec: &mut SpawnMemberSpec,
) -> Result<(), MobError> {
if ctx.spawn_source == SpawnSource::PolicySpawn {
spec.override_profile = Some(self.override_profile.clone());
spec.external_tools = Some(Arc::new(MultiToolDispatcher::new(&[
"policy_override_tool",
])));
}
Ok(())
}
}
#[tokio::test]
async fn test_policy_auto_spawn_ignores_customized_effective_profile_after_authority() {
let definition = sample_definition();
let mut override_profile = definition
.profiles
.get(&ProfileName::from("worker"))
.expect("worker profile")
.as_inline()
.expect("worker profile should be inline")
.clone();
override_profile.external_addressable = true;
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(PolicyAddressabilityCustomizer {
override_profile,
}))
.create()
.await
.expect("create mob");
handle
.set_spawn_policy(Some(Arc::new(StaticWorkerSpawnPolicy)))
.await
.expect("set spawn policy");
let target = AgentIdentity::from("policy-customized-addressable");
let result = handle
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new(
"queued for customized policy spawn".to_string(),
WorkOrigin::External,
),
)
.await;
assert!(
matches!(result, Err(MobError::NotExternallyAddressable(_))),
"policy auto-spawn must not let a customizer mutate post-authority effective profile: {result:?}"
);
assert!(
handle.get_member(&target).await.unwrap().is_none(),
"rejected policy auto-spawn must not create the target member"
);
assert_eq!(
service.recorded_create_requests().await.len(),
0,
"rejected policy auto-spawn must not provision from customized material"
);
}
#[tokio::test]
async fn test_spawn_member_customizer_skips_policy_auto_spawn_and_covers_respawn_builds() {
let customizer = RecordingSpawnCustomizer::new();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(sample_definition(), MobStorage::in_memory())
.with_session_service(service.clone())
.with_spawn_member_customizer(Arc::new(customizer.clone()))
.create()
.await
.expect("create mob");
let target = AgentIdentity::from("policy-auto");
handle
.set_spawn_policy(Some(Arc::new(StaticLeadSpawnPolicy)))
.await
.expect("set spawn policy");
handle
.submit_work(
AgentRuntimeId::initial(target.clone()),
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new("queued for policy spawn".to_string(), WorkOrigin::External),
)
.await
.expect("policy auto-spawn submit work");
let policy_session_id = handle
.resolve_bridge_session_id(&target)
.await
.expect("policy bridge session");
assert!(
!service
.external_tool_names(&policy_session_id)
.await
.contains(&"customizer_policy_spawn".to_string()),
"policy auto-spawn must not apply customizer-attached external tools after machine resolution"
);
let policy_session = service
.live_session_clone(&policy_session_id)
.await
.expect("policy live session");
assert!(
!matches!(
policy_session.messages().first(),
Some(Message::System(system)) if system.content == "custom prompt for policy_spawn"
),
"policy auto-spawn must not apply customizer system prompt replacement"
);
handle
.respawn(target.clone(), None)
.await
.expect("respawn policy member");
let respawn_session_id = handle
.resolve_bridge_session_id(&target)
.await
.expect("respawn bridge session");
assert_ne!(
policy_session_id, respawn_session_id,
"respawn should create a replacement session"
);
assert!(
service
.external_tool_names(&respawn_session_id)
.await
.contains(&"customizer_respawn".to_string()),
"respawn replacement build must apply customizer-attached external tools"
);
let respawn_session = service
.live_session_clone(&respawn_session_id)
.await
.expect("respawn live session");
assert!(
matches!(
respawn_session.messages().first(),
Some(Message::System(system)) if system.content == "custom prompt for respawn"
),
"respawn replacement build must apply customizer system prompt replacement"
);
let calls = customizer.calls();
assert!(
calls
.iter()
.all(|ctx| ctx.spawn_source != SpawnSource::PolicySpawn),
"policy auto-spawn must not invoke the build customizer after generated resolution"
);
assert!(
calls
.iter()
.any(|ctx| ctx.spawn_source == SpawnSource::Respawn
&& ctx.spawner_identity.is_none()
&& ctx.requested_profile.as_str() == "lead"),
"respawn replacement build must be customized with explicit source and no spawner"
);
}
#[tokio::test]
async fn test_external_tools_late_registration() {
let tool_defs: Arc<std::sync::RwLock<Vec<&str>>> = Arc::new(std::sync::RwLock::new(Vec::new()));
let tool_defs_clone = tool_defs.clone();
let provider: crate::ExternalToolsProvider = Arc::new(move || {
let names = tool_defs_clone.read().expect("lock");
if names.is_empty() {
return None;
}
let defs: Vec<Arc<ToolDef>> = names
.iter()
.map(|name| {
Arc::new(ToolDef {
name: (*name).into(),
description: format!("Tool {name}"),
input_schema: serde_json::Value::Object(serde_json::Map::new()),
provenance: None,
})
})
.collect();
struct DynDispatcher(Arc<[Arc<ToolDef>]>);
#[async_trait]
impl AgentToolDispatcher for DynDispatcher {
fn tools(&self) -> Arc<[Arc<ToolDef>]> {
Arc::clone(&self.0)
}
async fn dispatch(
&self,
call: ToolCallView<'_>,
) -> Result<ToolDispatchOutcome, ToolError> {
Ok(ToolResult::new(call.id.to_string(), "{}".to_string(), false).into())
}
}
Some(Arc::new(DynDispatcher(defs.into())) as Arc<dyn AgentToolDispatcher>)
});
let definition = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.with_default_external_tools_provider(Some(provider))
.create()
.await
.expect("create mob");
// Spawn before registering tools — provider returns None
let _ref1 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-before"),
None,
)
.await
.expect("spawn");
let flags = service.recorded_external_tools_flags().await;
assert!(
!flags[0],
"spawn before registration should have no external tools"
);
// Now register tools
{
let mut names = tool_defs.write().expect("lock");
names.push("late_tool_a");
names.push("late_tool_b");
}
// Spawn after registration — provider returns tools
let ref2 = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-after"),
None,
)
.await
.expect("spawn");
let flags = service.recorded_external_tools_flags().await;
assert!(
flags[1],
"spawn after registration should have external tools"
);
let session_id = ref2.bridge_session_id().expect("session-backed").clone();
let tool_names = service.external_tool_names(&session_id).await;
assert!(
tool_names.contains(&"late_tool_a".to_string()),
"late-registered tool should appear"
);
assert!(
tool_names.contains(&"late_tool_b".to_string()),
"late-registered tool should appear"
);
}
#[tokio::test]
async fn test_restored_member_gets_external_tools() {
let provider: crate::ExternalToolsProvider =
Arc::new(|| Some(Arc::new(EchoBundleDispatcher) as Arc<dyn AgentToolDispatcher>));
let definition = sample_definition();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage::in_memory();
let events = storage.events.clone();
let runtime_metadata = storage.runtime_metadata.clone();
// Create mob and spawn a worker
let handle = MobBuilder::new(definition.clone(), storage)
.with_session_service(service.clone())
.with_default_external_tools_provider(Some(provider.clone()))
.create()
.await
.expect("create mob");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn");
let old_sid = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("roster entry")
.bridge_session_id()
.cloned()
.expect("session-backed member");
// Stop and archive the session to simulate a stale state
handle.stop().await.expect("stop");
// Drop the live session while keeping the durable snapshot loadable
// (archive is terminal for durable snapshots).
MobSessionService::discard_live_session(service.as_ref(), &old_sid)
.await
.expect("discard live session");
// Resume with provider — restored member should get external tools
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata,
))
.with_session_service(service.clone())
.with_default_external_tools_provider(Some(provider))
.resume()
.await
.expect("resume");
resumed
.resume()
.await
.expect("explicit resume should rebuild stopped member sessions");
// The restore path creates a new session for the member.
// Check that the create_session call included external tools.
let flags = service.recorded_external_tools_flags().await;
// flags[0] = original spawn, flags[1] = restored session
assert!(
flags.len() >= 2,
"should have at least 2 create_session calls (original + restore)"
);
assert!(
flags[1],
"restored member should have external tools from provider"
);
}
/// Integration test: a flow with `root: Some(FrameSpec { ... })` executes frame nodes
/// via the FlowFrameEngine path (not the flat-step path).
///
/// This validates the wiring added in task #16 — FlowSpec.root dispatches to
/// FlowFrameEngine with a scripted step executor.
#[tokio::test]
async fn test_flow_with_root_frame_spec_executes_frame_nodes() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::{FlowNodeId, FrameId};
use crate::run::FlowContext;
use crate::runtime::flow_frame_engine::{FlowFrameEngine, FrameStepExecutor, FrameStepResult};
/// Scripted executor that returns pre-configured output per node.
struct ScriptedExecutor {
outputs: std::collections::HashMap<String, serde_json::Value>,
}
#[async_trait]
impl FrameStepExecutor for ScriptedExecutor {
async fn execute_step(
&self,
_run_id: &crate::ids::RunId,
_frame_id: &FrameId,
node_id: &FlowNodeId,
_step_id: &crate::ids::StepId,
_context: &FlowContext,
) -> Result<FrameStepResult, MobError> {
self.outputs
.get(&node_id.to_string())
.cloned()
.map(FrameStepResult::Completed)
.ok_or_else(|| MobError::Internal(format!("no scripted output for {node_id}")))
}
}
// Build store + run.
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create_run");
// Build a root FrameSpec: setup -> finalize
let root_spec = {
let mut nodes = IndexMap::new();
nodes.insert(
FlowNodeId::from("setup-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: crate::ids::StepId::from("setup"),
depends_on: vec![],
depends_on_mode: crate::definition::DependencyMode::All,
branch: None,
}),
);
nodes.insert(
FlowNodeId::from("finalize-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: crate::ids::StepId::from("finalize"),
depends_on: vec![FlowNodeId::from("setup-node")],
depends_on_mode: crate::definition::DependencyMode::All,
branch: None,
}),
);
FrameSpec { nodes }
};
let mut scripted_outputs = std::collections::HashMap::new();
scripted_outputs.insert(
"setup-node".to_string(),
serde_json::json!({"initialized": true}),
);
scripted_outputs.insert(
"finalize-node".to_string(),
serde_json::json!({"complete": true}),
);
let executor = Arc::new(ScriptedExecutor {
outputs: scripted_outputs,
});
let definition = sample_definition();
let (handle, _) = create_test_mob_with_run_store(definition, store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let engine = FlowFrameEngine::new(store.clone(), executor, handle, 0, 0);
let context = FlowContext {
run_id: run_id.clone(),
activation_params: serde_json::json!({}),
step_outputs: IndexMap::new(),
loop_outputs: IndexMap::new(),
};
let frame_id = FrameId::from(format!("{run_id}-root").as_str());
let outputs = engine
.execute_frame(&run_id, &frame_id, &root_spec, &context)
.await
.expect("frame-based execution should complete");
assert!(
outputs
.outputs
.contains_key(&crate::ids::StepId::from("setup")),
"setup output missing; outputs: {outputs:?}"
);
assert!(
outputs
.outputs
.contains_key(&crate::ids::StepId::from("finalize")),
"finalize output missing; outputs: {outputs:?}"
);
assert_eq!(
outputs.outputs[&crate::ids::StepId::from("finalize")],
serde_json::json!({"complete": true}),
);
// Verify frame state is Completed in the store.
let run = store
.get_run(&run_id)
.await
.expect("get_run")
.expect("run exists");
let frame_snap = run.frames.get(&frame_id).expect("frame snapshot");
assert_eq!(
frame_snap.kernel_state.phase,
crate::run::flow_frame::Phase::Completed,
"frame should be in Completed phase"
);
}
#[tokio::test]
async fn test_root_frame_condition_skip_emits_single_skip_projection() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_single_step_flow(500, 8);
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
let step = flow.steps.get_mut(&step_id("start")).expect("start step");
step.condition = Some(ConditionExpr::Eq {
path: "params.ok".to_string(),
value: serde_json::json!(true),
});
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, _service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({ "ok": false }))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
assert_eq!(
run.step_ledger
.iter()
.filter(|entry| {
entry.step_id.as_str() == "start" && entry.status == StepRunStatus::Skipped
})
.count(),
1,
"root-frame skipped step should appear exactly once in the step ledger"
);
let events = handle.events().replay_all().await.expect("replay");
assert_eq!(
events
.iter()
.filter(|event| {
matches!(
&event.kind,
MobEventKind::StepSkipped { run_id: id, step_id, .. }
if id == &run_id && step_id.as_str() == "start"
)
})
.count(),
1,
"root-frame skipped step should emit exactly one StepSkipped event"
);
}
#[tokio::test]
async fn test_step_condition_referencing_absent_root_skips_not_fails() {
// Regression (worktree-ultimate-dogma): a step condition that references a
// context root which has not been produced yet (e.g. a loop-iteration step
// output, or an `Eq`/`In` against an absent step) must resolve to a definite
// "not present" and SKIP the step, exactly as it did when the evaluator
// returned a plain `bool`. The fail-closed rewrite over-broadened and turned
// an absent root into a step/flow FAULT, collapsing every conditional flow
// step to `Failed`. Structural faults (typo'd keys inside a present root,
// unparsable references) must still fail closed; this only restores the
// absent-root case.
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_single_step_flow(500, 8);
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
let step = flow.steps.get_mut(&step_id("start")).expect("start step");
// This loop-iteration root is never produced by this flow, mirroring the
// smoke-test `loops.{loop}.iterations.0.steps.{step}.done` references that
// gate first-vs-subsequent loop-body behavior before the referenced step
// has run.
step.condition = Some(ConditionExpr::Eq {
path: "loops.never.iterations.0.steps.absent.done".to_string(),
value: serde_json::json!(true),
});
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, _service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({ "ok": false }))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(
terminal.status,
MobRunStatus::Completed,
"an absent-root condition must skip the step and complete the run, not fail it: {terminal:?}"
);
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
assert_eq!(
run.step_ledger
.iter()
.filter(|entry| {
entry.step_id.as_str() == "start" && entry.status == StepRunStatus::Skipped
})
.count(),
1,
"absent-root condition must skip the step exactly once"
);
}
#[tokio::test]
async fn test_root_frame_step_failure_does_not_abort_independent_siblings() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition();
let mut steps = IndexMap::new();
steps.insert(step_id("needs_lead"), flow_step("lead", "Lead-only task"));
steps.insert(step_id("worker_task"), flow_step("worker", "Worker task"));
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("needs-lead-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("needs_lead"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
root_nodes.insert(
FlowNodeId::from("worker-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("worker_task"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
definition.flows = BTreeMap::from([(
flow_id("demo"),
FlowSpec::new(
Some("independent root siblings".to_string()),
steps,
Some(FrameSpec { nodes: root_nodes }),
),
)]);
let (handle, service) = create_test_mob(definition).await;
let lead_session_id = handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed lead")
.clone();
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service
.set_flow_turn_fail_for_session(&lead_session_id, true)
.await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert!(
terminal.step_ledger.iter().any(|entry| {
entry.step_id.as_str() == "worker_task" && entry.status == StepRunStatus::Completed
}),
"independent worker sibling should still complete even if another root node fails"
);
assert!(
terminal
.failure_ledger
.iter()
.any(|entry| entry.step_id.as_str() == "needs_lead"),
"root-frame failure should project a failure ledger entry for the failed node"
);
let events = handle.events().replay_all().await.expect("replay");
assert!(
events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::StepCompleted { run_id: id, step_id }
if id == &run_id && step_id.as_str() == "worker_task"
)
}),
"independent worker sibling should still emit StepCompleted"
);
}
#[tokio::test]
async fn test_root_frame_step_failure_records_failure_ledger_once() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_single_step_flow(500, 8);
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
flow.root = FrameSpec {
nodes: IndexMap::from([(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
)]),
};
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert_eq!(
terminal
.failure_ledger
.iter()
.filter(|entry| entry.step_id.as_str() == "start")
.count(),
1,
"frame-mode target failures should record exactly one failure ledger row per failed step"
);
}
#[tokio::test]
async fn test_root_frame_supervisor_threshold_is_respected_before_reset() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_supervisor_threshold(2);
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead supervisor");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_fail(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
!events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::SupervisorEscalation { run_id: id, .. } if id == &run_id
)
}),
"frame-root failures should not escalate before the configured threshold"
);
assert_eq!(
handle.list_members().await.len(),
2,
"force_reset should not retire members before the escalation threshold is crossed"
);
}
#[tokio::test]
async fn test_root_frame_fan_in_persists_canonical_completed_aggregate_output() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition =
sample_definition_with_dispatch_mode_and_policy(DispatchMode::FanIn, CollectionPolicy::All);
let flow = definition
.flows
.get_mut(&FlowId::from("dispatch"))
.expect("dispatch flow");
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("dispatch-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("dispatch"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, _service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-2"),
None,
)
.await
.expect("spawn w-2");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn w-1");
let run_id = handle
.run_flow(FlowId::from("dispatch"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Completed);
let aggregate_output = terminal
.step_ledger
.iter()
.find(|entry| {
entry.step_id.as_str() == "dispatch"
&& entry.agent_identity == crate::runtime::flow_system_member_id()
&& entry.status == StepRunStatus::Completed
})
.and_then(|entry| entry.output.clone())
.expect("frame-root fan_in should persist canonical aggregate completed output");
assert_eq!(
aggregate_output,
serde_json::json!([
{"target":"w-1","output":"Turn completed"},
{"target":"w-2","output":"Turn completed"}
])
);
}
#[tokio::test]
async fn test_resume_running_loop_node_completes_instead_of_failing() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec, RepeatUntilSpec};
use crate::ids::{FlowNodeId, FrameId, LoopId};
use crate::run::FlowContext;
use crate::runtime::flow_frame_engine::FlowFrameMutator;
use crate::runtime::flow_frame_engine::{FlowFrameEngine, FrameStepExecutor, FrameStepResult};
struct ScriptedExecutor;
#[async_trait]
impl FrameStepExecutor for ScriptedExecutor {
async fn execute_step(
&self,
_run_id: &crate::ids::RunId,
_frame_id: &FrameId,
_node_id: &FlowNodeId,
_step_id: &crate::ids::StepId,
_context: &FlowContext,
) -> Result<FrameStepResult, MobError> {
Ok(FrameStepResult::Completed(
serde_json::json!({"done": true}),
))
}
}
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create_run");
let mut body_nodes = IndexMap::new();
body_nodes.insert(
FlowNodeId::from("body-step"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: crate::ids::StepId::from("body"),
depends_on: vec![],
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
let body_spec = FrameSpec { nodes: body_nodes };
let root_spec = {
let mut nodes = IndexMap::new();
nodes.insert(
FlowNodeId::from("loop-node"),
FlowNodeSpec::RepeatUntil(RepeatUntilSpec {
loop_id: LoopId::from("retry"),
depends_on: vec![],
depends_on_mode: DependencyMode::All,
body: body_spec.clone(),
until: ConditionExpr::Eq {
path: "steps.body.done".to_string(),
value: serde_json::json!(true),
},
max_iterations: 3,
}),
);
FrameSpec { nodes }
};
let definition = sample_definition();
let (handle, _) = create_test_mob_with_run_store(definition, store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let frame_kernel =
crate::runtime::flow_frame_engine::FlowFrameKernel::new(store.clone(), handle.clone());
let frame_id = FrameId::from(format!("{run_id}-root").as_str());
frame_kernel
.start_frame(&run_id, &frame_id, &root_spec)
.await
.expect("start frame");
frame_kernel
.admit_next_ready_node_with_retry(&run_id, &frame_id, 5)
.await
.expect("admit loop node")
.expect("loop node effects");
let loop_instance_id =
crate::ids::LoopInstanceId::from(format!("{frame_id}::loop-node").as_str());
seed_test_loop_in_mob_machine(
&handle,
&loop_instance_id,
&frame_id,
&crate::ids::FlowNodeId::from("loop-node"),
&crate::ids::LoopId::from("retry"),
1,
3,
)
.await;
let desired_loop = crate::run::LoopSnapshot {
kernel_state: crate::run::loop_iteration::State {
phase: crate::run::loop_iteration::Phase::Running,
loop_instance_id: loop_instance_id.clone(),
parent_frame_id: frame_id.clone(),
parent_node_id: crate::ids::FlowNodeId::from("loop-node"),
loop_id: crate::ids::LoopId::from("retry"),
depth: 1,
stage: crate::run::loop_iteration::LoopIterationStage::AwaitingBodyFrame,
current_iteration: 0,
last_completed_iteration: 0,
max_iterations: 3,
active_body_frame_id: None,
},
};
let current_run = store
.get_run(&run_id)
.await
.expect("get run for loop seed")
.expect("run exists");
let loop_seed_input =
MobRun::create_loop_seed_input(&desired_loop).expect("loop seed authority input");
let seeded = if let Some(current_loop) = current_run.loops.get(&loop_instance_id).cloned() {
store
.cas_loop_request_body_frame_with_authority(
&run_id,
&loop_instance_id,
¤t_loop,
desired_loop,
¤t_run.flow_state,
current_run.flow_state.clone(),
vec![loop_seed_input],
)
.await
.expect("seed running loop snapshot")
} else {
let current_frame = current_run
.frames
.get(&frame_id)
.cloned()
.expect("root frame should exist before loop snapshot seed");
store
.cas_start_loop_with_authority(
&run_id,
&loop_instance_id,
¤t_run.flow_state,
current_run.flow_state.clone(),
&frame_id,
¤t_frame,
current_frame.clone(),
desired_loop,
vec![loop_seed_input],
)
.await
.expect("seed missing running loop snapshot")
};
assert!(
seeded,
"loop snapshot seed should succeed through authority store path"
);
let engine = FlowFrameEngine::new(
store.clone(),
Arc::new(ScriptedExecutor),
handle.clone(),
0,
0,
);
let context = FlowContext {
run_id: run_id.clone(),
activation_params: serde_json::json!({}),
step_outputs: IndexMap::new(),
loop_outputs: IndexMap::new(),
};
let outputs = engine
.execute_frame(&run_id, &frame_id, &root_spec, &context)
.await
.expect("resume running loop node");
assert_eq!(
outputs.outputs.get(&crate::ids::StepId::from("body")),
Some(&serde_json::json!({"done": true}))
);
let run = store
.get_run(&run_id)
.await
.expect("get_run")
.expect("run exists");
let frame_snap = run.frames.get(&frame_id).expect("frame snapshot");
let node_status = &frame_snap.kernel_state.node_status;
assert!(
matches!(
node_status.get("loop-node"),
Some(variant) if *variant == crate::run::flow_frame::NodeRunStatus::Completed
),
"running loop node should resume to completion instead of being failed on restart"
);
}
#[tokio::test]
async fn test_resume_running_loop_node_does_not_duplicate_iteration_ledger_entry() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec, RepeatUntilSpec};
use crate::ids::{FlowNodeId, FrameId, LoopId};
use crate::run::FlowContext;
use crate::runtime::flow_frame_engine::FlowFrameMutator;
use crate::runtime::flow_frame_engine::{FlowFrameEngine, FrameStepExecutor, FrameStepResult};
struct ScriptedExecutor;
#[async_trait]
impl FrameStepExecutor for ScriptedExecutor {
async fn execute_step(
&self,
_run_id: &crate::ids::RunId,
_frame_id: &FrameId,
_node_id: &FlowNodeId,
_step_id: &crate::ids::StepId,
_context: &FlowContext,
) -> Result<FrameStepResult, MobError> {
Ok(FrameStepResult::Completed(
serde_json::json!({"done": true}),
))
}
}
let store = Arc::new(InMemoryMobRunStore::new());
let run = authority_backed_empty_test_run("test-mob", "test-flow");
let run_id = run.run_id.clone();
store.create_run(run).await.expect("create_run");
let mut body_nodes = IndexMap::new();
body_nodes.insert(
FlowNodeId::from("body-step"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: crate::ids::StepId::from("body"),
depends_on: vec![],
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
let body_spec = FrameSpec { nodes: body_nodes };
let root_spec = {
let mut nodes = IndexMap::new();
nodes.insert(
FlowNodeId::from("loop-node"),
FlowNodeSpec::RepeatUntil(RepeatUntilSpec {
loop_id: LoopId::from("retry"),
depends_on: vec![],
depends_on_mode: DependencyMode::All,
body: body_spec.clone(),
until: ConditionExpr::Eq {
path: "steps.body.done".to_string(),
value: serde_json::json!(true),
},
max_iterations: 3,
}),
);
FrameSpec { nodes }
};
let definition = sample_definition();
let (handle, _) = create_test_mob_with_run_store(definition, store.clone()).await;
seed_test_run_in_mob_machine(&handle, &run_id).await;
let frame_kernel =
crate::runtime::flow_frame_engine::FlowFrameKernel::new(store.clone(), handle.clone());
let frame_id = FrameId::from(format!("{run_id}-root").as_str());
frame_kernel
.start_frame(&run_id, &frame_id, &root_spec)
.await
.expect("start frame");
frame_kernel
.admit_next_ready_node_with_retry(&run_id, &frame_id, 5)
.await
.expect("admit loop node")
.expect("loop node effects");
let loop_instance_id =
crate::ids::LoopInstanceId::from(format!("{frame_id}::loop-node").as_str());
let body_frame_id = FrameId::from(format!("{loop_instance_id}::iter-0").as_str());
seed_test_loop_in_mob_machine(
&handle,
&loop_instance_id,
&frame_id,
&crate::ids::FlowNodeId::from("loop-node"),
&crate::ids::LoopId::from("retry"),
1,
3,
)
.await;
let desired_loop = crate::run::LoopSnapshot {
kernel_state: crate::run::loop_iteration::State {
phase: crate::run::loop_iteration::Phase::Running,
loop_instance_id: loop_instance_id.clone(),
parent_frame_id: frame_id.clone(),
parent_node_id: crate::ids::FlowNodeId::from("loop-node"),
loop_id: crate::ids::LoopId::from("retry"),
depth: 1,
stage: crate::run::loop_iteration::LoopIterationStage::BodyFrameActive,
current_iteration: 0,
last_completed_iteration: 0,
max_iterations: 3,
active_body_frame_id: Some(body_frame_id.clone()),
},
};
let current_run = store
.get_run(&run_id)
.await
.expect("get run for loop seed")
.expect("run exists");
if !current_run.loops.contains_key(&loop_instance_id) {
let initial_loop = crate::run::LoopSnapshot {
kernel_state: crate::run::loop_iteration::State {
phase: crate::run::loop_iteration::Phase::Running,
loop_instance_id: loop_instance_id.clone(),
parent_frame_id: frame_id.clone(),
parent_node_id: crate::ids::FlowNodeId::from("loop-node"),
loop_id: crate::ids::LoopId::from("retry"),
depth: 1,
stage: crate::run::loop_iteration::LoopIterationStage::AwaitingBodyFrame,
current_iteration: 0,
last_completed_iteration: 0,
max_iterations: 3,
active_body_frame_id: None,
},
};
let loop_seed_input =
MobRun::create_loop_seed_input(&initial_loop).expect("loop seed authority input");
let current_frame = current_run
.frames
.get(&frame_id)
.cloned()
.expect("root frame should exist before loop snapshot seed");
assert!(
store
.cas_start_loop_with_authority(
&run_id,
&loop_instance_id,
¤t_run.flow_state,
current_run.flow_state.clone(),
&frame_id,
¤t_frame,
current_frame.clone(),
initial_loop,
vec![loop_seed_input],
)
.await
.expect("seed missing loop snapshot before body frame seed"),
"loop snapshot seed should succeed through authority store path"
);
}
let current_run = store
.get_run(&run_id)
.await
.expect("get run after loop seed")
.expect("run exists");
let current_loop = current_run
.loops
.get(&loop_instance_id)
.cloned()
.expect("loop node admission should persist loop snapshot");
let ledger_entry = crate::run::LoopIterationLedgerEntry {
loop_instance_id: loop_instance_id.clone(),
iteration: 0,
frame_id: body_frame_id.clone(),
};
let body_frame_seed_input = MobRun::create_frame_seed_input(
&run_id,
&body_frame_id,
Some(&loop_instance_id),
0,
crate::machines::mob_machine::FrameScope::Body,
&body_spec,
&[FlowNodeId::from("body-step")],
)
.expect("body frame seed input");
handle
.project_machine_input(body_frame_seed_input.clone())
.await
.expect("seed body frame in MobMachine");
let body_node_id = FlowNodeId::from("body-step");
let body_frame = crate::run::FrameSnapshot {
kernel_state: crate::run::flow_frame::State {
phase: crate::run::flow_frame::Phase::Running,
frame_id: body_frame_id.clone(),
frame_scope: crate::run::flow_frame::FrameScope::Body,
loop_instance_id: loop_instance_id.clone(),
iteration: 0,
last_admitted_node: FlowNodeId::from(String::new()),
tracked_nodes: [body_node_id.clone()].into_iter().collect(),
ordered_nodes: vec![body_node_id.clone()],
node_kind: [(
body_node_id.clone(),
crate::run::flow_frame::FlowNodeKind::Step,
)]
.into_iter()
.collect(),
node_dependencies: [(body_node_id.clone(), Vec::new())].into_iter().collect(),
node_dependency_modes: [(
body_node_id.clone(),
crate::run::flow_frame::DependencyMode::All,
)]
.into_iter()
.collect(),
node_branches: [(body_node_id.clone(), None)].into_iter().collect(),
branch_winners: Default::default(),
node_status: [(
body_node_id.clone(),
crate::run::flow_frame::NodeRunStatus::Ready,
)]
.into_iter()
.collect(),
ready_queue: vec![body_node_id.clone()],
output_recorded: [(body_node_id.clone(), false)].into_iter().collect(),
node_condition_results: [(body_node_id, None)].into_iter().collect(),
},
};
assert!(
store
.cas_grant_body_frame_start_with_authority(
&run_id,
&loop_instance_id,
¤t_loop,
desired_loop,
&body_frame_id,
body_frame,
ledger_entry,
¤t_run.flow_state,
current_run.flow_state.clone(),
vec![body_frame_seed_input],
)
.await
.expect("seed running loop snapshot with ledger entry"),
"body frame snapshot seed should succeed through authority store path"
);
let engine = FlowFrameEngine::new(
store.clone(),
Arc::new(ScriptedExecutor),
handle.clone(),
0,
0,
);
let context = FlowContext {
run_id: run_id.clone(),
activation_params: serde_json::json!({}),
step_outputs: IndexMap::new(),
loop_outputs: IndexMap::new(),
};
engine
.execute_frame(&run_id, &frame_id, &root_spec, &context)
.await
.expect("resume running loop node");
let run = store
.get_run(&run_id)
.await
.expect("get_run")
.expect("run exists");
assert_eq!(
run.loop_iteration_ledger
.iter()
.filter(|entry| {
entry.loop_instance_id == loop_instance_id
&& entry.iteration == 0
&& entry.frame_id == body_frame_id
})
.count(),
1,
"resume retries must not duplicate the logical loop iteration ledger row"
);
}
#[tokio::test]
async fn test_root_frame_timeout_cleans_up_inflight_node() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_single_step_flow(60_000, 8);
definition.limits = Some(LimitsSpec {
max_flow_duration_ms: Some(25),
max_step_retries: None,
max_orphaned_turns: Some(8),
cancel_grace_timeout_ms: None,
..Default::default()
});
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
let events = handle.events().replay_all().await.expect("replay");
assert!(
terminal
.failure_ledger
.iter()
.any(|entry| { entry.reason.contains("max flow duration exceeded") })
|| events.iter().any(|event| {
matches!(
&event.kind,
MobEventKind::FlowFailed { run_id: id, reason, .. }
if id == &run_id && reason.contains("max flow duration exceeded")
)
}),
"root-frame timeout should surface an explicit flow-duration failure reason"
);
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
let frame_id = crate::ids::FrameId::from(format!("{run_id}-root").as_str());
let mut start_status = run
.frames
.get(&frame_id)
.expect("root frame snapshot")
.kernel_state
.node_status
.get("start-node")
.copied()
.expect("start node status");
let deadline = Instant::now() + Duration::from_secs(30);
while start_status == crate::run::flow_frame::NodeRunStatus::Running
&& Instant::now() < deadline
{
tokio::time::sleep(Duration::from_millis(20)).await;
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
start_status = run
.frames
.get(&frame_id)
.expect("root frame snapshot")
.kernel_state
.node_status
.get("start-node")
.copied()
.expect("start node status");
}
assert!(
start_status != crate::run::flow_frame::NodeRunStatus::Running,
"timed-out root-frame step should not remain Running after terminalization: {start_status:?}"
);
}
#[tokio::test]
async fn test_root_frame_max_active_nodes_limits_nested_body_step_admission() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec, LimitsSpec, RepeatUntilSpec};
use crate::ids::{FlowNodeId, LoopId};
let mut definition = sample_definition();
definition.limits = Some(LimitsSpec {
max_active_nodes: Some(1),
max_orphaned_turns: Some(8),
..Default::default()
});
let mut steps = IndexMap::new();
steps.insert(step_id("body"), flow_step("worker", "Loop body step"));
steps.insert(
step_id("sibling"),
flow_step("lead", "Independent sibling step"),
);
let body = FrameSpec {
nodes: IndexMap::from([(
FlowNodeId::from("body-step-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("body"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
)]),
};
let root = FrameSpec {
nodes: IndexMap::from([
(
FlowNodeId::from("loop-node"),
FlowNodeSpec::RepeatUntil(RepeatUntilSpec {
loop_id: LoopId::from("test-loop"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
body,
until: ConditionExpr::Eq {
path: "steps.body.done".to_string(),
value: serde_json::json!(true),
},
max_iterations: 3,
}),
),
(
FlowNodeId::from("sibling-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("sibling"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
),
]),
};
definition.flows = BTreeMap::from([(
flow_id("demo"),
FlowSpec::new(
Some("nested body-frame admission obeys max_active_nodes".to_string()),
steps,
Some(root),
),
)]);
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
handle
.spawn(
ProfileName::from("lead"),
AgentIdentity::from("lead-1"),
None,
)
.await
.expect("spawn lead");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let deadline = Instant::now() + Duration::from_secs(3);
loop {
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
let active_node_count = run.flow_state.active_node_count;
let ready_frames_len = run.flow_state.ready_frames.len() as u64;
let has_body_frame = run.frames.values().any(|frame| {
frame.kernel_state.frame_scope == crate::run::flow_frame::FrameScope::Body
});
if has_body_frame && active_node_count > 0 && ready_frames_len > 0 {
assert_eq!(
active_node_count, 1,
"run scheduler should admit at most one active node when max_active_nodes=1; run={run:?}"
);
break;
}
assert!(
Instant::now() < deadline,
"timed out waiting for nested body frame admission under max_active_nodes; last run={run:?}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
}
#[tokio::test]
async fn test_root_frame_cancel_cleans_up_inflight_node() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition_with_single_step_flow(60_000, 8);
let flow = definition
.flows
.get_mut(&FlowId::from("demo"))
.expect("demo flow");
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("start-node"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("start"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
flow.root = FrameSpec { nodes: root_nodes };
let (handle, service) = create_test_mob(definition).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
service.set_flow_turn_never_terminal(true);
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
tokio::time::sleep(Duration::from_millis(25)).await;
handle
.cancel_flow(run_id.clone())
.await
.expect("cancel flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(8)).await;
assert_eq!(terminal.status, MobRunStatus::Canceled);
let run = handle
.flow_status(run_id.clone())
.await
.expect("flow status")
.expect("run exists");
let frame_id = crate::ids::FrameId::from(format!("{run_id}-root").as_str());
let frame = run.frames.get(&frame_id).expect("root frame snapshot");
let start_status = frame
.kernel_state
.node_status
.get("start-node")
.expect("start node status");
assert!(
*start_status != crate::run::flow_frame::NodeRunStatus::Running,
"canceled root-frame step should not remain Running after terminalization: {start_status:?}"
);
}
#[tokio::test]
async fn test_root_loop_body_failure_stops_after_first_failed_iteration() {
use crate::definition::{FlowNodeSpec, FrameSpec, FrameStepSpec, RepeatUntilSpec};
use crate::ids::FlowNodeId;
let mut definition = sample_definition();
let mut steps = IndexMap::new();
steps.insert(
step_id("needs_lead"),
flow_step("lead", "Lead-only loop body"),
);
let mut body_nodes = IndexMap::new();
body_nodes.insert(
FlowNodeId::from("body-step"),
FlowNodeSpec::Step(FrameStepSpec {
step_id: step_id("needs_lead"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
branch: None,
}),
);
let mut root_nodes = IndexMap::new();
root_nodes.insert(
FlowNodeId::from("retry-loop"),
FlowNodeSpec::RepeatUntil(RepeatUntilSpec {
loop_id: crate::ids::LoopId::from("retry"),
depends_on: Vec::new(),
depends_on_mode: DependencyMode::All,
body: FrameSpec { nodes: body_nodes },
until: ConditionExpr::Eq {
path: "steps.needs_lead.done".to_string(),
value: serde_json::json!(true),
},
max_iterations: 3,
}),
);
definition.flows = BTreeMap::from([(
flow_id("demo"),
FlowSpec::new(
Some("loop failure should stop immediately".to_string()),
steps,
Some(FrameSpec { nodes: root_nodes }),
),
)]);
let (handle, _service) = create_test_mob(definition).await;
let run_id = handle
.run_flow(FlowId::from("demo"), serde_json::json!({}))
.await
.expect("run flow");
let terminal = wait_for_run_terminal(&handle, &run_id, Duration::from_secs(3)).await;
assert_eq!(terminal.status, MobRunStatus::Failed);
assert_eq!(
terminal.loop_iteration_ledger.len(),
1,
"failed loop body should stop loop execution instead of advancing to extra iterations"
);
}
// -----------------------------------------------------------------------
// T2.2: RealmRef profile resolution in spawn path
// -----------------------------------------------------------------------
/// Definition that has a RealmRef profile binding (references realm store).
fn sample_definition_with_realm_ref_profile() -> MobDefinition {
let mut def = sample_definition();
def.profiles.insert(
ProfileName::from("realm-worker"),
ProfileBinding::RealmRef {
realm_profile: "shared-worker".into(),
},
);
def
}
async fn create_test_mob_with_realm_store(
definition: MobDefinition,
realm_store: Arc<dyn crate::store::RealmProfileStore>,
) -> (MobHandle, Arc<MockSessionService>) {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage {
events: Arc::new(InMemoryMobEventStore::new()),
runs: Arc::new(InMemoryMobRunStore::new()),
specs: Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata: Arc::new(InMemoryMobRuntimeMetadataStore::new()),
identity: Arc::new(InMemoryMobIdentityStore::new()),
identity_member: None,
identity_status: Arc::new(InMemoryMobIdentityStatusStore::new()),
realm_profiles: Some(realm_store),
};
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob with realm store");
(handle, service)
}
#[tokio::test]
async fn test_spawn_inline_profile_still_works() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let result = handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await;
assert!(result.is_ok(), "inline profile spawn should succeed");
}
#[tokio::test]
async fn test_spawn_realm_ref_resolves_from_store() {
use crate::store::InMemoryRealmProfileStore;
let realm_store = Arc::new(InMemoryRealmProfileStore::new());
let worker_profile = Profile {
model: "claude-sonnet-4-5".into(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: vec![],
tools: ToolConfig {
comms: true,
..ToolConfig::default()
},
peer_description: "realm worker".into(),
external_addressable: false,
backend: None,
runtime_mode: crate::MobRuntimeMode::AutonomousHost,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
};
realm_store
.create("shared-worker", &worker_profile)
.await
.expect("seed realm profile");
let (handle, _service) =
create_test_mob_with_realm_store(sample_definition_with_realm_ref_profile(), realm_store)
.await;
let result = handle
.spawn(
ProfileName::from("realm-worker"),
AgentIdentity::from("rw-1"),
None,
)
.await;
assert!(result.is_ok(), "realm ref profile spawn should succeed");
}
#[tokio::test]
async fn test_spawn_realm_ref_nonexistent_returns_profile_not_found() {
use crate::store::InMemoryRealmProfileStore;
let realm_store = Arc::new(InMemoryRealmProfileStore::new());
// Do NOT seed "shared-worker" — it should fail.
let (handle, _service) =
create_test_mob_with_realm_store(sample_definition_with_realm_ref_profile(), realm_store)
.await;
let result = handle
.spawn(
ProfileName::from("realm-worker"),
AgentIdentity::from("rw-1"),
None,
)
.await;
assert!(
matches!(result, Err(MobError::ProfileNotFound(_))),
"missing realm profile should return ProfileNotFound, got: {result:?}"
);
}
#[tokio::test]
async fn test_spawn_realm_ref_without_store_returns_error() {
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let storage = MobStorage {
events: Arc::new(InMemoryMobEventStore::new()),
runs: Arc::new(InMemoryMobRunStore::new()),
specs: Arc::new(InMemoryMobSpecStore::new()),
runtime_metadata: Arc::new(InMemoryMobRuntimeMetadataStore::new()),
identity: Arc::new(InMemoryMobIdentityStore::new()),
identity_member: None,
identity_status: Arc::new(InMemoryMobIdentityStatusStore::new()),
realm_profiles: None, // no realm store
};
let handle = MobBuilder::new(sample_definition_with_realm_ref_profile(), storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
let result = handle
.spawn(
ProfileName::from("realm-worker"),
AgentIdentity::from("rw-1"),
None,
)
.await;
assert!(
result.is_err(),
"realm ref without store should fail: {result:?}"
);
let err = result.unwrap_err();
assert!(
matches!(err, MobError::Internal(_)),
"expected Internal error for missing store, got: {err:?}"
);
}
// ---------------------------------------------------------------------------
// Work lane (C5)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn test_submit_work_internal_origin_succeeds() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
let runtime_id = entry.agent_runtime_id.clone();
let fence = entry.fence_token;
let receipt = handle
.submit_work(
runtime_id.clone(),
fence,
WorkRef::new(),
WorkSpec::new("do work".to_string(), WorkOrigin::Internal),
)
.await
.expect("submit_work should succeed with valid fence token");
assert_eq!(receipt.runtime_id, runtime_id);
}
#[tokio::test]
async fn test_submit_work_external_origin_succeeds() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
let entry = handle
.get_member(&AgentIdentity::from("l-1"))
.await
.unwrap()
.expect("member exists");
let receipt = handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("user message".to_string(), WorkOrigin::External),
)
.await
.expect("submit_work external should succeed for externally addressable member");
assert_eq!(receipt.runtime_id, entry.agent_runtime_id);
}
#[tokio::test]
async fn test_wire_external_peer_not_blocked_by_delayed_turn_driven_submit_work() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("l-busy-turn-driven");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let entry = handle
.get_member(&AgentIdentity::from(member_id.as_str()))
.await
.unwrap()
.expect("member exists");
service.set_start_turn_delay_ms(600_000);
let submit_handle = handle.clone();
let submit_runtime_id = entry.agent_runtime_id.clone();
let submit_fence = entry.fence_token;
let submit = tokio::spawn(async move {
submit_handle
.submit_work(
submit_runtime_id,
submit_fence,
WorkRef::new(),
WorkSpec::new("long running work".to_string(), WorkOrigin::External),
)
.await
});
let deadline = Instant::now() + Duration::from_secs(2);
while service.start_turn_call_count() == 0 {
assert!(
Instant::now() < deadline,
"delayed turn-driven work should reach the runtime executor"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
let external = test_trusted_peer_descriptor(
"remote-mob/worker/busy-peer",
"inproc://remote-mob/worker/busy-peer",
);
tokio::time::timeout(
Duration::from_millis(100),
handle.wire(
AgentIdentity::from(member_id.as_str()),
PeerTarget::External(external.clone()),
),
)
.await
.expect("wire must not wait behind turn completion")
.expect("wire external peer");
let trusted = entry
.member_ref
.bridge_session_id()
.map(|session_id| service.trusted_peer_names(session_id));
if let Some(trusted) = trusted {
let trusted = trusted.await;
assert!(
trusted.iter().any(|name| name == external.name.as_str()),
"external peer trust should be installed after wire"
);
}
tokio::time::timeout(Duration::from_millis(100), submit)
.await
.expect("submit_work should return after ingress admission")
.expect("submit task should not panic")
.expect("submit_work should be accepted");
}
#[tokio::test]
async fn test_internal_turn_completed_reply_does_not_block_actor_operations() {
let (handle, service) = create_test_mob(sample_definition()).await;
let member_id = AgentIdentity::from("l-busy-internal-turn");
handle
.spawn_with_options(
ProfileName::from("lead"),
member_id.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven lead");
let baseline_start_turn_calls = service.start_turn_call_count();
service.set_start_turn_delay_ms(250);
let turn_handle = handle.clone();
let turn_identity = AgentIdentity::from(member_id.as_str());
let internal_turn = tokio::spawn(async move {
turn_handle
.member(&turn_identity)
.await?
.internal_turn("long internal turn")
.await
});
let deadline = Instant::now() + Duration::from_secs(2);
while service.start_turn_call_count() < baseline_start_turn_calls + 1 {
assert!(
Instant::now() < deadline,
"delayed internal turn should reach the runtime executor"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
tokio::time::timeout(
Duration::from_millis(100),
handle.list_members_including_retiring(),
)
.await
.expect("member listing must not wait behind TurnCompleted runtime completion");
tokio::time::timeout(
Duration::from_millis(100),
handle.spawn(
ProfileName::from("worker"),
AgentIdentity::from("worker-after-internal-turn"),
None,
),
)
.await
.expect("spawn must not wait behind TurnCompleted runtime completion")
.expect("spawn worker while internal turn is still running");
tokio::time::timeout(Duration::from_secs(2), internal_turn)
.await
.expect("internal_turn should complete once the runtime turn completes")
.expect("internal_turn task should not panic")
.expect("internal_turn result should succeed");
}
#[tokio::test]
async fn test_submit_work_stale_fence_token_rejected() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
let runtime_id = entry.agent_runtime_id.clone();
let stale_fence = FenceToken::new(entry.fence_token.get() + 999);
let result = handle
.submit_work(
runtime_id,
stale_fence,
WorkRef::new(),
WorkSpec::new("stale work".to_string(), WorkOrigin::Internal),
)
.await;
assert!(
matches!(result, Err(MobError::StaleFenceToken { .. })),
"submit_work must reject stale fence token: {result:?}"
);
}
#[tokio::test]
async fn test_submit_work_unknown_member_fails() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let runtime_id = AgentRuntimeId::initial(AgentIdentity::from("nonexistent"));
let result = handle
.submit_work(
runtime_id,
FenceToken::new(0),
WorkRef::new(),
WorkSpec::new("orphan work".to_string(), WorkOrigin::Internal),
)
.await;
assert!(
matches!(result, Err(MobError::MemberNotFound(_))),
"submit_work to nonexistent member must fail: {result:?}"
);
}
#[tokio::test]
async fn test_cancel_work_returns_unsupported() {
// #328 dogma gate: the advertised `mob/cancel_work` surface must fail
// closed with a typed `WorkCancellationUnsupported` rather than a phantom
// `WorkNotFound` (which lies about having searched a work-tracking ledger
// that does not exist). This keeps advertise == deliver: there is no real
// per-unit cancellation authority, so the surface honestly reports
// "unsupported" instead of a search-and-miss that never happened.
let (handle, _service) = create_test_mob(sample_definition()).await;
let work_ref = WorkRef::new();
let result = handle.cancel_work(work_ref.clone()).await;
match result {
Err(MobError::WorkCancellationUnsupported(returned)) => {
assert_eq!(
returned, work_ref,
"unsupported error must echo the requested work ref"
);
}
other => panic!(
"cancel_work must fail closed with WorkCancellationUnsupported (no phantom WorkNotFound/ok): {other:?}"
),
}
}
/// #210 dogma gate: the flow run-cancel verdict is owned by the MobMachine
/// (`CancelFlow`'s `run_known` guard over `run_status`) via
/// `apply_command_admission`, NOT by the shell's `run_cancel_tokens` /
/// `run_tasks` / `flow_streams` execution-handle maps.
///
/// A freshly-running mob has EMPTY tracker maps and the machine has no
/// `run_status` entry for any run. Cancelling an unknown run must therefore
/// be REJECTED by machine admission (a typed `InvalidTransition` verdict),
/// not silently no-op to `Ok(())` as it would if the shell decided the
/// outcome purely from `run_cancel_tokens` map absence. This proves the
/// maps are execution handles only and the semantic verdict is
/// machine-owned. It fails-old if the shell ever short-circuits cancel on
/// map presence/absence before machine admission.
#[tokio::test]
async fn cancel_flow_verdict_comes_from_machine_admission_not_tracker_maps() {
let (handle, _service) = create_test_mob(sample_definition()).await;
assert_eq!(handle.status().await.unwrap(), MobState::Running);
// Unknown run: tracker maps are empty AND the machine has no run_status
// entry. If map-absence decided the outcome, this would be a silent
// `Ok(())` no-op. The machine `run_known` guard rejects it instead.
let unknown_run = RunId::new();
let result = handle.cancel_flow(unknown_run).await;
match result {
Err(MobError::InvalidTransition {
from: MobState::Running,
to: MobState::Running,
}) => {}
other => panic!(
"cancel_flow on an unknown run must be rejected by MobMachine admission \
(InvalidTransition), not a shell map-presence no-op: {other:?}"
),
}
}
#[tokio::test]
async fn test_cancel_all_work_validates_fence_token() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
let stale_fence = FenceToken::new(entry.fence_token.get() + 1);
let result = handle
.cancel_all_work(entry.agent_runtime_id.clone(), stale_fence)
.await;
assert!(
matches!(result, Err(MobError::StaleFenceToken { .. })),
"cancel_all_work must reject stale fence token: {result:?}"
);
}
#[tokio::test]
async fn test_cancel_all_work_valid_fence_succeeds() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
let entry = handle
.get_member(&AgentIdentity::from("w-1"))
.await
.unwrap()
.expect("member exists");
let result = handle
.cancel_all_work(entry.agent_runtime_id.clone(), entry.fence_token)
.await;
assert!(
result.is_ok(),
"cancel_all_work with valid fence should succeed: {result:?}"
);
}
// -----------------------------------------------------------------------
// Identity-first comms pipeline verification
// -----------------------------------------------------------------------
/// Verify that spawning members with role-based wiring rules creates correct
/// identity-keyed roster entries, valid bridge sessions, and bidirectional
/// wiring — the exact flow that the pictionary e2e-smoke test relies on.
#[tokio::test]
async fn test_identity_first_spawn_with_role_wiring_creates_valid_roster_and_sessions() {
let mut def = sample_definition_with_cross_role_wiring();
// Ensure comms are enabled on both profiles so wiring creates trust edges.
for binding in def.profiles.values_mut() {
if let crate::profile::ProfileBinding::Inline(profile) = binding {
profile.tools.comms = true;
}
}
let (handle, _service) = create_test_mob(def).await;
// Spawn a "lead" and a "worker" — the cross_role_wiring rule wires lead<->worker.
let lead_result = handle
.spawn_spec(SpawnMemberSpec::new("lead", AgentIdentity::from("lead-1")))
.await
.expect("spawn lead");
assert_eq!(lead_result.agent_identity, AgentIdentity::from("lead-1"));
let worker_result = handle
.spawn_spec(SpawnMemberSpec::new(
"worker",
AgentIdentity::from("worker-1"),
))
.await
.expect("spawn worker");
assert_eq!(
worker_result.agent_identity,
AgentIdentity::from("worker-1")
);
// Verify roster entries exist via identity lookup.
let lead_entry = handle
.get_member(&AgentIdentity::from("lead-1"))
.await
.unwrap()
.expect("lead roster entry should exist");
assert_eq!(lead_entry.agent_identity, AgentIdentity::from("lead-1"));
assert_eq!(lead_entry.role, ProfileName::from("lead"));
let worker_entry = handle
.get_member(&AgentIdentity::from("worker-1"))
.await
.unwrap()
.expect("worker roster entry should exist");
assert_eq!(worker_entry.agent_identity, AgentIdentity::from("worker-1"));
assert_eq!(worker_entry.role, ProfileName::from("worker"));
// Verify resolve_bridge_session_id returns valid sessions.
let lead_session = handle
.resolve_bridge_session_id(&AgentIdentity::from("lead-1"))
.await;
assert!(
lead_session.is_some(),
"lead should have a bridge session after spawn"
);
let worker_session = handle
.resolve_bridge_session_id(&AgentIdentity::from("worker-1"))
.await;
assert!(
worker_session.is_some(),
"worker should have a bridge session after spawn"
);
// Sessions must be distinct.
assert_ne!(
lead_session, worker_session,
"lead and worker should have different bridge sessions"
);
// Verify wiring: worker should be wired to lead (via cross-role rule).
let worker_wired_to = &worker_entry.wired_to;
assert!(
worker_wired_to.contains(&AgentIdentity::from("lead-1")),
"worker should be wired to lead via cross-role wiring rule, got: {worker_wired_to:?}"
);
// Verify wiring is bidirectional.
// Re-fetch lead entry since wiring might have been set after initial spawn.
let lead_entry = handle
.get_member(&AgentIdentity::from("lead-1"))
.await
.unwrap()
.expect("lead roster entry should still exist");
assert!(
lead_entry
.wired_to
.contains(&AgentIdentity::from("worker-1")),
"lead should be wired to worker (bidirectional), got: {:?}",
lead_entry.wired_to
);
}
/// Verify that MemberHandle::send delivers content to a wired member
/// by checking the external turn dispatch resolves correctly.
#[tokio::test]
async fn test_identity_first_member_handle_send_routes_through_identity() {
let (handle, _service) = create_test_mob(sample_definition()).await;
// Spawn a member.
handle
.spawn_spec(
SpawnMemberSpec::new("worker", AgentIdentity::from("sender"))
.with_runtime_mode(crate::MobRuntimeMode::TurnDriven),
)
.await
.expect("spawn sender");
// Acquire a MemberHandle via identity.
let member = handle
.member(&AgentIdentity::from("sender"))
.await
.expect("member handle should resolve by identity");
assert_eq!(member.identity(), AgentIdentity::from("sender"));
// Verify the member has a valid bridge session.
let session = handle
.resolve_bridge_session_id(&AgentIdentity::from("sender"))
.await;
assert!(
session.is_some(),
"member should have a bridge session for turn delivery"
);
}
/// Verify that list_members returns identity-native entries with correct
/// agent_identity, agent_runtime_id, and fence_token fields.
#[tokio::test]
async fn test_identity_first_list_members_returns_identity_native_entries() {
let (handle, _service) = create_test_mob(sample_definition()).await;
handle
.spawn_spec(SpawnMemberSpec::new("worker", AgentIdentity::from("alice")))
.await
.expect("spawn alice");
handle
.spawn_spec(SpawnMemberSpec::new("worker", AgentIdentity::from("bob")))
.await
.expect("spawn bob");
let members = handle.list_members().await;
assert_eq!(members.len(), 2, "should have 2 members");
let identities: std::collections::BTreeSet<_> =
members.iter().map(|m| m.agent_identity.clone()).collect();
assert!(identities.contains(&AgentIdentity::from("alice")));
assert!(identities.contains(&AgentIdentity::from("bob")));
// Each member should have a valid agent_runtime_id and fence_token.
for member in &members {
let (agent_runtime_id, fence_token) = member
.binding_atoms()
.expect("spawned list member should have MobMachine binding atoms");
assert_eq!(
agent_runtime_id.identity, member.agent_identity,
"runtime_id identity should match agent_identity"
);
assert!(
fence_token.get() > 0,
"fence_token should be non-zero after spawn"
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum MobRuntimeParityClassification {
SameSurface,
DifferentSurface,
LeftOnly,
RightOnly,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum MobRuntimeParityPhase {
Running,
Stopped,
Completed,
}
impl MobRuntimeParityPhase {
fn schema_name(self) -> &'static str {
match self {
Self::Running => "Running",
Self::Stopped => "Stopped",
Self::Completed => "Completed",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum MobRuntimeParityOutcomeKind {
Ok,
Err,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
struct MobRuntimeParitySnapshotSummary {
phase: String,
destroy_admitted: bool,
flow_authority_schema_version: u64,
owner_bridge_session_id: Option<String>,
owner_bridge_destroy_on_archive: bool,
implicit_delegation_mob: bool,
live_intent_identities: BTreeSet<String>,
live_runtime_ids: BTreeSet<String>,
externally_addressable_runtime_ids: BTreeSet<String>,
runtime_fence_tokens: BTreeMap<String, u64>,
active_member_count: usize,
all_member_count: usize,
coordinator_bound: Option<bool>,
pending_spawn_count: Option<u32>,
active_flow_count: Option<u32>,
topology_revision: Option<u32>,
supervisor_active: Option<bool>,
representative_agent_identity: Option<String>,
representative_runtime_id: Option<String>,
representative_fence_token: Option<u64>,
formal_available_fields: BTreeMap<String, String>,
formal_unavailable_fields: Vec<String>,
// T2 DSL state fields — stubbed here as empty collections so the
// formal-fields coverage gate passes; full projection is a follow-up.
member_state_markers: BTreeMap<String, String>,
wiring_edges: BTreeSet<String>,
external_peer_edges: BTreeSet<String>,
external_peer_edges_by_key: BTreeMap<String, String>,
pending_respawn_topology: BTreeSet<String>,
abandoned_respawn_topology: BTreeMap<String, u64>,
identity_to_runtime: BTreeMap<String, String>,
identity_runtime_generations: BTreeMap<String, u64>,
identity_runtime_fence_tokens: BTreeMap<String, u64>,
member_peer_ids: BTreeMap<String, String>,
member_peer_endpoints: BTreeMap<String, String>,
member_prior_peer_endpoints: BTreeMap<String, BTreeSet<String>>,
member_restore_failures: BTreeMap<String, String>,
member_restore_failure_codes: BTreeMap<String, String>,
runtime_retire_refusal_codes: BTreeMap<String, String>,
runtime_retire_refusal_reasons: BTreeMap<String, String>,
runtime_retire_pending_sessions: BTreeMap<String, String>,
remote_runtime_retired_ids: BTreeSet<String>,
remote_supervisor_revoked_ids: BTreeSet<String>,
member_revival_pending: BTreeSet<String>,
supervisor_authority_peer_id: Option<String>,
supervisor_authority_signing_key: Option<String>,
supervisor_authority_epoch: Option<u64>,
supervisor_authority_protocol_version: Option<String>,
supervisor_pending_authority_peer_id: Option<String>,
supervisor_pending_authority_signing_key: Option<String>,
supervisor_pending_authority_epoch: Option<u64>,
supervisor_pending_authority_protocol_version: Option<String>,
supervisor_pending_authority_operation_id: Option<String>,
supervisor_pending_authority_member_target_names: BTreeMap<String, String>,
supervisor_pending_authority_member_target_addresses: BTreeMap<String, String>,
supervisor_pending_authority_accepted_peer_ids: BTreeSet<String>,
// Dogma row R044: machine-owned trust-install-before-terminality
// obligation window for supervisor-bridge recipients.
pending_recipient_trust: BTreeSet<String>,
host_binding_generations: BTreeMap<String, u64>,
host_binding_generation_highwater: BTreeMap<String, u64>,
confirmed_host_binding_revocations: BTreeSet<String>,
replacement_host_bind_endpoints: BTreeMap<String, String>,
replacement_host_binding_generations: BTreeMap<String, u64>,
// W3-H-1: canonical identity→bridge-session binding map. Stubbed as an
// empty BTreeMap for the parity evaluator; full projection through the
// runtime-parity snapshot is a follow-up to the observer wiring PR.
member_session_bindings: BTreeMap<String, String>,
spawn_exec_phase: BTreeMap<String, String>,
member_profile_names: BTreeMap<String, String>,
member_runtime_modes: BTreeMap<String, String>,
pending_spawn_sessions: BTreeMap<String, String>,
pending_session_ingress_detach_runtime_ids: BTreeSet<String>,
// Track-B (R5): monotonically increasing topology epoch. Incremented on
// every mutation of `wiring_edges` or `member_session_bindings`. Stubbed
// here as 0 for the parity evaluator; full projection lands with the
// observer wiring PR alongside `member_session_bindings`.
topology_epoch: u64,
spawn_policy_enabled: bool,
spawn_policy_revision: u64,
spawn_policy_resolution_revision: BTreeMap<String, u64>,
spawn_policy_resolution_profiles: BTreeMap<String, String>,
spawn_policy_resolution_runtime_modes: BTreeMap<String, String>,
spawn_policy_resolution_absent: BTreeSet<String>,
spawn_profile_authority_profile_names: BTreeMap<String, String>,
spawn_profile_authority_models: BTreeMap<String, String>,
spawn_profile_authority_material_digests: BTreeMap<String, String>,
spawn_profile_authority_tool_config_digests: BTreeMap<String, String>,
spawn_profile_authority_skills_digests: BTreeMap<String, String>,
spawn_profile_authority_provider_params_digests: BTreeMap<String, String>,
spawn_profile_authority_output_schema_digests: BTreeMap<String, String>,
spawn_profile_authority_external_addressable: BTreeMap<String, bool>,
// Row #320 / #293 / #314 / #351: machine-owned state folded into MobMachine.
orphan_budget: u64,
topology_default_policy: String,
external_member_rebind_capability: BTreeMap<String, String>,
desired_members: BTreeSet<String>,
members_to_spawn: BTreeSet<String>,
members_to_retire: BTreeSet<String>,
member_kickoff_objective_ids: BTreeMap<String, String>,
objective_outcomes: BTreeMap<String, String>,
concluded_objective_ids: BTreeSet<String>,
member_run_open: BTreeMap<String, bool>,
member_in_flight_work: BTreeMap<String, u64>,
member_progress_tokens: BTreeMap<String, String>,
member_last_observed_at_ms: BTreeMap<String, u64>,
member_last_progress_at_ms: BTreeMap<String, u64>,
member_last_progress_event: BTreeMap<String, String>,
member_health_class: BTreeMap<String, String>,
pending_placed_spawn_ids: BTreeMap<String, String>,
pending_placed_spawn_generations: BTreeMap<String, u64>,
pending_placed_spawn_fence_tokens: BTreeMap<String, u64>,
pending_placed_spawn_hosts: BTreeMap<String, String>,
pending_autonomous_placed_spawns: BTreeSet<String>,
pending_placed_spawn_host_binding_generations: BTreeMap<String, u64>,
pending_placed_spawn_spec_digests: BTreeMap<String, String>,
pending_placed_spawn_provision_operation_ids: BTreeMap<String, String>,
pending_placed_spawn_operation_owner_session_ids: BTreeMap<String, String>,
current_placed_spawn_ids: BTreeMap<String, String>,
current_placed_spawn_host_binding_generations: BTreeMap<String, u64>,
current_placed_spawn_provision_operation_ids: BTreeMap<String, String>,
current_placed_spawn_operation_owner_session_ids: BTreeMap<String, String>,
pending_placed_carrier_cleanup: BTreeSet<String>,
remote_turn_dispatch_sequence: u64,
committed_remote_turn_outcomes: BTreeSet<String>,
resolved_remote_turn_outcomes: BTreeSet<String>,
}
/// Lock-in test for T2 DSL field projection in the runtime parity snapshot.
///
/// The `MobMachine` DSL owns topology and membership fields that the runtime
/// parity snapshot must project: `member_state_markers`, `wiring_edges`,
/// `external_peer_edges`, and `identity_to_runtime`. The projector at
/// [`mob_runtime_parity_snapshot_summary`] reads them from the DSL authority
/// via the `debug_dsl_t2_snapshot()` command-channel seam (dogma #1: one
/// owner, #13: projection rebuilt from explicit DSL source).
#[tokio::test]
async fn parity_snapshot_projects_t2_dsl_fields() {
let (handle, _service) = create_test_mob(sample_definition()).await;
// Two members: one lead (externally addressable), one worker.
handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.expect("spawn lead");
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.expect("spawn worker");
// Wire the pair — populates DSL `wiring_edges`.
handle
.wire(AgentIdentity::from("l-1"), AgentIdentity::from("w-1"))
.await
.expect("wire");
let snap = mob_runtime_parity_snapshot_summary(&handle)
.await
.expect("parity snapshot must be Some for a running mob");
// Spawn writes identity → runtime into the DSL (via SpawnRunning update
// block). Two spawns must produce two entries.
assert!(
snap.identity_to_runtime.len() >= 2,
"DSL `identity_to_runtime` must project both spawned members; got {:?}",
snap.identity_to_runtime
);
// `member_state_markers` is populated only during the retire-drain window
// (DSL inserts `Retiring` on Retire and removes on observe-retired). At
// steady state the map is empty — just assert the field is reachable.
// `wiring_edges` is plumbed through the DSL state but the Wire input is
// currently field-less, so population is a Phase 5H DSL-widen follow-up.
assert!(snap.member_state_markers.len() <= all_members_len(&snap));
assert!(snap.wiring_edges.len() < 1000);
}
fn all_members_len(snap: &MobRuntimeParitySnapshotSummary) -> usize {
snap.all_member_count
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct MobRuntimeParityObservableSnapshot {
phase: String,
active_member_count: usize,
all_member_count: usize,
coordinator_bound: Option<bool>,
pending_spawn_count: Option<u32>,
active_flow_count: Option<u32>,
topology_revision: Option<u32>,
supervisor_active: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct MobRuntimeParityObservableSurface {
outcome_kind: MobRuntimeParityOutcomeKind,
result_summary: String,
after: Option<MobRuntimeParityObservableSnapshot>,
}
#[derive(Debug, Clone, Serialize)]
struct MobRuntimeParityInvocationReport {
phase: String,
setup_tags: Vec<String>,
before: Option<MobRuntimeParitySnapshotSummary>,
outcome_kind: MobRuntimeParityOutcomeKind,
result_summary: String,
after: Option<MobRuntimeParitySnapshotSummary>,
}
impl MobRuntimeParityInvocationReport {
fn observable_surface(&self) -> MobRuntimeParityObservableSurface {
MobRuntimeParityObservableSurface {
outcome_kind: self.outcome_kind,
result_summary: self.result_summary.clone(),
after: self
.after
.as_ref()
.map(|after| MobRuntimeParityObservableSnapshot {
phase: after.phase.clone(),
active_member_count: after.active_member_count,
all_member_count: after.all_member_count,
coordinator_bound: after.coordinator_bound,
pending_spawn_count: after.pending_spawn_count,
active_flow_count: after.active_flow_count,
topology_revision: after.topology_revision,
supervisor_active: after.supervisor_active,
}),
}
}
}
#[derive(Debug, Clone, Serialize)]
struct MobRuntimeParitySchemaTransitionSummary {
transition: String,
to_phase: String,
binding_names: Vec<String>,
guard_names: Vec<String>,
update_count: usize,
effect_variants: Vec<String>,
}
#[derive(Debug, Clone)]
struct MobRuntimeParitySchemaRow {
input_variant: SchemaMobMachineInputVariant,
classification: MobRuntimeParityClassification,
left: Vec<MobRuntimeParitySchemaTransitionSummary>,
right: Vec<MobRuntimeParitySchemaTransitionSummary>,
}
#[derive(Debug, Serialize)]
struct MobRuntimeParityProbeReport {
schema_classification: MobRuntimeParityClassification,
runtime_classification: MobRuntimeParityClassification,
agrees_with_schema: bool,
schema_left: Option<MobModeledStateSchemaReport>,
schema_right: Option<MobModeledStateSchemaReport>,
left: MobRuntimeParityInvocationReport,
right: MobRuntimeParityInvocationReport,
}
#[derive(Debug, Serialize)]
struct MobRuntimeParityRowReport {
input_variant: String,
probe_required: bool,
schema_classification: MobRuntimeParityClassification,
schema_left: Vec<MobRuntimeParitySchemaTransitionSummary>,
schema_right: Vec<MobRuntimeParitySchemaTransitionSummary>,
probe: Option<MobRuntimeParityProbeReport>,
note: Option<String>,
}
#[derive(Debug, Default, Serialize)]
struct MobRuntimeParityPairSummary {
interesting_rows: usize,
probed_rows: usize,
aligned_rows: usize,
mismatched_rows: usize,
unprobed_rows: usize,
surface_only_unprobed_rows: usize,
runtime_internal_unprobed_rows: usize,
}
#[derive(Debug, Serialize)]
struct MobRuntimeParityPairReport {
left_phase: String,
right_phase: String,
summary: MobRuntimeParityPairSummary,
rows: Vec<MobRuntimeParityRowReport>,
}
#[derive(Debug, Default, Serialize)]
struct MobRuntimeParityAuditSummary {
pair_count: usize,
interesting_rows: usize,
probed_rows: usize,
aligned_rows: usize,
mismatched_rows: usize,
unprobed_rows: usize,
surface_only_unprobed_rows: usize,
runtime_internal_unprobed_rows: usize,
}
#[derive(Debug, Serialize)]
struct MobRuntimeParityAuditReport {
machine: String,
generated_at: String,
summary: MobRuntimeParityAuditSummary,
pairs: Vec<MobRuntimeParityPairReport>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
enum MobModeledStateOutcomeKind {
Ok,
Err,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
struct MobModeledStateSummary {
phase: String,
formal_fields: BTreeMap<String, String>,
unavailable_fields: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
struct MobModeledStateRuntimeReport {
phase: String,
outcome_kind: MobModeledStateOutcomeKind,
before: Option<MobModeledStateSummary>,
after: Option<MobModeledStateSummary>,
result_summary: String,
}
#[derive(Debug, Clone, Serialize)]
struct MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind,
after: Option<MobModeledStateSummary>,
detail: String,
result_summary: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
struct MobModeledStateRowReport {
phase: String,
input_variant: String,
aligned: bool,
differing_keys: Vec<String>,
runtime: MobModeledStateRuntimeReport,
schema: MobModeledStateSchemaReport,
}
#[derive(Debug, Default, Serialize)]
struct MobModeledStateAuditSummary {
row_count: usize,
aligned_rows: usize,
mismatched_rows: usize,
unprobed_rows: usize,
}
#[derive(Debug, Serialize)]
struct MobModeledStateAuditReport {
machine: String,
generated_at: String,
summary: MobModeledStateAuditSummary,
rows: Vec<MobModeledStateRowReport>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MobRuntimeParityProbeInput {
Spawn,
AuthorizeSpawnProfile,
ClassifyMemberWait,
ClassifyRetirePendingSpawnDisposition,
EnsureMember,
Reconcile,
SubmitWork,
RunFlow,
CancelFlow,
Retire,
Respawn,
RetireAll,
WireMembers,
WireMembersWithTrust,
UnwireMembers,
WireExternalPeer,
UnwireExternalPeer,
AuthorizeMemberPeerRebind,
AuthorizeMemberPeerOverlay,
ExternalTurn,
InternalTurn,
CancelWork,
CancelAllWork,
Stop,
Resume,
Complete,
Reset,
Destroy,
SubscribeAgentEvents,
SubscribeAllAgentEvents,
SubscribeMobEvents,
RecordOperatorActionProvenance,
SetSpawnPolicy,
Shutdown,
ForceCancel,
ConcludeObjective,
}
struct MobRuntimeParityFixture {
handle: MobHandle,
service: Arc<MockSessionService>,
worker_identity: AgentIdentity,
lead_identity: AgentIdentity,
cancel_identity: AgentIdentity,
flow_run_id: Option<RunId>,
submitted_work_ref: Option<WorkRef>,
wired_external: bool,
objective_id: meerkat_core::interaction::ObjectiveId,
}
impl MobRuntimeParityFixture {
async fn cleanup(self) {
self.service.set_flow_turn_never_terminal(false);
let _ = self.handle.destroy().await;
tokio::time::sleep(Duration::from_millis(10)).await;
}
async fn ensure_worker(&self) -> Result<(), String> {
if self
.handle
.get_member(&self.worker_identity)
.await
.unwrap()
.is_none()
{
self.handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("w-1"),
None,
)
.await
.map_err(|error| format!("spawn worker: {error:?}"))?;
}
Ok(())
}
async fn ensure_lead(&self) -> Result<(), String> {
if self
.handle
.get_member(&self.lead_identity)
.await
.unwrap()
.is_none()
{
self.handle
.spawn(ProfileName::from("lead"), AgentIdentity::from("l-1"), None)
.await
.map_err(|error| format!("spawn lead: {error:?}"))?;
}
Ok(())
}
async fn lead_peer_endpoint(
&self,
) -> Result<crate::machines::mob_machine::MemberPeerEndpoint, String> {
self.ensure_lead().await?;
let dsl_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&self.lead_identity);
self.handle
.debug_dsl_t2_snapshot()
.await
.map_err(|error| format!("debug dsl snapshot: {error:?}"))?
.member_peer_endpoints
.get(&dsl_identity)
.cloned()
.ok_or_else(|| {
format!(
"lead member {} missing generated member peer endpoint",
self.lead_identity
)
})
}
async fn ensure_force_cancel_member(&self) -> Result<(), String> {
if self
.handle
.get_member(&self.cancel_identity)
.await
.unwrap()
.is_none()
{
self.handle
.spawn_with_options(
ProfileName::from("worker"),
AgentIdentity::from(self.cancel_identity.as_str()),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.map_err(|error| format!("spawn force-cancel member: {error:?}"))?;
}
Ok(())
}
async fn worker_entry(&self) -> Result<RosterEntry, String> {
self.ensure_worker().await?;
self.handle
.get_member(&self.worker_identity)
.await
.unwrap()
.ok_or_else(|| "worker member missing after spawn".to_string())
}
async fn ensure_wired_edge(&mut self) -> Result<(), String> {
if self.wired_external {
return Ok(());
}
self.ensure_worker().await?;
self.ensure_lead().await?;
self.handle
.wire(
self.worker_identity.clone(),
AgentIdentity::from(self.lead_identity.as_str()),
)
.await
.map_err(|error| format!("wire worker: {error:?}"))?;
self.wired_external = true;
Ok(())
}
async fn ensure_demo_run(&mut self) -> Result<RunId, String> {
if let Some(run_id) = &self.flow_run_id {
return Ok(run_id.clone());
}
self.ensure_worker().await?;
self.service.set_flow_turn_never_terminal(true);
let run_id = self
.handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({"source":"mob-runtime-parity"}),
)
.await
.map_err(|error| format!("run flow: {error:?}"))?;
tokio::time::sleep(Duration::from_millis(20)).await;
self.flow_run_id = Some(run_id.clone());
Ok(run_id)
}
async fn ensure_submitted_work(&mut self) -> Result<WorkRef, String> {
if let Some(work_ref) = &self.submitted_work_ref {
return Ok(work_ref.clone());
}
let entry = self.worker_entry().await?;
let work_ref = WorkRef::new();
self.handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
work_ref.clone(),
WorkSpec::new("mob-runtime-parity".to_string(), WorkOrigin::Internal),
)
.await
.map_err(|error| format!("submit work: {error:?}"))?;
self.submitted_work_ref = Some(work_ref.clone());
Ok(work_ref)
}
}
fn mob_runtime_parity_report_path() -> PathBuf {
std::env::temp_dir().join("mob-runtime-phase-parity.json")
}
fn mob_runtime_parity_full_report_path() -> PathBuf {
std::env::temp_dir().join("mob-runtime-phase-full-parity.json")
}
fn mob_modeled_state_report_path() -> PathBuf {
std::env::temp_dir().join("mob-runtime-modeled-state-parity.json")
}
fn mob_runtime_parity_target_pairs() -> &'static [(MobRuntimeParityPhase, MobRuntimeParityPhase)] {
&[
(
MobRuntimeParityPhase::Running,
MobRuntimeParityPhase::Stopped,
),
(
MobRuntimeParityPhase::Completed,
MobRuntimeParityPhase::Running,
),
(
MobRuntimeParityPhase::Completed,
MobRuntimeParityPhase::Stopped,
),
]
}
fn mob_schema_input_variant_id(input_variant: SchemaMobMachineInputVariant) -> InputVariantId {
InputVariantId::parse(input_variant.as_str()).expect("schema MobMachine input variant id")
}
fn mob_schema_input_variant_set(
input_ids: &[InputVariantId],
) -> BTreeSet<SchemaMobMachineInputVariant> {
let input_ids = input_ids.iter().cloned().collect::<BTreeSet<_>>();
SchemaMobMachineInput::variant_manifest()
.iter()
.copied()
.filter(|input_variant| input_ids.contains(&mob_schema_input_variant_id(*input_variant)))
.collect()
}
fn mob_runtime_internal_input_variant_set() -> BTreeSet<SchemaMobMachineInputVariant> {
crate::canonical_mob_machine_runtime_internal_input_variant_manifest()
.into_iter()
.collect()
}
fn mob_runtime_parity_probe_required(
input_variant: SchemaMobMachineInputVariant,
surface_only_inputs: &BTreeSet<SchemaMobMachineInputVariant>,
runtime_internal_inputs: &BTreeSet<SchemaMobMachineInputVariant>,
) -> bool {
!surface_only_inputs.contains(&input_variant)
&& !runtime_internal_inputs.contains(&input_variant)
}
const MOB_RUNTIME_PARITY_SURFACE_ONLY_PROBE_NOT_REQUIRED: &str =
"surface-only input: runtime probe not required";
const MOB_RUNTIME_PARITY_RUNTIME_INTERNAL_PROBE_NOT_REQUIRED: &str =
"runtime-internal input: runtime probe not required";
fn mob_runtime_parity_probe_for_input_variant(
input_variant: SchemaMobMachineInputVariant,
) -> Option<MobRuntimeParityProbeInput> {
match input_variant {
// The runtime spawn probe drives the full spawn-exec ladder; its
// membership-establishing `CommitSpawnMembership` step is the
// probe-required executable variant. `BeginSpawnExec`,
// `CommitSpawnActivation`, and `AbortSpawnExec` are runtime-internal
// sub-steps (see the runtime-internal manifest) and need no probe.
SchemaMobMachineInputVariant::CommitSpawnMembership => {
Some(MobRuntimeParityProbeInput::Spawn)
}
SchemaMobMachineInputVariant::AuthorizeSpawnProfile => {
Some(MobRuntimeParityProbeInput::AuthorizeSpawnProfile)
}
SchemaMobMachineInputVariant::ClassifyMemberWait => {
Some(MobRuntimeParityProbeInput::ClassifyMemberWait)
}
SchemaMobMachineInputVariant::ClassifyRetirePendingSpawnDisposition => {
Some(MobRuntimeParityProbeInput::ClassifyRetirePendingSpawnDisposition)
}
SchemaMobMachineInputVariant::EnsureMember => {
Some(MobRuntimeParityProbeInput::EnsureMember)
}
SchemaMobMachineInputVariant::Reconcile => Some(MobRuntimeParityProbeInput::Reconcile),
SchemaMobMachineInputVariant::SubmitWork => Some(MobRuntimeParityProbeInput::SubmitWork),
SchemaMobMachineInputVariant::RunFlow => Some(MobRuntimeParityProbeInput::RunFlow),
SchemaMobMachineInputVariant::CancelFlow => Some(MobRuntimeParityProbeInput::CancelFlow),
SchemaMobMachineInputVariant::Retire => Some(MobRuntimeParityProbeInput::Retire),
SchemaMobMachineInputVariant::Respawn => Some(MobRuntimeParityProbeInput::Respawn),
SchemaMobMachineInputVariant::RetireAll => Some(MobRuntimeParityProbeInput::RetireAll),
SchemaMobMachineInputVariant::WireMembers => Some(MobRuntimeParityProbeInput::WireMembers),
SchemaMobMachineInputVariant::WireMembersWithTrust => {
Some(MobRuntimeParityProbeInput::WireMembersWithTrust)
}
SchemaMobMachineInputVariant::UnwireMembers => {
Some(MobRuntimeParityProbeInput::UnwireMembers)
}
SchemaMobMachineInputVariant::WireExternalPeer => {
Some(MobRuntimeParityProbeInput::WireExternalPeer)
}
SchemaMobMachineInputVariant::UnwireExternalPeer => {
Some(MobRuntimeParityProbeInput::UnwireExternalPeer)
}
SchemaMobMachineInputVariant::AuthorizeMemberPeerRebind => {
Some(MobRuntimeParityProbeInput::AuthorizeMemberPeerRebind)
}
SchemaMobMachineInputVariant::AuthorizeMemberPeerOverlay => {
Some(MobRuntimeParityProbeInput::AuthorizeMemberPeerOverlay)
}
SchemaMobMachineInputVariant::CancelWork => Some(MobRuntimeParityProbeInput::CancelWork),
SchemaMobMachineInputVariant::CancelAllWork => {
Some(MobRuntimeParityProbeInput::CancelAllWork)
}
SchemaMobMachineInputVariant::Stop => Some(MobRuntimeParityProbeInput::Stop),
SchemaMobMachineInputVariant::Resume => Some(MobRuntimeParityProbeInput::Resume),
SchemaMobMachineInputVariant::Complete => Some(MobRuntimeParityProbeInput::Complete),
SchemaMobMachineInputVariant::Reset => Some(MobRuntimeParityProbeInput::Reset),
SchemaMobMachineInputVariant::Destroy => Some(MobRuntimeParityProbeInput::Destroy),
SchemaMobMachineInputVariant::SubscribeAgentEvents => {
Some(MobRuntimeParityProbeInput::SubscribeAgentEvents)
}
SchemaMobMachineInputVariant::SubscribeAllAgentEvents => {
Some(MobRuntimeParityProbeInput::SubscribeAllAgentEvents)
}
SchemaMobMachineInputVariant::SubscribeMobEvents => {
Some(MobRuntimeParityProbeInput::SubscribeMobEvents)
}
SchemaMobMachineInputVariant::RecordOperatorActionProvenance => {
Some(MobRuntimeParityProbeInput::RecordOperatorActionProvenance)
}
SchemaMobMachineInputVariant::SetSpawnPolicy => {
Some(MobRuntimeParityProbeInput::SetSpawnPolicy)
}
SchemaMobMachineInputVariant::Shutdown => Some(MobRuntimeParityProbeInput::Shutdown),
SchemaMobMachineInputVariant::ForceCancel => Some(MobRuntimeParityProbeInput::ForceCancel),
SchemaMobMachineInputVariant::ConcludeObjective => {
Some(MobRuntimeParityProbeInput::ConcludeObjective)
}
_ => None,
}
}
async fn mob_runtime_parity_snapshot_summary(
handle: &MobHandle,
) -> Option<MobRuntimeParitySnapshotSummary> {
let phase = handle.status().await.unwrap();
let active_members = handle.list_members().await;
let all_members = handle.list_all_members().await;
let orchestrator = handle.debug_orchestrator_snapshot().await.ok();
let lifecycle = handle.debug_lifecycle_snapshot().await.ok();
let dsl_t2 = handle.debug_dsl_t2_snapshot().await.ok();
let flow_authority_schema_version = dsl_t2
.as_ref()
.map(|snapshot| snapshot.flow_authority_schema_version)
.unwrap_or_default();
let owner_bridge_session_id = dsl_t2
.as_ref()
.and_then(|snapshot| snapshot.owner_bridge_session_id.as_ref())
.map(|session_id| format!("{session_id:?}"));
let owner_bridge_destroy_on_archive = dsl_t2
.as_ref()
.is_some_and(|snapshot| snapshot.owner_bridge_destroy_on_archive);
let implicit_delegation_mob = dsl_t2
.as_ref()
.is_some_and(|snapshot| snapshot.implicit_delegation_mob);
let representative = active_members
.iter()
.min_by(|left, right| left.agent_identity.cmp(&right.agent_identity));
let representative_agent_identity = representative.map(|entry| {
serde_json::to_string(&entry.agent_identity).unwrap_or_else(|_| "\"<agent>\"".into())
});
let representative_runtime_id = representative.and_then(|entry| {
let (runtime_id, _) = entry.binding_atoms()?;
Some(mob_modeled_normalize_formal_string(
&serde_json::to_string(&runtime_id).unwrap_or_else(|_| "\"<runtime-id>\"".into()),
))
});
let representative_fence_token =
representative.and_then(|entry| entry.binding_atoms().map(|(_, fence)| fence.get()));
// Durable voice-intent was a separate domain kernel; it's now owned at the
// transport layer via capability-driven realtime (Phase 5G), so the parity
// snapshot reports an empty set for this projection.
let live_intent_identities: BTreeSet<String> = BTreeSet::new();
let live_intent_identity_values = live_intent_identities
.iter()
.map(|raw| {
serde_json::from_str::<serde_json::Value>(raw)
.unwrap_or_else(|_| serde_json::Value::String(raw.clone()))
})
.collect::<Vec<_>>();
let live_runtime_ids = active_members
.iter()
.filter_map(|entry| {
let (runtime_id, _) = entry.binding_atoms()?;
Some(mob_modeled_normalize_formal_string(
&serde_json::to_string(&runtime_id)
.expect("serialize live runtime id for parity snapshot"),
))
})
.collect::<BTreeSet<_>>();
let live_runtime_id_values = live_runtime_ids
.iter()
.map(|raw| {
serde_json::from_str::<serde_json::Value>(raw)
.unwrap_or_else(|_| serde_json::Value::String(raw.clone()))
})
.collect::<Vec<_>>();
let runtime_fence_tokens = active_members
.iter()
.filter_map(|entry| {
let (runtime_id, fence_token) = entry.binding_atoms()?;
Some((
mob_modeled_normalize_formal_string(
&serde_json::to_string(&runtime_id)
.expect("serialize runtime id for fence parity snapshot"),
),
fence_token.get(),
))
})
.collect::<BTreeMap<_, _>>();
let mut externally_addressable_runtime_ids = BTreeSet::new();
for entry in &active_members {
let profile = handle
.definition
.resolve_profile(&entry.role, None)
.await
.ok();
if profile
.as_ref()
.is_some_and(|profile| profile.external_addressable)
&& let Some((runtime_id, _)) = entry.binding_atoms()
{
externally_addressable_runtime_ids.insert(mob_modeled_normalize_formal_string(
&serde_json::to_string(&runtime_id)
.expect("serialize runtime id for external addressability parity snapshot"),
));
}
}
let externally_addressable_runtime_id_values = externally_addressable_runtime_ids
.iter()
.map(|raw| {
serde_json::from_str::<serde_json::Value>(raw)
.unwrap_or_else(|_| serde_json::Value::String(raw.clone()))
})
.collect::<Vec<_>>();
let mut formal_available_fields = BTreeMap::new();
formal_available_fields.insert(
"live_intent_identities".into(),
serde_json::to_string(&live_intent_identity_values)
.expect("serialize live_intent_identities"),
);
formal_available_fields.insert(
"live_runtime_ids".into(),
serde_json::to_string(&live_runtime_id_values).expect("serialize live_runtime_ids"),
);
formal_available_fields.insert(
"externally_addressable_runtime_ids".into(),
serde_json::to_string(&externally_addressable_runtime_id_values)
.expect("serialize externally_addressable_runtime_ids"),
);
formal_available_fields.insert(
"runtime_fence_tokens".into(),
serde_json::to_string(&runtime_fence_tokens).expect("serialize runtime_fence_tokens"),
);
formal_available_fields.insert(
"active_run_count".into(),
serde_json::to_string(
&lifecycle
.as_ref()
.map(|snapshot| snapshot.active_run_count)
.unwrap_or_default(),
)
.expect("serialize active_run_count"),
);
formal_available_fields.insert(
"pending_spawn_count".into(),
serde_json::to_string(
&orchestrator
.as_ref()
.map(|snapshot| snapshot.pending_spawn_count)
.unwrap_or_default(),
)
.expect("serialize pending_spawn_count"),
);
formal_available_fields.insert(
"coordinator_bound".into(),
serde_json::to_string(
&orchestrator
.as_ref()
.map(|snapshot| snapshot.coordinator_bound)
.unwrap_or(false),
)
.expect("serialize coordinator_bound"),
);
if let Some(dsl) = dsl_t2.as_ref() {
for (name, value) in [
(
"member_kickoff_objective_ids",
serde_json::to_value(&dsl.member_kickoff_objective_ids),
),
(
"objective_owner_ids",
serde_json::to_value(&dsl.objective_owner_ids),
),
(
"objective_outcomes",
serde_json::to_value(&dsl.objective_outcomes),
),
(
"concluded_objective_ids",
serde_json::to_value(&dsl.concluded_objective_ids),
),
(
"member_run_open",
serde_json::to_value(&dsl.member_run_open),
),
(
"member_in_flight_work",
serde_json::to_value(&dsl.member_in_flight_work),
),
(
"member_progress_tokens",
serde_json::to_value(&dsl.member_progress_tokens),
),
(
"member_last_observed_at_ms",
serde_json::to_value(&dsl.member_last_observed_at_ms),
),
(
"member_last_progress_at_ms",
serde_json::to_value(&dsl.member_last_progress_at_ms),
),
(
"member_last_progress_event",
serde_json::to_value(
dsl.member_last_progress_event
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect::<BTreeMap<_, _>>(),
),
),
(
"member_health_class",
serde_json::to_value(
dsl.member_health_class
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect::<BTreeMap<_, _>>(),
),
),
] {
formal_available_fields.insert(
name.to_string(),
serde_json::to_string(&value.expect("serialize MobMachine authority field"))
.expect("serialize MobMachine authority JSON value"),
);
}
}
let mut formal_unavailable_fields = schema_mob_machine()
.state
.fields
.iter()
.filter(|field| !formal_available_fields.contains_key(field.name.as_str()))
.map(|field| field.name.as_str().to_string())
.collect::<Vec<_>>();
formal_unavailable_fields.sort();
// T2 DSL fields — projected directly from the `MobMachineAuthority`
// state via the `debug_dsl_t2_snapshot()` command-channel seam (dogma
// #1: one owner, #13: projection rebuilt from explicit DSL source).
let member_restore_failure_codes = dsl_t2
.as_ref()
.map(|snap| {
snap.member_restore_failure_codes
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let runtime_retire_refusal_codes = dsl_t2
.as_ref()
.map(|snap| {
snap.runtime_retire_refusal_codes
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let pending_respawn_topology = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_respawn_topology
.iter()
.map(|identity| format!("{identity:?}"))
.collect()
})
.unwrap_or_default();
let abandoned_respawn_topology = dsl_t2
.as_ref()
.map(|snap| {
snap.abandoned_respawn_topology
.iter()
.map(|(identity, generation)| (format!("{identity:?}"), generation.0))
.collect()
})
.unwrap_or_default();
let runtime_retire_refusal_reasons = dsl_t2
.as_ref()
.map(|snap| {
snap.runtime_retire_refusal_reasons
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let runtime_retire_pending_sessions = dsl_t2
.as_ref()
.map(|snap| {
snap.runtime_retire_pending_sessions
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let remote_runtime_retired_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.remote_runtime_retired_ids
.iter()
.map(|runtime_id| format!("{runtime_id:?}"))
.collect()
})
.unwrap_or_default();
let remote_supervisor_revoked_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.remote_supervisor_revoked_ids
.iter()
.map(|runtime_id| format!("{runtime_id:?}"))
.collect()
})
.unwrap_or_default();
let supervisor_pending_authority_operation_id = dsl_t2
.as_ref()
.and_then(|snap| snap.supervisor_pending_authority_operation_id.clone());
let supervisor_pending_authority_member_target_names = dsl_t2
.as_ref()
.map(|snap| {
snap.supervisor_pending_authority_member_target_names
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let supervisor_pending_authority_member_target_addresses = dsl_t2
.as_ref()
.map(|snap| {
snap.supervisor_pending_authority_member_target_addresses
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let member_kickoff_objective_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.member_kickoff_objective_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let objective_outcomes = dsl_t2
.as_ref()
.map(|snap| snap.objective_outcomes.clone())
.unwrap_or_default();
let concluded_objective_ids = dsl_t2
.as_ref()
.map(|snap| snap.concluded_objective_ids.clone())
.unwrap_or_default();
let member_run_open = dsl_t2
.as_ref()
.map(|snap| {
snap.member_run_open
.iter()
.map(|(key, value)| (format!("{key:?}"), *value))
.collect()
})
.unwrap_or_default();
let member_in_flight_work = dsl_t2
.as_ref()
.map(|snap| {
snap.member_in_flight_work
.iter()
.map(|(key, value)| (format!("{key:?}"), *value))
.collect()
})
.unwrap_or_default();
let member_progress_tokens = dsl_t2
.as_ref()
.map(|snap| {
snap.member_progress_tokens
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let member_last_observed_at_ms = dsl_t2
.as_ref()
.map(|snap| {
snap.member_last_observed_at_ms
.iter()
.map(|(key, value)| (format!("{key:?}"), *value))
.collect()
})
.unwrap_or_default();
let member_last_progress_at_ms = dsl_t2
.as_ref()
.map(|snap| {
snap.member_last_progress_at_ms
.iter()
.map(|(key, value)| (format!("{key:?}"), *value))
.collect()
})
.unwrap_or_default();
let member_last_progress_event = dsl_t2
.as_ref()
.map(|snap| {
snap.member_last_progress_event
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let host_binding_generations = dsl_t2
.as_ref()
.map(|snap| {
snap.host_binding_generations
.iter()
.map(|(key, generation)| (format!("{key:?}"), *generation))
.collect()
})
.unwrap_or_default();
let host_binding_generation_highwater = dsl_t2
.as_ref()
.map(|snap| {
snap.host_binding_generation_highwater
.iter()
.map(|(key, generation)| (format!("{key:?}"), *generation))
.collect()
})
.unwrap_or_default();
let confirmed_host_binding_revocations = dsl_t2
.as_ref()
.map(|snap| {
snap.confirmed_host_binding_revocations
.iter()
.map(|tombstone| format!("{tombstone:?}"))
.collect()
})
.unwrap_or_default();
let replacement_host_bind_endpoints = dsl_t2
.as_ref()
.map(|snap| {
snap.replacement_host_bind_endpoints
.iter()
.map(|(key, endpoint)| (format!("{key:?}"), format!("{endpoint:?}")))
.collect()
})
.unwrap_or_default();
let replacement_host_binding_generations = dsl_t2
.as_ref()
.map(|snap| {
snap.replacement_host_binding_generations
.iter()
.map(|(key, generation)| (format!("{key:?}"), *generation))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let member_health_class = dsl_t2
.as_ref()
.map(|snap| {
snap.member_health_class
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_generations = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_generations
.iter()
.map(|(key, value)| (format!("{key:?}"), value.0))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_fence_tokens = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_fence_tokens
.iter()
.map(|(key, value)| (format!("{key:?}"), value.0))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_hosts = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_hosts
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let pending_autonomous_placed_spawns = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_autonomous_placed_spawns
.iter()
.map(|identity| format!("{identity:?}"))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_host_binding_generations = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_host_binding_generations
.iter()
.map(|(key, generation)| (format!("{key:?}"), *generation))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_spec_digests = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_spec_digests
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_provision_operation_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_provision_operation_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let pending_placed_spawn_operation_owner_session_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_spawn_operation_owner_session_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let current_placed_spawn_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.current_placed_spawn_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let current_placed_spawn_host_binding_generations = dsl_t2
.as_ref()
.map(|snap| {
snap.current_placed_spawn_host_binding_generations
.iter()
.map(|(key, generation)| (format!("{key:?}"), *generation))
.collect()
})
.unwrap_or_default();
let current_placed_spawn_provision_operation_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.current_placed_spawn_provision_operation_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), value.clone()))
.collect()
})
.unwrap_or_default();
let current_placed_spawn_operation_owner_session_ids = dsl_t2
.as_ref()
.map(|snap| {
snap.current_placed_spawn_operation_owner_session_ids
.iter()
.map(|(key, value)| (format!("{key:?}"), format!("{value:?}")))
.collect()
})
.unwrap_or_default();
let pending_placed_carrier_cleanup = dsl_t2
.as_ref()
.map(|snap| {
snap.pending_placed_carrier_cleanup
.iter()
.map(|obligation| format!("{obligation:?}"))
.collect()
})
.unwrap_or_default();
let remote_turn_dispatch_sequence = dsl_t2
.as_ref()
.map_or(0, |snap| snap.remote_turn_dispatch_sequence);
let committed_remote_turn_outcomes = dsl_t2
.as_ref()
.map(|snap| {
snap.committed_remote_turn_outcomes
.iter()
.map(|obligation| format!("{obligation:?}"))
.collect()
})
.unwrap_or_default();
let resolved_remote_turn_outcomes = dsl_t2
.as_ref()
.map(|snap| {
snap.resolved_remote_turn_outcomes
.iter()
.map(|obligation| format!("{obligation:?}"))
.collect()
})
.unwrap_or_default();
let (
destroy_admitted,
member_state_markers,
wiring_edges,
external_peer_edges,
external_peer_edges_by_key,
identity_to_runtime,
identity_runtime_generations,
identity_runtime_fence_tokens,
member_peer_ids,
member_peer_endpoints,
member_prior_peer_endpoints,
member_restore_failures,
member_revival_pending,
supervisor_authority_peer_id,
supervisor_authority_signing_key,
supervisor_authority_epoch,
supervisor_authority_protocol_version,
supervisor_pending_authority_peer_id,
supervisor_pending_authority_signing_key,
supervisor_pending_authority_epoch,
supervisor_pending_authority_protocol_version,
supervisor_pending_authority_accepted_peer_ids,
pending_recipient_trust,
member_session_bindings,
spawn_exec_phase,
member_profile_names,
member_runtime_modes,
pending_spawn_sessions,
pending_session_ingress_detach_runtime_ids,
topology_epoch,
spawn_policy_enabled,
spawn_policy_revision,
spawn_policy_resolution_revision,
spawn_policy_resolution_profiles,
spawn_policy_resolution_runtime_modes,
spawn_policy_resolution_absent,
spawn_profile_authority_profile_names,
spawn_profile_authority_models,
spawn_profile_authority_material_digests,
spawn_profile_authority_tool_config_digests,
spawn_profile_authority_skills_digests,
spawn_profile_authority_provider_params_digests,
spawn_profile_authority_output_schema_digests,
spawn_profile_authority_external_addressable,
orphan_budget,
topology_default_policy,
external_member_rebind_capability,
desired_members,
members_to_spawn,
members_to_retire,
) = dsl_t2
.map(|snap| {
(
snap.destroy_admitted,
snap.member_state_markers
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.wiring_edges
.into_iter()
.map(|edge| format!("{edge:?}"))
.collect::<BTreeSet<_>>(),
snap.external_peer_edges
.into_iter()
.map(|edge| format!("{edge:?}"))
.collect::<BTreeSet<_>>(),
snap.external_peer_edges_by_key
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.identity_to_runtime
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.identity_runtime_generations
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v.0))
.collect::<BTreeMap<_, _>>(),
snap.identity_runtime_fence_tokens
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v.0))
.collect::<BTreeMap<_, _>>(),
snap.member_peer_ids
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.member_peer_endpoints
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.member_prior_peer_endpoints
.into_iter()
.map(|(k, endpoints)| {
(
format!("{k:?}"),
endpoints
.into_iter()
.map(|endpoint| format!("{endpoint:?}"))
.collect::<BTreeSet<_>>(),
)
})
.collect::<BTreeMap<_, _>>(),
snap.member_restore_failures
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.member_revival_pending
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
snap.supervisor_authority_peer_id
.map(|peer_id| format!("{peer_id:?}")),
snap.supervisor_authority_signing_key
.map(|signing_key| format!("{signing_key:?}")),
snap.supervisor_authority_epoch,
snap.supervisor_authority_protocol_version
.map(|protocol_version| format!("{protocol_version:?}")),
snap.supervisor_pending_authority_peer_id
.map(|peer_id| format!("{peer_id:?}")),
snap.supervisor_pending_authority_signing_key
.map(|signing_key| format!("{signing_key:?}")),
snap.supervisor_pending_authority_epoch,
snap.supervisor_pending_authority_protocol_version
.map(|protocol_version| format!("{protocol_version:?}")),
snap.supervisor_pending_authority_accepted_peer_ids
.into_iter()
.map(|peer_id| format!("{peer_id:?}"))
.collect::<BTreeSet<_>>(),
snap.pending_recipient_trust
.into_iter()
.map(|peer_id| format!("{peer_id:?}"))
.collect::<BTreeSet<_>>(),
snap.member_session_bindings
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.spawn_exec_phase
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.member_profile_names
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.member_runtime_modes
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.pending_spawn_sessions
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.pending_session_ingress_detach_runtime_ids
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
snap.topology_epoch,
snap.spawn_policy_enabled,
snap.spawn_policy_revision,
snap.spawn_policy_resolution_revision
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_policy_resolution_profiles
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_policy_resolution_runtime_modes
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.spawn_policy_resolution_absent
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
snap.spawn_profile_authority_profile_names
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_models
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_material_digests
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_tool_config_digests
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_skills_digests
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_provider_params_digests
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_output_schema_digests
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.spawn_profile_authority_external_addressable
.into_iter()
.map(|(k, v)| (format!("{k:?}"), v))
.collect::<BTreeMap<_, _>>(),
snap.orphan_budget,
format!("{:?}", snap.topology_default_policy),
snap.external_member_rebind_capability
.into_iter()
.map(|(k, v)| (format!("{k:?}"), format!("{v:?}")))
.collect::<BTreeMap<_, _>>(),
snap.desired_members
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
snap.members_to_spawn
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
snap.members_to_retire
.into_iter()
.map(|id| format!("{id:?}"))
.collect::<BTreeSet<_>>(),
)
})
.unwrap_or_else(|| {
(
false,
BTreeMap::new(),
BTreeSet::new(),
BTreeSet::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeSet::new(),
None,
None,
None,
None,
None,
None,
None,
None,
BTreeSet::new(),
BTreeSet::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeSet::new(),
0,
false,
0,
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeSet::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
BTreeMap::new(),
0,
format!(
"{:?}",
crate::machines::mob_machine::PolicyDecision::default()
),
BTreeMap::new(),
BTreeSet::new(),
BTreeSet::new(),
BTreeSet::new(),
)
});
Some(MobRuntimeParitySnapshotSummary {
phase: phase.as_str().to_string(),
destroy_admitted,
flow_authority_schema_version,
owner_bridge_session_id,
owner_bridge_destroy_on_archive,
implicit_delegation_mob,
live_intent_identities,
live_runtime_ids,
externally_addressable_runtime_ids,
runtime_fence_tokens,
active_member_count: active_members.len(),
all_member_count: all_members.len(),
coordinator_bound: orchestrator
.as_ref()
.map(|snapshot| snapshot.coordinator_bound),
pending_spawn_count: orchestrator
.as_ref()
.map(|snapshot| snapshot.pending_spawn_count),
active_flow_count: lifecycle.as_ref().map(|snapshot| snapshot.active_run_count),
topology_revision: orchestrator
.as_ref()
.map(|snapshot| snapshot.topology_revision),
supervisor_active: orchestrator
.as_ref()
.map(|snapshot| snapshot.supervisor_active),
representative_agent_identity,
representative_runtime_id,
representative_fence_token,
formal_available_fields,
formal_unavailable_fields,
member_state_markers,
wiring_edges,
external_peer_edges,
external_peer_edges_by_key,
pending_respawn_topology,
abandoned_respawn_topology,
identity_to_runtime,
identity_runtime_generations,
identity_runtime_fence_tokens,
member_peer_ids,
member_peer_endpoints,
member_prior_peer_endpoints,
member_restore_failures,
member_restore_failure_codes,
runtime_retire_refusal_codes,
runtime_retire_refusal_reasons,
runtime_retire_pending_sessions,
remote_runtime_retired_ids,
remote_supervisor_revoked_ids,
member_revival_pending,
supervisor_authority_peer_id,
supervisor_authority_signing_key,
supervisor_authority_epoch,
supervisor_authority_protocol_version,
supervisor_pending_authority_peer_id,
supervisor_pending_authority_signing_key,
supervisor_pending_authority_epoch,
supervisor_pending_authority_protocol_version,
supervisor_pending_authority_operation_id,
supervisor_pending_authority_member_target_names,
supervisor_pending_authority_member_target_addresses,
supervisor_pending_authority_accepted_peer_ids,
pending_recipient_trust,
host_binding_generations,
host_binding_generation_highwater,
confirmed_host_binding_revocations,
replacement_host_bind_endpoints,
replacement_host_binding_generations,
member_session_bindings,
spawn_exec_phase,
member_profile_names,
member_runtime_modes,
pending_spawn_sessions,
pending_session_ingress_detach_runtime_ids,
topology_epoch,
spawn_policy_enabled,
spawn_policy_revision,
spawn_policy_resolution_revision,
spawn_policy_resolution_profiles,
spawn_policy_resolution_runtime_modes,
spawn_policy_resolution_absent,
spawn_profile_authority_profile_names,
spawn_profile_authority_models,
spawn_profile_authority_material_digests,
spawn_profile_authority_tool_config_digests,
spawn_profile_authority_skills_digests,
spawn_profile_authority_provider_params_digests,
spawn_profile_authority_output_schema_digests,
spawn_profile_authority_external_addressable,
orphan_budget,
topology_default_policy,
external_member_rebind_capability,
desired_members,
members_to_spawn,
members_to_retire,
member_kickoff_objective_ids,
objective_outcomes,
concluded_objective_ids,
member_run_open,
member_in_flight_work,
member_progress_tokens,
member_last_observed_at_ms,
member_last_progress_at_ms,
member_last_progress_event,
member_health_class,
pending_placed_spawn_ids,
pending_placed_spawn_generations,
pending_placed_spawn_fence_tokens,
pending_placed_spawn_hosts,
pending_autonomous_placed_spawns,
pending_placed_spawn_host_binding_generations,
pending_placed_spawn_spec_digests,
pending_placed_spawn_provision_operation_ids,
pending_placed_spawn_operation_owner_session_ids,
current_placed_spawn_ids,
current_placed_spawn_host_binding_generations,
current_placed_spawn_provision_operation_ids,
current_placed_spawn_operation_owner_session_ids,
pending_placed_carrier_cleanup,
remote_turn_dispatch_sequence,
committed_remote_turn_outcomes,
resolved_remote_turn_outcomes,
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum MobRuntimeParityExprValue {
Bool(bool),
U64(u64),
String(String),
Set(BTreeSet<String>),
Map(BTreeMap<String, u64>),
None,
}
fn mob_runtime_parity_field_value(
snapshot: &MobRuntimeParitySnapshotSummary,
field: &str,
) -> Option<MobRuntimeParityExprValue> {
match field {
"destroy_admitted" => Some(MobRuntimeParityExprValue::Bool(snapshot.destroy_admitted)),
"flow_authority_schema_version" => Some(MobRuntimeParityExprValue::U64(
snapshot.flow_authority_schema_version,
)),
"owner_bridge_session_id" => Some(
snapshot
.owner_bridge_session_id
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"owner_bridge_destroy_on_archive" => Some(MobRuntimeParityExprValue::Bool(
snapshot.owner_bridge_destroy_on_archive,
)),
"implicit_delegation_mob" => Some(MobRuntimeParityExprValue::Bool(
snapshot.implicit_delegation_mob,
)),
"live_intent_identities" => Some(MobRuntimeParityExprValue::Set(
snapshot.live_intent_identities.clone(),
)),
"live_runtime_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.live_runtime_ids.clone(),
)),
"member_state_markers" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_state_markers
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"wiring_edges" => Some(MobRuntimeParityExprValue::Set(
snapshot.wiring_edges.clone(),
)),
"external_peer_edges" => Some(MobRuntimeParityExprValue::Set(
snapshot.external_peer_edges.clone(),
)),
"external_peer_edges_by_key" => Some(MobRuntimeParityExprValue::Map(
snapshot
.external_peer_edges_by_key
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"pending_respawn_topology" => Some(MobRuntimeParityExprValue::Set(
snapshot.pending_respawn_topology.clone(),
)),
"abandoned_respawn_topology" => Some(MobRuntimeParityExprValue::Map(
snapshot.abandoned_respawn_topology.clone(),
)),
"identity_to_runtime" => Some(MobRuntimeParityExprValue::Map(
snapshot
.identity_to_runtime
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"identity_runtime_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot.identity_runtime_generations.clone(),
)),
"identity_runtime_fence_tokens" => Some(MobRuntimeParityExprValue::Map(
snapshot.identity_runtime_fence_tokens.clone(),
)),
"member_peer_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_peer_ids
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_peer_endpoints" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_peer_endpoints
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_prior_peer_endpoints" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_prior_peer_endpoints
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_restore_failures" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_restore_failures
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_restore_failure_codes" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_restore_failure_codes
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"runtime_retire_refusal_codes" => Some(MobRuntimeParityExprValue::Map(
snapshot
.runtime_retire_refusal_codes
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"runtime_retire_refusal_reasons" => Some(MobRuntimeParityExprValue::Map(
snapshot
.runtime_retire_refusal_reasons
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"runtime_retire_pending_sessions" => Some(MobRuntimeParityExprValue::Map(
snapshot
.runtime_retire_pending_sessions
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"remote_runtime_retired_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.remote_runtime_retired_ids.clone(),
)),
"remote_supervisor_revoked_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.remote_supervisor_revoked_ids.clone(),
)),
"member_revival_pending" => Some(MobRuntimeParityExprValue::Set(
snapshot.member_revival_pending.clone(),
)),
"member_kickoff_objective_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_kickoff_objective_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"member_kickoff_input_ids"
| "retained_placed_kickoff_obligations"
| "member_placed_kickoff_outcome_kinds"
| "member_placed_kickoff_outcome_errors"
| "member_placed_kickoff_closure_kinds" => {
Some(MobRuntimeParityExprValue::Map(BTreeMap::new()))
}
"pending_placed_kickoff_outcomes" | "resolved_placed_kickoff_outcomes" => {
Some(MobRuntimeParityExprValue::Set(BTreeSet::new()))
}
"objective_owner_ids" => Some(MobRuntimeParityExprValue::Map(BTreeMap::new())),
"objective_outcomes" => Some(MobRuntimeParityExprValue::Map(
snapshot
.objective_outcomes
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"concluded_objective_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.concluded_objective_ids.clone(),
)),
"member_run_open" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_run_open
.iter()
.map(|(key, value)| (key.clone(), u64::from(*value)))
.collect(),
)),
"member_in_flight_work" => Some(MobRuntimeParityExprValue::Map(
snapshot.member_in_flight_work.clone(),
)),
"member_progress_tokens" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_progress_tokens
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"member_last_observed_at_ms" => Some(MobRuntimeParityExprValue::Map(
snapshot.member_last_observed_at_ms.clone(),
)),
"member_last_progress_at_ms" => Some(MobRuntimeParityExprValue::Map(
snapshot.member_last_progress_at_ms.clone(),
)),
"member_last_progress_event" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_last_progress_event
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"member_health_class" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_health_class
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"supervisor_authority_peer_id" => Some(
snapshot
.supervisor_authority_peer_id
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_authority_signing_key" => Some(
snapshot
.supervisor_authority_signing_key
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_authority_epoch" => Some(
snapshot
.supervisor_authority_epoch
.map(MobRuntimeParityExprValue::U64)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_authority_protocol_version" => Some(
snapshot
.supervisor_authority_protocol_version
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_peer_id" => Some(
snapshot
.supervisor_pending_authority_peer_id
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_signing_key" => Some(
snapshot
.supervisor_pending_authority_signing_key
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_epoch" => Some(
snapshot
.supervisor_pending_authority_epoch
.map(MobRuntimeParityExprValue::U64)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_protocol_version" => Some(
snapshot
.supervisor_pending_authority_protocol_version
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_operation_id" => Some(
snapshot
.supervisor_pending_authority_operation_id
.clone()
.map(MobRuntimeParityExprValue::String)
.unwrap_or(MobRuntimeParityExprValue::None),
),
"supervisor_pending_authority_member_target_names" => Some(MobRuntimeParityExprValue::Map(
snapshot
.supervisor_pending_authority_member_target_names
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"supervisor_pending_authority_member_target_addresses" => {
Some(MobRuntimeParityExprValue::Map(
snapshot
.supervisor_pending_authority_member_target_addresses
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
))
}
"supervisor_pending_authority_accepted_peer_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot
.supervisor_pending_authority_accepted_peer_ids
.clone(),
)),
"pending_recipient_trust" => Some(MobRuntimeParityExprValue::Set(
snapshot.pending_recipient_trust.clone(),
)),
"host_tracked_input_cancel" => Some(MobRuntimeParityExprValue::Map(BTreeMap::new())),
"member_session_bindings" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_session_bindings
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_exec_phase" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_exec_phase
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_profile_names" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_profile_names
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"member_runtime_modes" => Some(MobRuntimeParityExprValue::Map(
snapshot
.member_runtime_modes
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"pending_spawn_sessions" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_spawn_sessions
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"pending_session_ingress_detach_runtime_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.pending_session_ingress_detach_runtime_ids.clone(),
)),
"topology_epoch" => Some(MobRuntimeParityExprValue::U64(snapshot.topology_epoch)),
"spawn_policy_enabled" => Some(MobRuntimeParityExprValue::Bool(
snapshot.spawn_policy_enabled,
)),
"spawn_policy_revision" => Some(MobRuntimeParityExprValue::U64(
snapshot.spawn_policy_revision,
)),
"spawn_policy_resolution_revision" => Some(MobRuntimeParityExprValue::Map(
snapshot.spawn_policy_resolution_revision.clone(),
)),
"spawn_policy_resolution_profiles" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_policy_resolution_profiles
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_policy_resolution_runtime_modes" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_policy_resolution_runtime_modes
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_policy_resolution_absent" => Some(MobRuntimeParityExprValue::Set(
snapshot.spawn_policy_resolution_absent.clone(),
)),
"spawn_profile_authority_profile_names" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_profile_names
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_models" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_models
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_material_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_material_digests
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_tool_config_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_tool_config_digests
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_skills_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_skills_digests
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_provider_params_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_provider_params_digests
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_output_schema_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_output_schema_digests
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"spawn_profile_authority_external_addressable" => Some(MobRuntimeParityExprValue::Map(
snapshot
.spawn_profile_authority_external_addressable
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
"externally_addressable_runtime_ids" => Some(MobRuntimeParityExprValue::Set(
snapshot.externally_addressable_runtime_ids.clone(),
)),
"runtime_fence_tokens" => Some(MobRuntimeParityExprValue::Map(
snapshot.runtime_fence_tokens.clone(),
)),
"active_run_count" => Some(MobRuntimeParityExprValue::U64(
snapshot.active_flow_count.unwrap_or_default() as u64,
)),
"pending_spawn_count" => Some(MobRuntimeParityExprValue::U64(
snapshot.pending_spawn_count.unwrap_or_default() as u64,
)),
"coordinator_bound" => snapshot
.coordinator_bound
.map(MobRuntimeParityExprValue::Bool),
"run_ready_frame_membership_flat"
| "run_pending_body_frame_loop_membership_flat"
| "member_startup_binding_requested"
| "member_startup_runtime_ready"
| "member_startup_ready"
| "member_kickoff_pending"
| "member_kickoff_starting"
| "member_kickoff_callback_pending"
| "member_kickoff_started"
| "member_kickoff_failed"
| "member_kickoff_cancelled" => Some(MobRuntimeParityExprValue::Set(BTreeSet::new())),
"adaptive_active_run" => Some(MobRuntimeParityExprValue::None),
"adaptive_run_phase"
| "adaptive_stop_reason"
| "adaptive_limit_max_depth"
| "adaptive_limit_max_total_decisions"
| "adaptive_limit_max_repair_attempts"
| "adaptive_limit_max_layer_failures"
| "adaptive_limit_max_attempts_per_layer"
| "adaptive_limit_max_members_per_layer"
| "adaptive_limit_max_total_spawned_members"
| "adaptive_limit_max_active_members"
| "adaptive_limit_max_retained_layer_mobs"
| "adaptive_limit_max_aggregate_tokens"
| "adaptive_limit_max_aggregate_tool_calls"
| "adaptive_limit_allowed_model_classes"
| "adaptive_limit_allowed_tool_classes"
| "adaptive_limit_allowed_skill_identities"
| "adaptive_limit_allowed_auth_binding_refs"
| "adaptive_deadline_ms"
| "adaptive_depth"
| "adaptive_total_decisions"
| "adaptive_repair_attempts"
| "adaptive_layer_failures"
| "adaptive_total_spawned_members"
| "adaptive_active_members"
| "adaptive_retained_layer_mobs"
| "adaptive_aggregate_token_reserved"
| "adaptive_aggregate_token_actual"
| "adaptive_aggregate_tool_call_reserved"
| "adaptive_aggregate_tool_call_actual"
| "adaptive_active_layer"
| "adaptive_layer_adaptive_run"
| "adaptive_layer_phase"
| "adaptive_layer_attempt"
| "adaptive_layer_member_count"
| "adaptive_layer_plan_digest"
| "adaptive_layer_child_mob_id"
| "adaptive_layer_token_reservation"
| "adaptive_layer_tool_call_reservation"
| "adaptive_layer_run_id"
| "adaptive_layer_result_digest"
| "adaptive_layer_fault"
| "adaptive_layer_disposition"
| "adaptive_missing_body_digest" => Some(MobRuntimeParityExprValue::Map(BTreeMap::new())),
"run_status"
| "run_ordered_steps"
| "run_tracked_steps"
| "run_step_status"
| "run_output_recorded"
| "run_step_condition_results"
| "run_step_has_conditions"
| "run_step_dependencies"
| "run_step_dependency_modes"
| "run_step_branches"
| "run_step_collection_policies"
| "run_step_quorum_thresholds"
| "run_step_target_counts"
| "run_step_target_success_counts"
| "run_step_target_terminal_failure_counts"
| "run_target_retry_counts"
| "run_failure_count"
| "run_consecutive_failure_count"
| "run_escalation_threshold"
| "run_max_step_retries"
| "run_ready_frames"
| "run_ready_frame_membership"
| "run_pending_body_frame_loops"
| "run_pending_body_frame_loop_membership"
| "run_active_node_count"
| "run_active_frame_count"
| "run_last_granted_frame"
| "run_last_granted_loop"
| "run_max_active_nodes"
| "run_max_active_frames"
| "run_max_frame_depth"
| "frame_scope"
| "frame_phase"
| "frame_run"
| "frame_parent_loop"
| "frame_iteration"
| "frame_tracked_nodes"
| "frame_ordered_nodes"
| "frame_node_kind"
| "frame_node_dependencies"
| "frame_node_dependency_modes"
| "frame_node_step_ids"
| "frame_node_loop_ids"
| "frame_node_status"
| "frame_ready_queue"
| "frame_output_recorded"
| "frame_last_admitted_node"
| "frame_node_condition_results"
| "frame_node_branches"
| "loop_phase"
| "loop_parent_frame"
| "loop_parent_node"
| "loop_definition"
| "loop_depth"
| "loop_stage"
| "loop_current_iteration"
| "loop_last_completed_iteration"
| "loop_max_iterations"
| "loop_active_body_frame"
| "work_intent_status"
| "work_intent_revision"
| "work_intent_resources"
| "work_intent_owner_present"
| "work_intent_expires_at_ms"
| "resource_claim_status"
| "resource_claim_kind"
| "resource_claim_revision"
| "resource_claim_resources"
| "resource_claim_owner_present"
| "resource_claim_expires_at_ms"
| "member_kickoff_error" => Some(MobRuntimeParityExprValue::Map(BTreeMap::new())),
// Coordination event cursor is a machine-owned monotonic counter that
// initializes to 1 in a fresh mob (no coordination inputs applied yet);
// mirrors the default-projection treatment of the run_*/frame_* fields.
"coordination_event_next_sequence" => Some(MobRuntimeParityExprValue::U64(1)),
// Row #320: machine-owned orphan budget for hard turn timeouts.
"orphan_budget" => Some(MobRuntimeParityExprValue::U64(snapshot.orphan_budget)),
// Row #293: machine-owned default topology policy for unmatched edges.
"topology_default_policy" => Some(MobRuntimeParityExprValue::String(
snapshot.topology_default_policy.clone(),
)),
// Row #314: machine-owned per-member external rebind capability map
// (key-presence projection, mirroring the other DSL maps).
"external_member_rebind_capability" => Some(MobRuntimeParityExprValue::Map(
snapshot
.external_member_rebind_capability
.keys()
.map(|k| (k.clone(), 0u64))
.collect(),
)),
// Row #351: machine-owned reconcile membership sets.
"desired_members" => Some(MobRuntimeParityExprValue::Set(
snapshot.desired_members.clone(),
)),
"members_to_spawn" => Some(MobRuntimeParityExprValue::Set(
snapshot.members_to_spawn.clone(),
)),
"members_to_retire" => Some(MobRuntimeParityExprValue::Set(
snapshot.members_to_retire.clone(),
)),
"pending_placed_spawn_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"pending_placed_spawn_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot.pending_placed_spawn_generations.clone(),
)),
"pending_placed_spawn_fence_tokens" => Some(MobRuntimeParityExprValue::Map(
snapshot.pending_placed_spawn_fence_tokens.clone(),
)),
"pending_placed_spawn_hosts" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_hosts
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"pending_autonomous_placed_spawns" => Some(MobRuntimeParityExprValue::Set(
snapshot.pending_autonomous_placed_spawns.clone(),
)),
"pending_placed_spawn_host_binding_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_host_binding_generations
.clone(),
)),
"pending_placed_spawn_spec_digests" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_spec_digests
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"pending_placed_spawn_provision_operation_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_provision_operation_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"pending_placed_spawn_operation_owner_session_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.pending_placed_spawn_operation_owner_session_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"current_placed_spawn_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.current_placed_spawn_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"current_placed_spawn_host_binding_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot
.current_placed_spawn_host_binding_generations
.clone(),
)),
"current_placed_spawn_provision_operation_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.current_placed_spawn_provision_operation_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"current_placed_spawn_operation_owner_session_ids" => Some(MobRuntimeParityExprValue::Map(
snapshot
.current_placed_spawn_operation_owner_session_ids
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"pending_placed_carrier_cleanup" => Some(MobRuntimeParityExprValue::Set(
snapshot.pending_placed_carrier_cleanup.clone(),
)),
"remote_turn_dispatch_sequence" => Some(MobRuntimeParityExprValue::U64(
snapshot.remote_turn_dispatch_sequence,
)),
"placed_completion_dispatch_sequence" => Some(MobRuntimeParityExprValue::U64(0)),
"placed_completion_lifecycle_quiescing" => Some(MobRuntimeParityExprValue::Bool(false)),
"placed_completion_lifecycle_intent" => Some(MobRuntimeParityExprValue::None),
"pending_placed_completion_outcomes"
| "cancel_requested_placed_completion_outcomes"
| "resolved_placed_completion_outcomes" => {
Some(MobRuntimeParityExprValue::Set(BTreeSet::new()))
}
"committed_remote_turn_outcomes" => Some(MobRuntimeParityExprValue::Set(
snapshot.committed_remote_turn_outcomes.clone(),
)),
"resolved_remote_turn_outcomes" => Some(MobRuntimeParityExprValue::Set(
snapshot.resolved_remote_turn_outcomes.clone(),
)),
// Multi-host phase 1: machine-owned host/placement/obligation/grant
// state with no runtime producer yet (hosts bind in phase 2, placement
// lands in phase 3) — evaluated at the initial empty values, the
// kickoff/adaptive machine-only precedent above.
"mob_hosts" | "pending_remote_turn_outcomes" | "pending_route_installs" => {
Some(MobRuntimeParityExprValue::Set(BTreeSet::new()))
}
"confirmed_host_binding_revocations" => Some(MobRuntimeParityExprValue::Set(
snapshot.confirmed_host_binding_revocations.clone(),
)),
"host_binding_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot.host_binding_generations.clone(),
)),
"host_binding_generation_highwater" => Some(MobRuntimeParityExprValue::Map(
snapshot.host_binding_generation_highwater.clone(),
)),
"replacement_host_bind_endpoints" => Some(MobRuntimeParityExprValue::Map(
snapshot
.replacement_host_bind_endpoints
.keys()
.map(|key| (key.clone(), 0))
.collect(),
)),
"replacement_host_binding_generations" => Some(MobRuntimeParityExprValue::Map(
snapshot.replacement_host_binding_generations.clone(),
)),
"host_public_keys"
| "host_endpoints"
| "host_authority_epochs"
| "host_bind_phase"
| "host_protocol_min"
| "host_protocol_max"
| "host_engine_versions"
| "host_durable_sessions"
| "host_autonomous_members"
| "host_hard_cancel_member"
| "host_memory_store"
| "host_mcp"
| "host_resolvable_providers"
| "host_approval_forwarding"
| "host_live_endpoints"
| "member_placement"
| "member_materialization_failures"
| "operator_grant_scopes"
| "operator_grant_expiries"
| "spawn_profile_authority_resolved_spec_digests" => {
Some(MobRuntimeParityExprValue::Map(BTreeMap::new()))
}
_ => None,
}
}
fn mob_runtime_parity_eval_expr(
expr: &Expr,
snapshot: &MobRuntimeParitySnapshotSummary,
) -> Option<MobRuntimeParityExprValue> {
match expr {
Expr::Bool(value) => Some(MobRuntimeParityExprValue::Bool(*value)),
Expr::U64(value) => Some(MobRuntimeParityExprValue::U64(*value)),
Expr::String(value) => Some(MobRuntimeParityExprValue::String(value.clone())),
Expr::None => Some(MobRuntimeParityExprValue::None),
Expr::CurrentPhase => Some(MobRuntimeParityExprValue::String(snapshot.phase.clone())),
Expr::Phase(phase) => Some(MobRuntimeParityExprValue::String(
phase.as_str().to_string(),
)),
Expr::Field(field) => mob_runtime_parity_field_value(snapshot, field.as_str()),
Expr::Not(inner) => mob_runtime_parity_eval_bool(inner, snapshot)
.map(|value| MobRuntimeParityExprValue::Bool(!value)),
Expr::And(items) => match items
.iter()
.map(|item| mob_runtime_parity_eval_bool(item, snapshot))
.collect::<Vec<_>>()
.as_slice()
{
values if values.contains(&Some(false)) => Some(MobRuntimeParityExprValue::Bool(false)),
values if values.iter().all(|value| *value == Some(true)) => {
Some(MobRuntimeParityExprValue::Bool(true))
}
_ => None,
},
Expr::Or(items) => match items
.iter()
.map(|item| mob_runtime_parity_eval_bool(item, snapshot))
.collect::<Vec<_>>()
.as_slice()
{
values if values.contains(&Some(true)) => Some(MobRuntimeParityExprValue::Bool(true)),
values if values.iter().all(|value| *value == Some(false)) => {
Some(MobRuntimeParityExprValue::Bool(false))
}
_ => None,
},
Expr::Eq(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
Some(MobRuntimeParityExprValue::Bool(left == right))
}
Expr::Neq(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
Some(MobRuntimeParityExprValue::Bool(left != right))
}
Expr::Gt(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
match (left, right) {
(MobRuntimeParityExprValue::U64(left), MobRuntimeParityExprValue::U64(right)) => {
Some(MobRuntimeParityExprValue::Bool(left > right))
}
_ => None,
}
}
Expr::Gte(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
match (left, right) {
(MobRuntimeParityExprValue::U64(left), MobRuntimeParityExprValue::U64(right)) => {
Some(MobRuntimeParityExprValue::Bool(left >= right))
}
_ => None,
}
}
Expr::Lt(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
match (left, right) {
(MobRuntimeParityExprValue::U64(left), MobRuntimeParityExprValue::U64(right)) => {
Some(MobRuntimeParityExprValue::Bool(left < right))
}
_ => None,
}
}
Expr::Lte(left, right) => {
let left = mob_runtime_parity_eval_expr(left, snapshot)?;
let right = mob_runtime_parity_eval_expr(right, snapshot)?;
match (left, right) {
(MobRuntimeParityExprValue::U64(left), MobRuntimeParityExprValue::U64(right)) => {
Some(MobRuntimeParityExprValue::Bool(left <= right))
}
_ => None,
}
}
_ => None,
}
}
fn mob_runtime_parity_eval_bool(
expr: &Expr,
snapshot: &MobRuntimeParitySnapshotSummary,
) -> Option<bool> {
match mob_runtime_parity_eval_expr(expr, snapshot) {
Some(MobRuntimeParityExprValue::Bool(value)) => Some(value),
_ => None,
}
}
fn summarize_mob_runtime_success(probe: MobRuntimeParityProbeInput, summary: &str) -> String {
format!("{probe:?}:{summary}")
}
fn summarize_mob_runtime_error(error: &MobError) -> String {
match error {
MobError::MobNotFound(_) => "mob_not_found".to_string(),
MobError::ProfileNotFound(_) => "profile_not_found".to_string(),
MobError::MemberNotFound(_) => "meerkat_not_found".to_string(),
MobError::SessionUnavailableForResume { reason, .. } => {
format!("session_unavailable_for_resume:{reason:?}")
}
MobError::MemberAlreadyExists(_) => "meerkat_already_exists".to_string(),
MobError::NotExternallyAddressable(_) => "not_externally_addressable".to_string(),
MobError::InvalidTransition { from, to } => {
format!("invalid_transition:{}->{}", from.as_str(), to.as_str())
}
MobError::MobMachineRejected { context, .. } => {
format!("mob_machine_rejected:{context}")
}
MobError::WiringError(_) => "wiring_error".to_string(),
MobError::ExternalMemberCleanupUncertain { .. } => {
"external_member_cleanup_uncertain".to_string()
}
MobError::RetirementTopologyIncomplete(_) => "retirement_topology_incomplete".to_string(),
MobError::SupervisorRotationIncomplete { .. } => {
"supervisor_rotation_incomplete".to_string()
}
MobError::SupervisorEpochExhausted { .. } => "supervisor_epoch_exhausted".to_string(),
MobError::BridgeCommandRejected { cause, .. } => {
format!("bridge_command_rejected:{cause:?}")
}
MobError::MemberProvisionFailed { cause } => {
format!("member_provision_failed:{cause:?}")
}
MobError::MemberRestoreFailed { .. } => "member_restore_failed".to_string(),
MobError::SpawnMemberAdmissionDenied { .. } => "spawn_member_admission_denied".to_string(),
MobError::BridgeRequestTimedOut { .. } => "bridge_request_timed_out".to_string(),
MobError::ForkSourceUnavailable { .. } => "fork_source_unavailable".to_string(),
MobError::RuntimeEffectRefused {
kind, refusal_code, ..
} => format!("runtime_effect_refused:{kind}:{refusal_code}"),
MobError::KickoffWaitTimedOut { .. } => "kickoff_wait_timed_out".to_string(),
MobError::ReadyWaitTimedOut { .. } => "ready_wait_timed_out".to_string(),
MobError::DefinitionError(_) => "definition_error".to_string(),
MobError::FlowNotFound(_) => "flow_not_found".to_string(),
MobError::FlowFailed { .. } => "flow_failed".to_string(),
MobError::RunNotFound(_) => "run_not_found".to_string(),
MobError::RunCanceled(_) => "run_canceled".to_string(),
MobError::FlowTurnTimedOut => "flow_turn_timed_out".to_string(),
MobError::FrameDepthLimitExceeded { .. } => "frame_depth_limit_exceeded".to_string(),
MobError::FrameAtomicPersistenceUnavailable { .. } => {
"frame_atomic_persistence_unavailable".to_string()
}
MobError::SpecRevisionConflict { .. } => "spec_revision_conflict".to_string(),
MobError::SchemaValidation { .. } => "schema_validation".to_string(),
MobError::InsufficientTargets { .. } => "insufficient_targets".to_string(),
MobError::TopologyViolation { .. } => "topology_violation".to_string(),
MobError::BridgeDeliveryRejected { cause, .. } => {
format!("bridge_delivery_rejected:{cause}")
}
MobError::SupervisorEscalation(_) => "supervisor_escalation".to_string(),
MobError::UnsupportedForMode { .. } => "unsupported_for_mode".to_string(),
MobError::MissingMemberCapability { capability, .. } => {
format!("missing_member_capability:{capability}")
}
MobError::ResetBarrier => "reset_barrier".to_string(),
MobError::StorageError(_) => "storage_error".to_string(),
MobError::SessionError(_) => "session_error".to_string(),
MobError::CommsError(_) => "comms_error".to_string(),
MobError::CallbackPending { .. } | MobError::CallbackBatchPending { .. } => {
"callback_pending".to_string()
}
MobError::StaleFenceToken { .. } => "stale_fence_token".to_string(),
MobError::StaleMemberOperatorAuthority { .. } => {
"stale_member_operator_authority".to_string()
}
MobError::StaleEventCursor { .. } => "stale_event_cursor".to_string(),
MobError::WorkNotFound(_) => "work_not_found".to_string(),
MobError::WorkCancellationUnsupported(_) => "work_cancellation_unsupported".to_string(),
MobError::InjectedContextUndeliverable { .. } => {
"injected_context_undeliverable".to_string()
}
MobError::PlacedInteractionIdAlreadyUsed { .. } => {
"placed_interaction_id_already_used".to_string()
}
MobError::InvalidPlacedInteractionId { .. } => "invalid_placed_interaction_id".to_string(),
MobError::PlacedCompletionDeliveryRejected => {
"placed_completion_delivery_rejected".to_string()
}
MobError::PlacedCompletionHostNoEffect => "placed_completion_host_no_effect".to_string(),
MobError::PlacedCompletionHostCancelled => "placed_completion_host_cancelled".to_string(),
MobError::PlacedCompletionDisposed => "placed_completion_disposed".to_string(),
MobError::PlacedCompletionCleanupPending { .. } => {
"placed_completion_cleanup_pending".to_string()
}
MobError::PlacedKickoffCleanupPending { .. } => {
"placed_kickoff_cleanup_pending".to_string()
}
MobError::AutonomousStopInterruptsPending { .. } => {
"autonomous_stop_interrupts_pending".to_string()
}
MobError::LifecycleOperationPending { .. } => "lifecycle_operation_pending".to_string(),
MobError::ActorCommandChannelClosed => "actor_command_channel_closed".to_string(),
MobError::ActorReplyChannelClosed => "actor_reply_channel_closed".to_string(),
MobError::BridgeSessionNotInLiveAuthority { .. } => {
"bridge_session_not_in_live_authority".to_string()
}
MobError::MemberCommsName(_) => "member_comms_name".to_string(),
MobError::ConditionEval { .. } => "condition_eval".to_string(),
MobError::HostCapabilityContractViolation { .. } => {
"host_capability_contract_violation".to_string()
}
MobError::Internal(reason) => format!("internal:{reason}"),
MobError::ScopeDenied(denial) => format!("scope_denied:{:?}", denial.required),
MobError::FlowStepDispatchRejected { kind, .. } => {
format!("flow_step_dispatch_rejected:{kind}")
}
}
}
fn mob_modeled_schema_result_summary(
before: &MobRuntimeParitySnapshotSummary,
probe: MobRuntimeParityProbeInput,
) -> Option<String> {
match probe {
MobRuntimeParityProbeInput::Spawn => Some(summarize_mob_runtime_success(probe, "spawned")),
MobRuntimeParityProbeInput::SubmitWork => {
Some(summarize_mob_runtime_success(probe, "work_receipt"))
}
MobRuntimeParityProbeInput::RunFlow => Some(summarize_mob_runtime_success(probe, "run_id")),
MobRuntimeParityProbeInput::EnsureMember
| MobRuntimeParityProbeInput::AuthorizeSpawnProfile
| MobRuntimeParityProbeInput::ClassifyMemberWait
| MobRuntimeParityProbeInput::ClassifyRetirePendingSpawnDisposition
| MobRuntimeParityProbeInput::Reconcile
| MobRuntimeParityProbeInput::CancelFlow
| MobRuntimeParityProbeInput::Retire
| MobRuntimeParityProbeInput::RetireAll
| MobRuntimeParityProbeInput::WireMembers
| MobRuntimeParityProbeInput::WireMembersWithTrust
| MobRuntimeParityProbeInput::UnwireMembers
| MobRuntimeParityProbeInput::WireExternalPeer
| MobRuntimeParityProbeInput::UnwireExternalPeer
| MobRuntimeParityProbeInput::AuthorizeMemberPeerRebind
| MobRuntimeParityProbeInput::AuthorizeMemberPeerOverlay
| MobRuntimeParityProbeInput::CancelWork
| MobRuntimeParityProbeInput::CancelAllWork
| MobRuntimeParityProbeInput::Stop
| MobRuntimeParityProbeInput::Resume
| MobRuntimeParityProbeInput::Complete
| MobRuntimeParityProbeInput::Reset
| MobRuntimeParityProbeInput::Destroy
| MobRuntimeParityProbeInput::RecordOperatorActionProvenance
| MobRuntimeParityProbeInput::SetSpawnPolicy
| MobRuntimeParityProbeInput::Shutdown
| MobRuntimeParityProbeInput::ForceCancel => {
Some(summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::ConcludeObjective => {
Some(summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::Respawn => {
Some(summarize_mob_runtime_success(probe, "respawned"))
}
MobRuntimeParityProbeInput::ExternalTurn => {
Some(summarize_mob_runtime_success(probe, "bridge_session"))
}
MobRuntimeParityProbeInput::InternalTurn => {
Some(summarize_mob_runtime_success(probe, "member_delivery"))
}
MobRuntimeParityProbeInput::SubscribeAgentEvents => {
Some(summarize_mob_runtime_success(probe, "event_stream"))
}
MobRuntimeParityProbeInput::SubscribeAllAgentEvents => Some(summarize_mob_runtime_success(
probe,
&format!("all_agent_streams:{}", before.all_member_count),
)),
MobRuntimeParityProbeInput::SubscribeMobEvents => {
Some(summarize_mob_runtime_success(probe, "mob_event_router"))
}
}
}
async fn build_mob_runtime_parity_fixture() -> MobRuntimeParityFixture {
let definition = sample_definition_with_single_step_flow(60_000, 8);
let (handle, service) = create_test_mob(definition).await;
let _ = service.enable_runtime_adapter();
service.set_flow_turn_delay_ms(150);
MobRuntimeParityFixture {
handle,
service,
worker_identity: AgentIdentity::from("w-1"),
lead_identity: AgentIdentity::from("l-1"),
cancel_identity: AgentIdentity::from("cancel-target"),
flow_run_id: None,
submitted_work_ref: None,
wired_external: false,
objective_id: meerkat_core::interaction::ObjectiveId::new(),
}
}
fn mob_runtime_parity_external_peer() -> TrustedPeerDescriptor {
test_trusted_peer_descriptor(
"mob-runtime-parity/external/agent",
"inproc://mob-runtime-parity/external/agent",
)
}
async fn mob_runtime_parity_prepare_probe(
target_phase: MobRuntimeParityPhase,
probe: MobRuntimeParityProbeInput,
fixture: &mut MobRuntimeParityFixture,
setup_tags: &mut Vec<String>,
) -> Result<(), String> {
match probe {
MobRuntimeParityProbeInput::SubmitWork
| MobRuntimeParityProbeInput::Retire
| MobRuntimeParityProbeInput::Respawn
| MobRuntimeParityProbeInput::InternalTurn
| MobRuntimeParityProbeInput::CancelAllWork => {
fixture.ensure_worker().await?;
setup_tags.push("worker_spawned".to_string());
}
MobRuntimeParityProbeInput::CancelWork => {
fixture.ensure_worker().await?;
setup_tags.push("worker_spawned".to_string());
let _ = fixture.ensure_submitted_work().await?;
setup_tags.push("work_submitted".to_string());
}
MobRuntimeParityProbeInput::RunFlow => {
fixture.ensure_worker().await?;
setup_tags.push("worker_spawned".to_string());
}
MobRuntimeParityProbeInput::CancelFlow => {
if target_phase == MobRuntimeParityPhase::Running {
let _ = fixture.ensure_demo_run().await?;
setup_tags.push("flow_run_started".to_string());
} else {
fixture.flow_run_id = Some(RunId::new());
setup_tags.push("synthetic_run_id".to_string());
}
}
MobRuntimeParityProbeInput::RetireAll => {
fixture.ensure_worker().await?;
setup_tags.push("worker_spawned".to_string());
}
MobRuntimeParityProbeInput::WireMembers
| MobRuntimeParityProbeInput::WireMembersWithTrust => {
fixture.ensure_worker().await?;
fixture.ensure_lead().await?;
setup_tags.push("worker_spawned".to_string());
setup_tags.push("lead_spawned".to_string());
}
MobRuntimeParityProbeInput::UnwireMembers => {
fixture.ensure_wired_edge().await?;
setup_tags.push("wired_external_edge".to_string());
}
MobRuntimeParityProbeInput::WireExternalPeer => {
fixture.ensure_lead().await?;
setup_tags.push("lead_spawned".to_string());
}
MobRuntimeParityProbeInput::AuthorizeMemberPeerRebind
| MobRuntimeParityProbeInput::AuthorizeMemberPeerOverlay => {
fixture.ensure_lead().await?;
setup_tags.push("lead_spawned".to_string());
}
MobRuntimeParityProbeInput::UnwireExternalPeer => {
fixture.ensure_lead().await?;
fixture
.handle
.wire(
fixture.lead_identity.clone(),
PeerTarget::External(mob_runtime_parity_external_peer()),
)
.await
.map_err(|error| format!("wire external peer: {error:?}"))?;
setup_tags.push("external_peer_wired".to_string());
}
MobRuntimeParityProbeInput::ExternalTurn
| MobRuntimeParityProbeInput::SubscribeAgentEvents
| MobRuntimeParityProbeInput::SubscribeAllAgentEvents => {
fixture.ensure_lead().await?;
setup_tags.push("lead_spawned".to_string());
}
MobRuntimeParityProbeInput::ForceCancel => {
fixture.ensure_force_cancel_member().await?;
setup_tags.push("turn_driven_member_spawned".to_string());
}
MobRuntimeParityProbeInput::ConcludeObjective => {
fixture.ensure_worker().await?;
let snapshot = fixture
.handle
.member_status(&fixture.worker_identity)
.await
.map_err(|error| format!("project worker kickoff objective: {error:?}"))?;
fixture.objective_id = snapshot
.kickoff
.and_then(|kickoff| kickoff.objective_id)
.ok_or_else(|| "spawned worker has no kickoff objective".to_string())?;
setup_tags.push("worker_objective_resolved".to_string());
}
MobRuntimeParityProbeInput::Spawn
| MobRuntimeParityProbeInput::AuthorizeSpawnProfile
| MobRuntimeParityProbeInput::ClassifyMemberWait
| MobRuntimeParityProbeInput::ClassifyRetirePendingSpawnDisposition
| MobRuntimeParityProbeInput::Stop
| MobRuntimeParityProbeInput::Resume
| MobRuntimeParityProbeInput::Complete
| MobRuntimeParityProbeInput::Reset
| MobRuntimeParityProbeInput::Destroy
| MobRuntimeParityProbeInput::SubscribeMobEvents
| MobRuntimeParityProbeInput::RecordOperatorActionProvenance
| MobRuntimeParityProbeInput::SetSpawnPolicy
| MobRuntimeParityProbeInput::Shutdown
| MobRuntimeParityProbeInput::EnsureMember
| MobRuntimeParityProbeInput::Reconcile => {}
}
match target_phase {
MobRuntimeParityPhase::Running => {}
MobRuntimeParityPhase::Stopped => {
if fixture.handle.status().await.unwrap() != MobState::Stopped {
fixture
.handle
.stop()
.await
.map_err(|error| format!("stop to target phase: {error:?}"))?;
setup_tags.push("transitioned_to_stopped".to_string());
}
}
MobRuntimeParityPhase::Completed => {
if fixture.handle.status().await.unwrap() != MobState::Completed {
fixture
.handle
.complete()
.await
.map_err(|error| format!("complete to target phase: {error:?}"))?;
setup_tags.push("transitioned_to_completed".to_string());
}
}
}
Ok(())
}
async fn mob_runtime_parity_execute_probe(
fixture: &mut MobRuntimeParityFixture,
probe: MobRuntimeParityProbeInput,
) -> Result<String, MobError> {
match probe {
MobRuntimeParityProbeInput::Spawn => fixture
.handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("parity-spawn"),
None,
)
.await
.map(|_| summarize_mob_runtime_success(probe, "spawned")),
MobRuntimeParityProbeInput::AuthorizeSpawnProfile => fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: crate::machines::mob_machine::AgentIdentity::from(
"parity-authorize-profile",
),
profile_name: "worker".to_string(),
model: "test-model".to_string(),
profile_material_digest: "parity-profile-material".to_string(),
tool_config_digest: "parity-tool-config".to_string(),
skills_digest: "parity-skills".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: false,
resolved_spec_digest: None,
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::ClassifyMemberWait => fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::ClassifyMemberWait {
agent_identity: crate::machines::mob_machine::AgentIdentity::from(
fixture.worker_identity.as_str(),
),
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::ClassifyRetirePendingSpawnDisposition => fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::ClassifyRetirePendingSpawnDisposition {
agent_identity: crate::machines::mob_machine::AgentIdentity::from(
fixture.worker_identity.as_str(),
),
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::EnsureMember => fixture
.handle
.ensure_member(SpawnMemberSpec::new("worker", "parity-ensure"))
.await
.map(|_| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Reconcile => fixture
.handle
.reconcile(
vec![SpawnMemberSpec::new("worker", "parity-reconcile")],
crate::runtime::reconcile::ReconcileOptions::default(),
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::SubmitWork => {
let entry = fixture.worker_entry().await.map_err(MobError::Internal)?;
fixture
.handle
.submit_work(
entry.agent_runtime_id.clone(),
entry.fence_token,
WorkRef::new(),
WorkSpec::new("mob-runtime-parity".to_string(), WorkOrigin::Internal),
)
.await
.map(|_| summarize_mob_runtime_success(probe, "work_receipt"))
}
MobRuntimeParityProbeInput::RunFlow => fixture
.handle
.run_flow(
FlowId::from("demo"),
serde_json::json!({"source":"mob-runtime-parity"}),
)
.await
.map(|_| summarize_mob_runtime_success(probe, "run_id")),
MobRuntimeParityProbeInput::CancelFlow => {
let run_id = match &fixture.flow_run_id {
Some(run_id) => run_id.clone(),
None => RunId::new(),
};
fixture
.handle
.cancel_flow(run_id)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::Retire => fixture
.handle
.retire(fixture.worker_identity.clone())
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Respawn => fixture
.handle
.respawn(fixture.worker_identity.clone(), None)
.await
.map(|_| summarize_mob_runtime_success(probe, "respawned"))
.map_err(|error| MobError::Internal(error.to_string())),
MobRuntimeParityProbeInput::RetireAll => fixture
.handle
.retire_all()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::WireMembers => fixture
.handle
.wire(
fixture.worker_identity.clone(),
AgentIdentity::from(fixture.lead_identity.as_str()),
)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::WireMembersWithTrust => {
let edge = crate::machines::mob_machine::WiringEdge::new(
crate::machines::mob_machine::AgentIdentity::from_domain(&fixture.worker_identity),
crate::machines::mob_machine::AgentIdentity::from_domain(&fixture.lead_identity),
);
fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::WireMembersWithTrust {
a_identity: edge.a.clone(),
b_identity: edge.b.clone(),
edge,
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::UnwireMembers => fixture
.handle
.unwire(
fixture.worker_identity.clone(),
AgentIdentity::from(fixture.lead_identity.as_str()),
)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::WireExternalPeer => fixture
.handle
.wire(
fixture.lead_identity.clone(),
PeerTarget::External(mob_runtime_parity_external_peer()),
)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::UnwireExternalPeer => fixture
.handle
.unwire(
fixture.lead_identity.clone(),
AgentIdentity::from(mob_runtime_parity_external_peer().name.as_str()),
)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::AuthorizeMemberPeerRebind => {
let agent_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&fixture.lead_identity);
let expected_peer_endpoint = fixture
.lead_peer_endpoint()
.await
.map_err(MobError::Internal)?;
fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::AuthorizeMemberPeerRebind {
agent_identity,
expected_peer_endpoint,
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::AuthorizeMemberPeerOverlay => {
let agent_identity =
crate::machines::mob_machine::AgentIdentity::from_domain(&fixture.lead_identity);
let expected_peer_endpoint = fixture
.lead_peer_endpoint()
.await
.map_err(MobError::Internal)?;
fixture
.handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::AuthorizeMemberPeerOverlay {
agent_identity,
expected_peer_endpoint,
},
)
.await
.map(|_| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::ExternalTurn => fixture
.handle
.external_turn_for_member(
AgentIdentity::from(fixture.lead_identity.as_str()),
meerkat_core::types::ContentInput::from("mob runtime parity external turn"),
meerkat_core::types::HandlingMode::Queue,
None,
None,
super::handle::MemberTurnObservers::default(),
)
.await
.map(|_| summarize_mob_runtime_success(probe, "member_delivery")),
MobRuntimeParityProbeInput::InternalTurn => fixture
.handle
.member(&fixture.worker_identity)
.await?
.internal_turn("mob runtime parity internal turn")
.await
.map(|_| summarize_mob_runtime_success(probe, "member_delivery")),
MobRuntimeParityProbeInput::CancelWork => fixture
.handle
.cancel_work(fixture.submitted_work_ref.clone().unwrap_or_default())
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::CancelAllWork => {
let entry = fixture.worker_entry().await.map_err(MobError::Internal)?;
fixture
.handle
.cancel_all_work(entry.agent_runtime_id.clone(), entry.fence_token)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::Stop => fixture
.handle
.stop()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Resume => fixture
.handle
.resume()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Complete => fixture
.handle
.complete()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Reset => fixture
.handle
.reset()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Destroy => fixture
.handle
.destroy()
.await
.map(|_| summarize_mob_runtime_success(probe, "destroy_report"))
.map_err(|error| MobError::Internal(error.to_string())),
MobRuntimeParityProbeInput::SubscribeAgentEvents => fixture
.handle
.subscribe_agent_events(&fixture.lead_identity)
.await
.map(|_| summarize_mob_runtime_success(probe, "event_stream")),
MobRuntimeParityProbeInput::SubscribeAllAgentEvents => {
let streams = fixture.handle.subscribe_all_agent_events().await?;
Ok(summarize_mob_runtime_success(
probe,
&format!("all_agent_streams:{}", streams.len()),
))
}
MobRuntimeParityProbeInput::SubscribeMobEvents => {
let router = fixture
.handle
.subscribe_mob_events_with_config(MobEventRouterConfig {
poll_interval: Duration::from_millis(10),
channel_capacity: 32,
})
.await?;
router.cancel();
Ok(summarize_mob_runtime_success(probe, "mob_event_router"))
}
MobRuntimeParityProbeInput::RecordOperatorActionProvenance => {
let authority = meerkat_runtime::mob_operator_authority::with_audit_invocation_id(
&meerkat_runtime::mob_operator_authority::create_only_mob_operator_authority()
.expect("generated create-only mob authority should be accepted"),
"mob-runtime-parity",
)
.expect("generated mob authority should attach audit invocation id");
fixture
.handle
.record_operator_action_provenance("spawn_member", &authority)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit"))
}
MobRuntimeParityProbeInput::SetSpawnPolicy => fixture
.handle
.set_spawn_policy(None)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::Shutdown => fixture
.handle
.shutdown()
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::ForceCancel => fixture
.handle
.force_cancel_member(fixture.cancel_identity.clone())
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
MobRuntimeParityProbeInput::ConcludeObjective => fixture
.handle
.conclude_objective(
&fixture.worker_identity,
fixture.objective_id,
"mob runtime parity objective complete",
)
.await
.map(|()| summarize_mob_runtime_success(probe, "unit")),
}
}
async fn execute_mob_runtime_parity_probe(
phase: MobRuntimeParityPhase,
probe: MobRuntimeParityProbeInput,
) -> Result<MobRuntimeParityInvocationReport, String> {
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
let mut fixture = build_mob_runtime_parity_fixture().await;
let mut setup_tags = Vec::new();
let setup_result = tokio::time::timeout(
PROBE_TIMEOUT,
mob_runtime_parity_prepare_probe(phase, probe, &mut fixture, &mut setup_tags),
)
.await;
let setup_error = match setup_result {
Ok(Ok(())) => None,
Ok(Err(error)) => Some(error),
Err(_) => Some(format!(
"setup timed out for {} {:?}",
phase.schema_name(),
probe
)),
};
if let Some(error) = setup_error {
fixture.cleanup().await;
return Err(error);
}
let before = mob_runtime_parity_snapshot_summary(&fixture.handle).await;
let result = tokio::time::timeout(
PROBE_TIMEOUT,
mob_runtime_parity_execute_probe(&mut fixture, probe),
)
.await;
let after = mob_runtime_parity_snapshot_summary(&fixture.handle).await;
fixture.cleanup().await;
let (outcome_kind, result_summary) = match result {
Ok(Ok(summary)) => (MobRuntimeParityOutcomeKind::Ok, summary),
Ok(Err(error)) => (
MobRuntimeParityOutcomeKind::Err,
summarize_mob_runtime_error(&error),
),
Err(_) => (
MobRuntimeParityOutcomeKind::Err,
format!("probe_timeout:{probe:?}"),
),
};
Ok(MobRuntimeParityInvocationReport {
phase: phase.schema_name().to_string(),
setup_tags,
before,
outcome_kind,
result_summary,
after,
})
}
fn classify_mob_runtime_parity_probe_pair(
left: &MobRuntimeParityInvocationReport,
right: &MobRuntimeParityInvocationReport,
) -> MobRuntimeParityClassification {
match (left.outcome_kind, right.outcome_kind) {
(MobRuntimeParityOutcomeKind::Ok, MobRuntimeParityOutcomeKind::Err) => {
MobRuntimeParityClassification::LeftOnly
}
(MobRuntimeParityOutcomeKind::Err, MobRuntimeParityOutcomeKind::Ok) => {
MobRuntimeParityClassification::RightOnly
}
_ if left.observable_surface() == right.observable_surface() => {
MobRuntimeParityClassification::SameSurface
}
_ => MobRuntimeParityClassification::DifferentSurface,
}
}
async fn probe_mob_runtime_parity_row(
left_phase: MobRuntimeParityPhase,
right_phase: MobRuntimeParityPhase,
probe: MobRuntimeParityProbeInput,
schema_classification: MobRuntimeParityClassification,
) -> Result<MobRuntimeParityProbeReport, String> {
let left = execute_mob_runtime_parity_probe(left_phase, probe).await?;
let right = execute_mob_runtime_parity_probe(right_phase, probe).await?;
let runtime_classification = classify_mob_runtime_parity_probe_pair(&left, &right);
Ok(MobRuntimeParityProbeReport {
schema_classification,
runtime_classification,
agrees_with_schema: runtime_classification == schema_classification,
schema_left: None,
schema_right: None,
left,
right,
})
}
async fn probe_mob_runtime_full_parity_row(
schema: &MachineSchema,
left_phase: MobRuntimeParityPhase,
right_phase: MobRuntimeParityPhase,
probe: MobRuntimeParityProbeInput,
) -> Result<MobRuntimeParityProbeReport, String> {
let left = execute_mob_runtime_parity_probe(left_phase, probe).await?;
let right = execute_mob_runtime_parity_probe(right_phase, probe).await?;
let schema_left = mob_modeled_schema_report(schema, &left, probe);
let schema_right = mob_modeled_schema_report(schema, &right, probe);
let schema_classification = classify_mob_modeled_schema_pair(&schema_left, &schema_right);
let runtime_classification = classify_mob_runtime_parity_probe_pair(&left, &right);
Ok(MobRuntimeParityProbeReport {
schema_classification,
runtime_classification,
agrees_with_schema: runtime_classification == schema_classification,
schema_left: Some(schema_left),
schema_right: Some(schema_right),
left,
right,
})
}
fn mob_runtime_parity_transition_enabled(
transition: &meerkat_machine_schema::TransitionSchema,
representative: Option<&MobRuntimeParitySnapshotSummary>,
) -> bool {
representative.is_none_or(|snapshot| {
transition
.guards
.iter()
.all(|guard| mob_runtime_parity_eval_bool(&guard.expr, snapshot) != Some(false))
})
}
fn mob_runtime_parity_schema_transition_summaries_for_phase_input(
schema: &MachineSchema,
phase: &str,
input_variant: SchemaMobMachineInputVariant,
representative: Option<&MobRuntimeParitySnapshotSummary>,
) -> Vec<MobRuntimeParitySchemaTransitionSummary> {
let input_variant_name = input_variant.as_str();
let mut summaries = schema
.transitions
.iter()
.filter(|transition| {
transition.on.kind() == TriggerKind::Input
&& transition.on.variant_str() == input_variant_name
&& transition.from.iter().any(|from| from.as_str() == phase)
&& mob_runtime_parity_transition_enabled(transition, representative)
})
.map(|transition| MobRuntimeParitySchemaTransitionSummary {
transition: transition.name.as_str().to_string(),
to_phase: transition.to.as_str().to_string(),
binding_names: transition
.on
.bindings()
.iter()
.map(|b| b.as_str().to_string())
.collect(),
guard_names: transition
.guards
.iter()
.map(|guard| guard.name.to_string())
.collect(),
update_count: transition.updates.len(),
effect_variants: transition
.emit
.iter()
.map(|effect| effect.variant.as_str().to_string())
.collect(),
})
.collect::<Vec<_>>();
summaries.sort_by(|left, right| left.transition.cmp(&right.transition));
summaries
}
fn classify_mob_runtime_parity_schema_row(
left: &[MobRuntimeParitySchemaTransitionSummary],
right: &[MobRuntimeParitySchemaTransitionSummary],
) -> MobRuntimeParityClassification {
if left.is_empty() && !right.is_empty() {
return MobRuntimeParityClassification::RightOnly;
}
if !left.is_empty() && right.is_empty() {
return MobRuntimeParityClassification::LeftOnly;
}
let left_surface = left
.iter()
.map(|summary| {
(
summary.to_phase.clone(),
summary.binding_names.clone(),
summary.guard_names.clone(),
summary.update_count,
summary.effect_variants.clone(),
)
})
.collect::<std::collections::BTreeSet<_>>();
let right_surface = right
.iter()
.map(|summary| {
(
summary.to_phase.clone(),
summary.binding_names.clone(),
summary.guard_names.clone(),
summary.update_count,
summary.effect_variants.clone(),
)
})
.collect::<std::collections::BTreeSet<_>>();
if left_surface == right_surface {
MobRuntimeParityClassification::SameSurface
} else {
MobRuntimeParityClassification::DifferentSurface
}
}
fn mob_runtime_parity_schema_row_for_input(
schema: &MachineSchema,
left_phase: MobRuntimeParityPhase,
right_phase: MobRuntimeParityPhase,
input_variant: SchemaMobMachineInputVariant,
left_representative: Option<&MobRuntimeParitySnapshotSummary>,
right_representative: Option<&MobRuntimeParitySnapshotSummary>,
) -> Option<MobRuntimeParitySchemaRow> {
let static_left = mob_runtime_parity_schema_transition_summaries_for_phase_input(
schema,
left_phase.schema_name(),
input_variant,
None,
);
let static_right = mob_runtime_parity_schema_transition_summaries_for_phase_input(
schema,
right_phase.schema_name(),
input_variant,
None,
);
if static_left.is_empty() && static_right.is_empty() {
return None;
}
let left = mob_runtime_parity_schema_transition_summaries_for_phase_input(
schema,
left_phase.schema_name(),
input_variant,
left_representative,
);
let right = mob_runtime_parity_schema_transition_summaries_for_phase_input(
schema,
right_phase.schema_name(),
input_variant,
right_representative,
);
Some(MobRuntimeParitySchemaRow {
input_variant,
classification: classify_mob_runtime_parity_schema_row(&left, &right),
left,
right,
})
}
async fn build_mob_runtime_parity_pair_report(
schema: &MachineSchema,
left_phase: MobRuntimeParityPhase,
right_phase: MobRuntimeParityPhase,
) -> MobRuntimeParityPairReport {
let mut rows = Vec::new();
let surface_only_inputs = mob_schema_input_variant_set(&schema.surface_only_inputs);
let runtime_internal_inputs = mob_runtime_internal_input_variant_set();
for input_variant in SchemaMobMachineInput::variant_manifest().iter().copied() {
let input_variant_name = input_variant.as_str();
println!(
"probing {} <-> {} on {}",
left_phase.schema_name(),
right_phase.schema_name(),
input_variant_name,
);
let probe_required = mob_runtime_parity_probe_required(
input_variant,
&surface_only_inputs,
&runtime_internal_inputs,
);
let (mut probe, note) = match mob_runtime_parity_probe_for_input_variant(input_variant) {
Some(probe_input) => {
match probe_mob_runtime_parity_row(
left_phase,
right_phase,
probe_input,
MobRuntimeParityClassification::SameSurface,
)
.await
{
Ok(probe) => (Some(probe), None),
Err(error) => (None, Some(format!("probe setup failed: {error}"))),
}
}
None if probe_required => (
None,
Some(
"required runtime probe missing; non-surface-only omissions fail closed"
.to_string(),
),
),
None if surface_only_inputs.contains(&input_variant) => (
None,
Some(MOB_RUNTIME_PARITY_SURFACE_ONLY_PROBE_NOT_REQUIRED.to_string()),
),
None => (
None,
Some(MOB_RUNTIME_PARITY_RUNTIME_INTERNAL_PROBE_NOT_REQUIRED.to_string()),
),
};
assert!(
!probe_required || probe.is_some(),
"required MobMachine runtime parity probe for {input_variant_name} was missing or failed setup before schema row classification: {note:?}"
);
let Some(schema_row) = mob_runtime_parity_schema_row_for_input(
schema,
left_phase,
right_phase,
input_variant,
probe.as_ref().and_then(|probe| probe.left.before.as_ref()),
probe.as_ref().and_then(|probe| probe.right.before.as_ref()),
) else {
continue;
};
if let Some(probe) = &mut probe {
probe.schema_classification = schema_row.classification;
probe.agrees_with_schema = probe.runtime_classification == schema_row.classification;
}
rows.push(MobRuntimeParityRowReport {
input_variant: schema_row.input_variant.as_str().to_string(),
probe_required,
schema_classification: schema_row.classification,
schema_left: schema_row.left,
schema_right: schema_row.right,
probe,
note,
});
}
let summary = rows.iter().fold(
MobRuntimeParityPairSummary {
interesting_rows: rows.len(),
..Default::default()
},
|mut summary, row| {
match &row.probe {
Some(probe) => {
summary.probed_rows += 1;
if probe.agrees_with_schema {
summary.aligned_rows += 1;
} else {
summary.mismatched_rows += 1;
}
}
None if row.probe_required => summary.unprobed_rows += 1,
None if row.note.as_deref()
== Some(MOB_RUNTIME_PARITY_SURFACE_ONLY_PROBE_NOT_REQUIRED) =>
{
summary.surface_only_unprobed_rows += 1;
}
None => summary.runtime_internal_unprobed_rows += 1,
}
summary
},
);
assert_eq!(
summary.unprobed_rows, 0,
"Mob runtime parity audit must fail closed when a required typed probe is missing"
);
MobRuntimeParityPairReport {
left_phase: left_phase.schema_name().to_string(),
right_phase: right_phase.schema_name().to_string(),
summary,
rows,
}
}
async fn build_mob_runtime_full_pair_report(
schema: &MachineSchema,
left_phase: MobRuntimeParityPhase,
right_phase: MobRuntimeParityPhase,
) -> MobRuntimeParityPairReport {
let mut rows = Vec::new();
let surface_only_inputs = mob_schema_input_variant_set(&schema.surface_only_inputs);
let runtime_internal_inputs = mob_runtime_internal_input_variant_set();
for input_variant in SchemaMobMachineInput::variant_manifest().iter().copied() {
let input_variant_name = input_variant.as_str();
println!(
"probing full {} <-> {} on {}",
left_phase.schema_name(),
right_phase.schema_name(),
input_variant_name,
);
let probe_required = mob_runtime_parity_probe_required(
input_variant,
&surface_only_inputs,
&runtime_internal_inputs,
);
let (probe, note) = match mob_runtime_parity_probe_for_input_variant(input_variant) {
Some(probe_input) => {
match probe_mob_runtime_full_parity_row(
schema,
left_phase,
right_phase,
probe_input,
)
.await
{
Ok(probe) => (Some(probe), None),
Err(error) => (None, Some(format!("probe setup failed: {error}"))),
}
}
None if probe_required => (
None,
Some(
"required runtime probe missing; non-surface-only omissions fail closed"
.to_string(),
),
),
None if surface_only_inputs.contains(&input_variant) => (
None,
Some(MOB_RUNTIME_PARITY_SURFACE_ONLY_PROBE_NOT_REQUIRED.to_string()),
),
None => (
None,
Some(MOB_RUNTIME_PARITY_RUNTIME_INTERNAL_PROBE_NOT_REQUIRED.to_string()),
),
};
assert!(
!probe_required || probe.is_some(),
"required MobMachine full runtime parity probe for {input_variant_name} was missing or failed setup before schema row classification: {note:?}"
);
let Some(schema_row) = mob_runtime_parity_schema_row_for_input(
schema,
left_phase,
right_phase,
input_variant,
probe.as_ref().and_then(|probe| probe.left.before.as_ref()),
probe.as_ref().and_then(|probe| probe.right.before.as_ref()),
) else {
continue;
};
let effective_schema_classification = probe
.as_ref()
.map(|probe| probe.schema_classification)
.unwrap_or(schema_row.classification);
let note = match (&probe, note) {
(Some(probe), None) if probe.schema_classification != schema_row.classification => {
Some(format!(
"static schema classified {:?}, modeled schema classified {:?}",
schema_row.classification, probe.schema_classification
))
}
(_, note) => note,
};
rows.push(MobRuntimeParityRowReport {
input_variant: schema_row.input_variant.as_str().to_string(),
probe_required,
schema_classification: effective_schema_classification,
schema_left: schema_row.left,
schema_right: schema_row.right,
probe,
note,
});
}
let summary = rows.iter().fold(
MobRuntimeParityPairSummary {
interesting_rows: rows.len(),
..Default::default()
},
|mut summary, row| {
match &row.probe {
Some(probe) => {
summary.probed_rows += 1;
if probe.agrees_with_schema {
summary.aligned_rows += 1;
} else {
summary.mismatched_rows += 1;
}
}
None if row.probe_required => summary.unprobed_rows += 1,
None if row.note.as_deref()
== Some(MOB_RUNTIME_PARITY_SURFACE_ONLY_PROBE_NOT_REQUIRED) =>
{
summary.surface_only_unprobed_rows += 1;
}
None => summary.runtime_internal_unprobed_rows += 1,
}
summary
},
);
assert_eq!(
summary.unprobed_rows, 0,
"Mob runtime full parity audit must fail closed when a required typed probe is missing"
);
MobRuntimeParityPairReport {
left_phase: left_phase.schema_name().to_string(),
right_phase: right_phase.schema_name().to_string(),
summary,
rows,
}
}
fn mob_runtime_parity_probe_input_variant(
probe: MobRuntimeParityProbeInput,
) -> Option<SchemaMobMachineInputVariant> {
match probe {
MobRuntimeParityProbeInput::Spawn => {
Some(SchemaMobMachineInputVariant::CommitSpawnMembership)
}
MobRuntimeParityProbeInput::AuthorizeSpawnProfile => {
Some(SchemaMobMachineInputVariant::AuthorizeSpawnProfile)
}
MobRuntimeParityProbeInput::ClassifyMemberWait => {
Some(SchemaMobMachineInputVariant::ClassifyMemberWait)
}
MobRuntimeParityProbeInput::ClassifyRetirePendingSpawnDisposition => {
Some(SchemaMobMachineInputVariant::ClassifyRetirePendingSpawnDisposition)
}
MobRuntimeParityProbeInput::EnsureMember => {
Some(SchemaMobMachineInputVariant::EnsureMember)
}
MobRuntimeParityProbeInput::Reconcile => Some(SchemaMobMachineInputVariant::Reconcile),
MobRuntimeParityProbeInput::SubmitWork => Some(SchemaMobMachineInputVariant::SubmitWork),
MobRuntimeParityProbeInput::RunFlow => Some(SchemaMobMachineInputVariant::RunFlow),
MobRuntimeParityProbeInput::CancelFlow => Some(SchemaMobMachineInputVariant::CancelFlow),
MobRuntimeParityProbeInput::Retire => Some(SchemaMobMachineInputVariant::Retire),
MobRuntimeParityProbeInput::Respawn => Some(SchemaMobMachineInputVariant::Respawn),
MobRuntimeParityProbeInput::RetireAll => Some(SchemaMobMachineInputVariant::RetireAll),
MobRuntimeParityProbeInput::WireMembers => Some(SchemaMobMachineInputVariant::WireMembers),
MobRuntimeParityProbeInput::WireMembersWithTrust => {
Some(SchemaMobMachineInputVariant::WireMembersWithTrust)
}
MobRuntimeParityProbeInput::UnwireMembers => {
Some(SchemaMobMachineInputVariant::UnwireMembers)
}
MobRuntimeParityProbeInput::WireExternalPeer => {
Some(SchemaMobMachineInputVariant::WireExternalPeer)
}
MobRuntimeParityProbeInput::UnwireExternalPeer => {
Some(SchemaMobMachineInputVariant::UnwireExternalPeer)
}
MobRuntimeParityProbeInput::AuthorizeMemberPeerRebind => {
Some(SchemaMobMachineInputVariant::AuthorizeMemberPeerRebind)
}
MobRuntimeParityProbeInput::AuthorizeMemberPeerOverlay => {
Some(SchemaMobMachineInputVariant::AuthorizeMemberPeerOverlay)
}
MobRuntimeParityProbeInput::ExternalTurn | MobRuntimeParityProbeInput::InternalTurn => None,
MobRuntimeParityProbeInput::CancelWork => Some(SchemaMobMachineInputVariant::CancelWork),
MobRuntimeParityProbeInput::CancelAllWork => {
Some(SchemaMobMachineInputVariant::CancelAllWork)
}
MobRuntimeParityProbeInput::Stop => Some(SchemaMobMachineInputVariant::Stop),
MobRuntimeParityProbeInput::Resume => Some(SchemaMobMachineInputVariant::Resume),
MobRuntimeParityProbeInput::Complete => Some(SchemaMobMachineInputVariant::Complete),
MobRuntimeParityProbeInput::Reset => Some(SchemaMobMachineInputVariant::Reset),
MobRuntimeParityProbeInput::Destroy => Some(SchemaMobMachineInputVariant::Destroy),
MobRuntimeParityProbeInput::SubscribeAgentEvents => {
Some(SchemaMobMachineInputVariant::SubscribeAgentEvents)
}
MobRuntimeParityProbeInput::SubscribeAllAgentEvents => {
Some(SchemaMobMachineInputVariant::SubscribeAllAgentEvents)
}
MobRuntimeParityProbeInput::SubscribeMobEvents => {
Some(SchemaMobMachineInputVariant::SubscribeMobEvents)
}
MobRuntimeParityProbeInput::RecordOperatorActionProvenance => {
Some(SchemaMobMachineInputVariant::RecordOperatorActionProvenance)
}
MobRuntimeParityProbeInput::SetSpawnPolicy => {
Some(SchemaMobMachineInputVariant::SetSpawnPolicy)
}
MobRuntimeParityProbeInput::Shutdown => Some(SchemaMobMachineInputVariant::Shutdown),
MobRuntimeParityProbeInput::ForceCancel => Some(SchemaMobMachineInputVariant::ForceCancel),
MobRuntimeParityProbeInput::ConcludeObjective => {
Some(SchemaMobMachineInputVariant::ConcludeObjective)
}
}
}
#[test]
fn mob_runtime_parity_probe_manifest_round_trips_through_typed_generated_variants() {
for input_variant in SchemaMobMachineInput::variant_manifest().iter().copied() {
let Some(probe) = mob_runtime_parity_probe_for_input_variant(input_variant) else {
continue;
};
assert_eq!(
mob_runtime_parity_probe_input_variant(probe).expect("typed probe variant"),
input_variant,
"mob runtime parity probe mapping must round-trip through typed generated input variants"
);
}
}
#[test]
fn mob_runtime_parity_probe_inventory_closes_required_executable_probes() {
let schema = schema_mob_machine();
let surface_only_inputs = mob_schema_input_variant_set(&schema.surface_only_inputs);
let runtime_internal_inputs = mob_runtime_internal_input_variant_set();
let unprobed_required = SchemaMobMachineInput::variant_manifest()
.iter()
.copied()
.filter(|input_variant| {
mob_runtime_parity_probe_required(
*input_variant,
&surface_only_inputs,
&runtime_internal_inputs,
) && mob_runtime_parity_probe_for_input_variant(*input_variant).is_none()
})
.collect::<Vec<_>>();
assert!(
unprobed_required.is_empty(),
"every non-surface MobMachine input that requires a runtime parity probe must be backed by the executable typed probe map; missing {unprobed_required:?}"
);
}
#[tokio::test]
async fn mob_runtime_parity_pair_report_executes_required_probe_inventory() {
let schema = schema_mob_machine();
let report = build_mob_runtime_parity_pair_report(
&schema,
MobRuntimeParityPhase::Running,
MobRuntimeParityPhase::Stopped,
)
.await;
assert_eq!(
report.summary.unprobed_rows, 0,
"mob runtime parity report must fail closed on required probes that are missing or fail setup"
);
assert!(
report.summary.probed_rows > 0,
"mob runtime parity report must execute the typed probe map, not only inspect it"
);
}
#[tokio::test]
async fn mob_runtime_parity_audit_writers_execute_required_probe_inventory() {
let full_report =
write_mob_runtime_full_parity_audit_report(mob_runtime_parity_full_report_path()).await;
assert_eq!(
full_report.summary.unprobed_rows, 0,
"full mob runtime parity audit must execute or explicitly exempt every typed input"
);
assert!(
full_report.summary.probed_rows > 0,
"full mob runtime parity audit must execute the typed probe map, not only inspect it"
);
let modeled_report =
write_mob_runtime_modeled_state_audit_report(mob_modeled_state_report_path()).await;
assert_eq!(
modeled_report.summary.unprobed_rows, 0,
"modeled-state audit must execute or explicitly exempt every typed input"
);
assert!(
modeled_report.summary.row_count > 0,
"modeled-state audit must exercise required typed runtime probes"
);
}
fn mob_modeled_normalize_json_value(value: serde_json::Value) -> serde_json::Value {
match value {
serde_json::Value::Array(items) => serde_json::Value::Array(
items
.into_iter()
.map(mob_modeled_normalize_json_value)
.collect(),
),
serde_json::Value::Object(entries) => {
let mut normalized = serde_json::Map::new();
let mut keys: Vec<_> = entries.into_iter().collect();
keys.sort_by(|(left, _), (right, _)| left.cmp(right));
for (key, value) in keys {
normalized.insert(key, mob_modeled_normalize_json_value(value));
}
serde_json::Value::Object(normalized)
}
other => other,
}
}
fn mob_modeled_normalize_formal_string(raw: &str) -> String {
serde_json::from_str(raw)
.map(mob_modeled_normalize_json_value)
.and_then(|value| serde_json::to_string(&value))
.unwrap_or_else(|_| raw.to_string())
}
fn mob_modeled_default_kernel_value(
ty: &meerkat_machine_schema::TypeRef,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
match ty {
meerkat_machine_schema::TypeRef::Bool => {
meerkat_machine_kernels::test_oracle::KernelValue::Bool(false)
}
meerkat_machine_schema::TypeRef::U32 | meerkat_machine_schema::TypeRef::U64 => {
meerkat_machine_kernels::test_oracle::KernelValue::U64(0)
}
meerkat_machine_schema::TypeRef::String => {
meerkat_machine_kernels::test_oracle::KernelValue::String(String::new())
}
meerkat_machine_schema::TypeRef::Named(name) => mob_modeled_named_value(
name,
if mob_modeled_named_type_is_u64(name.as_str()) {
meerkat_machine_kernels::test_oracle::KernelValue::U64(0)
} else {
meerkat_machine_kernels::test_oracle::KernelValue::String(String::new())
},
),
meerkat_machine_schema::TypeRef::Enum(name) => {
meerkat_machine_kernels::test_oracle::KernelValue::NamedVariant {
enum_name: name.clone(),
variant: meerkat_machine_schema::identity::EnumVariantId::parse("Unknown")
.expect("valid slug"),
}
}
meerkat_machine_schema::TypeRef::Option(_) => {
meerkat_machine_kernels::test_oracle::KernelValue::None
}
meerkat_machine_schema::TypeRef::Set(_) => {
meerkat_machine_kernels::test_oracle::KernelValue::Set(BTreeSet::new())
}
meerkat_machine_schema::TypeRef::Seq(_) => {
meerkat_machine_kernels::test_oracle::KernelValue::Seq(Vec::new())
}
meerkat_machine_schema::TypeRef::Map(_, _) => {
meerkat_machine_kernels::test_oracle::KernelValue::Map(BTreeMap::new())
}
}
}
fn mob_modeled_named_type_is_u64(name: &str) -> bool {
matches!(name, "FenceToken" | "Generation")
}
fn mob_modeled_named_value(
type_name: &meerkat_machine_schema::identity::NamedTypeId,
value: meerkat_machine_kernels::test_oracle::KernelValue,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
if matches!(
&value,
meerkat_machine_kernels::test_oracle::KernelValue::Named {
type_name: existing,
..
} if existing == type_name
) {
return value;
}
let value = if mob_modeled_named_type_is_u64(type_name.as_str()) {
match value {
meerkat_machine_kernels::test_oracle::KernelValue::U64(value) => {
meerkat_machine_kernels::test_oracle::KernelValue::U64(value)
}
meerkat_machine_kernels::test_oracle::KernelValue::String(value) => {
meerkat_machine_kernels::test_oracle::KernelValue::U64(
serde_json::from_str::<u64>(&value)
.ok()
.or_else(|| value.parse::<u64>().ok())
.unwrap_or_default(),
)
}
_ => meerkat_machine_kernels::test_oracle::KernelValue::U64(0),
}
} else {
match value {
meerkat_machine_kernels::test_oracle::KernelValue::String(value) => {
meerkat_machine_kernels::test_oracle::KernelValue::String(value)
}
meerkat_machine_kernels::test_oracle::KernelValue::U64(value) => {
meerkat_machine_kernels::test_oracle::KernelValue::String(value.to_string())
}
other => meerkat_machine_kernels::test_oracle::KernelValue::String(
serde_json::to_string(&mob_modeled_json_from_kernel_value(&other))
.unwrap_or_default(),
),
}
};
meerkat_machine_kernels::test_oracle::KernelValue::Named {
type_name: type_name.clone(),
value: Box::new(value),
}
}
fn mob_modeled_coerce_value_to_type(
ty: &meerkat_machine_schema::TypeRef,
value: meerkat_machine_kernels::test_oracle::KernelValue,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
match ty {
meerkat_machine_schema::TypeRef::Named(name) => mob_modeled_named_value(name, value),
meerkat_machine_schema::TypeRef::Option(inner) => match value {
meerkat_machine_kernels::test_oracle::KernelValue::None => {
meerkat_machine_kernels::test_oracle::KernelValue::None
}
meerkat_machine_kernels::test_oracle::KernelValue::Map(mut entries)
if entries.len() == 1
&& entries.contains_key(
&meerkat_machine_kernels::test_oracle::KernelValue::String(
"value".to_string(),
),
) =>
{
let key =
meerkat_machine_kernels::test_oracle::KernelValue::String("value".to_string());
let inner_value = entries
.remove(&key)
.unwrap_or(meerkat_machine_kernels::test_oracle::KernelValue::None);
mob_modeled_option_some(mob_modeled_coerce_value_to_type(inner, inner_value))
}
other => mob_modeled_option_some(mob_modeled_coerce_value_to_type(inner, other)),
},
meerkat_machine_schema::TypeRef::Set(inner) => match value {
meerkat_machine_kernels::test_oracle::KernelValue::Set(items) => {
meerkat_machine_kernels::test_oracle::KernelValue::Set(
items
.into_iter()
.map(|item| mob_modeled_coerce_value_to_type(inner, item))
.collect(),
)
}
other => other,
},
meerkat_machine_schema::TypeRef::Seq(inner) => match value {
meerkat_machine_kernels::test_oracle::KernelValue::Seq(items) => {
meerkat_machine_kernels::test_oracle::KernelValue::Seq(
items
.into_iter()
.map(|item| mob_modeled_coerce_value_to_type(inner, item))
.collect(),
)
}
other => other,
},
meerkat_machine_schema::TypeRef::Map(key_ty, value_ty) => match value {
meerkat_machine_kernels::test_oracle::KernelValue::Map(entries) => {
meerkat_machine_kernels::test_oracle::KernelValue::Map(
entries
.into_iter()
.map(|(key, value)| {
(
mob_modeled_coerce_value_to_type(key_ty, key),
mob_modeled_coerce_value_to_type(value_ty, value),
)
})
.collect(),
)
}
other => other,
},
_ => value,
}
}
fn mob_modeled_option_some(
value: meerkat_machine_kernels::test_oracle::KernelValue,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
meerkat_machine_kernels::test_oracle::KernelValue::Map(BTreeMap::from([(
meerkat_machine_kernels::test_oracle::KernelValue::String("value".to_string()),
value,
)]))
}
fn mob_modeled_json_value_from_raw(raw: &str) -> serde_json::Value {
serde_json::from_str(raw).unwrap_or_else(|_| serde_json::Value::String(raw.to_string()))
}
fn mob_modeled_kernel_value_from_json(
ty: &meerkat_machine_schema::TypeRef,
value: &serde_json::Value,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
match ty {
meerkat_machine_schema::TypeRef::Bool => {
meerkat_machine_kernels::test_oracle::KernelValue::Bool(
value.as_bool().unwrap_or(false),
)
}
meerkat_machine_schema::TypeRef::U32 | meerkat_machine_schema::TypeRef::U64 => {
meerkat_machine_kernels::test_oracle::KernelValue::U64(value.as_u64().unwrap_or(0))
}
meerkat_machine_schema::TypeRef::String => {
meerkat_machine_kernels::test_oracle::KernelValue::String(
value
.as_str()
.map(str::to_owned)
.unwrap_or_else(|| serde_json::to_string(value).unwrap_or_default()),
)
}
meerkat_machine_schema::TypeRef::Named(name)
if matches!(name.as_str(), "FenceToken" | "Generation") =>
{
meerkat_machine_kernels::test_oracle::KernelValue::Named {
type_name: name.clone(),
value: Box::new(meerkat_machine_kernels::test_oracle::KernelValue::U64(
value.as_u64().unwrap_or(0),
)),
}
}
meerkat_machine_schema::TypeRef::Named(name) => {
let normalized = value
.as_str()
.and_then(|raw| serde_json::from_str::<serde_json::Value>(raw).ok())
.unwrap_or_else(|| value.clone());
meerkat_machine_kernels::test_oracle::KernelValue::Named {
type_name: name.clone(),
value: Box::new(meerkat_machine_kernels::test_oracle::KernelValue::String(
serde_json::to_string(&normalized).unwrap_or_else(|_| "null".into()),
)),
}
}
meerkat_machine_schema::TypeRef::Enum(name) => {
meerkat_machine_kernels::test_oracle::KernelValue::NamedVariant {
enum_name: name.clone(),
variant: meerkat_machine_schema::identity::EnumVariantId::parse(
value.as_str().unwrap_or("Unknown"),
)
.unwrap_or_else(|_| {
meerkat_machine_schema::identity::EnumVariantId::parse("Unknown")
.expect("valid slug")
}),
}
}
meerkat_machine_schema::TypeRef::Option(inner) => {
if value.is_null() {
meerkat_machine_kernels::test_oracle::KernelValue::None
} else {
mob_modeled_option_some(mob_modeled_kernel_value_from_json(inner, value))
}
}
meerkat_machine_schema::TypeRef::Set(inner) => {
let values = value
.as_array()
.map(|items| {
items
.iter()
.map(|item| mob_modeled_kernel_value_from_json(inner, item))
.collect()
})
.unwrap_or_default();
meerkat_machine_kernels::test_oracle::KernelValue::Set(values)
}
meerkat_machine_schema::TypeRef::Seq(inner) => {
meerkat_machine_kernels::test_oracle::KernelValue::Seq(
value
.as_array()
.map(|items| {
items
.iter()
.map(|item| mob_modeled_kernel_value_from_json(inner, item))
.collect()
})
.unwrap_or_default(),
)
}
meerkat_machine_schema::TypeRef::Map(key_ty, value_ty) => {
let mut entries = BTreeMap::new();
if let Some(object) = value.as_object() {
for (key, item) in object {
let key_value = mob_modeled_kernel_value_from_json(
key_ty,
&serde_json::Value::String(key.clone()),
);
let value_value = mob_modeled_kernel_value_from_json(value_ty, item);
entries.insert(key_value, value_value);
}
}
meerkat_machine_kernels::test_oracle::KernelValue::Map(entries)
}
}
}
fn mob_modeled_kernel_value_from_raw(
ty: &meerkat_machine_schema::TypeRef,
raw: &str,
) -> meerkat_machine_kernels::test_oracle::KernelValue {
mob_modeled_kernel_value_from_json(ty, &mob_modeled_json_value_from_raw(raw))
}
fn mob_modeled_json_from_kernel_value(
value: &meerkat_machine_kernels::test_oracle::KernelValue,
) -> serde_json::Value {
match value {
meerkat_machine_kernels::test_oracle::KernelValue::Bool(value) => {
serde_json::Value::Bool(*value)
}
meerkat_machine_kernels::test_oracle::KernelValue::U64(value) => {
serde_json::Value::Number(serde_json::Number::from(*value))
}
meerkat_machine_kernels::test_oracle::KernelValue::String(value) => {
serde_json::from_str(value).unwrap_or_else(|_| serde_json::Value::String(value.clone()))
}
meerkat_machine_kernels::test_oracle::KernelValue::Named { value, .. } => {
mob_modeled_json_from_kernel_value(value)
}
meerkat_machine_kernels::test_oracle::KernelValue::NamedVariant { variant, .. } => {
serde_json::Value::String(variant.as_str().to_string())
}
meerkat_machine_kernels::test_oracle::KernelValue::Seq(items) => serde_json::Value::Array(
items
.iter()
.map(mob_modeled_json_from_kernel_value)
.collect(),
),
meerkat_machine_kernels::test_oracle::KernelValue::Set(items) => serde_json::Value::Array(
items
.iter()
.map(mob_modeled_json_from_kernel_value)
.collect(),
),
meerkat_machine_kernels::test_oracle::KernelValue::Map(entries)
if entries.len() == 1
&& entries.contains_key(
&meerkat_machine_kernels::test_oracle::KernelValue::String("value".to_string()),
) =>
{
mob_modeled_json_from_kernel_value(
entries
.get(&meerkat_machine_kernels::test_oracle::KernelValue::String(
"value".to_string(),
))
.expect("value key present"),
)
}
meerkat_machine_kernels::test_oracle::KernelValue::Map(entries) => {
let mut object = serde_json::Map::new();
for (key, value) in entries {
let key_json = mob_modeled_json_from_kernel_value(key);
let key_string = key_json
.as_str()
.map(str::to_owned)
.unwrap_or_else(|| serde_json::to_string(&key_json).unwrap_or_default());
object.insert(key_string, mob_modeled_json_from_kernel_value(value));
}
serde_json::Value::Object(object)
}
meerkat_machine_kernels::test_oracle::KernelValue::None => serde_json::Value::Null,
}
}
fn mob_modeled_formal_string_from_kernel_value(
value: &meerkat_machine_kernels::test_oracle::KernelValue,
) -> String {
mob_modeled_normalize_formal_string(
&serde_json::to_string(&mob_modeled_json_from_kernel_value(value))
.unwrap_or_else(|_| "null".into()),
)
}
fn mob_modeled_summary_from_runtime_snapshot(
snapshot: Option<&MobRuntimeParitySnapshotSummary>,
) -> Option<MobModeledStateSummary> {
snapshot.map(|snapshot| MobModeledStateSummary {
phase: snapshot.phase.clone(),
formal_fields: snapshot.formal_available_fields.clone(),
unavailable_fields: snapshot.formal_unavailable_fields.clone(),
})
}
fn mob_modeled_summary_from_kernel_state(
schema: &MachineSchema,
state: &meerkat_machine_kernels::test_oracle::KernelState,
runtime_reference: &MobRuntimeParitySnapshotSummary,
) -> MobModeledStateSummary {
let formal_fields = schema
.state
.fields
.iter()
.filter(|field| {
runtime_reference
.formal_available_fields
.contains_key(field.name.as_str())
})
.map(|field| {
let value = state
.fields
.get(&field.name)
.map(mob_modeled_formal_string_from_kernel_value)
.unwrap_or_else(|| "null".to_string());
(field.name.as_str().to_string(), value)
})
.collect();
MobModeledStateSummary {
phase: state.phase.as_str().to_string(),
formal_fields,
unavailable_fields: runtime_reference.formal_unavailable_fields.clone(),
}
}
fn mob_modeled_differing_keys(
runtime_after: &Option<MobModeledStateSummary>,
schema_after: &Option<MobModeledStateSummary>,
) -> Vec<String> {
let mut keys = BTreeSet::new();
if runtime_after.as_ref().map(|summary| summary.phase.as_str())
!= schema_after.as_ref().map(|summary| summary.phase.as_str())
{
keys.insert("phase".to_string());
}
let runtime_fields = runtime_after
.as_ref()
.map(|summary| &summary.formal_fields)
.cloned()
.unwrap_or_default();
let schema_fields = schema_after
.as_ref()
.map(|summary| &summary.formal_fields)
.cloned()
.unwrap_or_default();
for key in runtime_fields
.keys()
.chain(schema_fields.keys())
.collect::<BTreeSet<_>>()
{
if runtime_fields.get(key) != schema_fields.get(key) {
keys.insert(key.clone());
}
}
keys.into_iter().collect()
}
fn mob_modeled_kernel_state(
schema: &MachineSchema,
before: &MobRuntimeParitySnapshotSummary,
) -> meerkat_machine_kernels::test_oracle::KernelState {
let mut fields = BTreeMap::new();
for field in &schema.state.fields {
let value = before
.formal_available_fields
.get(field.name.as_str())
.map(|raw| mob_modeled_kernel_value_from_raw(&field.ty, raw))
.unwrap_or_else(|| mob_modeled_default_kernel_value(&field.ty));
fields.insert(
field.name.clone(),
mob_modeled_coerce_value_to_type(&field.ty, value),
);
}
meerkat_machine_kernels::test_oracle::KernelState {
phase: meerkat_machine_schema::identity::PhaseId::parse(before.phase.as_str())
.expect("valid phase slug"),
fields,
}
}
fn mob_modeled_named_string(value: String) -> meerkat_machine_kernels::test_oracle::KernelValue {
meerkat_machine_kernels::test_oracle::KernelValue::String(value)
}
fn mob_modeled_representative_identity_value(before: &MobRuntimeParitySnapshotSummary) -> String {
before
.representative_agent_identity
.clone()
.unwrap_or_else(|| "\"<agent-identity>\"".to_string())
}
fn mob_modeled_representative_runtime_value(before: &MobRuntimeParitySnapshotSummary) -> String {
before
.representative_runtime_id
.clone()
.unwrap_or_else(|| "\"<runtime-id>\"".to_string())
}
fn mob_modeled_spawn_runtime_value(identity: &str, generation: u64) -> String {
serde_json::json!({
"identity": identity,
"generation": generation,
})
.to_string()
}
fn mob_modeled_kernel_input(
schema: &MachineSchema,
before: &MobRuntimeParitySnapshotSummary,
probe: MobRuntimeParityProbeInput,
) -> Result<meerkat_machine_kernels::test_oracle::KernelInput, String> {
let input_variant = mob_runtime_parity_probe_input_variant(probe)
.ok_or_else(|| format!("{probe:?} has no MobMachine input variant"))?;
let variant = meerkat_machine_schema::identity::InputVariantId::parse(input_variant.as_str())
.expect("valid variant slug");
let input_variant = schema
.inputs
.variant_named(&variant)
.map_err(|err| err.to_string())?;
let mut fields = BTreeMap::new();
for field in &input_variant.fields {
let value = match field.name.as_str() {
"agent_identity" => match probe {
MobRuntimeParityProbeInput::Spawn => {
mob_modeled_named_string("\"parity-spawn\"".to_string())
}
_ => mob_modeled_named_string(mob_modeled_representative_identity_value(before)),
},
"agent_runtime_id" => match probe {
MobRuntimeParityProbeInput::Spawn => {
mob_modeled_named_string(mob_modeled_spawn_runtime_value("parity-spawn", 0))
}
_ => mob_modeled_named_string(mob_modeled_representative_runtime_value(before)),
},
"fence_token" => {
let value = match probe {
MobRuntimeParityProbeInput::Spawn => 1,
MobRuntimeParityProbeInput::Respawn => {
before.representative_fence_token.unwrap_or_default() + 1
}
_ => before.representative_fence_token.unwrap_or_default(),
};
meerkat_machine_kernels::test_oracle::KernelValue::U64(value)
}
"generation" => {
let value = match probe {
MobRuntimeParityProbeInput::Spawn => 0,
MobRuntimeParityProbeInput::Respawn => 0,
_ => 0,
};
meerkat_machine_kernels::test_oracle::KernelValue::U64(value)
}
"work_id" => mob_modeled_named_string("\"<work-id>\"".to_string()),
_ => mob_modeled_default_kernel_value(&field.ty),
};
fields.insert(
field.name.clone(),
mob_modeled_coerce_value_to_type(&field.ty, value),
);
}
Ok(meerkat_machine_kernels::test_oracle::KernelInput { variant, fields })
}
fn mob_modeled_transition_refusal_detail(
error: &meerkat_machine_kernels::test_oracle::TransitionRefusal,
) -> String {
match error {
meerkat_machine_kernels::test_oracle::TransitionRefusal::UnknownInputVariant {
variant,
..
} => {
format!("unknown_input:{variant}")
}
meerkat_machine_kernels::test_oracle::TransitionRefusal::UnknownSignalVariant {
variant,
..
} => {
format!("unknown_signal:{variant}")
}
meerkat_machine_kernels::test_oracle::TransitionRefusal::InvalidInputPayload {
reason,
..
} => {
format!("invalid_input:{reason}")
}
meerkat_machine_kernels::test_oracle::TransitionRefusal::InvalidSignalPayload {
reason,
..
} => {
format!("invalid_signal:{reason}")
}
meerkat_machine_kernels::test_oracle::TransitionRefusal::NoMatchingTransition {
phase,
trigger,
..
} => {
format!("no_match:{phase}:{}", trigger.as_str())
}
meerkat_machine_kernels::test_oracle::TransitionRefusal::AmbiguousTransition {
phase,
trigger,
transitions,
..
} => format!("ambiguous:{phase}:{}:{transitions:?}", trigger.as_str()),
meerkat_machine_kernels::test_oracle::TransitionRefusal::EvaluationError {
transition,
reason,
..
} => format!("evaluation:{transition}:{reason}"),
}
}
fn mob_modeled_schema_report(
schema: &MachineSchema,
runtime: &MobRuntimeParityInvocationReport,
probe: MobRuntimeParityProbeInput,
) -> MobModeledStateSchemaReport {
let Some(before) = runtime.before.as_ref() else {
return MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind::Err,
after: None,
detail: "missing runtime pre-state".to_string(),
result_summary: None,
};
};
let state = mob_modeled_kernel_state(schema, before);
let input = match mob_modeled_kernel_input(schema, before, probe) {
Ok(input) => input,
Err(detail) => {
return MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind::Err,
after: None,
detail,
result_summary: None,
};
}
};
match meerkat_machine_kernels::test_oracle::GeneratedMachineKernel::new(
meerkat_machine_kernels::generated::mob::schema(),
)
.transition(&state, &input)
{
Ok(outcome) => MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind::Ok,
after: Some(mob_modeled_summary_from_kernel_state(
schema,
&outcome.next_state,
before,
)),
detail: outcome.transition.as_str().to_string(),
result_summary: mob_modeled_schema_result_summary(before, probe),
},
Err(error) => MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind::Err,
after: Some(MobModeledStateSummary {
phase: before.phase.clone(),
formal_fields: before.formal_available_fields.clone(),
unavailable_fields: before.formal_unavailable_fields.clone(),
}),
detail: mob_modeled_transition_refusal_detail(&error),
result_summary: None,
},
}
}
fn classify_mob_modeled_schema_pair(
left: &MobModeledStateSchemaReport,
right: &MobModeledStateSchemaReport,
) -> MobRuntimeParityClassification {
match (left.outcome_kind, right.outcome_kind) {
(MobModeledStateOutcomeKind::Ok, MobModeledStateOutcomeKind::Err) => {
MobRuntimeParityClassification::LeftOnly
}
(MobModeledStateOutcomeKind::Err, MobModeledStateOutcomeKind::Ok) => {
MobRuntimeParityClassification::RightOnly
}
(MobModeledStateOutcomeKind::Ok, MobModeledStateOutcomeKind::Ok)
if left.after == right.after && left.result_summary == right.result_summary =>
{
MobRuntimeParityClassification::SameSurface
}
(MobModeledStateOutcomeKind::Err, MobModeledStateOutcomeKind::Err)
if left.after == right.after && left.detail == right.detail =>
{
MobRuntimeParityClassification::SameSurface
}
_ => MobRuntimeParityClassification::DifferentSurface,
}
}
fn mob_modeled_runtime_report(
runtime: &MobRuntimeParityInvocationReport,
) -> MobModeledStateRuntimeReport {
let before = mob_modeled_summary_from_runtime_snapshot(runtime.before.as_ref());
let after = mob_modeled_summary_from_runtime_snapshot(runtime.after.as_ref()).or_else(|| {
(runtime.outcome_kind == MobRuntimeParityOutcomeKind::Err)
.then(|| before.clone())
.flatten()
});
MobModeledStateRuntimeReport {
phase: runtime.phase.clone(),
outcome_kind: match runtime.outcome_kind {
MobRuntimeParityOutcomeKind::Ok => MobModeledStateOutcomeKind::Ok,
MobRuntimeParityOutcomeKind::Err => MobModeledStateOutcomeKind::Err,
},
before,
after,
result_summary: runtime.result_summary.clone(),
}
}
async fn write_mob_runtime_modeled_state_audit_report(path: PathBuf) -> MobModeledStateAuditReport {
let schema = meerkat_machine_kernels::generated::mob::schema();
let surface_only_inputs = mob_schema_input_variant_set(&schema.surface_only_inputs);
let runtime_internal_inputs = mob_runtime_internal_input_variant_set();
let mut rows = Vec::new();
for phase in [
MobRuntimeParityPhase::Running,
MobRuntimeParityPhase::Stopped,
MobRuntimeParityPhase::Completed,
] {
for input_variant in SchemaMobMachineInput::variant_manifest().iter().copied() {
let input_variant_name = input_variant.as_str();
if !mob_runtime_parity_probe_required(
input_variant,
&surface_only_inputs,
&runtime_internal_inputs,
) {
continue;
}
let Some(probe) = mob_runtime_parity_probe_for_input_variant(input_variant) else {
rows.push(MobModeledStateRowReport {
phase: phase.schema_name().to_string(),
input_variant: input_variant_name.to_string(),
aligned: false,
differing_keys: vec!["unprobed".to_string()],
runtime: MobModeledStateRuntimeReport {
phase: phase.schema_name().to_string(),
outcome_kind: MobModeledStateOutcomeKind::Err,
before: None,
after: None,
result_summary:
"required runtime probe missing; non-surface-only omissions fail closed"
.to_string(),
},
schema: MobModeledStateSchemaReport {
outcome_kind: MobModeledStateOutcomeKind::Err,
after: None,
detail:
"required runtime probe missing; non-surface-only omissions fail closed"
.to_string(),
result_summary: None,
},
});
continue;
};
let runtime = execute_mob_runtime_parity_probe(phase, probe)
.await
.expect("mob runtime modeled-state probe should succeed");
let schema_report = mob_modeled_schema_report(&schema, &runtime, probe);
let runtime_report = mob_modeled_runtime_report(&runtime);
let differing_keys =
mob_modeled_differing_keys(&runtime_report.after, &schema_report.after);
let aligned = runtime_report.outcome_kind == schema_report.outcome_kind
&& differing_keys.is_empty();
rows.push(MobModeledStateRowReport {
phase: phase.schema_name().to_string(),
input_variant: input_variant_name.to_string(),
aligned,
differing_keys,
runtime: runtime_report,
schema: schema_report,
});
}
}
let summary = rows.iter().fold(
MobModeledStateAuditSummary::default(),
|mut summary, row| {
summary.row_count += 1;
if row.runtime.result_summary
== "required runtime probe missing; non-surface-only omissions fail closed"
{
summary.unprobed_rows += 1;
} else if row.aligned {
summary.aligned_rows += 1;
} else {
summary.mismatched_rows += 1;
}
summary
},
);
assert_eq!(
summary.unprobed_rows, 0,
"Mob modeled-state audit must fail closed when a required typed probe is missing"
);
let report = MobModeledStateAuditReport {
machine: "MobMachine".to_string(),
generated_at: Utc::now().to_rfc3339(),
summary,
rows,
};
std::fs::write(
&path,
serde_json::to_vec_pretty(&report).expect("serialize modeled-state audit report"),
)
.expect("write modeled-state audit report");
report
}
async fn write_mob_runtime_full_parity_audit_report(path: PathBuf) -> MobRuntimeParityAuditReport {
let schema = schema_mob_machine();
let mut pairs = Vec::new();
for &(left_phase, right_phase) in mob_runtime_parity_target_pairs() {
pairs.push(build_mob_runtime_full_pair_report(&schema, left_phase, right_phase).await);
}
let summary = pairs.iter().fold(
MobRuntimeParityAuditSummary {
pair_count: pairs.len(),
..Default::default()
},
|mut summary, pair| {
summary.interesting_rows += pair.summary.interesting_rows;
summary.probed_rows += pair.summary.probed_rows;
summary.aligned_rows += pair.summary.aligned_rows;
summary.mismatched_rows += pair.summary.mismatched_rows;
summary.unprobed_rows += pair.summary.unprobed_rows;
summary.surface_only_unprobed_rows += pair.summary.surface_only_unprobed_rows;
summary.runtime_internal_unprobed_rows += pair.summary.runtime_internal_unprobed_rows;
summary
},
);
assert_eq!(
summary.unprobed_rows, 0,
"Mob runtime full parity audit must fail closed when a required typed probe is missing"
);
let report = MobRuntimeParityAuditReport {
machine: "MobMachine".to_string(),
generated_at: Utc::now().to_rfc3339(),
summary,
pairs,
};
std::fs::write(
&path,
serde_json::to_vec_pretty(&report).expect("serialize mob runtime full parity report"),
)
.expect("write mob runtime full parity report");
report
}
// ---------------------------------------------------------------------------
// T5d: declarative API composition tests (ensure_member / reconcile /
// list_members_matching)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn test_ensure_member_spawns_then_returns_existing() {
use crate::runtime::reconcile::EnsureMemberOutcome;
let (handle, _service) = create_test_mob(sample_definition()).await;
let spec1 = crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-ensure"),
);
let first = handle
.ensure_member(spec1)
.await
.expect("first ensure_member");
assert!(
matches!(first, EnsureMemberOutcome::Spawned(_)),
"first ensure_member on absent identity should Spawn"
);
let spec2 = crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-ensure"),
);
let second = handle
.ensure_member(spec2)
.await
.expect("second ensure_member");
match second {
EnsureMemberOutcome::Existed(entry) => {
assert_eq!(entry.agent_identity, AgentIdentity::from("w-ensure"));
}
other => panic!("second ensure_member should be Existed, got {other:?}"),
}
}
#[tokio::test]
async fn test_ensure_member_respawns_history_only_identity_at_next_generation() {
use crate::runtime::reconcile::EnsureMemberOutcome;
let (handle, _service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("w-ensure-retired");
let spec = || {
crate::runtime::handle::SpawnMemberSpec::new(ProfileName::from("worker"), identity.clone())
};
assert!(matches!(
handle.ensure_member(spec()).await.expect("initial ensure"),
EnsureMemberOutcome::Spawned(_)
));
let first_generation = handle
.get_member(&identity)
.await
.expect("query first incarnation")
.expect("first incarnation exists")
.generation;
handle
.retire(identity.clone())
.await
.expect("retire first incarnation");
let outcome = handle
.ensure_member(spec())
.await
.expect("ensure history-only identity");
assert!(
matches!(outcome, EnsureMemberOutcome::Spawned(_)),
"history-only identity is missing from current membership and must respawn"
);
let replacement = handle
.get_member(&identity)
.await
.expect("query replacement")
.expect("replacement exists");
assert_eq!(
replacement.generation,
first_generation
.next()
.expect("test generation has a successor")
);
}
#[tokio::test]
async fn test_concurrent_ensure_member_converges_on_exact_machine_identity() {
use crate::runtime::reconcile::EnsureMemberOutcome;
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_create_session_delay_ms(180);
let first_handle = handle.clone();
let second_handle = handle.clone();
let first = tokio::spawn(async move {
first_handle
.ensure_member(SpawnMemberSpec::new("worker", "adaptive-flowmaster"))
.await
});
tokio::time::sleep(Duration::from_millis(25)).await;
let second = tokio::spawn(async move {
second_handle
.ensure_member(SpawnMemberSpec::new("worker", "adaptive-flowmaster"))
.await
});
let first = first
.await
.expect("first ensure join")
.expect("first ensure");
let second = second
.await
.expect("second ensure join")
.expect("second ensure converges");
assert!(matches!(first, EnsureMemberOutcome::Spawned(_)));
assert!(matches!(second, EnsureMemberOutcome::Existed(_)));
let state = handle.query_machine_state().await.expect("machine state");
let identity = crate::machines::mob_machine::AgentIdentity::from("adaptive-flowmaster");
assert_eq!(
state.member_profile_name_for_identity(&identity),
Some("worker")
);
assert_eq!(
handle
.list_members()
.await
.into_iter()
.filter(|entry| entry.agent_identity.as_str() == "adaptive-flowmaster")
.count(),
1
);
assert_eq!(service.active_session_count().await, 1);
}
#[tokio::test]
async fn test_ensure_member_retain_converges_before_roster_projection() {
use crate::runtime::reconcile::EnsureMemberOutcome;
let (handle, _service) = create_test_mob(sample_definition()).await;
let identity = AgentIdentity::from("adaptive-flowmaster");
let dsl_identity = crate::machines::mob_machine::AgentIdentity::from_domain(&identity);
let runtime_id = AgentRuntimeId::initial(identity.clone());
let dsl_runtime_id = crate::machines::mob_machine::AgentRuntimeId::from_domain(&runtime_id);
let generation = crate::ids::Generation::INITIAL;
let fence_token = FenceToken::new(1);
let profile_digest = "ensure-retain-profile-digest".to_string();
handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::AuthorizeSpawnProfile {
agent_identity: dsl_identity.clone(),
profile_name: "worker".to_string(),
model: "test-model".to_string(),
profile_material_digest: profile_digest.clone(),
tool_config_digest: "ensure-retain-tool-digest".to_string(),
skills_digest: "ensure-retain-skills-digest".to_string(),
provider_params_digest: None,
output_schema_digest: None,
external_addressable: false,
resolved_spec_digest: None,
},
)
.await
.expect("authorize committed-before-roster member profile");
handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::BeginSpawnExec {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id.clone(),
fence_token: crate::machines::mob_machine::FenceToken::from_domain(fence_token),
generation: crate::machines::mob_machine::Generation::from_domain(generation),
profile_material_digest: profile_digest.clone(),
external_addressable: false,
runtime_mode: crate::machines::mob_machine::SpawnPolicyRuntimeMode::TurnDriven,
bridge_session_id: None,
replacing: None,
placement: None,
workgraph_required: false,
rust_bundles_present: false,
per_spawn_external_tools_present: false,
mob_default_external_tools_present: false,
default_llm_client_override_present: false,
host_surface_mcp_allowlist_present: false,
inherited_tool_filter_present: false,
shell_env_present: false,
mcp_stdio_env_present: false,
mcp_http_headers_present: false,
memory_required: false,
mcp_required: false,
resume_session_id: None,
placed_spawn_id: None,
placed_provision_operation_id: None,
placed_operation_owner_session_id: None,
effective_profile_override_present: false,
effective_model_override_present: false,
},
)
.await
.expect("open committed-before-roster spawn");
handle
.apply_machine_input_effects(
crate::machines::mob_machine::MobMachineInput::CommitSpawnMembership {
agent_identity: dsl_identity.clone(),
agent_runtime_id: dsl_runtime_id,
fence_token: crate::machines::mob_machine::FenceToken::from_domain(fence_token),
generation: crate::machines::mob_machine::Generation::from_domain(generation),
profile_material_digest: profile_digest,
external_addressable: false,
runtime_mode: crate::machines::mob_machine::SpawnPolicyRuntimeMode::TurnDriven,
bridge_session_id: None,
replacing: None,
member_peer_endpoint: None,
spec_digest_echo: None,
ack_engine_version: None,
placed_spawn_id: None,
provision_operation_id: None,
},
)
.await
.expect("commit membership before mechanical roster projection");
let committed = handle.machine_state_watch_rx.borrow().clone();
assert!(
committed.identity_to_runtime.contains_key(&dsl_identity),
"machine membership must be committed while the roster row is absent"
);
assert_eq!(
committed
.member_lifecycle_for_identity(&dsl_identity)
.status,
crate::machines::mob_machine::MobMemberLifecycleStatus::Active,
"the machine-owned current incarnation must already be active"
);
assert!(
handle.roster.read().await.get(&identity).is_none(),
"mechanical roster row must still be absent"
);
let outcome = handle
.ensure_member(SpawnMemberSpec::new("worker", "adaptive-flowmaster"))
.await
.expect("machine retain verdict must converge before roster publication");
match outcome {
EnsureMemberOutcome::Existed(entry) => assert_eq!(entry.agent_identity, identity),
other => panic!("committed member must be retained, got {other:?}"),
}
assert!(
handle.roster.read().await.get(&identity).is_none(),
"retain convergence must not manufacture a mechanical roster row"
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn pack_cleanup_physically_destroys_real_mob_handle() {
struct TestCancellationOwner;
impl crate::adaptive::AdaptiveLayerCancellationOwner<MobHandle> for TestCancellationOwner {
fn take_layer_for_cancellation(&self, _layer: MobHandle) {}
fn disarm_after_cleanup(&self) {}
}
let (layer, service) = create_test_mob(sample_definition()).await;
let mut runtime = crate::runtime::mobpack_execution::PackAdaptiveRuntime {
control_mob: layer.clone(),
session_service: service,
active_planning_run: None,
layer_created_probe: None,
};
let layer_lease =
crate::adaptive::AdaptiveLayerLease::new(layer.clone(), Arc::new(TestCancellationOwner));
let deadline = adaptive_test_operation_deadline();
let cleanup = <crate::runtime::mobpack_execution::PackAdaptiveRuntime as crate::adaptive::AdaptiveDriverRuntime>::cleanup_layer(
&mut runtime,
&layer_lease,
&crate::adaptive::LayerId::new("real-layer").expect("layer id"),
1,
&deadline,
)
.await
.expect("physical cleanup");
let _destroyed_layer = layer_lease.disarm();
assert_eq!(cleanup, crate::adaptive::AdaptiveLayerCleanup::Destroyed);
assert_eq!(
layer.status().await.expect("terminal status projection"),
crate::runtime::MobState::Destroyed
);
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test]
async fn aborting_pack_provision_destroys_child_and_cancels_machine_run() {
let (control_mob, service) = create_test_mob(sample_definition()).await;
let capability = control_mob
.initialize_adaptive_run(InitializeAdaptiveRunRequest {
adaptive_run_id: "adaptive-cancel-run".to_string(),
limits: adaptive_test_limits(),
})
.await
.expect("initialize adaptive cancellation run");
control_mob
.record_adaptive_planning_decision(&capability, AdaptivePlanningDecisionKind::RunLayer)
.await
.expect("record adaptive layer decision");
let layer_id =
crate::adaptive::LayerId::new("adaptive-cancel-run-layer").expect("valid layer id");
let mut admission = adaptive_admission_request(layer_id.as_str(), 1);
admission.child_mob_id = "adaptive-cancel-child".to_string();
control_mob
.resolve_adaptive_layer_admission(&capability, admission)
.await
.expect("admit adaptive cancellation layer");
let mut child_definition = sample_definition();
child_definition.id = MobId::from("adaptive-cancel-child");
let mut body_store = crate::adaptive::InMemoryBodyStore::default();
let plan_digest = body_store
.put_json(&serde_json::json!({"layer": "adaptive-cancel-run-layer"}))
.expect("digest cancellation test layer");
let compiled = crate::adaptive::CompiledLayer {
child_mob_id: child_definition.id.clone(),
definition: child_definition,
spawn_specs: vec![SpawnMemberSpec::new("worker", "cancel-worker")],
activation_params: BTreeMap::new(),
plan_digest,
policy_evidence: crate::adaptive::LayerPolicyEvidence::default(),
};
// Hold provisioning inside spawn_many after the child actor and its
// cancellation guardian have both been created.
service.set_create_session_delay_ms(5_000);
let (layer_created_tx, mut layer_created_rx) = tokio::sync::mpsc::unbounded_channel();
let mut runtime = crate::runtime::mobpack_execution::PackAdaptiveRuntime {
control_mob: control_mob.clone(),
session_service: service.clone(),
active_planning_run: None,
layer_created_probe: Some(layer_created_tx),
};
let task_capability = capability.clone();
let task_layer_id = layer_id.clone();
let deadline = adaptive_test_operation_deadline();
let provision = tokio::spawn(async move {
<crate::runtime::mobpack_execution::PackAdaptiveRuntime as crate::adaptive::AdaptiveDriverRuntime>::provision_layer(
&mut runtime,
&task_capability,
&task_layer_id,
1,
&compiled,
&deadline,
)
.await
});
tokio::time::timeout(Duration::from_secs(2), async {
while service.max_concurrent_create_session_calls() == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("spawn_many must enter session creation before abort");
let child_mob = layer_created_rx
.recv()
.await
.expect("provision must expose its created child to the test probe");
provision.abort();
let join_error = match provision.await {
Err(error) => error,
Ok(_) => panic!("provision task must abort"),
};
assert!(
join_error.is_cancelled(),
"join error must report cancellation"
);
let snapshot = tokio::time::timeout(Duration::from_secs(3), async {
loop {
let snapshot = control_mob
.adaptive_run_snapshot(&capability)
.await
.expect("read adaptive cancellation snapshot");
if snapshot.phase == Some(AdaptiveRunPhaseView::Canceled)
&& snapshot.active_members == 0
{
break snapshot;
}
tokio::task::yield_now().await;
}
})
.await
.expect("guardian must finish machine cancellation and physical teardown");
assert_eq!(
snapshot.stop_reason,
Some(AdaptiveStopReasonView::HostCancel)
);
assert_eq!(
snapshot.layers[layer_id.as_str()].phase,
AdaptiveLayerPhaseView::Canceled
);
let machine_state = control_mob
.query_machine_state()
.await
.expect("read cancellation machine state");
assert!(
!machine_state.adaptive_active_layer.contains_key(
&crate::machines::mob_machine::AdaptiveRunId::from("adaptive-cancel-run")
),
"host cancellation must clear the machine-owned active-layer pointer"
);
assert_eq!(
service.active_session_count().await,
0,
"aborted spawn_many must not leave a child session alive"
);
assert_eq!(
child_mob
.status()
.await
.expect("read canceled child terminal status"),
MobState::Destroyed,
"the cancellation guardian must physically destroy the child mob"
);
}
#[tokio::test]
async fn test_reconcile_spawns_missing_and_retires_stale() {
use crate::runtime::reconcile::{ReconcileOptions, ReconcileStage};
let (handle, _service) = create_test_mob(sample_definition()).await;
// Seed one member that is NOT in the desired set so retire_stale takes it.
handle
.spawn(
ProfileName::from("worker"),
AgentIdentity::from("stale-1"),
None,
)
.await
.expect("seed stale-1");
let desired = vec![
crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-new-1"),
),
crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-new-2"),
),
];
let options = ReconcileOptions { retire_stale: true };
let report = handle.reconcile(desired, options).await.expect("reconcile");
assert_eq!(report.desired.len(), 2, "two desired identities reported");
assert_eq!(report.spawned.len(), 2, "two new spawns committed");
assert_eq!(
report.retained.len(),
0,
"no desired identity was pre-existing"
);
assert_eq!(
report.retired,
vec![AgentIdentity::from("stale-1")],
"stale-1 was retired because retire_stale is set",
);
assert!(
report.failures.is_empty(),
"no failures expected in happy path, got {:?}",
report
.failures
.iter()
.map(|f| (f.agent_identity.clone(), f.stage, f.error.to_string()))
.collect::<Vec<_>>()
);
assert!(
report
.failures
.iter()
.all(|f| f.stage == ReconcileStage::Spawn || f.stage == ReconcileStage::Retire),
"ReconcileStage tags cover only Spawn and Retire"
);
}
#[tokio::test]
async fn test_reconcile_treats_retired_history_as_missing_not_retained_or_stale() {
use crate::runtime::reconcile::ReconcileOptions;
let (handle, _service) = create_test_mob(sample_definition()).await;
let desired_identity = AgentIdentity::from("reconcile-retired-desired");
let history_only = AgentIdentity::from("reconcile-history-only");
for identity in [&desired_identity, &history_only] {
handle
.spawn(ProfileName::from("worker"), identity.clone(), None)
.await
.expect("seed member");
handle
.retire(identity.clone())
.await
.expect("retire seeded member");
}
let report = handle
.reconcile(
vec![crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
desired_identity.clone(),
)],
ReconcileOptions { retire_stale: true },
)
.await
.expect("reconcile history-only identities");
assert!(
report.failures.is_empty(),
"unexpected failures: {:?}",
report.failures
);
assert!(
report.retained.is_empty(),
"retired desired identity must not be classified as retained"
);
assert_eq!(report.spawned.len(), 1);
assert_eq!(report.spawned[0].agent_identity, desired_identity);
assert!(
!report.retired.contains(&history_only),
"history-only identity must not be repeatedly retired as stale"
);
}
#[tokio::test]
async fn test_list_members_matching_filters_by_label_and_role() {
use crate::runtime::reconcile::MemberFilter;
let (handle, _service) = create_test_mob(sample_definition()).await;
let mut labels_a = std::collections::BTreeMap::new();
labels_a.insert("faction".to_string(), "north".to_string());
let mut labels_b = std::collections::BTreeMap::new();
labels_b.insert("faction".to_string(), "south".to_string());
handle
.ensure_member(
crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-north"),
)
.with_labels(labels_a),
)
.await
.expect("spawn w-north");
handle
.ensure_member(
crate::runtime::handle::SpawnMemberSpec::new(
ProfileName::from("worker"),
AgentIdentity::from("w-south"),
)
.with_labels(labels_b),
)
.await
.expect("spawn w-south");
let mut filter = MemberFilter::default();
filter
.labels
.insert("faction".to_string(), "north".to_string());
let matches = handle.list_members_matching(filter).await.unwrap();
assert_eq!(matches.len(), 1, "only one member matches faction=north");
assert_eq!(matches[0].agent_identity, AgentIdentity::from("w-north"));
// Empty filter matches every member.
let all = handle
.list_members_matching(MemberFilter::default())
.await
.unwrap();
assert_eq!(all.len(), 2, "empty filter returns both members");
}
/// #212 dogma gate: the MobMachine emits `EmitKickoffLifecycleNotice` for
/// EVERY kickoff phase. The shell must forward every machine-emitted intent
/// mechanically — it may not narrow the set to `Failed`/`Cancelled` by
/// mapping the other phases to `None` and silently dropping them. This test
/// pins `kickoff_notice_intent` as a TOTAL projection: every `KickoffIntent`
/// variant maps to a distinct, non-empty notice tag (no phase silently
/// dropped). It fails-old (the prior `Option<&'static str>` shape returned
/// `None` for Pending/Starting/Started/CallbackPending) and passes-new.
#[test]
fn kickoff_notice_intent_delivers_every_machine_emitted_phase() {
use crate::machines::mob_machine::KickoffIntent;
// Every DSL-emitted kickoff intent. Mirrors the closed `KickoffIntent`
// enum; if a variant is added, this list must grow with it (the
// exhaustive `match` in `kickoff_notice_intent` forces the projection
// to stay total).
let all_intents = [
KickoffIntent::Pending,
KickoffIntent::Starting,
KickoffIntent::Started,
KickoffIntent::CallbackPending,
KickoffIntent::Failed,
KickoffIntent::Cancelled,
];
let mut seen = std::collections::BTreeSet::new();
for intent in all_intents {
let tag = super::actor::MobActor::kickoff_notice_intent(intent);
assert!(
!tag.is_empty(),
"kickoff intent {intent:?} must map to a non-empty notice tag (no None-drop)"
);
assert!(
tag.starts_with("mob.kickoff_"),
"kickoff notice tag for {intent:?} must use the mob.kickoff_* namespace, got {tag:?}"
);
assert!(
seen.insert(tag),
"kickoff notice tag {tag:?} for {intent:?} must be distinct per phase"
);
}
assert_eq!(
seen.len(),
all_intents.len(),
"every machine-emitted kickoff phase must be forwarded with a distinct notice tag"
);
}
// ===========================================================================
// 0.7.2 disciplined shell inputs — L5 (mob-kickoff/spawn waiters vs teardown)
//
// Worklist rows 12/13/14: the autonomous-kickoff completion waiter
// (actor.rs `start_autonomous_member` spawn) and the spawn-provisioning
// waiter (actor.rs `enqueue_spawn` spawn) race retire/destroy teardown.
// These tests drive the racy orders deterministically and pin the typed
// benign contract: teardown treats the waiters as machine-owned drain
// obligations, and late arrivals are accepted typed no-ops (no error path,
// no stuck state, no silent machine-state divergence).
//
// Stage A expectation (DSL landed, shell unchanged):
// * test_retire_quiesces_machine_inflight_kickoff_without_shell_handle RED
// * test_destroy_quiesces_machine_inflight_kickoff_without_shell_handle RED
// (the old shell only cancels kickoff when its own
// `autonomous_initial_turns` handle probe hits — a shell-side probe the
// machine-owned `RequestKickoffQuiesce` obligation replaces in Stage B)
// * the late-arrival and pending-spawn tests pin contracts that the DSL
// change alone may already satisfy (regression pins, expected GREEN).
// ===========================================================================
#[tokio::test]
async fn test_retire_quiesces_machine_inflight_kickoff_without_shell_handle() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member = AgentIdentity::from("worker-kickoff-quiesce-retire");
handle
.spawn_with_options(
ProfileName::from("worker"),
member.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
// Machine-owned kickoff facts with NO shell waiter handle: this is the
// exact shape the old shell cannot quiesce (its cancel path probes the
// shell-side `autonomous_initial_turns` map instead of discharging the
// machine obligation).
let dsl_member = crate::machines::mob_machine::AgentIdentity::from_domain(&member);
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::KickoffMarkPending {
member_id: dsl_member.clone(),
objective_id: "00000000-0000-0000-0000-000000000001".into(),
},
)
.await
.expect("mark kickoff pending");
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::KickoffMarkStarting {
member_id: dsl_member.clone(),
},
)
.await
.expect("mark kickoff starting");
handle.retire(member.clone()).await.expect(
"retire must discharge the machine-owned kickoff quiesce obligation \
(RequestKickoffQuiesce -> KickoffQuiesced) before archival, instead of \
failing the archival guard for a kickoff the shell never owned a handle for",
);
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(
!state.member_kickoff_pending.contains(&dsl_member)
&& !state.member_kickoff_starting.contains(&dsl_member)
&& !state.member_kickoff_callback_pending.contains(&dsl_member),
"no kickoff may remain in flight after retire completed",
);
assert!(
state.member_kickoff_cancelled.contains(&dsl_member),
"teardown quiesce of an in-flight kickoff must be machine-recorded as Cancelled",
);
// Live retire removes the member from both current machine membership and
// the roster while preserving only incarnation high-water history.
assert!(
!handle
.list_members()
.await
.iter()
.any(|entry| entry.agent_identity == member),
"member must be fully retired from the live roster",
);
}
#[tokio::test]
async fn test_destroy_quiesces_machine_inflight_kickoff_without_shell_handle() {
let (handle, _service) = create_test_mob(sample_definition()).await;
let member = AgentIdentity::from("worker-kickoff-quiesce-destroy");
handle
.spawn_with_options(
ProfileName::from("worker"),
member.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn turn-driven worker");
let dsl_member = crate::machines::mob_machine::AgentIdentity::from_domain(&member);
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::KickoffMarkPending {
member_id: dsl_member.clone(),
objective_id: "00000000-0000-0000-0000-000000000001".into(),
},
)
.await
.expect("mark kickoff pending");
handle
.project_machine_input(
crate::machines::mob_machine::MobMachineInput::KickoffMarkStarting {
member_id: dsl_member.clone(),
},
)
.await
.expect("mark kickoff starting");
handle.destroy().await.expect(
"destroy must discharge every kickoff quiesce obligation before the \
Destroy transition (guarded on kickoff_waiters_quiesced) commits",
);
assert_eq!(
handle.status().await.expect("status"),
MobState::Destroyed,
"destroy must reach the Destroyed phase",
);
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(
!state.member_kickoff_pending.contains(&dsl_member)
&& !state.member_kickoff_starting.contains(&dsl_member)
&& !state.member_kickoff_callback_pending.contains(&dsl_member),
"no kickoff may remain in flight after destroy committed",
);
}
#[tokio::test]
async fn test_late_kickoff_outcome_after_retire_is_benign() {
let (handle, service) = create_test_mob(sample_definition()).await;
// Park the autonomous initial turn so the kickoff stays Starting and the
// real completion waiter never delivers before teardown.
service.set_start_turn_delay_ms(10_000);
let member = AgentIdentity::from("lead-late-outcome");
handle
.spawn(ProfileName::from("lead"), member.clone(), None)
.await
.expect("spawn autonomous lead");
let dsl_member = crate::machines::mob_machine::AgentIdentity::from_domain(&member);
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let state = handle
.query_machine_state()
.await
.expect("query machine state");
if state.member_kickoff_starting.contains(&dsl_member) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("autonomous kickoff must reach the machine-owned Starting state");
handle
.retire(member.clone())
.await
.expect("retire of a kickoff-in-flight autonomous member must succeed");
// The completion outcome lands AFTER teardown quiesced the kickoff —
// delivered through the real actor command path. It must be acknowledged
// as a typed benign late arrival: no error, no kickoff resurrection.
handle
.debug_inject_kickoff_outcome(
member.clone(),
Ok(meerkat_runtime::CompletionOutcome::CompletedWithoutResult),
)
.await
.expect("late kickoff outcome must be acknowledged, not surfaced as an error");
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(
state.member_kickoff_cancelled.contains(&dsl_member),
"machine-recorded cancellation must survive the late outcome",
);
assert!(
!state.member_kickoff_started.contains(&dsl_member),
"a late completion outcome must not resurrect a quiesced kickoff to Started",
);
// The mob is not stuck: full teardown still commits.
handle
.destroy()
.await
.expect("destroy must succeed after the benign late kickoff arrival");
assert_eq!(handle.status().await.expect("status"), MobState::Destroyed);
}
#[tokio::test]
async fn test_late_kickoff_failure_outcome_after_retire_is_benign() {
let (handle, service) = create_test_mob(sample_definition()).await;
service.set_start_turn_delay_ms(10_000);
let member = AgentIdentity::from("lead-late-failure");
handle
.spawn(ProfileName::from("lead"), member.clone(), None)
.await
.expect("spawn autonomous lead");
let dsl_member = crate::machines::mob_machine::AgentIdentity::from_domain(&member);
tokio::time::timeout(Duration::from_secs(2), async {
loop {
let state = handle
.query_machine_state()
.await
.expect("query machine state");
if state.member_kickoff_starting.contains(&dsl_member) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("autonomous kickoff must reach the machine-owned Starting state");
handle
.retire(member.clone())
.await
.expect("retire of a kickoff-in-flight autonomous member must succeed");
// Abandoned/terminated outcomes map to KickoffResolveFailed; after the
// quiesce they must be benign no-ops too (row 13).
handle
.debug_inject_kickoff_outcome(
member.clone(),
Ok(meerkat_runtime::CompletionOutcome::RuntimeTerminated {
reason: "runtime retired during teardown".to_string(),
error: meerkat_core::TurnErrorMetadata::runtime_apply_failure(
"runtime retired during teardown",
),
}),
)
.await
.expect("late kickoff failure outcome must be acknowledged, not surfaced as an error");
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(
state.member_kickoff_cancelled.contains(&dsl_member),
"machine-recorded cancellation must survive the late failure outcome",
);
assert!(
!state.member_kickoff_failed.contains(&dsl_member),
"a late failure outcome must not rewrite a quiesced kickoff to Failed",
);
assert!(
!state.member_kickoff_error.contains_key(&dsl_member),
"a late failure outcome must not record an error for a quiesced kickoff",
);
}
#[tokio::test]
async fn test_destroy_with_inflight_spawn_provisioning_quiesces_waiter() {
let _delay_guard = SpawnProvisionedCommandDelayGuard::set(500);
let (handle, service) = create_test_mob(sample_definition()).await;
let member = AgentIdentity::from("w-destroy-pending-spawn");
let comms_name = test_comms_name("worker", member.as_str());
let pending_spawn = {
let handle = handle.clone();
let member = member.clone();
tokio::spawn(async move {
handle
.spawn(ProfileName::from("worker"), member, None)
.await
})
};
// The provisioning task has provisioned the session and is parked in the
// pre-send delay window — the deterministic "in-flight spawn waiter".
let _session_id = wait_for_session_id_for_comms_name(&service, &comms_name).await;
handle.destroy().await.expect(
"destroy must drain the pending-spawn obligations (abort + await the \
provisioning waiter, typed CancelPendingSpawn closure) and commit",
);
assert_eq!(handle.status().await.expect("status"), MobState::Destroyed);
let spawn_error = pending_spawn
.await
.expect("spawn task join")
.expect_err("pending spawn must fail with the typed teardown cancellation");
assert!(
spawn_error.to_string().contains("mob is destroying"),
"pending spawn must surface the typed destroy cancellation reason, got: {spawn_error}"
);
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert_eq!(
state.pending_spawn_count, 0,
"no machine pending-spawn obligation may survive destroy",
);
assert!(
state.pending_spawn_sessions.is_empty(),
"no machine pending-spawn session may survive destroy",
);
// Destroy synchronously quiesces the actor-owned pending-spawn state and
// closes the caller's reply. If cancellation lands before the provisioning
// task publishes its exact cleanup anchor, however, ownership has already
// transferred to the process cleanup runtime and that exact cleanup may
// finish immediately after destroy returns. The durable-create contract
// does not require cancellation to make the create indistinguishable from
// never having happened, so assert bounded convergence rather than an
// incorrect return-boundary guarantee.
tokio::time::timeout(Duration::from_secs(2), async {
while service.active_session_count().await != 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("process-owned exact cleanup must remove the cancelled provisioned session");
}
// =========================================================================
// Multi-host mobs (§7.2 steps 2 & 4): controlling-side host bind ceremony.
//
// TDD lane W3: these rows drive `MobHandle::bind_host`/`revoke_host`, the
// actor-side ceremony (BeginHostBind → RequestHostBind realization →
// BindHost over the supervisor bridge → CommitHostBind), the FLAG-3(a)
// durable `MobHostAuthorityRecord` writers, builder recovery via
// `RecoverHostBinding`, and the §7.2 step 4 rotation fan-out to bound hosts
// (write-ahead pending membership + idempotent rebind-replay convergence).
// =========================================================================
use crate::machines::mob_machine as host_dsl;
/// In-process stand-in for a `rkat mob host` daemon's bridge responder: real
/// comms runtime, real signed envelopes, serving
/// `BindHost`/`RebindHost`/`RevokeHost`/`HostStatus` with
/// the host DSL's admission ladder in miniature and typed rejections for
/// everything else (DEC-P2-7).
struct HostDaemonStub {
address: String,
peer_id: meerkat_core::comms::PeerId,
pubkey: [u8; 32],
task: tokio::task::JoinHandle<()>,
bind_requests: Arc<RwLock<Vec<super::bridge_protocol::BridgeHostBindPayload>>>,
rebind_requests: Arc<RwLock<Vec<super::bridge_protocol::BridgeHostRebindPayload>>>,
revoke_requests: Arc<RwLock<Vec<super::bridge_protocol::BridgeHostRevokePayload>>>,
reject_next_bind: Arc<AtomicBool>,
ambiguous_next_bind_rejection:
Arc<RwLock<Option<super::bridge_protocol::BridgeRejectionCause>>>,
reject_rebinds: Arc<AtomicBool>,
bind_peer_id_override: Arc<RwLock<Option<String>>>,
}
impl HostDaemonStub {
/// The domain-side bind request a caller would build from this daemon's
/// binding descriptor.
fn bind_request(&self) -> super::handle::HostBindRequest {
super::handle::HostBindRequest {
expected_peer_id: self.peer_id,
pubkey: self.pubkey,
address: self.address.clone(),
bootstrap_token: super::bridge_protocol::BridgeBootstrapToken::from(
"host-ceremony-token",
),
live_endpoint: None,
}
}
}
impl Drop for HostDaemonStub {
fn drop(&mut self) {
self.task.abort();
}
}
fn host_stub_capabilities() -> super::bridge_protocol::BridgeCapabilities {
super::bridge_protocol::BridgeCapabilities {
durable_sessions: true,
autonomous_members: true,
hard_cancel_member: true,
tracked_input_cancel: true,
memory_store: false,
mcp: true,
engine_version: "host-stub-0.1".to_string(),
approval_forwarding: false,
resolvable_providers: vec![Provider::Anthropic],
..super::bridge_protocol::BridgeCapabilities::default()
}
}
async fn spawn_host_daemon_stub(label: &str, live_endpoint: Option<String>) -> HostDaemonStub {
let stub_name = format!("host-daemon-stub/{label}/{}", Uuid::new_v4().simple());
let runtime =
meerkat_comms::CommsRuntime::inproc_only(&stub_name).expect("create host stub runtime");
let address = runtime.advertised_address();
let runtime = Arc::new(runtime);
install_ephemeral_peer_request_response_authority(
&runtime,
&format!("host-daemon-stub-{label}"),
);
let peer_id = runtime.public_key().to_peer_id();
let pubkey = *runtime.public_key().as_bytes();
let bind_requests = Arc::new(RwLock::new(Vec::new()));
let rebind_requests = Arc::new(RwLock::new(Vec::new()));
let revoke_requests = Arc::new(RwLock::new(Vec::new()));
let reject_next_bind = Arc::new(AtomicBool::new(false));
let ambiguous_next_bind_rejection = Arc::new(RwLock::new(None));
let reject_rebinds = Arc::new(AtomicBool::new(false));
let bind_peer_id_override: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
let responder_runtime = runtime.clone();
let responder_address = address.clone();
let responder_peer_id = peer_id.to_string();
let responder_runtime_incarnation = super::bridge_protocol::BridgeHostRuntimeIncarnation::new();
let responder_live_endpoint = live_endpoint;
let responder_bind_requests = bind_requests.clone();
let responder_rebind_requests = rebind_requests.clone();
let responder_revoke_requests = revoke_requests.clone();
let responder_reject_next_bind = reject_next_bind.clone();
let responder_ambiguous_next_bind_rejection = ambiguous_next_bind_rejection.clone();
let responder_reject_rebinds = reject_rebinds.clone();
let responder_bind_peer_id_override = bind_peer_id_override.clone();
let task = tokio::spawn(async move {
// Recorded (supervisor peer id, epoch) per mob: the host DSL binding
// facts in miniature, driving the rebind ladder below.
let mut supervisors: BTreeMap<String, (String, u64)> = BTreeMap::new();
let mut revoked: BTreeMap<String, (String, u64)> = BTreeMap::new();
let inbox_notify = responder_runtime.inbox_notify();
loop {
let notified = inbox_notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let candidates = responder_runtime.drain_peer_input_candidates().await;
if candidates.is_empty() {
if responder_runtime.dismiss_received() {
break;
}
(&mut notified).await;
continue;
}
for candidate in candidates {
let meerkat_core::interaction::InteractionContent::Request { params, .. } =
&candidate.interaction.content
else {
continue;
};
responder_runtime
.peer_interaction_handle()
.expect("host stub semantic authority")
.request_received(
PeerCorrelationId::from_uuid(candidate.interaction.id.0),
candidate.interaction.handling_mode,
)
.expect("record inbound host stub request before response");
let to =
trust_candidate_sender_for_reply(responder_runtime.as_ref(), &candidate).await;
let sender_peer_id = candidate
.ingress
.route
.as_ref()
.map(|route| route.peer_id.to_string())
.or_else(|| {
candidate
.interaction
.from_route
.as_ref()
.map(std::string::ToString::to_string)
});
let response = match serde_json::from_value::<super::bridge_protocol::BridgeCommand>(
params.clone(),
) {
Ok(super::bridge_protocol::BridgeCommand::BindHost(payload)) => {
responder_bind_requests.write().await.push(payload.clone());
let ambiguous_rejection =
responder_ambiguous_next_bind_rejection.write().await.take();
if let Some(cause) = ambiguous_rejection {
serde_json::to_value(super::bridge_protocol::BridgeReply::Rejected {
cause,
reason: "host stub: injected ambiguous bind rejection".to_string(),
})
.expect("serialize ambiguous bind rejection")
} else if responder_reject_next_bind.swap(false, Ordering::Relaxed) {
serde_json::to_value(super::bridge_protocol::BridgeReply::Rejected {
// Provably pre-persistence in the real host
// ladder, so the controller may roll back its
// provisional recipient trust.
cause: super::bridge_protocol::BridgeRejectionCause::InvalidBootstrapToken,
reason: "host stub: injected pre-write bind rejection".to_string(),
})
.expect("serialize bind rejection")
} else {
revoked.remove(&payload.mob_id);
supervisors.insert(
payload.mob_id.clone(),
(payload.supervisor.peer_id.clone(), payload.epoch),
);
let host_peer_id = responder_bind_peer_id_override
.read()
.await
.clone()
.unwrap_or_else(|| responder_peer_id.clone());
serde_json::to_value(super::bridge_protocol::BridgeReply::BindHost(
super::bridge_protocol::BridgeHostBindResponse {
host_peer_id,
address: responder_address.clone(),
binding_generation: payload.binding_generation,
capabilities: host_stub_capabilities(),
live_endpoint: responder_live_endpoint.clone(),
},
))
.expect("serialize bind reply")
}
}
Ok(super::bridge_protocol::BridgeCommand::RebindHost(payload)) => {
responder_rebind_requests
.write()
.await
.push(payload.clone());
let reply = if responder_reject_rebinds.load(Ordering::Relaxed) {
super::bridge_protocol::BridgeReply::Rejected {
cause:
super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: "host stub: injected rebind rejection".to_string(),
}
} else {
// Mirror the host DSL rebind ladder: the sender
// must be the supervisor the payload declares; a
// strictly higher epoch rotates; the recorded
// (peer, epoch) pair acks idempotently (the
// FLAG-2(ii) replay arm); anything else is a
// typed rejection.
let sender_matches = sender_peer_id.as_deref()
== Some(payload.supervisor.peer_id.as_str());
match supervisors.get(&payload.mob_id).cloned() {
_ if !sender_matches => {
super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::SenderMismatch,
reason: "host stub: rebind sender does not match declared supervisor".to_string(),
}
}
Some((_, recorded_epoch)) if payload.epoch > recorded_epoch => {
supervisors.insert(
payload.mob_id.clone(),
(payload.supervisor.peer_id.clone(), payload.epoch),
);
super::bridge_protocol::BridgeReply::HostRebound(
super::bridge_protocol::BridgeHostReboundResponse {
host_peer_id: responder_peer_id.clone(),
binding_generation: payload.binding_generation,
capabilities: host_stub_capabilities(),
live_endpoint: responder_live_endpoint.clone(),
},
)
}
Some((recorded_peer, recorded_epoch))
if payload.epoch == recorded_epoch
&& recorded_peer == payload.supervisor.peer_id =>
{
super::bridge_protocol::BridgeReply::HostRebound(
super::bridge_protocol::BridgeHostReboundResponse {
host_peer_id: responder_peer_id.clone(),
binding_generation: payload.binding_generation,
capabilities: host_stub_capabilities(),
live_endpoint: responder_live_endpoint.clone(),
},
)
}
Some(_) => super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: "host stub: rebind epoch is not newer than the recorded binding".to_string(),
},
None => super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
reason: "host stub: no binding recorded for mob".to_string(),
},
}
};
serde_json::to_value(reply).expect("serialize rebind reply")
}
Ok(super::bridge_protocol::BridgeCommand::RevokeHost(payload)) => {
responder_revoke_requests
.write()
.await
.push(payload.clone());
let sender_matches =
sender_peer_id.as_deref() == Some(payload.supervisor.peer_id.as_str());
let reply = match supervisors.get(&payload.mob_id).cloned() {
Some((recorded_peer, recorded_epoch))
if sender_matches
&& recorded_peer == payload.supervisor.peer_id
&& recorded_epoch == payload.epoch =>
{
supervisors.remove(&payload.mob_id);
revoked.insert(
payload.mob_id.clone(),
(recorded_peer, recorded_epoch),
);
super::bridge_protocol::BridgeReply::HostRevoked(
super::bridge_protocol::BridgeHostRevokedResponse {
host_peer_id: responder_peer_id.clone(),
mob_id: payload.mob_id,
epoch: payload.epoch,
binding_generation: payload.binding_generation,
released_members: Vec::new(),
},
)
}
Some((recorded_peer, _))
if !sender_matches
|| recorded_peer != payload.supervisor.peer_id =>
{
super::bridge_protocol::BridgeReply::Rejected {
cause:
super::bridge_protocol::BridgeRejectionCause::SenderMismatch,
reason: "host stub: revoke sender mismatch".to_string(),
}
}
Some(_) => super::bridge_protocol::BridgeReply::Rejected {
cause:
super::bridge_protocol::BridgeRejectionCause::StaleSupervisor,
reason: "host stub: revoke epoch mismatch".to_string(),
},
None if revoked.get(&payload.mob_id).is_some_and(
|(peer_id, epoch)| {
peer_id == &payload.supervisor.peer_id
&& *epoch == payload.epoch
},
) && sender_matches =>
{
super::bridge_protocol::BridgeReply::HostRevoked(
super::bridge_protocol::BridgeHostRevokedResponse {
host_peer_id: responder_peer_id.clone(),
mob_id: payload.mob_id,
epoch: payload.epoch,
binding_generation: payload.binding_generation,
released_members: Vec::new(),
},
)
}
None => super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::NotBound,
reason: "host stub: no binding or matching revoke receipt"
.to_string(),
},
};
serde_json::to_value(reply).expect("serialize revoke reply")
}
Ok(super::bridge_protocol::BridgeCommand::HostStatus(_)) => {
serde_json::to_value(super::bridge_protocol::BridgeReply::HostStatus(
super::bridge_protocol::BridgeHostStatusResponse {
runtime_incarnation: responder_runtime_incarnation,
members: Vec::new(),
capabilities: host_stub_capabilities(),
},
))
.expect("serialize host status reply")
}
_ => serde_json::to_value(super::bridge_protocol::BridgeReply::Rejected {
cause: super::bridge_protocol::BridgeRejectionCause::Unsupported,
reason: "host stub serves BindHost/RebindHost/RevokeHost/HostStatus only"
.to_string(),
})
.expect("serialize unsupported rejection"),
};
let send_deadline =
tokio::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
match responder_runtime
.send(CommsCommand::PeerResponse {
objective_id: None,
content_taint: None,
to: to.clone(),
in_reply_to: candidate.interaction.id,
status: meerkat_core::interaction::ResponseStatus::Completed,
result: response.clone(),
blocks: None,
handling_mode: None,
})
.await
{
Ok(_) => break,
Err(meerkat_core::comms::SendError::PeerNotFound(_))
if tokio::time::Instant::now() < send_deadline =>
{
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
Err(error) => {
panic!("send host stub response failed: {error}");
}
}
}
}
}
});
HostDaemonStub {
address,
peer_id,
pubkey,
task,
bind_requests,
rebind_requests,
revoke_requests,
reject_next_bind,
ambiguous_next_bind_rejection,
reject_rebinds,
bind_peer_id_override,
}
}
async fn create_host_ceremony_mob(
label: &str,
) -> (
MobHandle,
MobId,
Arc<dyn crate::store::MobRuntimeMetadataStore>,
) {
let definition = with_unique_mob_id(sample_definition_with_external_backend(), label);
let mob_id = definition.id.clone();
let storage = MobStorage::in_memory();
let runtime_metadata = storage.runtime_metadata.clone();
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service)
.create()
.await
.expect("create mob");
(handle, mob_id, runtime_metadata)
}
/// §11 ceremony rows: a committed bind records identity, endpoint, the
/// supervisor authority epoch (DEC-P2-9), and the full declared capability
/// record in the machine AND the durable host authority record (FLAG-3(a)),
/// with the wire payload carrying explicit mob identity (FLAG-1) and the
/// descriptor bootstrap token.
#[tokio::test]
async fn test_bind_host_commits_machine_facts_and_persists_record() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) = create_host_ceremony_mob("bind-host-commit").await;
let stub =
spawn_host_daemon_stub("bind-commit", Some("wss://host-stub/live".to_string())).await;
let report = handle
.bind_host(stub.bind_request())
.await
.expect("bind host");
let supervisor = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor authority")
.expect("supervisor authority record");
assert_eq!(report.host_id, stub.peer_id.to_string());
assert_eq!(
report.epoch, supervisor.epoch,
"BindHost stamps the supervisor authority epoch (DEC-P2-9), never a second counter"
);
assert!(report.capabilities.durable_sessions);
assert!(report.capabilities.mcp);
assert!(!report.capabilities.memory_store);
assert_eq!(report.capabilities.engine_version, "host-stub-0.1");
assert!(
report
.capabilities
.resolvable_providers
.contains("anthropic")
);
assert_eq!(
report.capabilities.live_endpoint.as_deref(),
Some("wss://host-stub/live")
);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(state.mob_hosts.contains(&host_id));
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Bound)
);
assert_eq!(
state.host_authority_epochs.get(&host_id).copied(),
Some(supervisor.epoch)
);
assert_eq!(
state.host_durable_sessions.get(&host_id).copied(),
Some(true)
);
assert_eq!(
state.host_engine_versions.get(&host_id).cloned(),
Some("host-stub-0.1".to_string())
);
assert_eq!(
state.host_live_endpoints.get(&host_id).cloned(),
Some(host_dsl::LiveWsEndpointUrl::from(
"wss://host-stub/live".to_string()
))
);
let records = runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records");
assert_eq!(records.len(), 1, "one durable record per bound host");
assert_eq!(records[0].host_id, stub.peer_id.to_string());
assert_eq!(records[0].signing_key, stub.pubkey);
assert_eq!(records[0].authority_epoch, supervisor.epoch);
assert_eq!(
records[0].live_endpoint.as_deref(),
Some("wss://host-stub/live")
);
let binds = stub.bind_requests.read().await;
assert_eq!(binds.len(), 1);
assert_eq!(
binds[0].mob_id,
mob_id.to_string(),
"FLAG-1: host-addressed payloads carry explicit mob identity, never name-derived"
);
assert_eq!(binds[0].expected_host_peer_id, stub.peer_id.to_string());
assert_eq!(binds[0].epoch, supervisor.epoch);
drop(binds);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&stub.peer_id.to_string()),
"a committed bind keeps the host's recipient trust installed"
);
assert!(
pending_recipient_trust_snapshot(&handle).is_empty(),
"the pending recipient-trust obligation must be resolved on confirmed bind"
);
}
/// A reply whose host identity does not match the descriptor identity arrives
/// after the host may already have durably committed BindHost. The controlling
/// side must retain trust plus the Requested/pending-obligation quarantine and
/// fail-stop before any later command can falsely treat Requested as no-effect.
#[tokio::test]
async fn test_bind_host_reply_identity_mismatch_quarantines_and_fail_stops() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) =
create_host_ceremony_mob("bind-host-identity-mismatch").await;
let stub = spawn_host_daemon_stub("identity-mismatch", None).await;
*stub.bind_peer_id_override.write().await = Some("host-imposter".to_string());
let error = handle
.bind_host(stub.bind_request())
.await
.expect_err("mismatched reply identity must fail the ceremony");
assert!(
error.to_string().contains("identity mismatch"),
"unexpected error: {error}"
);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle.machine_state_watch_rx.borrow().clone();
assert!(!state.mob_hosts.contains(&host_id));
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Requested),
"the exact host identity remains quarantined in the Requested bind window"
);
assert!(
runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records")
.is_empty(),
"no durable host record may exist for an uncommitted bind"
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&stub.peer_id.to_string()),
"post-send uncertainty must retain recipient trust for exact replay"
);
assert_eq!(
pending_recipient_trust_snapshot(&handle),
vec![stub.peer_id.to_string()],
"post-send uncertainty must retain the machine cleanup obligation"
);
*stub.bind_peer_id_override.write().await = None;
let retry = handle.bind_host(stub.bind_request()).await;
assert!(retry.is_err(), "fail-stopped actor must reject live retry");
assert_eq!(
stub.bind_requests.read().await.len(),
1,
"no second BindHost may escape before cold exact replay"
);
}
#[tokio::test]
async fn test_bind_host_post_ack_persistence_failure_cold_recovers_from_confirmed_anchor() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"bind-host-post-ack-persist-failure",
);
let mob_id = definition.id.clone();
let events = Arc::new(InMemoryMobEventStore::new());
let runtime_metadata = Arc::new(FaultInjectedRuntimeMetadataStore::new());
let storage =
MobStorage::with_events_and_runtime_metadata(events.clone(), runtime_metadata.clone());
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let stub = spawn_host_daemon_stub("post-ack-persist-failure", None).await;
runtime_metadata.fail_next_put_host_authority();
let error = handle
.bind_host(stub.bind_request())
.await
.expect_err("post-ACK authority persistence failure must fail the ceremony");
assert!(
error.to_string().contains("host authority write failure"),
"unexpected error: {error}",
);
assert_eq!(stub.bind_requests.read().await.len(), 1);
assert_eq!(stub.bind_requests.read().await[0].binding_generation, 1);
assert!(
handle.revoke_host(&stub.peer_id.to_string()).await.is_err(),
"fail-stopped actor must not locally clear an ACKed Requested binding",
);
assert!(
stub.revoke_requests.read().await.is_empty(),
"no revoke may escape from the quarantined live actor",
);
assert_eq!(
pending_recipient_trust_snapshot(&handle),
vec![stub.peer_id.to_string()],
"the trust obligation remains pending until durable local commit",
);
drop(handle);
let resumed = MobBuilder::for_resume(MobStorage::with_events_and_runtime_metadata(
events,
runtime_metadata.clone(),
))
.with_session_service(service)
.resume()
.await
.expect("cold resume after quarantined bind");
let requests = stub.bind_requests.read().await;
assert_eq!(
requests.len(),
1,
"authenticated Confirmed authority reconstructs local Bound without token replay",
);
drop(requests);
let record = runtime_metadata
.load_mob_host_authority(&mob_id, &stub.peer_id.to_string())
.await
.expect("load host authority")
.expect("cold replay persisted host authority");
assert_eq!(record.binding_generation, 1);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = resumed
.query_machine_state()
.await
.expect("query recovered machine state");
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Bound),
);
}
/// W4.3: a typed daemon-side rejection that certifies pre-persistence
/// non-acceptance surfaces to the caller, rolls back provisional trust, and
/// permits an exact live retry.
#[tokio::test]
async fn test_bind_host_surfaces_typed_rejection_and_retries() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) =
create_host_ceremony_mob("bind-host-typed-rejection").await;
let stub = spawn_host_daemon_stub("typed-rejection", None).await;
stub.reject_next_bind.store(true, Ordering::Relaxed);
let error = handle
.bind_host(stub.bind_request())
.await
.expect_err("typed daemon rejection must surface");
assert!(
error.to_string().contains("pre-write bind rejection"),
"the typed rejection reason must pass through: {error}"
);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Requested)
);
assert!(
runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records")
.is_empty()
);
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&stub.peer_id.to_string()),
"a rejected bind must roll back the recipient trust it installed"
);
let report = handle
.bind_host(stub.bind_request())
.await
.expect("bind retry after rejection");
assert_eq!(report.host_id, stub.peer_id.to_string());
}
/// `AlreadyBound` and `Internal` do not certify non-acceptance: both can be
/// returned after the exact binding became durable (lost prior ACK or
/// post-persist supervisor-trust installation failure). The controller must
/// retain its trust/pending obligation and fail-stop for cold exact replay.
#[tokio::test]
async fn test_bind_host_ambiguous_rejections_quarantine_instead_of_rolling_back() {
let _serial = lock_real_comms_tests();
for (label, cause) in [
(
"already-bound",
super::bridge_protocol::BridgeRejectionCause::AlreadyBound,
),
(
"internal",
super::bridge_protocol::BridgeRejectionCause::Internal,
),
] {
let (handle, mob_id, runtime_metadata) =
create_host_ceremony_mob(&format!("bind-host-ambiguous-{label}")).await;
let stub = spawn_host_daemon_stub(&format!("ambiguous-{label}"), None).await;
*stub.ambiguous_next_bind_rejection.write().await = Some(cause);
let error = handle
.bind_host(stub.bind_request())
.await
.expect_err("ambiguous host rejection must quarantine the bind");
assert!(
error
.to_string()
.contains("injected ambiguous bind rejection"),
"unexpected ambiguous rejection error: {error}",
);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle.machine_state_watch_rx.borrow().clone();
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Requested),
"ambiguous rejection must retain the exact Requested generation",
);
assert!(
runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records")
.is_empty(),
"the controller has not received proof needed to persist Bound",
);
assert!(
supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&stub.peer_id.to_string()),
"ambiguous rejection must retain recipient trust for cold replay",
);
assert_eq!(
pending_recipient_trust_snapshot(&handle),
vec![stub.peer_id.to_string()],
);
assert!(
handle.bind_host(stub.bind_request()).await.is_err(),
"the quarantined live actor must reject retry until cold recovery",
);
assert_eq!(
stub.bind_requests.read().await.len(),
1,
"fail-stop must prevent a second live BindHost from escaping",
);
assert!(
handle.revoke_host(&stub.peer_id.to_string()).await.is_err(),
"the quarantined actor must not locally clear uncertain authority",
);
assert!(stub.revoke_requests.read().await.is_empty());
}
}
/// W4.3: `revoke_host` clears every machine host fact, deletes the durable
/// host authority record under the transition witness, and removes the
/// host's recipient trust; a fresh bind after revoke succeeds.
#[tokio::test]
async fn test_revoke_host_clears_machine_facts_and_deletes_record() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) = create_host_ceremony_mob("revoke-host").await;
let stub = spawn_host_daemon_stub("revoke", None).await;
let report = handle
.bind_host(stub.bind_request())
.await
.expect("bind host");
let revoke_report = handle
.revoke_host(&report.host_id)
.await
.expect("revoke host");
// Phase 7 (ADJ-P7-2): the typed revoke report names the revoked host
// and the released placements (none in this fixture — no member was
// placed on the stub host).
assert_eq!(revoke_report.host_id, report.host_id);
assert!(revoke_report.released_members.is_empty());
let revokes = stub.revoke_requests.read().await;
assert_eq!(revokes.len(), 1, "controller must invoke host revoke first");
assert_eq!(revokes[0].mob_id, mob_id.to_string());
assert_eq!(revokes[0].epoch, report.epoch);
drop(revokes);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert!(!state.mob_hosts.contains(&host_id));
assert!(!state.host_bind_phase.contains_key(&host_id));
assert!(!state.host_public_keys.contains_key(&host_id));
assert!(!state.host_endpoints.contains_key(&host_id));
assert!(!state.host_authority_epochs.contains_key(&host_id));
assert!(!state.host_live_endpoints.contains_key(&host_id));
assert!(
runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records")
.is_empty(),
"revoke must delete the durable host authority record"
);
assert!(
!supervisor_bridge_trusted_peer_ids(&handle)
.await
.contains(&stub.peer_id.to_string()),
"revoke must remove the host's recipient trust"
);
let rebound = handle
.bind_host(stub.bind_request())
.await
.expect("fresh bind after revoke");
assert_eq!(rebound.host_id, stub.peer_id.to_string());
}
/// Destroy drains host authority through the authenticated host terminal,
/// rather than merely clearing the mob's event and metadata stores. Leaving
/// `mob_runtime_host_authorities` behind would let recovery resurrect a host
/// binding from the destroyed incarnation.
#[tokio::test]
async fn test_destroy_revokes_bound_host_and_deletes_authority_record() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) =
create_host_ceremony_mob("destroy-revokes-host-authority").await;
let stub = spawn_host_daemon_stub("destroy-revoke", None).await;
let bind = handle
.bind_host(stub.bind_request())
.await
.expect("bind host before destroy");
handle
.destroy()
.await
.expect("destroy should durably revoke every tracked host first");
let revokes = stub.revoke_requests.read().await;
assert_eq!(
revokes.len(),
1,
"destroy must invoke host revoke exactly once"
);
assert_eq!(revokes[0].mob_id, mob_id.to_string());
assert_eq!(revokes[0].epoch, bind.epoch);
drop(revokes);
assert!(
runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("list host authority records after destroy")
.is_empty(),
"destroy must not leave a recoverable controlling-side host authority row"
);
}
#[test]
fn test_destroy_drain_retries_certified_failed_replacement_bind_window() {
let active = host_dsl::HostId::from("host-active");
let replacement = host_dsl::HostId::from("host-replacement");
let mut state = host_dsl::MobMachineState::default();
state
.host_bind_phase
.insert(active.clone(), host_dsl::HostBindPhase::Bound);
// A provably pre-write rejection leaves this exact request open for
// retry, but replacement requests intentionally have no phase row.
state.replacement_host_bind_endpoints.insert(
replacement.clone(),
host_dsl::PeerAddress::from("tcp://host-replacement"),
);
state
.replacement_host_binding_generations
.insert(replacement.clone(), 2);
let targets = MobActor::host_ids_requiring_destroy_revoke(&state);
assert_eq!(
targets,
BTreeSet::from([
active.as_str().to_string(),
replacement.as_str().to_string(),
]),
"Destroy must retry both committed authority and the phase-less replacement window",
);
}
/// §7.2 step 4: supervisor rotation fans out to bound hosts via `RebindHost`
/// sent AS the attempted authority; on acceptance the controlling machine
/// records `HostRebound` at the new epoch with the re-declared capability
/// record, the durable host record is rewritten, and only then does the
/// rotation commit.
#[tokio::test]
async fn test_rotate_supervisor_rebinds_bound_hosts_and_commits() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) = create_host_ceremony_mob("rotate-host-commit").await;
let stub = spawn_host_daemon_stub("rotate-commit", None).await;
let bind_report = handle
.bind_host(stub.bind_request())
.await
.expect("bind host");
let report = handle.rotate_supervisor().await.expect("rotate supervisor");
assert_eq!(report.current_epoch, bind_report.epoch + 1);
let rebinds = stub.rebind_requests.read().await;
assert!(
!rebinds.is_empty(),
"rotation must fan out a RebindHost to every bound host"
);
assert_eq!(
rebinds[0].mob_id,
mob_id.to_string(),
"FLAG-2(i): the rebind carrier is mob-scoped"
);
assert_eq!(rebinds[0].epoch, report.current_epoch);
assert_eq!(rebinds[0].supervisor.peer_id, report.public_peer_id);
drop(rebinds);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert_eq!(
state.host_authority_epochs.get(&host_id).copied(),
Some(report.current_epoch),
"controlling-side HostRebound must record the committed epoch"
);
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Bound)
);
let rotated = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load rotated supervisor")
.expect("rotated supervisor record");
assert_eq!(rotated.epoch, report.current_epoch);
assert!(
rotated.pending_rotation.is_none(),
"a committed rotation clears the pending-rotation memory"
);
let records = runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records");
assert_eq!(records.len(), 1);
assert_eq!(
records[0].authority_epoch, report.current_epoch,
"the durable host record must be rewritten at the committed epoch"
);
}
/// §7.2 step 4 fail-closed: a bound host rejecting the rebind leaves the
/// persisted supervisor authority at the pre-rotation epoch with the
/// attempted authority retained as pending-rotation metadata (host peer in
/// the accepted membership — write-ahead, since host epochs cannot be rolled
/// back), and a retry converges through the idempotent rebind path.
#[tokio::test]
async fn test_rotate_supervisor_fails_closed_when_host_rejects_rebind() {
let _serial = lock_real_comms_tests();
let (handle, mob_id, runtime_metadata) =
create_host_ceremony_mob("rotate-host-fail-closed").await;
let stub = spawn_host_daemon_stub("rotate-fail-closed", None).await;
let bind_report = handle
.bind_host(stub.bind_request())
.await
.expect("bind host");
stub.reject_rebinds.store(true, Ordering::Relaxed);
let error = handle
.rotate_supervisor()
.await
.expect_err("rotation must fail closed when a bound host rejects the rebind");
let MobError::SupervisorRotationIncomplete {
previous_epoch,
attempted_epoch,
pending_authority_recorded,
..
} = &error
else {
panic!("expected SupervisorRotationIncomplete, got: {error}");
};
assert_eq!(*previous_epoch, bind_report.epoch);
assert_eq!(*attempted_epoch, bind_report.epoch + 1);
assert!(
*pending_authority_recorded,
"the attempted authority must be durably recorded for host retry convergence"
);
let durable = runtime_metadata
.load_supervisor_authority(&mob_id)
.await
.expect("load supervisor authority")
.expect("supervisor authority record");
assert_eq!(
durable.epoch, bind_report.epoch,
"local supervisor authority must stay at the pre-rotation epoch"
);
let pending = durable
.pending_rotation
.as_ref()
.expect("pending rotation metadata must be retained");
assert!(
pending
.accepted_peer_ids
.contains(&stub.peer_id.to_string()),
"the bound host must be recorded in the pending membership (write-ahead)"
);
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = handle
.query_machine_state()
.await
.expect("query machine state");
assert_eq!(
state.host_authority_epochs.get(&host_id).copied(),
Some(bind_report.epoch),
"a rejected rebind must not advance the controlling-side host epoch"
);
stub.reject_rebinds.store(false, Ordering::Relaxed);
let report = handle
.rotate_supervisor()
.await
.expect("retry must converge by re-probing the recorded host");
assert_eq!(report.previous_epoch, bind_report.epoch);
assert_eq!(report.current_epoch, bind_report.epoch + 1);
let state = handle
.query_machine_state()
.await
.expect("query machine state after retry");
assert_eq!(
state.host_authority_epochs.get(&host_id).copied(),
Some(report.current_epoch)
);
let records = runtime_metadata
.list_mob_host_authorities(&mob_id)
.await
.expect("load host authority records");
assert_eq!(records[0].authority_epoch, report.current_epoch);
}
/// FLAG-3(a) recovery row: a controlling restart rebuilds the machine's host
/// binding facts from the durable `MobHostAuthorityRecord` family via the
/// `RecoverHostBinding` signal (the `RecoverSupervisorAuthority` sibling) —
/// without it, §9's controlling-restart contract is violated.
#[tokio::test]
async fn test_resume_recovers_host_bindings_from_durable_records() {
let _serial = lock_real_comms_tests();
let definition = with_unique_mob_id(
sample_definition_with_external_backend(),
"resume-host-recovery",
);
let mob_id = definition.id.clone();
let events: Arc<dyn crate::store::MobEventStore> = Arc::new(InMemoryMobEventStore::new());
let runs = Arc::new(InMemoryMobRunStore::new());
let specs = Arc::new(InMemoryMobSpecStore::new());
let runtime_metadata = Arc::new(InMemoryMobRuntimeMetadataStore::new());
let identity = Arc::new(InMemoryMobIdentityStore::new());
let identity_status = Arc::new(InMemoryMobIdentityStatusStore::new());
let storage = MobStorage::custom_with_runtime_metadata(
events.clone(),
runs.clone(),
specs.clone(),
runtime_metadata.clone(),
identity.clone(),
identity_status.clone(),
);
let service = Arc::new(MockSessionService::new());
let _ = service.enable_runtime_adapter();
let handle = MobBuilder::new(definition, storage)
.with_session_service(service.clone())
.create()
.await
.expect("create mob");
let stub = spawn_host_daemon_stub(
"resume-recovery",
Some("wss://host-stub/resume-live".to_string()),
)
.await;
let report = handle
.bind_host(stub.bind_request())
.await
.expect("bind host");
handle.shutdown().await.expect("shutdown mob");
let resumed_storage = MobStorage::custom_with_runtime_metadata(
events,
runs,
specs,
runtime_metadata.clone(),
identity,
identity_status,
);
let resumed = MobBuilder::for_resume(resumed_storage)
.with_session_service(service)
.resume()
.await
.expect("resume mob");
let host_id = host_dsl::HostId::from(stub.peer_id.to_string());
let state = resumed
.query_machine_state()
.await
.expect("query machine state after resume");
assert!(
state.mob_hosts.contains(&host_id),
"resume must recover the bound host from the durable record"
);
assert_eq!(
state.host_bind_phase.get(&host_id).copied(),
Some(host_dsl::HostBindPhase::Bound)
);
assert_eq!(
state.host_authority_epochs.get(&host_id).copied(),
Some(report.epoch)
);
assert_eq!(
state.host_public_keys.get(&host_id).copied(),
Some(host_dsl::PeerSigningKey::from(stub.pubkey))
);
assert_eq!(
state.host_durable_sessions.get(&host_id).copied(),
Some(true)
);
assert_eq!(
state.host_live_endpoints.get(&host_id).cloned(),
Some(host_dsl::LiveWsEndpointUrl::from(
"wss://host-stub/resume-live".to_string()
))
);
}
// ===========================================================================
// Phase 5 — T-LS4: authority-lane bindings (§15.2 lane separation)
// ===========================================================================
/// The builder-launched handle is the OWNER console (explicit launch-site
/// mint, A16), while the member-tool dispatcher rebinds onto the agent
/// authority lane — a forgotten rebind running member tools as Owner is the
/// invisible-in-v1 failure this pins.
#[tokio::test]
async fn dispatch_handles_bind_their_authority_lanes() {
let (handle, _service) = create_test_mob(sample_definition()).await;
assert!(
matches!(
handle.command_authority_kind(),
crate::control_policy::CommandAuthorityKind::Principal
),
"MobBuilder::launch binds the owner principal console lane"
);
let dispatcher = super::tools::MobOperatorToolDispatcher::new(
handle.clone(),
false,
generated_mob_operator_authority_with_scope(handle.definition().id.as_str()),
);
assert!(
matches!(
dispatcher.command_authority_kind(),
crate::control_policy::CommandAuthorityKind::AgentLane
),
"member operator tools run on the agent authority lane (T-LS4)"
);
let fenced_dispatcher = super::tools::MobOperatorToolDispatcher::new(
handle.clone().with_command_authority(
crate::control_policy::CommandAuthority::remote_member_operator(
crate::control_policy::MemberOperatorExecutionFence {
agent_identity: "remote-member".to_string(),
host_id: "host-a".to_string(),
host_binding_generation: 1,
requester_member_session_id: "member-session-a".to_string(),
generation: 0,
fence_token: 1,
},
),
),
false,
generated_mob_operator_authority_with_scope(handle.definition().id.as_str()),
);
assert!(
fenced_dispatcher.has_member_operator_execution_fence(),
"dispatcher construction must preserve the exact upcall execution fence"
);
handle.shutdown().await.expect("shutdown test mob");
}
// Runtime-backed tracked-turn LLM identity and terminal-event regressions.
#[tokio::test]
async fn test_batched_steer_turns_fan_out_complete_commit_gated_event_streams() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-batched-steer-observers");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn turn-driven lead")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_emit_runtime_event_before_completion(true);
service.set_runtime_event_delta_count(2);
service.set_block_runtime_turns(true);
// Pause after the first admission wakes the loop but before it acquires
// queue authority. The second real public Steer admission then shares the
// same generated same-boundary batch deterministically.
let (queue_gap_entered, queue_gap_release) = service
.runtime_adapter
.arm_runtime_loop_before_queue_authority_test_hook(session_id.clone());
let member = handle.member(&identity).await.expect("member handle");
let (event_a_tx, mut event_a_rx) = tokio::sync::mpsc::channel(1);
let mut turn_a = member
.start_turn(
ContentInput::Text("first batched steer".into()),
HandlingMode::Steer,
crate::MemberTurnOptions::default(),
Some(event_a_tx),
)
.await
.expect("first Steer admission");
tokio::time::timeout(Duration::from_secs(2), queue_gap_entered)
.await
.expect("runtime loop should pause before selecting the Steer batch")
.expect("queue-authority hook should remain armed");
let (event_b_tx, mut event_b_rx) = tokio::sync::mpsc::channel(1);
let mut turn_b = member
.start_turn(
ContentInput::Text("second batched steer".into()),
HandlingMode::Steer,
crate::MemberTurnOptions::default(),
Some(event_b_tx),
)
.await
.expect("second Steer admission");
assert_eq!(
turn_a
.wait_for_applied_llm_identity()
.await
.expect("first Steer identity acknowledgement"),
None
);
assert_eq!(
turn_b
.wait_for_applied_llm_identity()
.await
.expect("second Steer identity acknowledgement"),
None
);
let runtime_started = service.runtime_turn_started.notified();
queue_gap_release
.send(())
.expect("release runtime-loop queue-authority gap");
tokio::time::timeout(Duration::from_secs(2), runtime_started)
.await
.expect("batched Steer primitive should reach runtime apply");
let contributors = service
.applied_runtime_contributing_input_ids(&session_id)
.await;
assert_eq!(contributors.len(), 1, "both Steers must form one primitive");
assert_eq!(
contributors[0].len(),
2,
"the primitive must retain both contributing input identities"
);
// Capacity-one external receivers force delayed/backpressured delivery
// while the service attempts two deltas and both terminal frames. The
// fanout must preserve the complete ordered live sequence for each; the
// direct router regression separately proves the exact Full boundary.
for expected_seq in 1..=2 {
let event_a = tokio::time::timeout(Duration::from_secs(2), event_a_rx.recv())
.await
.expect("first observer should receive live delta")
.expect("first observer remains connected");
let event_b = tokio::time::timeout(Duration::from_secs(2), event_b_rx.recv())
.await
.expect("second observer should receive live delta")
.expect("second observer remains connected");
assert_eq!(event_a.seq, expected_seq);
assert_eq!(event_b.seq, expected_seq);
assert_eq!(event_a.event_id, event_b.event_id);
assert_eq!(event_a.source, event_b.source);
assert_eq!(event_a.source.session_id(), Some(&session_id));
match (&event_a.payload, &event_b.payload) {
(
AgentEvent::TextDelta { delta: delta_a },
AgentEvent::TextDelta { delta: delta_b },
) => {
assert_eq!(delta_a, delta_b);
}
events => panic!("both observers should receive matching live deltas: {events:?}"),
}
}
assert!(
tokio::time::timeout(Duration::from_millis(50), event_a_rx.recv())
.await
.is_err(),
"first observer terminal must remain commit-gated"
);
assert!(
tokio::time::timeout(Duration::from_millis(50), event_b_rx.recv())
.await
.is_err(),
"second observer terminal must remain commit-gated"
);
service.set_block_runtime_turns(false);
service.release_runtime_turns();
let (completion_a, completion_b) = tokio::join!(
tokio::time::timeout(Duration::from_secs(2), turn_a.wait()),
tokio::time::timeout(Duration::from_secs(2), turn_b.wait()),
);
completion_a
.expect("first Steer completion should resolve")
.expect("first Steer should succeed");
completion_b
.expect("second Steer completion should resolve")
.expect("second Steer should succeed");
for expected_seq in 3..=4 {
let event_a = tokio::time::timeout(Duration::from_secs(2), event_a_rx.recv())
.await
.expect("first observer should receive committed terminal")
.expect("first observer remains connected");
let event_b = tokio::time::timeout(Duration::from_secs(2), event_b_rx.recv())
.await
.expect("second observer should receive committed terminal")
.expect("second observer remains connected");
assert_eq!(event_a.seq, expected_seq);
assert_eq!(event_b.seq, expected_seq);
assert_eq!(event_a.event_id, event_b.event_id);
assert_eq!(event_a.source, event_b.source);
assert_eq!(event_a.source.session_id(), Some(&session_id));
match (expected_seq, &event_a.payload, &event_b.payload) {
(
3,
AgentEvent::RunCompleted {
session_id: session_a,
..
},
AgentEvent::RunCompleted {
session_id: session_b,
..
},
) => {
assert_eq!(session_a, &session_id);
assert_eq!(session_b, &session_id);
}
(4, AgentEvent::InteractionComplete { .. }, AgentEvent::InteractionComplete { .. }) => {
}
events => {
panic!("both observers should receive matching ordered terminals: {events:?}")
}
}
}
}
#[tokio::test]
async fn test_runtime_backed_member_turn_hot_swap_preserves_binding_and_commit_truth() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-hot-swap");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let before = handle
.member_status(&identity)
.await
.expect("member status before turn");
let before_runtime_id = before
.agent_runtime_id
.clone()
.expect("active member runtime id");
let before_fence = before.fence_token.expect("active member fence");
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host.clone());
service.set_block_runtime_turns(true);
service.set_emit_runtime_event_before_completion(true);
let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(8);
let requested_options = crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"hot-swap-model",
)),
provider: Some(Provider::OpenAI),
provider_params: Some(meerkat_core::lifecycle::run_primitive::TurnMetadataOverride::Clear),
..Default::default()
};
let member = handle.member(&identity).await.expect("member handle");
let (turn, runtime_started) = tokio::join!(
member.start_turn(
ContentInput::Text("use the hot-swap model".into()),
HandlingMode::Queue,
requested_options.clone(),
Some(event_tx),
),
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
),
);
let turn = turn.expect("runtime-backed hot-swap turn should be admitted");
runtime_started.expect("runtime-backed turn should reach the core apply boundary");
assert_eq!(turn.receipt().agent_runtime_id, before_runtime_id);
assert_eq!(turn.receipt().fence_token, before_fence);
assert_eq!(
handle.resolve_bridge_session_id(&identity).await.as_ref(),
Some(&session_id),
"per-turn model selection must not replace the member session"
);
let applied = service.applied_runtime_turn_metadata(&session_id).await;
let applied = applied
.last()
.and_then(Option::as_ref)
.expect("runtime turn metadata should reach the canonical core boundary");
assert_eq!(applied.model, requested_options.model);
assert_eq!(applied.provider, requested_options.provider);
assert_eq!(
applied.provider_params, requested_options.provider_params,
"the preserve/set/clear provider-params tri-state must remain typed"
);
let live_event = tokio::time::timeout(Duration::from_secs(2), event_rx.recv())
.await
.expect("nonterminal event should stream before runtime completion")
.expect("event sender should remain connected");
assert!(matches!(live_event.payload, AgentEvent::TextDelta { .. }));
assert!(
tokio::time::timeout(Duration::from_millis(50), event_rx.recv())
.await
.is_err(),
"terminal events must remain commit-gated"
);
let live_identity = llm_host.current_identity();
assert_eq!(live_identity.model, "hot-swap-model");
assert_eq!(live_identity.provider, Provider::OpenAI);
assert_eq!(live_identity.provider_params, None);
assert_eq!(llm_host.live_apply_count.load(Ordering::SeqCst), 1);
let mut completion = Box::pin(turn.wait());
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut completion)
.await
.is_err(),
"completion handle must remain pending until the runtime commits"
);
service.set_block_runtime_turns(false);
service.release_runtime_turns();
let delivery = tokio::time::timeout(Duration::from_secs(2), completion)
.await
.expect("completion handle should resolve after runtime completion")
.expect("committed runtime turn should succeed");
assert_eq!(delivery.agent_runtime_id, before_runtime_id);
assert_eq!(delivery.fence_token, before_fence);
let terminal_event = tokio::time::timeout(Duration::from_secs(2), event_rx.recv())
.await
.expect("terminal event should unblock after machine completion")
.expect("event sender should remain connected");
assert!(matches!(
terminal_event.payload,
AgentEvent::RunCompleted { .. }
));
let after = handle
.member_status(&identity)
.await
.expect("member status after turn");
assert_eq!(after.agent_runtime_id.as_ref(), Some(&before_runtime_id));
assert_eq!(after.fence_token, Some(before_fence));
assert_eq!(after.current_bridge_session_id.as_ref(), Some(&session_id));
}
#[tokio::test]
async fn test_member_turn_llm_reconfigure_does_not_reacquire_held_finalization_boundary() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-non-reentrant-llm-boundary");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn turn-driven lead");
// The runtime loop acquires this service-owned gate before CoreExecutor::apply.
// The LLM host deliberately exposes the exact same non-reentrant gate so a
// call through the public control-plane reconfigure seam would self-deadlock.
let gate = service.install_non_reentrant_turn_finalization_gate();
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::with_turn_finalization_gate(gate));
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host.clone());
let mut turn = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("switch under the already-held boundary".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"boundary-safe-model",
)),
..Default::default()
},
None,
)
.await
.expect("turn should be admitted");
let applied =
tokio::time::timeout(Duration::from_secs(2), turn.wait_for_applied_llm_identity())
.await
.expect("LLM reconfiguration must not deadlock on the held finalization boundary")
.expect("LLM reconfiguration should succeed")
.expect("identity override should produce an applied identity");
assert_eq!(applied.model, "boundary-safe-model");
assert_eq!(
llm_host.boundary_acquire_count.load(Ordering::SeqCst),
0,
"the runtime executor must use the under-boundary reconfigure seam"
);
tokio::time::timeout(Duration::from_secs(2), turn.wait())
.await
.expect("turn completion must not remain blocked on a nested guard")
.expect("turn should complete successfully");
}
#[tokio::test]
async fn test_member_turn_reconfigure_failure_resolves_after_admission_without_applying_prompt() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-reconfigure-failure");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let prompts_before = service.applied_runtime_prompts(&session_id).await;
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
llm_host.set_fail_live_apply(true);
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host.clone());
let member = handle.member(&identity).await.expect("member handle");
let mut turn = member
.start_turn(
ContentInput::Text("admit but do not apply this turn".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"unavailable-model",
)),
..Default::default()
},
None,
)
.await
.expect("MobMachine ingress must remain admitted before executor realization");
let identity_error = turn
.wait_for_applied_llm_identity()
.await
.expect_err("failed reconfiguration must fail the applied-identity acknowledgement");
assert!(
identity_error
.to_string()
.contains("injected live LLM apply failure")
);
let error = turn
.wait()
.await
.expect_err("failed live reconfiguration must fail the completion boundary");
assert!(
error
.to_string()
.contains("injected live LLM apply failure"),
"unexpected reconfiguration failure: {error}"
);
assert_eq!(llm_host.live_apply_count.load(Ordering::SeqCst), 0);
assert_eq!(llm_host.current_identity().model, "baseline-model");
assert_eq!(
service.applied_runtime_prompts(&session_id).await,
prompts_before,
"runtime prompt must not be applied after live reconfiguration fails"
);
}
#[tokio::test]
async fn test_direct_session_member_rejects_llm_identity_override_before_mob_admission() {
let (handle, _service) = create_test_mob_with_real_comms(sample_definition()).await;
let identity = AgentIdentity::from("lead-direct-no-reconfigure");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn direct-session turn-driven member");
let error = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("must not run on the old model".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"unsupported-direct-model",
)),
..Default::default()
},
None,
)
.await
.expect_err("direct session service must not silently ignore model override");
assert!(matches!(
error,
MobError::MissingMemberCapability {
ref member_id,
capability: crate::error::MobMemberCapability::SessionLlmReconfigure,
context: "member turn LLM identity override",
} if member_id == &identity
));
let server_only_error = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("must not silently ignore an exact local route".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
self_hosted_server_id: Some("local-b".to_string()),
..Default::default()
},
None,
)
.await
.expect_err("server-only identity override requires the runtime executor boundary");
assert!(matches!(
server_only_error,
MobError::MissingMemberCapability {
ref member_id,
capability: crate::error::MobMemberCapability::SessionLlmReconfigure,
context: "member turn LLM identity override",
} if member_id == &identity
));
}
#[tokio::test]
async fn test_default_ephemeral_session_service_rejects_llm_override_without_host() {
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
PersistentMockBuilder,
16,
));
assert!(
crate::runtime::MobSessionService::supports_runtime_turn_apply(service.as_ref()),
"the production ephemeral service can apply generic runtime turns"
);
let runtime_adapter = crate::runtime::MobSessionService::runtime_adapter(service.as_ref())
.expect("the production ephemeral service exposes a runtime adapter");
assert!(
!runtime_adapter.has_session_llm_reconfigure_host(),
"a default ephemeral adapter must not claim an uninstalled LLM host"
);
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(ProfileBinding::as_inline_mut)
.expect("worker profile")
.external_addressable = true;
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create mob with production ephemeral service");
let identity = AgentIdentity::from("w-ephemeral-no-llm-host");
handle
.spawn_with_options(
ProfileName::from("worker"),
identity.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn production ephemeral worker");
let error = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("must reject before runtime admission".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::new().with_model(
meerkat_core::lifecycle::run_primitive::ModelId::new("unavailable-model"),
),
None,
)
.await
.expect_err("generic runtime-turn support is not LLM-reconfigure support");
assert!(matches!(
error,
MobError::MissingMemberCapability {
ref member_id,
capability: crate::error::MobMemberCapability::SessionLlmReconfigure,
context: "member turn LLM identity override",
} if member_id == &identity
));
handle.shutdown().await.expect("shutdown test mob");
}
#[tokio::test]
async fn test_ephemeral_session_service_accepts_llm_override_with_installed_host() {
let provider_visible_tools = Arc::new(std::sync::Mutex::new(Vec::new()));
let provider_turn_overlays = Arc::new(std::sync::Mutex::new(Vec::new()));
let service = Arc::new(meerkat_session::EphemeralSessionService::new(
OverlayProbeSessionAgentBuilder {
provider_visible_tools,
provider_turn_overlays,
},
16,
));
let runtime_adapter = crate::runtime::MobSessionService::runtime_adapter(service.as_ref())
.expect("the production ephemeral service exposes a runtime adapter");
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
runtime_adapter.set_session_llm_reconfigure_host(llm_host.clone());
assert!(runtime_adapter.has_session_llm_reconfigure_host());
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("worker"))
.and_then(ProfileBinding::as_inline_mut)
.expect("worker profile")
.external_addressable = true;
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service)
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create mob with configured production ephemeral service");
let identity = AgentIdentity::from("w-ephemeral-with-llm-host");
handle
.spawn_with_options(
ProfileName::from("worker"),
identity.clone(),
None,
Some(crate::MobRuntimeMode::TurnDriven),
None,
)
.await
.expect("spawn production ephemeral worker");
let mut turn = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("apply through the installed host".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::new().with_model(
meerkat_core::lifecycle::run_primitive::ModelId::new("configured-host-model"),
),
None,
)
.await
.expect("installed host should make the member capability realizable");
let applied = turn
.wait_for_applied_llm_identity()
.await
.expect("installed host should apply the requested identity")
.expect("model override should yield an applied identity");
assert_eq!(applied.model, "configured-host-model");
turn.wait()
.await
.expect("turn should complete through the production ephemeral service");
assert_eq!(llm_host.live_apply_count.load(Ordering::SeqCst), 1);
assert_eq!(llm_host.current_identity().model, "configured-host-model");
handle.shutdown().await.expect("shutdown test mob");
}
#[tokio::test]
async fn test_queued_member_turns_apply_their_own_llm_identity_at_execution() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-queued-models");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
service.observe_runtime_llm_identity_from(&llm_host);
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host.clone());
service.set_block_runtime_turns(true);
let member = handle.member(&identity).await.expect("member handle");
let (turn_a, turn_b, runtime_a_started) = tokio::join!(
member.start_turn(
ContentInput::Text("run A".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"shared-local-model",
)),
provider: Some(Provider::SelfHosted),
self_hosted_server_id: Some("local-a".to_string()),
..Default::default()
},
None,
),
member.start_turn(
ContentInput::Text("run B".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"shared-local-model",
)),
provider: Some(Provider::SelfHosted),
self_hosted_server_id: Some("local-b".to_string()),
..Default::default()
},
None,
),
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified()
),
);
let mut turn_a = turn_a.expect("turn A admission");
let mut turn_b = turn_b.expect("turn B simultaneous admission");
runtime_a_started.expect("turn A should enter actual session apply");
let applied_a = turn_a
.wait_for_applied_llm_identity()
.await
.expect("turn A identity application")
.expect("turn A requested an identity");
assert_eq!(applied_a.model, "shared-local-model");
assert_eq!(applied_a.provider, Provider::SelfHosted);
assert_eq!(applied_a.self_hosted_server_id.as_deref(), Some("local-a"));
let mut applied_b = Box::pin(turn_b.wait_for_applied_llm_identity());
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut applied_b)
.await
.is_err(),
"turn B identity must not apply while turn A owns the executor"
);
let observed_while_a_blocked = service.observed_runtime_llm_identities(&session_id).await;
assert_eq!(observed_while_a_blocked.len(), 1);
assert_eq!(observed_while_a_blocked[0].model, "shared-local-model");
assert_eq!(observed_while_a_blocked[0].provider, Provider::SelfHosted);
assert_eq!(
observed_while_a_blocked[0].self_hosted_server_id.as_deref(),
Some("local-a")
);
assert_eq!(
llm_host.current_identity().self_hosted_server_id.as_deref(),
Some("local-a"),
"queued turn B must not mutate the shared session while A executes"
);
service.release_one_runtime_turn();
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if service
.observed_runtime_llm_identities(&session_id)
.await
.len()
== 2
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("turn B should enter actual session apply after A commits");
let observed = service.observed_runtime_llm_identities(&session_id).await;
assert_eq!(observed[1].model, "shared-local-model");
assert_eq!(observed[1].provider, Provider::SelfHosted);
assert_eq!(
observed[1].self_hosted_server_id.as_deref(),
Some("local-b")
);
let applied_b = tokio::time::timeout(Duration::from_secs(2), applied_b)
.await
.expect("turn B identity acknowledgement")
.expect("turn B identity application")
.expect("turn B requested an identity");
assert_eq!(applied_b.model, "shared-local-model");
assert_eq!(applied_b.provider, Provider::SelfHosted);
assert_eq!(applied_b.self_hosted_server_id.as_deref(), Some("local-b"));
service.release_one_runtime_turn();
turn_a.wait().await.expect("turn A completion");
turn_b.wait().await.expect("turn B completion");
}
#[derive(Clone)]
struct HeadCanonicalQueueGateClient {
state: Arc<HeadCanonicalQueueGateState>,
}
struct HeadCanonicalQueueGateState {
request_count: AtomicUsize,
release: tokio::sync::watch::Sender<bool>,
}
impl HeadCanonicalQueueGateClient {
fn new() -> Self {
let (release, _release_rx) = tokio::sync::watch::channel(false);
Self {
state: Arc::new(HeadCanonicalQueueGateState {
request_count: AtomicUsize::new(0),
release,
}),
}
}
fn request_count(&self) -> usize {
self.state.request_count.load(Ordering::SeqCst)
}
fn release(&self) {
self.state.release.send_replace(true);
}
async fn wait_for_request_count(&self, expected: usize, context: &'static str) {
tokio::time::timeout(Duration::from_secs(10), async {
while self.request_count() < expected {
tokio::task::yield_now().await;
}
})
.await
.unwrap_or_else(|_| {
panic!(
"{context}: observed {} of {expected} LLM requests",
self.request_count()
)
});
}
}
#[async_trait]
impl meerkat_client::LlmClient for HeadCanonicalQueueGateClient {
fn project_replay_messages(
&self,
messages: &[Message],
) -> Result<Vec<Message>, meerkat_client::LlmError> {
Ok(messages.to_vec())
}
fn stream<'a>(
&'a self,
_request: &'a meerkat_client::LlmRequest,
) -> meerkat_client::types::LlmStream<'a> {
self.state.request_count.fetch_add(1, Ordering::SeqCst);
let mut release = self.state.release.subscribe();
Box::pin(async_stream::try_stream! {
loop {
if *release.borrow_and_update() {
break;
}
if release.changed().await.is_err() {
break;
}
}
yield meerkat_client::LlmEvent::TextDelta {
delta: "ok".to_string(),
meta: None,
};
yield meerkat_client::LlmEvent::Done {
outcome: meerkat_client::LlmDoneOutcome::Success {
stop_reason: meerkat_core::StopReason::EndTurn,
},
};
})
}
fn provider(&self) -> Provider {
Provider::Other
}
async fn health_check(&self) -> Result<(), meerkat_client::LlmError> {
Ok(())
}
}
#[cfg(feature = "runtime-adapter")]
#[tokio::test(flavor = "multi_thread")]
async fn test_head_canonical_same_member_queue_admissions_serialize_committed_boundaries() {
const PIPELINED_TURNS: usize = 5;
let temp = tempfile::TempDir::new().expect("head-canonical queue tempdir");
let user_config_root = temp.path().join("user-config");
let runtime_root = temp.path().join("runtime");
let project_root = temp.path().join("project");
let context_root = temp.path().join("context");
let realm_db_path = temp.path().join("realm.db");
for path in [
&user_config_root,
&runtime_root,
&project_root,
&context_root,
] {
std::fs::create_dir_all(path).expect("create head-canonical queue test root");
}
let factory = meerkat::AgentFactory::new(runtime_root.join("factory-store"))
.user_config_root(user_config_root)
.runtime_root(runtime_root)
.project_root(project_root)
.context_root(context_root)
.builtins(false)
.comms(true);
let mut config = meerkat::Config::default();
config.compaction.auto_compact_threshold = 1_000_000;
let mut builder = meerkat::FactoryAgentBuilder::new(factory, config);
let session_store = Arc::new(
meerkat_store::SqliteSessionStore::open(&realm_db_path)
.expect("open co-tenant HeadCanonical SessionStore"),
);
builder.default_session_store = Some(Arc::new(meerkat_store::StoreAdapter::new(
session_store.clone(),
)));
let session_store_dyn: Arc<dyn meerkat::SessionStore> = session_store;
let runtime_store: Arc<dyn meerkat_runtime::RuntimeStore> = Arc::new(
meerkat_runtime::SqliteRuntimeStore::new_head_canonical(&realm_db_path)
.expect("open co-tenant HeadCanonical RuntimeStore"),
);
let blob_store: Arc<dyn meerkat_core::BlobStore> =
Arc::new(meerkat_store::MemoryBlobStore::new());
let service = Arc::new(meerkat_session::PersistentSessionService::new(
builder,
16,
session_store_dyn,
runtime_store,
blob_store,
));
let mut definition = with_unique_mob_id(sample_definition(), "head-canonical-queue");
let lead = definition
.profiles
.get_mut(&ProfileName::from("lead"))
.and_then(ProfileBinding::as_inline_mut)
.expect("inline lead profile");
lead.model = "gpt-5.5".to_string();
lead.runtime_mode = crate::MobRuntimeMode::TurnDriven;
lead.tools = ToolConfig {
comms: true,
..ToolConfig::default()
};
let client = HeadCanonicalQueueGateClient::new();
let handle = MobBuilder::new(definition, MobStorage::in_memory())
.with_session_service(service.clone())
.with_default_llm_client(Arc::new(client.clone()))
.create()
.await
.expect("create HeadCanonical queue mob");
let identity = AgentIdentity::from("head-canonical-queued-lead");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn HeadCanonical queue member")
.bridge_session_id()
.expect("turn-driven member has a bridge session")
.clone();
let member = handle
.member(&identity)
.await
.expect("queued member handle");
let first = tokio::time::timeout(
Duration::from_secs(30),
member.start_turn(
ContentInput::Text("queued-boundary-0".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
None,
),
)
.await
.expect("first queued turn admission timed out")
.expect("admit first queued turn");
client
.wait_for_request_count(1, "first queued turn must own the executor")
.await;
let mut turns = vec![first];
for index in 1..PIPELINED_TURNS {
turns.push(
tokio::time::timeout(
Duration::from_secs(30),
member.start_turn(
ContentInput::Text(format!("queued-boundary-{index}")),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
None,
),
)
.await
.unwrap_or_else(|_| panic!("queued turn {index} admission timed out"))
.unwrap_or_else(|error| panic!("admit queued turn {index}: {error}")),
);
}
tokio::task::yield_now().await;
assert_eq!(
client.request_count(),
1,
"back-to-back Queue admission must not start a second same-session turn \
while the first owns the executor"
);
client.release();
for (index, turn) in turns.into_iter().enumerate() {
tokio::time::timeout(Duration::from_secs(30), turn.wait())
.await
.unwrap_or_else(|_| panic!("queued turn {index} completion timed out"))
.unwrap_or_else(|error| {
panic!("queued turn {index} failed its committed boundary: {error}")
});
}
client
.wait_for_request_count(
PIPELINED_TURNS,
"every pipelined Queue admission must execute",
)
.await;
let follow_up = tokio::time::timeout(
Duration::from_secs(30),
member.start_turn(
ContentInput::Text("queued-boundary-follow-up".to_string()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
None,
),
)
.await
.expect("post-pipeline health turn admission timed out")
.expect("admit post-pipeline health turn");
tokio::time::timeout(Duration::from_secs(30), follow_up.wait())
.await
.expect("post-pipeline health turn completion timed out")
.expect("post-pipeline health turn commits");
client
.wait_for_request_count(
PIPELINED_TURNS + 1,
"post-pipeline HeadCanonical store remains writable",
)
.await;
let durable = tokio::time::timeout(
Duration::from_secs(30),
service.load_persisted_session(&session_id),
)
.await
.expect("durable HeadCanonical session load timed out")
.expect("load durable HeadCanonical session after queued turns")
.expect("queued member session remains durable");
let user_text = durable
.messages()
.iter()
.filter_map(|message| match message {
Message::User(user) => Some(user.text_content()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n");
for index in 0..PIPELINED_TURNS {
assert!(
user_text.contains(&format!("queued-boundary-{index}")),
"durable transcript lost queued turn {index}: {user_text}"
);
}
assert!(
user_text.contains("queued-boundary-follow-up"),
"durable transcript lost the post-pipeline health turn: {user_text}"
);
let connection =
rusqlite::Connection::open(&realm_db_path).expect("inspect co-tenant HeadCanonical DB");
let (runtime_token, runtime_revision): (String, i64) = connection
.query_row(
"SELECT committed_head_token, store_revision \
FROM runtime_session_authority WHERE session_id = ?1",
[session_id.to_string()],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.expect("read runtime HeadCanonical authority");
let (session_token, session_revision, message_count): (String, i64, i64) = connection
.query_row(
"SELECT cas_token, version, message_count \
FROM session_heads WHERE session_id = ?1",
[session_id.to_string()],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
)
.expect("read canonical session head");
assert_eq!(
runtime_token, session_token,
"runtime and SessionStore must converge on one exact HeadCanonical token"
);
assert!(
runtime_revision >= i64::try_from(PIPELINED_TURNS + 1).expect("turn count fits i64"),
"the HeadCanonical store revision must advance across every queued boundary: \
observed {runtime_revision}"
);
assert_eq!(
u32::try_from(session_revision).expect("canonical session version fits u32"),
durable.version(),
"materialized durable session must retain the canonical head schema version"
);
assert_eq!(
usize::try_from(message_count).expect("non-negative canonical message count"),
durable.messages().len(),
"materialized durable transcript must match the canonical head"
);
let whole_blob_rows: i64 = connection
.query_row(
"SELECT COUNT(*) FROM runtime_session_snapshots WHERE runtime_id = ?1",
[meerkat_runtime::LogicalRuntimeId::for_session(&session_id).to_string()],
|row| row.get(0),
)
.expect("count WholeBlob runtime snapshots");
assert_eq!(
whole_blob_rows, 0,
"HeadCanonical queue processing must not fall back to WholeBlob authority"
);
let provisional_tail_rows: i64 = connection
.query_row(
"SELECT COUNT(*) FROM runtime_head_canonical_provisional_tails \
WHERE session_id = ?1",
[session_id.to_string()],
|row| row.get(0),
)
.expect("count provisional HeadCanonical tails");
assert_eq!(
provisional_tail_rows, 0,
"all queued turn candidates must be finalized after committed completion"
);
tokio::time::timeout(Duration::from_secs(30), handle.shutdown())
.await
.expect("HeadCanonical queue mob shutdown timed out")
.expect("shutdown HeadCanonical queue mob");
}
#[tokio::test]
async fn test_finalization_failure_suppresses_buffered_success_terminal() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) =
create_test_mob_with_persistent_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-finalization-failure");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead");
service.set_emit_runtime_event_before_completion(true);
service.set_fail_runtime_boundary_acknowledgement(true);
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host);
let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(8);
let mut turn = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("fail finalization".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"finalization-model",
)),
..Default::default()
},
Some(event_tx),
)
.await
.expect("turn admission");
let applied = turn
.wait_for_applied_llm_identity()
.await
.expect("identity applies before finalization")
.expect("identity override");
assert_eq!(applied.model, "finalization-model");
let error = turn
.wait()
.await
.expect_err("checkpoint failure must fail tracked completion");
assert!(
error
.to_string()
.contains("runtime session checkpoint failed after commit"),
"unexpected finalization error: {error}"
);
let mut saw_delta = false;
while let Ok(Some(event)) =
tokio::time::timeout(Duration::from_millis(100), event_rx.recv()).await
{
saw_delta |= matches!(&event.payload, AgentEvent::TextDelta { .. });
assert!(
!matches!(
&event.payload,
AgentEvent::RunCompleted { .. }
| AgentEvent::ExtractionSucceeded { .. }
| AgentEvent::InteractionComplete { .. }
| AgentEvent::InteractionCallbackPending { .. }
),
"a success terminal must not escape when completion reports finalization failure"
);
}
assert!(saw_delta, "nonterminal output should still stream live");
}
#[tokio::test]
async fn test_extraction_failure_releases_only_extraction_failure_terminal() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-extraction-failure");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead");
service.set_emit_runtime_event_before_completion(true);
service.set_return_extraction_failure(true);
let llm_host = Arc::new(RecordingSessionLlmReconfigureHost::new());
service
.runtime_adapter
.set_session_llm_reconfigure_host(llm_host);
let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(8);
let mut turn = handle
.member(&identity)
.await
.expect("member handle")
.start_turn(
ContentInput::Text("fail structured extraction".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"extraction-model",
)),
..Default::default()
},
Some(event_tx),
)
.await
.expect("turn admission");
let applied = turn
.wait_for_applied_llm_identity()
.await
.expect("identity applies before extraction terminal")
.expect("identity override");
assert_eq!(applied.model, "extraction-model");
turn.wait()
.await
.expect("main turn completion remains successful after extraction failure");
let mut saw_extraction_failure = false;
while let Ok(Some(event)) =
tokio::time::timeout(Duration::from_millis(100), event_rx.recv()).await
{
saw_extraction_failure |= matches!(&event.payload, AgentEvent::ExtractionFailed { .. });
assert!(
!matches!(
&event.payload,
AgentEvent::RunCompleted { .. }
| AgentEvent::ExtractionSucceeded { .. }
| AgentEvent::InteractionComplete { .. }
| AgentEvent::InteractionCallbackPending { .. }
),
"extraction failure must not be projected as a success terminal"
);
}
assert!(
saw_extraction_failure,
"the sole canonical ExtractionFailed terminal must be released"
);
}
#[tokio::test]
async fn test_member_turn_rejects_conflicting_metadata_handling_mode_before_admission() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-handling-authority");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let prompts_before = service.applied_runtime_prompts(&session_id).await;
let error = handle
.external_turn_for_member(
identity,
ContentInput::Text("conflicting handling mode".into()),
HandlingMode::Queue,
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
handling_mode: Some(HandlingMode::Steer),
..Default::default()
},
),
None,
super::handle::MemberTurnObservers::default(),
)
.await
.expect_err("flat handling mode must remain the only admission authority");
assert!(matches!(
error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
..
}
));
assert_eq!(
service.applied_runtime_prompts(&session_id).await,
prompts_before,
"handling-mode conflicts must fail before runtime admission"
);
}
#[tokio::test]
async fn test_member_turn_handle_reports_post_admission_runtime_failure() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-tracked-failure");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn lead");
service
.runtime_adapter
.set_session_llm_reconfigure_host(Arc::new(RecordingSessionLlmReconfigureHost::new()));
service.set_block_runtime_turns(true);
service.set_fail_runtime_turns(true);
let member = handle.member(&identity).await.expect("member handle");
let (turn, runtime_started) = tokio::join!(
member.start_turn(
ContentInput::Text("fail after admission".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"applied-before-runtime-failure",
)),
..Default::default()
},
None,
),
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
),
);
let mut turn = turn.expect("runtime input should be admitted before execution failure");
runtime_started.expect("runtime turn should reach apply boundary");
assert!(
turn.session_id().is_some(),
"tracked admission should expose its exact bridge session"
);
let applied =
tokio::time::timeout(Duration::from_secs(2), turn.wait_for_applied_llm_identity())
.await
.expect("applied identity acknowledgement should resolve")
.expect("reconfigure should succeed before runtime apply failure")
.expect("turn requested an identity override");
assert_eq!(applied.model, "applied-before-runtime-failure");
service.release_one_runtime_turn();
let error = tokio::time::timeout(Duration::from_secs(2), turn.wait())
.await
.expect("completion should resolve")
.expect_err("post-admission runtime failure must reach the completion handle");
assert!(
error.to_string().contains("injected runtime turn failure"),
"unexpected completion error: {error}"
);
}
#[tokio::test]
async fn test_autonomous_member_rejects_unrepresentable_turn_overrides_before_admission() {
let (handle, service) =
create_test_mob_with_runtime_backed_real_comms(sample_definition()).await;
let identity = AgentIdentity::from("lead-autonomous-overrides");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn autonomous lead")
.bridge_session_id()
.expect("session-backed")
.clone();
let member = handle.member(&identity).await.expect("member handle");
let metadata_error = member
.start_turn(
ContentInput::Text("unsupported model override".into()),
HandlingMode::Queue,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"hot-swap-model",
)),
..Default::default()
},
None,
)
.await
.expect_err("autonomous inbox must not silently drop a model override");
assert!(matches!(
metadata_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::AutonomousHost,
..
}
));
let (event_tx, _event_rx) = tokio::sync::mpsc::channel(1);
let event_error = member
.start_turn(
ContentInput::Text("unsupported tracked turn".into()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
Some(event_tx),
)
.await
.expect_err("autonomous inbox must not silently drop a turn event sender");
assert!(matches!(
event_error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::AutonomousHost,
..
}
));
let applied_prompts = service.applied_runtime_prompts(&session_id).await;
for rejected in ["unsupported model override", "unsupported tracked turn"] {
assert!(
!applied_prompts
.iter()
.any(|prompt| matches!(prompt, ContentInput::Text(text) if text == rejected)),
"unsupported carrier prompt '{rejected}' must be rejected before MobMachine ingress admission"
);
}
}
#[tokio::test]
async fn test_running_steer_rejects_llm_identity_override_before_admission() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-running-steer-override");
let session_id = handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn turn-driven lead")
.bridge_session_id()
.expect("session-backed")
.clone();
service.set_block_runtime_turns(true);
let member = handle.member(&identity).await.expect("member handle");
let (active_turn, runtime_started) = tokio::join!(
member.start_turn(
ContentInput::Text("hold the executor boundary".into()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
None,
),
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
),
);
let active_turn = active_turn.expect("active turn should be admitted");
runtime_started.expect("active turn should reach the executor boundary");
let prompts_before = service.applied_runtime_prompts(&session_id).await;
let error = member
.start_turn(
ContentInput::Text("must not steer onto a different model".into()),
HandlingMode::Steer,
crate::MemberTurnOptions {
model: Some(meerkat_core::lifecycle::run_primitive::ModelId::new(
"forbidden-steer-model",
)),
..Default::default()
},
None,
)
.await
.expect_err("running Steer has no serialized LLM reconfiguration boundary");
assert!(matches!(
error,
MobError::UnsupportedForMode {
mode: crate::MobRuntimeMode::TurnDriven,
ref reason,
} if reason.contains("queued executor turn")
));
assert_eq!(
service.applied_runtime_prompts(&session_id).await,
prompts_before,
"rejected Steer override must not reach runtime execution"
);
service.release_one_runtime_turn();
tokio::time::timeout(Duration::from_secs(2), active_turn.wait())
.await
.expect("active turn should finish after release")
.expect("active turn should succeed");
}
#[tokio::test]
async fn test_running_steer_without_override_resolves_applied_identity_as_none() {
let mut definition = sample_definition();
definition
.profiles
.get_mut(&ProfileName::from("lead"))
.expect("lead profile")
.as_inline_mut()
.unwrap()
.runtime_mode = crate::MobRuntimeMode::TurnDriven;
let (handle, service) = create_test_mob_with_runtime_backed_real_comms(definition).await;
let identity = AgentIdentity::from("lead-running-steer-no-override");
handle
.spawn(ProfileName::from("lead"), identity.clone(), None)
.await
.expect("spawn turn-driven lead");
service.set_block_runtime_turns(true);
let member = handle.member(&identity).await.expect("member handle");
let (active_turn, runtime_started) = tokio::join!(
member.start_turn(
ContentInput::Text("hold the executor boundary".into()),
HandlingMode::Queue,
crate::MemberTurnOptions::default(),
None,
),
tokio::time::timeout(
Duration::from_secs(2),
service.runtime_turn_started.notified(),
),
);
let active_turn = active_turn.expect("active turn should be admitted");
runtime_started.expect("active turn should reach the executor boundary");
let mut steer = member
.start_turn(
ContentInput::Text("steer without changing identity".into()),
HandlingMode::Steer,
crate::MemberTurnOptions::default(),
None,
)
.await
.expect("plain running Steer should be admitted");
let applied_identity = tokio::time::timeout(
Duration::from_secs(2),
steer.wait_for_applied_llm_identity(),
)
.await
.expect("Steer applied-identity acknowledgement should resolve")
.expect("plain Steer acknowledgement should succeed");
assert_eq!(
applied_identity, None,
"a Steer without an identity override must resolve explicitly as no reconfiguration"
);
// The identity fact resolves at admission, while the completion-bearing
// handle remains correctly tied to the active run's committed terminal.
service.set_block_runtime_turns(false);
service.release_runtime_turns();
let (steer_completion, active_completion) = tokio::join!(
tokio::time::timeout(Duration::from_secs(2), steer.wait()),
tokio::time::timeout(Duration::from_secs(2), active_turn.wait()),
);
steer_completion
.expect("plain Steer completion should resolve with the active run")
.expect("plain Steer completion should succeed");
active_completion
.expect("active turn should finish after release")
.expect("active turn should succeed");
}